From a5f13f793db5560bf90f902745471ce3a9e47630 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 22 May 2018 02:21:30 -0700 Subject: [PATCH] batch_util.h is generally useful so moved to util/ from kernels/ where it will be included in the pip package. PiperOrigin-RevId: 197532524 --- tensorflow/contrib/makefile/tf_op_files.txt | 1 - tensorflow/core/BUILD | 1 + tensorflow/core/kernels/BUILD | 9 +--- tensorflow/core/kernels/batch_util.h | 39 ++------------ tensorflow/core/kernels/data/BUILD | 6 --- .../core/kernels/data/batch_dataset_op.cc | 2 +- .../kernels/data/padded_batch_dataset_op.cc | 2 +- .../core/kernels/data/slide_dataset_op.cc | 2 +- .../kernels/data/tensor_queue_dataset_op.cc | 2 +- .../kernels/data/tensor_slice_dataset_op.cc | 2 +- .../core/kernels/data/unbatch_dataset_op.cc | 2 +- tensorflow/core/kernels/fifo_queue.cc | 2 +- tensorflow/core/kernels/padding_fifo_queue.cc | 2 +- tensorflow/core/kernels/priority_queue.cc | 2 +- tensorflow/core/kernels/queue_base.cc | 2 +- .../core/kernels/random_shuffle_queue_op.cc | 2 +- .../core/{kernels => util}/batch_util.cc | 2 +- tensorflow/core/util/batch_util.h | 54 +++++++++++++++++++ 18 files changed, 73 insertions(+), 61 deletions(-) rename tensorflow/core/{kernels => util}/batch_util.cc (99%) create mode 100644 tensorflow/core/util/batch_util.h diff --git a/tensorflow/contrib/makefile/tf_op_files.txt b/tensorflow/contrib/makefile/tf_op_files.txt index d4c3f2eda8b..89db9ee2794 100644 --- a/tensorflow/contrib/makefile/tf_op_files.txt +++ b/tensorflow/contrib/makefile/tf_op_files.txt @@ -300,7 +300,6 @@ tensorflow/core/kernels/spacetobatch_op.cc tensorflow/core/kernels/batchtospace_op.cc tensorflow/core/kernels/warn_about_ints.cc tensorflow/core/kernels/segment_reduction_ops.cc -tensorflow/core/kernels/batch_util.cc tensorflow/core/ops/audio_ops.cc tensorflow/core/kernels/decode_proto_op.cc tensorflow/core/kernels/encode_proto_op.cc diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index 4146e264895..5d63cd68ae2 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -586,6 +586,7 @@ tf_cuda_library( "framework/types.h", "public/version.h", "util/activation_mode.h", + "util/batch_util.h", "util/bcast.h", "util/cuda_kernel_helper.h", "util/device_name_utils.h", diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index 7f00f5e6324..e1911361ce1 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -360,7 +360,6 @@ cc_library( srcs = ["queue_base.cc"], hdrs = ["queue_base.h"], deps = [ - ":batch_util", "//tensorflow/core:framework", "//tensorflow/core:lib", "//tensorflow/core:protos_all_cc", @@ -382,7 +381,6 @@ cc_library( srcs = ["priority_queue.cc"], hdrs = ["priority_queue.h"], deps = [ - ":batch_util", ":queue_base", ":typed_queue", "//tensorflow/core:framework", @@ -1697,7 +1695,6 @@ tf_kernel_library( name = "random_shuffle_queue_op", prefix = "random_shuffle_queue_op", deps = DATA_FLOW_DEPS + [ - ":batch_util", "//tensorflow/core:protos_all_cc", ], ) @@ -1890,7 +1887,6 @@ cc_library( hdrs = ["fifo_queue.h"], visibility = ["//visibility:private"], deps = [ - ":batch_util", ":queue_base", ":typed_queue", "//tensorflow/core:framework", @@ -1905,7 +1901,6 @@ cc_library( hdrs = ["padding_fifo_queue.h"], visibility = ["//visibility:private"], deps = [ - ":batch_util", ":fifo_queue", ":queue_base", ":typed_queue", @@ -5004,7 +4999,6 @@ filegroup( filegroup( name = "android_extended_ops_group2", srcs = [ - "batch_util.cc", "batchtospace_op.cc", "ctc_decoder_ops.cc", "decode_bmp_op.cc", @@ -6131,9 +6125,10 @@ tf_mkl_kernel_library( ], ) +# NOTE(lespeholt): This rule is deprecated, please use: +# tensorflow/core/util/batch_util.h cc_library( name = "batch_util", - srcs = ["batch_util.cc"], hdrs = ["batch_util.h"], deps = [ "//tensorflow/core:framework", diff --git a/tensorflow/core/kernels/batch_util.h b/tensorflow/core/kernels/batch_util.h index 69098fbd1d8..dad2ec4e103 100644 --- a/tensorflow/core/kernels/batch_util.h +++ b/tensorflow/core/kernels/batch_util.h @@ -12,43 +12,12 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ +// NOTE(lespeholt): This file is deprecated. Use +// "tensorflow/core/util/batch_util.h" instead. + #ifndef TENSORFLOW_CORE_KERNELS_BATCH_UTIL_H_ #define TENSORFLOW_CORE_KERNELS_BATCH_UTIL_H_ -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/lib/core/status.h" - -namespace tensorflow { -namespace batch_util { - -// Copies element into the index^th slice of parent (in the 0th dimension). -// -// NOTE(mrry): The `element` argument is taken by value. Use `std::move()` -// to move the `element` argument into this function, and the implementation -// may be able to optimize the copy to a move. This is particularly important -// for DT_STRING tensors. -Status CopyElementToSlice(Tensor element, Tensor* parent, int64 index); - -// Copies the index^th slice of parent (in the 0th dimension) into element. -Status CopySliceToElement(const Tensor& parent, Tensor* element, int64 index); - -// Copies the index^th slice of parent (in the 0th dimension) into element. -// -// NOTE(mrry): The implementation may be able to optimize the copy to a move. -// This is particularly important for DT_STRING tensors. -Status MaybeMoveSliceToElement(Tensor* parent, Tensor* element, int64 index); - -// Zero-initializes the tensor `element` using the scalar stored in `padding`. -// Both `element` and `padding` must have matching `dtype`. -Status SetElementZero(Tensor* element, const Tensor& padding); - -// Copies `element` into a (0th dimension) slice of `parent`, assuming -// the shape of `element` is strictly not larger along any axis than a -// slice. -Status CopyElementToLargerSlice(const Tensor& element, Tensor* parent, - int index); - -} // namespace batch_util -} // namespace tensorflow +#include "tensorflow/core/util/batch_util.h" #endif // TENSORFLOW_CORE_KERNELS_BATCH_UTIL_H_ diff --git a/tensorflow/core/kernels/data/BUILD b/tensorflow/core/kernels/data/BUILD index 9ded2667eb0..d35aad980de 100644 --- a/tensorflow/core/kernels/data/BUILD +++ b/tensorflow/core/kernels/data/BUILD @@ -81,7 +81,6 @@ tf_kernel_library( "//tensorflow/core:framework", "//tensorflow/core:lib", "//tensorflow/core:lib_internal", - "//tensorflow/core/kernels:batch_util", ], ) @@ -94,7 +93,6 @@ tf_kernel_library( "//tensorflow/core:framework", "//tensorflow/core:lib", "//tensorflow/core:lib_internal", - "//tensorflow/core/kernels:batch_util", ], ) @@ -107,7 +105,6 @@ tf_kernel_library( "//tensorflow/core:framework", "//tensorflow/core:lib", "//tensorflow/core:lib_internal", - "//tensorflow/core/kernels:batch_util", ], ) @@ -443,7 +440,6 @@ tf_kernel_library( "//tensorflow/core:framework", "//tensorflow/core:lib", "//tensorflow/core:lib_internal", - "//tensorflow/core/kernels:batch_util", ], ) @@ -456,7 +452,6 @@ tf_kernel_library( "//tensorflow/core:framework", "//tensorflow/core:lib", "//tensorflow/core:lib_internal", - "//tensorflow/core/kernels:batch_util", ], ) @@ -469,7 +464,6 @@ tf_kernel_library( "//tensorflow/core:framework", "//tensorflow/core:lib", "//tensorflow/core:lib_internal", - "//tensorflow/core/kernels:batch_util", ], ) diff --git a/tensorflow/core/kernels/data/batch_dataset_op.cc b/tensorflow/core/kernels/data/batch_dataset_op.cc index 7fa67efb9e2..3618c75827f 100644 --- a/tensorflow/core/kernels/data/batch_dataset_op.cc +++ b/tensorflow/core/kernels/data/batch_dataset_op.cc @@ -14,8 +14,8 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/core/framework/partial_tensor_shape.h" #include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/kernels/batch_util.h" #include "tensorflow/core/kernels/data/dataset.h" +#include "tensorflow/core/util/batch_util.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/data/padded_batch_dataset_op.cc b/tensorflow/core/kernels/data/padded_batch_dataset_op.cc index cfb4efda9a5..e41800a8069 100644 --- a/tensorflow/core/kernels/data/padded_batch_dataset_op.cc +++ b/tensorflow/core/kernels/data/padded_batch_dataset_op.cc @@ -15,8 +15,8 @@ limitations under the License. #include "tensorflow/core/framework/partial_tensor_shape.h" #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_util.h" -#include "tensorflow/core/kernels/batch_util.h" #include "tensorflow/core/kernels/data/dataset.h" +#include "tensorflow/core/util/batch_util.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/data/slide_dataset_op.cc b/tensorflow/core/kernels/data/slide_dataset_op.cc index 4f3537b6912..78c8363f91a 100644 --- a/tensorflow/core/kernels/data/slide_dataset_op.cc +++ b/tensorflow/core/kernels/data/slide_dataset_op.cc @@ -14,8 +14,8 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/core/framework/partial_tensor_shape.h" #include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/kernels/batch_util.h" #include "tensorflow/core/kernels/data/dataset.h" +#include "tensorflow/core/util/batch_util.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/data/tensor_queue_dataset_op.cc b/tensorflow/core/kernels/data/tensor_queue_dataset_op.cc index ff412a4671b..e271a42b2ac 100644 --- a/tensorflow/core/kernels/data/tensor_queue_dataset_op.cc +++ b/tensorflow/core/kernels/data/tensor_queue_dataset_op.cc @@ -20,8 +20,8 @@ limitations under the License. #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/variant.h" #include "tensorflow/core/framework/variant_encode_decode.h" -#include "tensorflow/core/kernels/batch_util.h" #include "tensorflow/core/kernels/data/dataset.h" +#include "tensorflow/core/util/batch_util.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/data/tensor_slice_dataset_op.cc b/tensorflow/core/kernels/data/tensor_slice_dataset_op.cc index d5be4c77807..95708cc01ce 100644 --- a/tensorflow/core/kernels/data/tensor_slice_dataset_op.cc +++ b/tensorflow/core/kernels/data/tensor_slice_dataset_op.cc @@ -14,8 +14,8 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/core/framework/partial_tensor_shape.h" #include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/kernels/batch_util.h" #include "tensorflow/core/kernels/data/dataset.h" +#include "tensorflow/core/util/batch_util.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/data/unbatch_dataset_op.cc b/tensorflow/core/kernels/data/unbatch_dataset_op.cc index 241b615aca1..2b383e50977 100644 --- a/tensorflow/core/kernels/data/unbatch_dataset_op.cc +++ b/tensorflow/core/kernels/data/unbatch_dataset_op.cc @@ -14,8 +14,8 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/core/framework/partial_tensor_shape.h" #include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/kernels/batch_util.h" #include "tensorflow/core/kernels/data/dataset.h" +#include "tensorflow/core/util/batch_util.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/fifo_queue.cc b/tensorflow/core/kernels/fifo_queue.cc index 479f7be4b50..a23478af5b5 100644 --- a/tensorflow/core/kernels/fifo_queue.cc +++ b/tensorflow/core/kernels/fifo_queue.cc @@ -23,13 +23,13 @@ limitations under the License. #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/types.h" -#include "tensorflow/core/kernels/batch_util.h" #include "tensorflow/core/kernels/fifo_queue.h" #include "tensorflow/core/kernels/queue_base.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/mutex.h" #include "tensorflow/core/platform/types.h" +#include "tensorflow/core/util/batch_util.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/padding_fifo_queue.cc b/tensorflow/core/kernels/padding_fifo_queue.cc index 9d35ecb66c0..ff553f11c9f 100644 --- a/tensorflow/core/kernels/padding_fifo_queue.cc +++ b/tensorflow/core/kernels/padding_fifo_queue.cc @@ -23,13 +23,13 @@ limitations under the License. #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/types.h" -#include "tensorflow/core/kernels/batch_util.h" #include "tensorflow/core/kernels/padding_fifo_queue.h" #include "tensorflow/core/kernels/queue_base.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/mutex.h" #include "tensorflow/core/platform/types.h" +#include "tensorflow/core/util/batch_util.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/priority_queue.cc b/tensorflow/core/kernels/priority_queue.cc index bab94f7f0ad..342e66a7c7d 100644 --- a/tensorflow/core/kernels/priority_queue.cc +++ b/tensorflow/core/kernels/priority_queue.cc @@ -22,7 +22,6 @@ limitations under the License. #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/types.h" -#include "tensorflow/core/kernels/batch_util.h" #include "tensorflow/core/kernels/priority_queue.h" #include "tensorflow/core/kernels/queue_base.h" #include "tensorflow/core/lib/core/errors.h" @@ -30,6 +29,7 @@ limitations under the License. #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/mutex.h" #include "tensorflow/core/platform/types.h" +#include "tensorflow/core/util/batch_util.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/queue_base.cc b/tensorflow/core/kernels/queue_base.cc index de495c19cba..acfbc02ee7a 100644 --- a/tensorflow/core/kernels/queue_base.cc +++ b/tensorflow/core/kernels/queue_base.cc @@ -18,10 +18,10 @@ limitations under the License. #include #include "tensorflow/core/framework/node_def.pb.h" #include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/kernels/batch_util.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/platform/mutex.h" #include "tensorflow/core/platform/types.h" +#include "tensorflow/core/util/batch_util.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/random_shuffle_queue_op.cc b/tensorflow/core/kernels/random_shuffle_queue_op.cc index 87fc9433316..31e8ce944fe 100644 --- a/tensorflow/core/kernels/random_shuffle_queue_op.cc +++ b/tensorflow/core/kernels/random_shuffle_queue_op.cc @@ -24,7 +24,6 @@ limitations under the License. #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/types.h" -#include "tensorflow/core/kernels/batch_util.h" #include "tensorflow/core/kernels/queue_op.h" #include "tensorflow/core/kernels/typed_queue.h" #include "tensorflow/core/lib/core/errors.h" @@ -36,6 +35,7 @@ limitations under the License. #include "tensorflow/core/platform/mutex.h" #include "tensorflow/core/platform/thread_annotations.h" #include "tensorflow/core/platform/types.h" +#include "tensorflow/core/util/batch_util.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/batch_util.cc b/tensorflow/core/util/batch_util.cc similarity index 99% rename from tensorflow/core/kernels/batch_util.cc rename to tensorflow/core/util/batch_util.cc index 1182ed42e7a..7ea8851e651 100644 --- a/tensorflow/core/kernels/batch_util.cc +++ b/tensorflow/core/util/batch_util.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/core/kernels/batch_util.h" +#include "tensorflow/core/util/batch_util.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/types.h" diff --git a/tensorflow/core/util/batch_util.h b/tensorflow/core/util/batch_util.h new file mode 100644 index 00000000000..eee0309fbc4 --- /dev/null +++ b/tensorflow/core/util/batch_util.h @@ -0,0 +1,54 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CORE_UTIL_BATCH_UTIL_H_ +#define TENSORFLOW_CORE_UTIL_BATCH_UTIL_H_ + +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/lib/core/status.h" + +namespace tensorflow { +namespace batch_util { + +// Copies element into the index^th slice of parent (in the 0th dimension). +// +// NOTE(mrry): The `element` argument is taken by value. Use `std::move()` +// to move the `element` argument into this function, and the implementation +// may be able to optimize the copy to a move. This is particularly important +// for DT_STRING tensors. +Status CopyElementToSlice(Tensor element, Tensor* parent, int64 index); + +// Copies the index^th slice of parent (in the 0th dimension) into element. +Status CopySliceToElement(const Tensor& parent, Tensor* element, int64 index); + +// Copies the index^th slice of parent (in the 0th dimension) into element. +// +// NOTE(mrry): The implementation may be able to optimize the copy to a move. +// This is particularly important for DT_STRING tensors. +Status MaybeMoveSliceToElement(Tensor* parent, Tensor* element, int64 index); + +// Zero-initializes the tensor `element` using the scalar stored in `padding`. +// Both `element` and `padding` must have matching `dtype`. +Status SetElementZero(Tensor* element, const Tensor& padding); + +// Copies `element` into a (0th dimension) slice of `parent`, assuming +// the shape of `element` is strictly not larger along any axis than a +// slice. +Status CopyElementToLargerSlice(const Tensor& element, Tensor* parent, + int index); + +} // namespace batch_util +} // namespace tensorflow + +#endif // TENSORFLOW_CORE_UTIL_BATCH_UTIL_H_