experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Extend tf.io.parse_sequence_example and tf.io.parse_single_sequence_example to handle ragged tensor features.

Browse Source 
tf.io.parse_single_sequence_example also now uses the fast-parsing code path that was written for parse_sequence_example, which should improve efficiency for large inputs.

PiperOrigin-RevId: 272639850
This commit is contained in:
Edward Loper 2019-10-03 05:24:37 -07:00 committed by TensorFlower Gardener
parent 402026fa78
commit d969c96e5c
15 changed files with 2298 additions and 746 deletions

147
tensorflow/core/api_def/base_api/api_def_ParseSequenceExampleV2.pbtxt Normal file
View File

@ -0,0 +1,147 @@
op {
graph_op_name: "ParseSequenceExampleV2"
in_arg {
name: "serialized"
description: <<END
A scalar or vector containing binary serialized SequenceExample protos.
END
}
in_arg {
name: "debug_name"
description: <<END
A scalar or vector containing the names of the serialized protos.
May contain, for example, table key (descriptive) name for the
corresponding serialized proto. This is purely useful for debugging
purposes, and the presence of values here has no effect on the output.
May also be an empty vector if no name is available.
END
}
in_arg {
name: "context_sparse_keys"
description: <<END
The keys expected in the Examples' features associated with context_sparse
values.
END
}
in_arg {
name: "context_dense_keys"
description: <<END
The keys expected in the SequenceExamples' context features associated with
dense values.
END
}
in_arg {
name: "context_ragged_keys"
description: <<END
The keys expected in the Examples' features associated with context_ragged
values.
END
}
in_arg {
name: "feature_list_sparse_keys"
description: <<END
The keys expected in the FeatureLists associated with sparse values.
END
}
in_arg {
name: "feature_list_dense_keys"
description: <<END
The keys expected in the SequenceExamples' feature_lists associated
with lists of dense values.
END
}
in_arg {
name: "feature_list_ragged_keys"
description: <<END
The keys expected in the FeatureLists associated with ragged values.
END
}
in_arg {
name: "feature_list_dense_missing_assumed_empty"
description: <<END
A vector corresponding 1:1 with featue_list_dense_keys, indicating which
features may be missing from the SequenceExamples. If the associated
FeatureList is missing, it is treated as empty.
END
}
in_arg {
name: "context_dense_defaults"
description: <<END
A list of Ncontext_dense Tensors (some may be empty).
context_dense_defaults[j] provides default values
when the SequenceExample's context map lacks context_dense_key[j].
If an empty Tensor is provided for context_dense_defaults[j],
then the Feature context_dense_keys[j] is required.
The input type is inferred from context_dense_defaults[j], even when it's
empty. If context_dense_defaults[j] is not empty, its shape must match
context_dense_shapes[j].
END
}
attr {
name: "context_sparse_types"
description: <<END
A list of Ncontext_sparse types; the data types of data in
each context Feature given in context_sparse_keys.
Currently the ParseSingleSequenceExample supports DT_FLOAT (FloatList),
DT_INT64 (Int64List), and DT_STRING (BytesList).
END
}
attr {
name: "context_dense_shapes"
description: <<END
A list of Ncontext_dense shapes; the shapes of data in
each context Feature given in context_dense_keys.
The number of elements in the Feature corresponding to context_dense_key[j]
must always equal context_dense_shapes[j].NumEntries().
The shape of context_dense_values[j] will match context_dense_shapes[j].
END
}
attr {
name: "feature_list_sparse_types"
description: <<END
A list of Nfeature_list_sparse types; the data types
of data in each FeatureList given in feature_list_sparse_keys.
Currently the ParseSingleSequenceExample supports DT_FLOAT (FloatList),
DT_INT64 (Int64List), and DT_STRING (BytesList).
END
}
attr {
name: "feature_list_dense_shapes"
description: <<END
A list of Nfeature_list_dense shapes; the shapes of
data in each FeatureList given in feature_list_dense_keys.
The shape of each Feature in the FeatureList corresponding to
feature_list_dense_key[j] must always equal
feature_list_dense_shapes[j].NumEntries().
END
}
attr {
name: "context_ragged_value_types"
description: <<END
RaggedTensor.value dtypes for the ragged context features.
END
}
attr {
name: "context_ragged_split_types"
description: <<END
RaggedTensor.row_split dtypes for the ragged context features.
END
}
attr {
name: "feature_list_ragged_value_types"
description: <<END
RaggedTensor.value dtypes for the ragged FeatureList features.
END
}
attr {
name: "feature_list_ragged_split_types"
description: <<END
RaggedTensor.row_split dtypes for the ragged FeatureList features.
END
}
summary: <<END
Transforms a vector of tf.io.SequenceExample protos (as strings) into
typed tensors.
END
}

4
tensorflow/core/api_def/python_api/api_def_ParseSequenceExampleV2.pbtxt Normal file
View File

@ -0,0 +1,4 @@
op {
graph_op_name: "ParseSequenceExampleV2"
visibility: HIDDEN
}

361
tensorflow/core/kernels/example_parsing_ops.cc
View File

@ -38,6 +38,7 @@ namespace tensorflow {
namespace {
constexpr char kParseExampleV2[] = "ParseExampleV2";
constexpr char kParseSequenceExampleV2[] = "ParseSequenceExampleV2";
} // namespace
// Note: this kernel is used by both the ParseExample op and the ParseExampleV2
@ -388,8 +389,10 @@ REGISTER_KERNEL_BUILDER(Name("ParseSingleExample").Device(DEVICE_CPU),
class ParseSequenceExampleOp : public OpKernel {
public:
explicit ParseSequenceExampleOp(OpKernelConstruction* ctx) : OpKernel(ctx) {
OP_REQUIRES_OK(ctx, attrs_.Init(ctx));
explicit ParseSequenceExampleOp(OpKernelConstruction* ctx)
: OpKernel(ctx),
op_version_(ctx->def().op() == kParseSequenceExampleV2 ? 2 : 1) {
OP_REQUIRES_OK(ctx, attrs_.Init(ctx, op_version_));
metrics::RecordParseDenseFeature(attrs_.context_dense_keys.size() +
attrs_.feature_list_dense_keys.size());
metrics::RecordParseSparseFeature(attrs_.context_sparse_keys.size() +
@ -405,92 +408,222 @@ class ParseSequenceExampleOp : public OpKernel {
OP_REQUIRES_OK(ctx, ctx->input("serialized", &serialized));
OP_REQUIRES_OK(ctx, ctx->input_list("context_dense_defaults",
&context_dense_defaults));
bool has_debug_name = (debug_name->NumElements() > 0);
if (has_debug_name) {
OP_REQUIRES(ctx, TensorShapeUtils::IsVector(debug_name->shape()),
errors::InvalidArgument(
"Expected debug_name to be a vector, got shape: ",
debug_name->shape().DebugString()));
const Tensor* context_dense_keys = nullptr;
const Tensor* context_sparse_keys = nullptr;
const Tensor* context_ragged_keys = nullptr;
const Tensor* feature_list_dense_keys = nullptr;
const Tensor* feature_list_sparse_keys = nullptr;
const Tensor* feature_list_ragged_keys = nullptr;
const Tensor* feature_list_dense_missing_assumed_empty = nullptr;
if (op_version_ == 2) {
OP_REQUIRES_OK(ctx,
ctx->input("feature_list_dense_missing_assumed_empty",
&feature_list_dense_missing_assumed_empty));
OP_REQUIRES_OK(ctx,
ctx->input("context_dense_keys", &context_dense_keys));
OP_REQUIRES_OK(ctx,
ctx->input("context_sparse_keys", &context_sparse_keys));
OP_REQUIRES_OK(ctx,
ctx->input("context_ragged_keys", &context_ragged_keys));
OP_REQUIRES_OK(
ctx, ctx->input("feature_list_dense_keys", &feature_list_dense_keys));
OP_REQUIRES_OK(ctx, ctx->input("feature_list_sparse_keys",
&feature_list_sparse_keys));
OP_REQUIRES_OK(ctx, ctx->input("feature_list_ragged_keys",
&feature_list_ragged_keys));
std::call_once(flag_, [&]() {
metrics::RecordParseDenseFeature(
context_dense_keys->NumElements() +
feature_list_dense_keys->NumElements());
metrics::RecordParseSparseFeature(
context_sparse_keys->NumElements() +
feature_list_sparse_keys->NumElements());
metrics::RecordParseRaggedFeature(
context_ragged_keys->NumElements() +
feature_list_ragged_keys->NumElements());
});
}
OP_REQUIRES(ctx, TensorShapeUtils::IsVector(serialized->shape()),
errors::InvalidArgument(
"Expected serialized to be a vector, got shape: ",
serialized->shape().DebugString()));
// Validate input tensor shapes.
OP_REQUIRES_OK(ctx, CheckInputShapes(
serialized, debug_name, context_dense_defaults,
context_dense_keys, context_sparse_keys,
context_ragged_keys, feature_list_dense_keys,
feature_list_sparse_keys, feature_list_ragged_keys,
feature_list_dense_missing_assumed_empty));
OP_REQUIRES(ctx, context_dense_defaults.size() == attrs_.num_context_dense,
errors::InvalidArgument("Expected len(context_dense_defaults) "
"== len(context_dense_keys) but got: ",
context_dense_defaults.size(), " vs. ",
attrs_.num_context_dense));
std::vector<bool> required(attrs_.num_context_dense);
for (int d = 0; d < attrs_.num_context_dense; ++d) {
const Tensor& def_value = context_dense_defaults[d];
required[d] = (def_value.NumElements() == 0); // No default provided.
if (def_value.NumElements() > 0) {
OP_REQUIRES(ctx, def_value.shape() == attrs_.context_dense_shapes[d],
errors::InvalidArgument(
"default_value[", d,
"].shape() == ", def_value.shape().DebugString(),
" != context_dense_shapes[", d,
"] == ", attrs_.context_dense_shapes[d].DebugString()));
OP_REQUIRES(
ctx, def_value.dtype() == attrs_.context_dense_types[d],
errors::InvalidArgument(
"context_dense_defaults[", d, "].dtype() == ",
DataTypeString(def_value.dtype()), " != context_dense_types[",
d, "] == ", DataTypeString(attrs_.context_dense_types[d])));
}
}
example::Result context_result, feature_list_result;
std::vector<Tensor> dense_feature_lengths;
example::FastParseExampleConfig context_config;
for (int d = 0; d < attrs_.num_context_dense; ++d) {
context_config.dense.push_back(
{attrs_.context_dense_keys[d], attrs_.context_dense_types[d],
attrs_.context_dense_shapes[d], context_dense_defaults[d],
false /* attrs_.context_variable_length[d] */,
0 /*attrs_.context_elements_per_stride[d] */});
}
for (int d = 0; d < attrs_.num_context_sparse; ++d) {
context_config.sparse.push_back(
{attrs_.context_sparse_keys[d], attrs_.context_sparse_types[d]});
}
example::FastParseExampleConfig feature_list_config;
for (int d = 0; d < attrs_.num_feature_list_dense; ++d) {
DataType dtype = attrs_.feature_list_dense_types[d];
Tensor default_value = Tensor(dtype, TensorShape({}));
feature_list_config.dense.push_back(
{attrs_.feature_list_dense_keys[d], dtype,
attrs_.feature_list_dense_shapes[d], default_value,
(attrs_.feature_list_dense_missing_assumed_empty.count(
attrs_.feature_list_dense_keys[d]) > 0),
0 /*attrs_.context_elements_per_stride[d] */});
}
for (int d = 0; d < attrs_.num_feature_list_sparse; ++d) {
feature_list_config.sparse.push_back(
{attrs_.feature_list_sparse_keys[d],
attrs_.feature_list_sparse_types[d]});
}
example::FastParseExampleConfig context_config =
MakeContextConfig(context_dense_keys, context_sparse_keys,
context_ragged_keys, context_dense_defaults);
example::FastParseExampleConfig feature_list_config = MakeFeatureListConfig(
feature_list_dense_keys, feature_list_sparse_keys,
feature_list_ragged_keys, feature_list_dense_missing_assumed_empty);
bool is_batch = TensorShapeUtils::IsVector(serialized->shape());
auto serialized_t = serialized->flat<tstring>();
auto debug_name_t = debug_name->flat<tstring>();
gtl::ArraySlice<tstring> slice(serialized_t.data(), serialized_t.size());
gtl::ArraySlice<tstring> names_slice(debug_name_t.data(),
debug_name_t.size());
example::Result context_result, feature_list_result;
std::vector<Tensor> dense_feature_lengths;
OP_REQUIRES_OK(
ctx,
FastParseSequenceExample(
context_config, feature_list_config, slice, names_slice,
ctx->device()->tensorflow_cpu_worker_threads()->workers,
&context_result, &feature_list_result, &dense_feature_lengths));
ctx, FastParseSequenceExample(
context_config, feature_list_config, slice, names_slice,
ctx->device()->tensorflow_cpu_worker_threads()->workers,
&context_result, &feature_list_result, &dense_feature_lengths,
is_batch));
OP_REQUIRES_OK(ctx, WriteOutput(context_result, feature_list_result,
dense_feature_lengths, ctx));
}
protected:
Status CheckInputShapes(
const Tensor* serialized, const Tensor* names,
const OpInputList& context_dense_defaults,
const Tensor* context_dense_keys, const Tensor* context_sparse_keys,
const Tensor* context_ragged_keys, const Tensor* feature_list_dense_keys,
const Tensor* feature_list_sparse_keys,
const Tensor* feature_list_ragged_keys,
const Tensor* feature_list_dense_missing_assumed_empty) const {
if (TensorShapeUtils::IsMatrixOrHigher(serialized->shape())) {
return errors::InvalidArgument(
"Expected serialized to be a scalar or vector, got shape: ",
serialized->shape().DebugString());
}
if (op_version_ > 1) {
if (context_dense_keys->NumElements() != attrs_.num_context_dense) {
return errors::InvalidArgument(
"Expected len(context_dense_keys) to match len(Tcontext_dense)");
}
if (context_sparse_keys->NumElements() != attrs_.num_context_sparse) {
return errors::InvalidArgument(
"Expected len(context_sparse_keys) to match Ncontext_sparse");
}
if (context_ragged_keys->NumElements() != attrs_.num_context_ragged) {
return errors::InvalidArgument(
"Expected len(context_ragged_keys) to match "
"len(context_ragged_value_types)");
}
if (feature_list_dense_keys->NumElements() !=
attrs_.num_feature_list_dense) {
return errors::InvalidArgument(
"Expected len(feature_list_dense_keys) to match "
"Nfeature_list_dense");
}
if (feature_list_dense_missing_assumed_empty->NumElements() !=
attrs_.num_feature_list_dense) {
return errors::InvalidArgument(
"Expected len(feature_list_dense_missing_assumed_empty to match "
"Nfeature_list_dense");
}
if (feature_list_sparse_keys->NumElements() !=
attrs_.num_feature_list_sparse) {
return errors::InvalidArgument(
"Expected len(feature_list_sparse_keys) to match "
"Nfeature_list_sparse");
}
if (feature_list_ragged_keys->NumElements() !=
attrs_.num_feature_list_ragged) {
return errors::InvalidArgument(
"Expected len(feature_list_ragged_keys) to match "
"len(feature_list_ragged_value_types)");
}
}
if (context_dense_defaults.size() != attrs_.num_context_dense) {
return errors::InvalidArgument(
"Expected len(context_dense_defaults) "
"== len(context_dense_keys) but got: ",
context_dense_defaults.size(), " vs. ", attrs_.num_context_dense);
}
for (int d = 0; d < attrs_.num_context_dense; ++d) {
const Tensor& def_value = context_dense_defaults[d];
if (def_value.NumElements() > 0) {
if (def_value.shape() != attrs_.context_dense_shapes[d]) {
return errors::InvalidArgument(
"default_value[", d,
"].shape() == ", def_value.shape().DebugString(),
" != context_dense_shapes[", d,
"] == ", attrs_.context_dense_shapes[d].DebugString());
}
if (def_value.dtype() != attrs_.context_dense_types[d]) {
return errors::InvalidArgument(
"context_dense_defaults[", d,
"].dtype() == ", DataTypeString(def_value.dtype()),
" != context_dense_types[", d,
"] == ", DataTypeString(attrs_.context_dense_types[d]));
}
}
}
return Status::OK();
}
example::FastParseExampleConfig MakeContextConfig(
const Tensor* dense_keys, const Tensor* sparse_keys,
const Tensor* ragged_keys,
const OpInputList& context_dense_defaults) const {
example::FastParseExampleConfig config;
for (int d = 0; d < attrs_.num_context_dense; ++d) {
const string& key = dense_keys ? dense_keys->flat<tstring>()(d)
: attrs_.context_dense_keys[d];
config.dense.push_back({key, attrs_.context_dense_types[d],
attrs_.context_dense_shapes[d],
context_dense_defaults[d],
false /* attrs_.context_variable_length[d] */,
0 /*attrs_.context_elements_per_stride[d] */});
}
for (int d = 0; d < attrs_.num_context_sparse; ++d) {
const string& key = sparse_keys ? sparse_keys->flat<tstring>()(d)
: attrs_.context_sparse_keys[d];
config.sparse.push_back({key, attrs_.context_sparse_types[d]});
}
for (int d = 0; d < attrs_.num_context_ragged; ++d) {
config.ragged.push_back({ragged_keys->flat<tstring>()(d),
attrs_.context_ragged_value_types[d],
attrs_.context_ragged_split_types[d]});
}
return config;
}
example::FastParseExampleConfig MakeFeatureListConfig(
const Tensor* dense_keys, const Tensor* sparse_keys,
const Tensor* ragged_keys,
const Tensor* feature_list_dense_missing_assumed_empty) const {
example::FastParseExampleConfig config;
for (int d = 0; d < attrs_.num_feature_list_dense; ++d) {
const string& key = dense_keys ? dense_keys->flat<tstring>()(d)
: attrs_.feature_list_dense_keys[d];
bool missing_assumed_empty =
feature_list_dense_missing_assumed_empty
? feature_list_dense_missing_assumed_empty->flat<bool>()(d)
: attrs_.feature_list_dense_missing_assumed_empty.count(key) > 0;
DataType dtype = attrs_.feature_list_dense_types[d];
Tensor default_value = Tensor(dtype, TensorShape({}));
config.dense.push_back(
{key, dtype, attrs_.feature_list_dense_shapes[d], default_value,
missing_assumed_empty,
0 /*attrs_.feature_list_elements_per_stride[d] */});
}
for (int d = 0; d < attrs_.num_feature_list_sparse; ++d) {
const string& key = sparse_keys ? sparse_keys->flat<tstring>()(d)
: attrs_.feature_list_sparse_keys[d];
config.sparse.push_back({key, attrs_.feature_list_sparse_types[d]});
}
for (int d = 0; d < attrs_.num_feature_list_ragged; ++d) {
config.ragged.push_back({ragged_keys->flat<tstring>()(d),
attrs_.feature_list_ragged_value_types[d],
attrs_.feature_list_ragged_split_types[d]});
}
return config;
}
Status WriteOutput(const example::Result& context_result,
const example::Result& feature_list_result,
const std::vector<Tensor>& dense_feature_lengths,
OpKernelContext* ctx) const {
OpOutputList context_sparse_indices;
OpOutputList context_sparse_values;
OpOutputList context_sparse_shapes;
@ -501,31 +634,29 @@ class ParseSequenceExampleOp : public OpKernel {
OpOutputList feature_list_dense_values;
OpOutputList feature_list_dense_lengths;
OP_REQUIRES_OK(ctx, ctx->output_list("context_sparse_indices",
&context_sparse_indices));
OP_REQUIRES_OK(
ctx, ctx->output_list("context_sparse_values", &context_sparse_values));
OP_REQUIRES_OK(
ctx, ctx->output_list("context_sparse_shapes", &context_sparse_shapes));
OP_REQUIRES_OK(
ctx, ctx->output_list("context_dense_values", &context_dense_values));
OP_REQUIRES_OK(ctx, ctx->output_list("context_sparse_indices",
&context_sparse_indices));
OP_REQUIRES_OK(ctx, ctx->output_list("feature_list_sparse_indices",
&feature_list_sparse_indices));
OP_REQUIRES_OK(ctx, ctx->output_list("feature_list_sparse_values",
&feature_list_sparse_values));
OP_REQUIRES_OK(ctx, ctx->output_list("feature_list_sparse_shapes",
&feature_list_sparse_shapes));
OP_REQUIRES_OK(ctx, ctx->output_list("feature_list_dense_values",
&feature_list_dense_values));
OP_REQUIRES_OK(ctx, ctx->output_list("feature_list_dense_lengths",
&feature_list_dense_lengths));
TF_RETURN_IF_ERROR(
ctx->output_list("context_sparse_indices", &context_sparse_indices));
TF_RETURN_IF_ERROR(
ctx->output_list("context_sparse_values", &context_sparse_values));
TF_RETURN_IF_ERROR(
ctx->output_list("context_sparse_shapes", &context_sparse_shapes));
TF_RETURN_IF_ERROR(
ctx->output_list("context_dense_values", &context_dense_values));
TF_RETURN_IF_ERROR(
ctx->output_list("context_sparse_indices", &context_sparse_indices));
TF_RETURN_IF_ERROR(ctx->output_list("feature_list_sparse_indices",
&feature_list_sparse_indices));
TF_RETURN_IF_ERROR(ctx->output_list("feature_list_sparse_values",
&feature_list_sparse_values));
TF_RETURN_IF_ERROR(ctx->output_list("feature_list_sparse_shapes",
&feature_list_sparse_shapes));
TF_RETURN_IF_ERROR(ctx->output_list("feature_list_dense_values",
&feature_list_dense_values));
TF_RETURN_IF_ERROR(ctx->output_list("feature_list_dense_lengths",
&feature_list_dense_lengths));
for (int d = 0; d < attrs_.num_context_dense; ++d) {
context_dense_values.set(d, context_result.dense_values[d]);
}
TensorShape lengths_shape;
lengths_shape.AddDim(serialized_t.size());
for (int d = 0; d < attrs_.num_feature_list_dense; ++d) {
feature_list_dense_values.set(d, feature_list_result.dense_values[d]);
feature_list_dense_lengths.set(d, dense_feature_lengths[d]);
@ -540,14 +671,46 @@ class ParseSequenceExampleOp : public OpKernel {
feature_list_sparse_values.set(d, feature_list_result.sparse_values[d]);
feature_list_sparse_shapes.set(d, feature_list_result.sparse_shapes[d]);
}
if (op_version_ == 2) {
OpOutputList context_ragged_values;
OpOutputList context_ragged_splits;
OpOutputList feature_list_ragged_values;
OpOutputList feature_list_ragged_inner_splits;
OpOutputList feature_list_ragged_outer_splits;
TF_RETURN_IF_ERROR(
ctx->output_list("context_ragged_values", &context_ragged_values));
TF_RETURN_IF_ERROR(ctx->output_list("context_ragged_row_splits",
&context_ragged_splits));
TF_RETURN_IF_ERROR(ctx->output_list("feature_list_ragged_values",
&feature_list_ragged_values));
TF_RETURN_IF_ERROR(ctx->output_list("feature_list_ragged_inner_splits",
&feature_list_ragged_inner_splits));
TF_RETURN_IF_ERROR(ctx->output_list("feature_list_ragged_outer_splits",
&feature_list_ragged_outer_splits));
for (int d = 0; d < attrs_.num_context_ragged; ++d) {
context_ragged_values.set(d, context_result.ragged_values[d]);
context_ragged_splits.set(d, context_result.ragged_splits[d]);
}
for (int d = 0; d < attrs_.num_feature_list_ragged; ++d) {
feature_list_ragged_values.set(d, feature_list_result.ragged_values[d]);
feature_list_ragged_outer_splits.set(
d, feature_list_result.ragged_outer_splits[d]);
feature_list_ragged_inner_splits.set(
d, feature_list_result.ragged_splits[d]);
}
}
return Status::OK();
}
protected:
ParseSequenceExampleAttrs attrs_;
int op_version_;
std::once_flag flag_;
};
REGISTER_KERNEL_BUILDER(Name("ParseSequenceExample").Device(DEVICE_CPU),
ParseSequenceExampleOp);
REGISTER_KERNEL_BUILDER(Name("ParseSequenceExampleV2").Device(DEVICE_CPU),
ParseSequenceExampleOp);
class ParseSingleSequenceExampleOp : public OpKernel {
public:

99
tensorflow/core/ops/parsing_ops.cc
View File

@ -71,16 +71,16 @@ Status AddRaggedOutputShapes(int num_ragged, bool ragged_rank_2,
for (int i = 0; i < num_ragged; ++i) {
c->set_output((*output_idx)++, c->Vector(c->UnknownDim()));
}
// Inner row_splits
// Outer row_splits.
for (int i = 0; i < num_ragged; ++i) {
c->set_output((*output_idx)++, c->Vector(num_splits));
}
// Inner row_splits (for ParseSequenceExample feature_list features)
if (ragged_rank_2) {
for (int i = 0; i < num_ragged; ++i) {
c->set_output((*output_idx)++, c->Vector(c->UnknownDim()));
}
}
// Outer row_splits.
for (int i = 0; i < num_ragged; ++i) {
c->set_output((*output_idx)++, c->Vector(num_splits));
}
return Status::OK();
}
@ -307,6 +307,95 @@ REGISTER_OP("ParseSequenceExample")
return Status::OK();
});
// Differences between ParseSequenceExample and ParseSequenceExampleV2:
// * Supports ragged features.
// * `serialized` may be a vector or a scalar. (With v1, `serialized` could
// only be a vector).
// * Each set of keys is passed with a vector instead of an attr list.
// * feature_list_dense_missing_assumed_empty is passed with as a boolean
// vector (aligned 1:1 w/ feature_list_dense_kyes) rather than an attrib
// containing a list of strings.
// * No Ncontext_dense attribute (not needed).
REGISTER_OP("ParseSequenceExampleV2")
.Input("serialized: string")
.Input("debug_name: string")
// Inputs: context features
.Input("context_sparse_keys: string")
.Input("context_dense_keys: string")
.Input("context_ragged_keys: string")
// Inputs: feature lists
.Input("feature_list_sparse_keys: string")
.Input("feature_list_dense_keys: string")
.Input("feature_list_ragged_keys: string")
.Input("feature_list_dense_missing_assumed_empty: bool")
.Input("context_dense_defaults: Tcontext_dense")
// Outputs: context features
.Output("context_sparse_indices: Ncontext_sparse * int64")
.Output("context_sparse_values: context_sparse_types")
.Output("context_sparse_shapes: Ncontext_sparse * int64")
.Output("context_dense_values: Tcontext_dense")
.Output("context_ragged_values: context_ragged_value_types")
.Output("context_ragged_row_splits: context_ragged_split_types")
// Outputs: feature lists
.Output("feature_list_sparse_indices: Nfeature_list_sparse * int64")
.Output("feature_list_sparse_values: feature_list_sparse_types")
.Output("feature_list_sparse_shapes: Nfeature_list_sparse * int64")
.Output("feature_list_dense_values: feature_list_dense_types")
.Output("feature_list_dense_lengths: Nfeature_list_dense * int64")
.Output("feature_list_ragged_values: feature_list_ragged_value_types")
.Output("feature_list_ragged_outer_splits: feature_list_ragged_split_types")
.Output("feature_list_ragged_inner_splits: feature_list_ragged_split_types")
// Attribs: context features
.Attr("Ncontext_sparse: int >= 0 = 0")
.Attr("Tcontext_dense: list({float,int64,string}) >= 0 = []") // inferred
.Attr("context_sparse_types: list({float,int64,string}) >= 0 = []")
.Attr("context_ragged_value_types: list({float,int64,string}) >= 0 = []")
.Attr("context_ragged_split_types: list({int32,int64}) >= 0 = []")
.Attr("context_dense_shapes: list(shape) >= 0 = []")
// Attribs: feature lists
.Attr("Nfeature_list_sparse: int >= 0 = 0")
.Attr("Nfeature_list_dense: int >= 0 = 0")
.Attr("feature_list_dense_types: list({float,int64,string}) >= 0 = []")
.Attr("feature_list_sparse_types: list({float,int64,string}) >= 0 = []")
.Attr(
"feature_list_ragged_value_types: list({float,int64,string}) >= 0 = []")
.Attr("feature_list_ragged_split_types: list({int32,int64}) >= 0 = []")
.Attr("feature_list_dense_shapes: list(shape) >= 0 = []")
.SetShapeFn([](InferenceContext* c) {
ParseSequenceExampleAttrs attrs;
TF_RETURN_IF_ERROR(attrs.Init(c, /*op_version=*/2));
ShapeHandle input;
TF_RETURN_IF_ERROR(c->WithRankAtMost(c->input(0), 1, &input));
ShapeHandle names;
TF_RETURN_IF_ERROR(c->WithRankAtMost(c->input(1), 1, &names));
ShapeHandle feature_list_dense_prefix;
TF_RETURN_IF_ERROR(c->Concatenate(input, c->UnknownShapeOfRank(1),
&feature_list_dense_prefix));
DimensionHandle num_examples = c->UnknownDim();
if (c->RankKnown(input) && c->Rank(input) == 1) {
num_examples = c->Dim(input, 0);
}
int output_idx = 0;
// Context outputs.
AddSparseOutputShapes(attrs.num_context_sparse, input, 1, c, &output_idx);
TF_RETURN_IF_ERROR(AddDenseOutputShapes(attrs.context_dense_shapes, input,
c, &output_idx));
TF_RETURN_IF_ERROR(AddRaggedOutputShapes(attrs.num_context_ragged, false,
num_examples, c, &output_idx));
// FeatureList outputs.
AddSparseOutputShapes(attrs.num_feature_list_sparse, input, 2, c,
&output_idx);
TF_RETURN_IF_ERROR(AddDenseOutputShapes(attrs.feature_list_dense_shapes,
feature_list_dense_prefix, c,
&output_idx));
AddDenseLengthsShapes(attrs.num_feature_list_dense, input, c,
&output_idx);
TF_RETURN_IF_ERROR(AddRaggedOutputShapes(
attrs.num_feature_list_ragged, true, num_examples, c, &output_idx));
return Status::OK();
});
REGISTER_OP("ParseSingleSequenceExample")
.Input("serialized: string")
.Input("feature_list_dense_missing_assumed_empty: string")

209
tensorflow/core/ops/parsing_ops_test.cc
View File

@ -226,7 +226,7 @@ TEST(ParsingOpsTest, ParseSequenceExample_ShapeFn) {
// Confirm an error from ParseSequenceExampleAttrs.Init().
set_outputs(1, 1, 1, 1, true /* add_extra_shape */);
INFER_ERROR(
"num_context_dense (1) must match the size of context_dense_keys (1), "
"num_context_dense (1) must match the size of "
"context_dense_types (1) and context_dense_shapes (2)",
op, "[?];[?];?");
}
@ -386,4 +386,211 @@ TEST(ParsingOpsTest, ParseExampleV2_ShapeFn) {
"?;?;?;?;?;?;?;?");
}
TEST(ParsingOpsTest, ParseSequenceExampleV2_ShapeFn) {
ShapeInferenceTestOp op("ParseSequenceExampleV2");
auto set_outputs = [&op](int num_context_sparse, int num_context_dense,
int num_context_ragged, int num_feature_list_sparse,
int num_feature_list_dense,
int num_feature_list_ragged,
bool add_extra_shape = false) {
using NodeOutList = std::vector<NodeDefBuilder::NodeOut>;
using DataTypeList = std::vector<DataType>;
string string_in("test");
NodeDefBuilder::NodeOut node_in{"a", 0, DT_STRING};
TF_ASSERT_OK(
NodeDefBuilder("test", "ParseSequenceExampleV2")
.Input("serialized", 0, DT_STRING)
.Input("debug_name", 0, DT_STRING)
.Input("context_sparse_keys", 0, DT_STRING)
.Input("context_dense_keys", 0, DT_STRING)
.Input("context_ragged_keys", 0, DT_STRING)
.Input("feature_list_sparse_keys", 0, DT_STRING)
.Input("feature_list_dense_keys", 0, DT_STRING)
.Input("feature_list_ragged_keys", 0, DT_STRING)
.Input("feature_list_dense_missing_assumed_empty", 0, DT_BOOL)
.Input(NodeOutList(num_context_dense, node_in))
.Attr("Ncontext_sparse", num_context_sparse)
.Attr("Nfeature_list_sparse", num_feature_list_sparse)
.Attr("Nfeature_list_dense", num_feature_list_dense)
.Attr("context_sparse_types",
DataTypeList(num_context_sparse, DT_FLOAT))
.Attr("context_dense_types",
DataTypeList(num_context_dense, DT_FLOAT))
.Attr("context_dense_shapes",
MakeDenseShapes(num_context_dense, add_extra_shape, 0))
.Attr("feature_list_sparse_types",
DataTypeList(num_feature_list_sparse, DT_FLOAT))
.Attr("feature_list_dense_types",
DataTypeList(num_feature_list_dense, DT_FLOAT))
.Attr("feature_list_dense_shapes",
MakeDenseShapes(num_feature_list_dense, add_extra_shape, 0))
.Attr("context_ragged_value_types",
DataTypeList(num_context_ragged, DT_FLOAT))
.Attr("context_ragged_split_types",
DataTypeList(num_context_ragged, DT_INT32))
.Attr("feature_list_ragged_value_types",
DataTypeList(num_feature_list_ragged, DT_FLOAT))
.Attr("feature_list_ragged_split_types",
DataTypeList(num_feature_list_ragged, DT_INT32))
.Finalize(&op.node_def));
};
// Verify inputs 'serialized' and 'debug_name'.
set_outputs(0, 0, 0, 0, 0, 0); // no features
INFER_OK(op, "?;[?];?;?;?;?;?;?;?", "");
INFER_OK(op, "[?];[?];?;?;?;?;?;?;?", "");
INFER_OK(op, "[8];[8];?;?;?;?;?;?;?", "");
INFER_OK(op, "[];[];?;?;?;?;?;?;?", "");
INFER_ERROR("must be at most rank 1", op, "[1,2];?;?;?;?;?;?;?;?");
INFER_ERROR("must be at most rank 1", op, "?;[2,3];?;?;?;?;?;?;?");
// context inputs with no feature_list inputs.
set_outputs(2 /* num_context_sparse */, 3 /* num_context_dense */,
4 /* num_ragged */, 0, 0, 0);
INFER_OK(op, "[?];[?];?;?;?;?;?;?;?;?;?;?", // Vector input, unknown size
("[?,2];[?,2];" // context sparse indices
"[?];[?];" // context sparse values
"[2];[2];" // context sparse dense_shapes
"[d0_0,1];[d0_0,1,2];[d0_0,1,2,3];" // context dense outputs
"[?];[?];[?];[?];" // context ragged values
"[?];[?];[?];[?]")); // context ragged row_splits
INFER_OK(op, "[5];[?];?;?;?;?;?;?;?;?;?;?", // Vector input, known size
("[?,2];[?,2];" // context sparse indices
"[?];[?];" // context sparse values
"[2];[2];" // context sparse dense_shapes
"[d0_0,1];[d0_0,1,2];[d0_0,1,2,3];" // context dense outputs
"[?];[?];[?];[?];" // context ragged values
"[6];[6];[6];[6]")); // context ragged row_splits
INFER_OK(op, "[];[?];?;?;?;?;?;?;?;?;?;?", // Scalar input
("[?,1];[?,1];" // context sparse indices
"[?];[?];" // context sparse values
"[1];[1];" // context sparse dense_shapes
"[1];[1,2];[1,2,3];" // context dense outputs
"[?];[?];[?];[?];" // context ragged values
"[?];[?];[?];[?]")); // context ragged row_splits
INFER_OK(op, "?;[?];?;?;?;?;?;?;?;?;?;?", // Unknown rank
("[?,?];[?,?];" // context sparse indices
"[?];[?];" // context sparse values
"[?];[?];" // context sparse dense_shapes
"?;?;?;" // context dense outputs
"[?];[?];[?];[?];" // context ragged values
"[?];[?];[?];[?]")); // context ragged row_splits
// feature_list inputs with no context inputs.
set_outputs(0, 0, 0, 2 /* num_context_sparse */, 3 /* num_context_dense */,
4 /* num_ragged */);
INFER_OK(op, "[?];[?];?;?;?;?;?;?;?", // Vector input, unknown size
("[?,3];[?,3];" // f_list sparse indices
"[?];[?];" // f_list sparse values
"[3];[3];" // f_list sparse dense_shapes
"[d0_0,?,1];[d0_0,?,1,2];" // f_list dense outputs
"[d0_0,?,1,2,3];" // (continued)
"in0;in0;in0;" // f_list dense lengths
"[?];[?];[?];[?];" // f_list ragged values
"[?];[?];[?];[?];" // f_list ragged outer_splits
"[?];[?];[?];[?]")); // f_list ragged inner_splits
INFER_OK(op, "[5];[?];?;?;?;?;?;?;?", // Vector input, known size
("[?,3];[?,3];" // f_list sparse indices
"[?];[?];" // f_list sparse values
"[3];[3];" // f_list sparse dense_shapes
"[d0_0,?,1];[d0_0,?,1,2];" // f_list dense outputs
"[d0_0,?,1,2,3];" // (continued)
"in0;in0;in0;" // f_list dense lengths
"[?];[?];[?];[?];" // f_list ragged values
"[6];[6];[6];[6];" // f_list ragged outer_splits
"[?];[?];[?];[?]")); // f_list ragged inner_splits
INFER_OK(op, "[];[?];?;?;?;?;?;?;?", // Scalar input
("[?,2];[?,2];" // f_list sparse indices
"[?];[?];" // f_list sparse values
"[2];[2];" // f_list sparse dense_shapes
"[?,1];[?,1,2];[?,1,2,3];" // f_list dense outputs
"in0;in0;in0;" // f_list dense lengths
"[?];[?];[?];[?];" // f_list ragged values
"[?];[?];[?];[?];" // f_list ragged outer_splits
"[?];[?];[?];[?]")); // f_list ragged inner_splits
INFER_OK(op, "?;[?];?;?;?;?;?;?;?", // Unknown rank
("[?,?];[?,?];" // f_list sparse indices
"[?];[?];" // f_list sparse values
"[?];[?];" // f_list sparse dense_shapes
"?;?;?;" // f_list dense outputs
"in0;in0;in0;" // f_list dense lengths
"[?];[?];[?];[?];" // f_list ragged values
"[?];[?];[?];[?];" // f_list ragged outer_splits
"[?];[?];[?];[?]")); // f_list ragged inner_splits
// Combine previous two test cases.
set_outputs(2 /* num_context_sparse */, 3 /* num_context_dense */,
4 /* num_ragged */, 2 /* num_context_sparse */,
3 /* num_context_dense */, 4 /* num_ragged */);
INFER_OK(op, "[?];[?];?;?;?;?;?;?;?;?;?;?", // Vector input, unknown size
("[?,2];[?,2];" // context sparse indices
"[?];[?];" // context sparse values
"[2];[2];" // context sparse dense_shapes
"[d0_0,1];[d0_0,1,2];[d0_0,1,2,3];" // context dense outputs
"[?];[?];[?];[?];" // context ragged values
"[?];[?];[?];[?];" // context ragged row_splits
"[?,3];[?,3];" // f_list sparse indices
"[?];[?];" // f_list sparse values
"[3];[3];" // f_list sparse dense_shapes
"[d0_0,?,1];[d0_0,?,1,2];" // f_list dense outputs
"[d0_0,?,1,2,3];" // (continued)
"in0;in0;in0;" // f_list dense lengths
"[?];[?];[?];[?];" // f_list ragged values
"[?];[?];[?];[?];" // f_list ragged outer_splits
"[?];[?];[?];[?]")); // f_list ragged inner_splits
INFER_OK(op, "[5];[?];?;?;?;?;?;?;?;?;?;?", // Vector input, known size
("[?,2];[?,2];" // context sparse indices
"[?];[?];" // context sparse values
"[2];[2];" // context sparse dense_shapes
"[d0_0,1];[d0_0,1,2];[d0_0,1,2,3];" // context dense outputs
"[?];[?];[?];[?];" // context ragged values
"[6];[6];[6];[6];" // context ragged row_splits
"[?,3];[?,3];" // f_list sparse indices
"[?];[?];" // f_list sparse values
"[3];[3];" // f_list sparse dense_shapes
"[d0_0,?,1];[d0_0,?,1,2];" // f_list dense outputs
"[d0_0,?,1,2,3];" // (continued)
"in0;in0;in0;" // f_list dense lengths
"[?];[?];[?];[?];" // f_list ragged values
"[6];[6];[6];[6];" // f_list ragged outer_splits
"[?];[?];[?];[?]")); // f_list ragged inner_splits
INFER_OK(op, "[];[?];?;?;?;?;?;?;?;?;?;?", // Scalar input
("[?,1];[?,1];" // context sparse indices
"[?];[?];" // context sparse values
"[1];[1];" // context sparse dense_shapes
"[1];[1,2];[1,2,3];" // context dense outputs
"[?];[?];[?];[?];" // context ragged values
"[?];[?];[?];[?];" // context ragged row_splits
"[?,2];[?,2];" // f_list sparse indices
"[?];[?];" // f_list sparse values
"[2];[2];" // f_list sparse dense_shapes
"[?,1];[?,1,2];[?,1,2,3];" // f_list dense outputs
"in0;in0;in0;" // f_list dense lengths
"[?];[?];[?];[?];" // f_list ragged values
"[?];[?];[?];[?];" // f_list ragged outer_splits
"[?];[?];[?];[?]")); // f_list ragged inner_splits
INFER_OK(op, "?;[?];?;?;?;?;?;?;?;?;?;?", // Unknown rank
("[?,?];[?,?];" // context sparse indices
"[?];[?];" // context sparse values
"[?];[?];" // context sparse dense_shapes
"?;?;?;" // context dense outputs
"[?];[?];[?];[?];" // context ragged values
"[?];[?];[?];[?];" // context ragged row_splits
"[?,?];[?,?];" // f_list sparse indices
"[?];[?];" // f_list sparse values
"[?];[?];" // f_list sparse dense_shapes
"?;?;?;" // f_list dense outputs
"in0;in0;in0;" // f_list dense lengths
"[?];[?];[?];[?];" // f_list ragged values
"[?];[?];[?];[?];" // f_list ragged outer_splits
"[?];[?];[?];[?]")); // f_list ragged inner_splits
// Confirm an error from ParseSequenceExampleAttrs.Init().
set_outputs(1, 1, 1, 1, 1, 1, true /* add_extra_shape */);
INFER_ERROR(
"num_context_dense (1) must match the size of "
"context_dense_types (1) and context_dense_shapes (2)",
op, "[?];[?];?;?;?;?;?;?;?;?");
}
} // end namespace tensorflow

1349
tensorflow/core/util/example_proto_fast_parsing.cc
View File

File diff suppressed because it is too large Load Diff

4
tensorflow/core/util/example_proto_fast_parsing.h
View File

@ -97,6 +97,7 @@ struct Result {
std::vector<Tensor> dense_values;
std::vector<Tensor> ragged_values;
std::vector<Tensor> ragged_splits;
std::vector<Tensor> ragged_outer_splits; // For SequenceExamples
// This vector will be populated with one element per example if
// `FastParseExampleConfig::collect_feature_stats` is set to `true`.
@ -122,13 +123,14 @@ Status FastParseSingleExample(const FastParseSingleExampleConfig& config,
// result according to given config.
// Given example names have to either be empty or the same size as serialized.
// example_names are used only for error messages.
// (If batch=true, then this parses a single SequenceExample.)
Status FastParseSequenceExample(
const example::FastParseExampleConfig& context_config,
const example::FastParseExampleConfig& feature_list_config,
gtl::ArraySlice<tstring> serialized, gtl::ArraySlice<tstring> example_names,
thread::ThreadPool* thread_pool, example::Result* context_result,
example::Result* feature_list_result,
std::vector<Tensor>* dense_feature_lengths);
std::vector<Tensor>* dense_feature_lengths, bool is_batch = true);
// This function parses serialized Example and populates given example.
// It uses the same specialized parser as FastParseExample which is efficient.

98
tensorflow/core/util/example_proto_helper.cc
View File

@ -472,43 +472,89 @@ Status ParseSingleExampleAttrs::FinishInit() {
return Status::OK();
}
Status ParseSequenceExampleAttrs::FinishInit() {
if (num_context_sparse != context_sparse_keys.size() ||
num_context_sparse != context_sparse_types.size()) {
Status ParseSequenceExampleAttrs::FinishInit(int op_version) {
switch (op_version) {
case 1:
num_context_ragged = 0;
num_feature_list_ragged = 0;
if (num_context_sparse != context_sparse_keys.size()) {
return errors::InvalidArgument(
"num_context_sparse (", num_context_sparse,
") must match the size of context_sparse_keys (",
context_sparse_keys.size(), ")");
}
if (num_context_dense != context_dense_keys.size()) {
return errors::InvalidArgument(
"num_context_dense (", num_context_dense,
") must match the size of context_dense_keys (",
context_dense_keys.size(), ")");
}
if (num_feature_list_sparse != feature_list_sparse_keys.size()) {
return errors::InvalidArgument(
"num_feature_list_sparse (", num_feature_list_sparse,
") must match the size of feature_list_sparse_keys (",
feature_list_sparse_keys.size(), ")");
}
if (num_feature_list_dense != feature_list_dense_keys.size()) {
return errors::InvalidArgument(
"num_feature_list_dense (", num_feature_list_dense,
") must match the size of feature_list_dense_keys (",
feature_list_dense_keys.size(), ")");
}
break;
case 2:
num_context_dense = context_dense_types.size();
num_context_ragged = context_ragged_value_types.size();
num_feature_list_ragged = feature_list_ragged_value_types.size();
break;
default:
return errors::InvalidArgument("Unexpected op_version", op_version);
}
if (num_context_sparse != context_sparse_types.size()) {
return errors::InvalidArgument(
"num_context_sparse (", num_context_sparse,
") must match the size of context_sparse_keys (",
context_sparse_keys.size(), ") and context_sparse_types (",
") must match the size of context_sparse_types (",
context_sparse_types.size(), ")");
}
if (num_context_dense != context_dense_keys.size() ||
num_context_dense != context_dense_types.size() ||
if (num_context_dense != context_dense_types.size() ||
num_context_dense != context_dense_shapes.size()) {
return errors::InvalidArgument(
"num_context_dense (", num_context_dense,
") must match the size of context_dense_keys (",
context_dense_keys.size(), "), context_dense_types (",
") must match the size of context_dense_types (",
context_dense_types.size(), ") and context_dense_shapes (",
context_dense_shapes.size(), ")");
}
if (num_feature_list_sparse != feature_list_sparse_keys.size() ||
num_feature_list_sparse != feature_list_sparse_types.size()) {
if ((num_context_ragged != context_ragged_value_types.size()) ||
(num_context_ragged != context_ragged_split_types.size())) {
return errors::InvalidArgument(
"num_context_ragged (", num_context_ragged,
") must match the size of context_ragged_value_types (",
context_ragged_value_types.size(), ") and context_ragged_split_types (",
context_ragged_split_types.size(), ")");
}
if (num_feature_list_sparse != feature_list_sparse_types.size()) {
return errors::InvalidArgument(
"num_feature_list_sparse (", num_feature_list_sparse,
") must match the size of feature_list_sparse_keys (",
feature_list_sparse_keys.size(), ") and feature_list_sparse_types (",
") must match the size of feature_list_sparse_types (",
feature_list_sparse_types.size(), ")");
}
if (num_feature_list_dense != feature_list_dense_keys.size() ||
num_feature_list_dense != feature_list_dense_types.size() ||
if (num_feature_list_dense != feature_list_dense_types.size() ||
num_feature_list_dense != feature_list_dense_shapes.size()) {
return errors::InvalidArgument(
"num_feature_list_dense (", num_feature_list_dense,
") must match the size of feature_list_dense_keys (",
feature_list_dense_keys.size(), "), feature_list_dense_types (",
") must match the size of feature_list_dense_types (",
feature_list_dense_types.size(), ") and feature_list_dense_shapes (",
feature_list_dense_shapes.size(), ")");
}
if ((num_feature_list_ragged != feature_list_ragged_value_types.size()) ||
(num_feature_list_ragged != feature_list_ragged_split_types.size())) {
return errors::InvalidArgument(
"num_feature_list_ragged (", num_feature_list_ragged,
") must match the size of feature_list_ragged_value_types (",
feature_list_ragged_value_types.size(),
") and feature_list_ragged_split_types (",
feature_list_ragged_split_types.size(), ")");
}
for (const DataType& type : context_dense_types) {
TF_RETURN_IF_ERROR(CheckValidType(type));
}
@ -521,6 +567,24 @@ Status ParseSequenceExampleAttrs::FinishInit() {
for (const DataType& type : feature_list_sparse_types) {
TF_RETURN_IF_ERROR(CheckValidType(type));
}
for (const DataType& type : context_ragged_value_types) {
TF_RETURN_IF_ERROR(CheckValidType(type));
}
for (const DataType& type : context_ragged_split_types) {
if (!(type == DT_INT64 || type == DT_INT32)) {
return errors::InvalidArgument("Invalid context_ragged_split_type: ",
DataTypeString(type));
}
}
for (const DataType& type : feature_list_ragged_value_types) {
TF_RETURN_IF_ERROR(CheckValidType(type));
}
for (const DataType& type : feature_list_ragged_split_types) {
if (!(type == DT_INT64 || type == DT_INT32)) {
return errors::InvalidArgument("Invalid feature_list_ragged_split_type: ",
DataTypeString(type));
}
}
return Status::OK();
}

57
tensorflow/core/util/example_proto_helper.h
View File

@ -243,24 +243,41 @@ struct ParseSingleExampleAttrs {
struct ParseSequenceExampleAttrs {
public:
template <typename ContextType>
Status Init(ContextType* ctx) {
std::vector<string> feature_list_dense_missing_assumed_empty_tmp;
TF_RETURN_IF_ERROR(
ctx->GetAttr("feature_list_dense_missing_assumed_empty",
&feature_list_dense_missing_assumed_empty_tmp));
for (const string& feature : feature_list_dense_missing_assumed_empty_tmp) {
feature_list_dense_missing_assumed_empty.insert(feature);
Status Init(ContextType* ctx, int op_version = 1) {
switch (op_version) {
case 1: {
std::vector<string> missing_empty_vector;
TF_RETURN_IF_ERROR(ctx->GetAttr(
"feature_list_dense_missing_assumed_empty", &missing_empty_vector));
for (const string& feature : missing_empty_vector) {
feature_list_dense_missing_assumed_empty.insert(feature);
}
}
TF_RETURN_IF_ERROR(
ctx->GetAttr("context_sparse_keys", &context_sparse_keys));
TF_RETURN_IF_ERROR(
ctx->GetAttr("context_dense_keys", &context_dense_keys));
TF_RETURN_IF_ERROR(ctx->GetAttr("feature_list_sparse_keys",
&feature_list_sparse_keys));
TF_RETURN_IF_ERROR(
ctx->GetAttr("feature_list_dense_keys", &feature_list_dense_keys));
TF_RETURN_IF_ERROR(ctx->GetAttr("Ncontext_dense", &num_context_dense));
break;
case 2:
TF_RETURN_IF_ERROR(ctx->GetAttr("context_ragged_value_types",
&context_ragged_value_types));
TF_RETURN_IF_ERROR(ctx->GetAttr("context_ragged_split_types",
&context_ragged_split_types));
TF_RETURN_IF_ERROR(ctx->GetAttr("feature_list_ragged_value_types",
&feature_list_ragged_value_types));
TF_RETURN_IF_ERROR(ctx->GetAttr("feature_list_ragged_split_types",
&feature_list_ragged_split_types));
break;
default:
return errors::InvalidArgument("Unexpected op_version", op_version);
}
TF_RETURN_IF_ERROR(
ctx->GetAttr("context_sparse_keys", &context_sparse_keys));
TF_RETURN_IF_ERROR(ctx->GetAttr("context_dense_keys", &context_dense_keys));
TF_RETURN_IF_ERROR(
ctx->GetAttr("feature_list_sparse_keys", &feature_list_sparse_keys));
TF_RETURN_IF_ERROR(
ctx->GetAttr("feature_list_dense_keys", &feature_list_dense_keys));
TF_RETURN_IF_ERROR(
ctx->GetAttr("context_sparse_types", &context_sparse_types));
TF_RETURN_IF_ERROR(ctx->GetAttr("Ncontext_dense", &num_context_dense));
TF_RETURN_IF_ERROR(
ctx->GetAttr("Nfeature_list_dense", &num_feature_list_dense));
TF_RETURN_IF_ERROR(ctx->GetAttr("Ncontext_sparse", &num_context_sparse));
@ -275,14 +292,16 @@ struct ParseSequenceExampleAttrs {
ctx->GetAttr("context_dense_shapes", &context_dense_shapes));
TF_RETURN_IF_ERROR(
ctx->GetAttr("feature_list_dense_shapes", &feature_list_dense_shapes));
return FinishInit();
return FinishInit(op_version);
}
std::unordered_set<string> feature_list_dense_missing_assumed_empty;
int64 num_context_sparse;
int64 num_context_dense;
int64 num_context_ragged;
int64 num_feature_list_sparse;
int64 num_feature_list_dense;
int64 num_feature_list_ragged;
std::vector<string> context_sparse_keys;
std::vector<string> context_dense_keys;
std::vector<string> feature_list_sparse_keys;
@ -293,9 +312,13 @@ struct ParseSequenceExampleAttrs {
std::vector<DataType> feature_list_sparse_types;
std::vector<DataType> feature_list_dense_types;
std::vector<TensorShape> feature_list_dense_shapes;
std::vector<DataType> context_ragged_value_types;
std::vector<DataType> context_ragged_split_types;
std::vector<DataType> feature_list_ragged_value_types;
std::vector<DataType> feature_list_ragged_split_types;
private:
Status FinishInit(); // for context-independent parts of Init.
Status FinishInit(int op_version); // for context-independent parts of Init.
};
// Parses the attributes passed to ParseSingleSequenceExample.

2
tensorflow/python/kernel_tests/BUILD
View File

@ -766,7 +766,7 @@ cuda_py_test(
tf_py_test(
name = "parsing_ops_test",
size = "small",
size = "medium",
srcs = ["parsing_ops_test.py"],
additional_deps = [
"//third_party/py/numpy",

340
tensorflow/python/kernel_tests/parsing_ops_test.py
View File

@ -1521,9 +1521,6 @@ class ParseSequenceExampleTest(test.TestCase):
kwargs["serialized"] = [kwargs.pop("serialized")]
kwargs["example_names"] = [kwargs.pop("example_name")
] if "example_name" in kwargs else None
# Disable error string matching; it's not consistent for batch mode.
if expected_err:
expected_err = (expected_err[0], "")
# Add a batch dimension to expected output
if expected_context_values:
@ -1531,7 +1528,7 @@ class ParseSequenceExampleTest(test.TestCase):
for k in expected_context_values:
v = expected_context_values[k]
if isinstance(kwargs["context_features"][k],
parsing_ops.FixedLenFeature):
(parsing_ops.FixedLenFeature, parsing_ops.RaggedFeature)):
new_values[k] = np.expand_dims(v, axis=0)
else:
# Sparse tensor.
@ -1548,6 +1545,9 @@ class ParseSequenceExampleTest(test.TestCase):
parsing_ops.FixedLenSequenceFeature):
expected_length_values[k] = [np.shape(v)[0]]
new_values[k] = np.expand_dims(v, axis=0)
elif isinstance(kwargs["sequence_features"][k],
parsing_ops.RaggedFeature):
new_values[k] = np.expand_dims(v, axis=0)
else:
# Sparse tensor.
new_values[k] = (np.insert(v[0], 0, 0, axis=1), v[1],
@ -1562,6 +1562,7 @@ class ParseSequenceExampleTest(test.TestCase):
expected_err=expected_err,
batch=True)
@test_util.with_forward_compatibility_horizons(None, [2019, 10, 31])
def testSequenceExampleWithSparseAndDenseContext(self):
original = sequence_example(
context=features({
@ -1605,6 +1606,7 @@ class ParseSequenceExampleTest(test.TestCase):
},
expected_context_values=expected_context_output)
@test_util.with_forward_compatibility_horizons(None, [2019, 10, 31])
def testSequenceExampleWithMultipleSizeFeatureLists(self):
original = sequence_example(
feature_lists=feature_lists({
@ -1668,6 +1670,7 @@ class ParseSequenceExampleTest(test.TestCase):
},
expected_feat_list_values=expected_feature_list_output)
@test_util.with_forward_compatibility_horizons(None, [2019, 10, 31])
def testSequenceExampleWithoutDebugName(self):
original = sequence_example(
feature_lists=feature_lists({
@ -1725,6 +1728,7 @@ class ParseSequenceExampleTest(test.TestCase):
},
expected_feat_list_values=expected_feature_list_output)
@test_util.with_forward_compatibility_horizons(None, [2019, 10, 31])
def testSequenceExampleWithSparseAndDenseFeatureLists(self):
original = sequence_example(
feature_lists=feature_lists({
@ -1783,6 +1787,7 @@ class ParseSequenceExampleTest(test.TestCase):
},
expected_feat_list_values=expected_feature_list_output)
@test_util.with_forward_compatibility_horizons(None, [2019, 10, 31])
def testSequenceExampleWithEmptyFeatureInFeatureLists(self):
original = sequence_example(
feature_lists=feature_lists({
@ -1815,6 +1820,7 @@ class ParseSequenceExampleTest(test.TestCase):
},
expected_feat_list_values=expected_feature_list_output)
@test_util.with_forward_compatibility_horizons(None, [2019, 10, 31])
def testSequenceExampleListWithInconsistentDataFails(self):
original = sequence_example(
feature_lists=feature_lists({
@ -1835,6 +1841,7 @@ class ParseSequenceExampleTest(test.TestCase):
expected_err=(errors_impl.OpError, "Feature list: a, Index: 1."
" Data types don't match. Expected type: int64"))
@test_util.with_forward_compatibility_horizons(None, [2019, 10, 31])
def testSequenceExampleListWithWrongDataTypeFails(self):
original = sequence_example(
feature_lists=feature_lists({
@ -1855,6 +1862,7 @@ class ParseSequenceExampleTest(test.TestCase):
"Feature list: a, Index: 0. Data types don't match."
" Expected type: int64"))
@test_util.with_forward_compatibility_horizons(None, [2019, 10, 31])
def testSequenceExampleListWithWrongSparseDataTypeFails(self):
original = sequence_example(
feature_lists=feature_lists({
@ -1878,9 +1886,9 @@ class ParseSequenceExampleTest(test.TestCase):
},
expected_err=(errors_impl.OpError,
"Name: in1, Feature list: a, Index: 2."
" Data types don't match. Expected type: int64"
" Feature is: float_list"))
" Data types don't match. Expected type: int64"))
@test_util.with_forward_compatibility_horizons(None, [2019, 10, 31])
def testSequenceExampleListWithWrongShapeFails(self):
original = sequence_example(
feature_lists=feature_lists({
@ -1899,10 +1907,44 @@ class ParseSequenceExampleTest(test.TestCase):
"a": parsing_ops.FixedLenSequenceFeature((2,), dtypes.int64)
}
},
expected_err=(errors_impl.OpError, r"Name: in1, Key: a, Index: 1."
r" Number of int64 values != expected."
r" values size: 3 but output shape: \[2\]"))
expected_err=(
errors_impl.OpError,
# message from ParseSingleExample.
r"Name: in1, Key: a, Index: 1."
r" Number of int64 values != expected."
r" values size: 3 but output shape: \[2\]"
# or message from FastParseSequenceExample
r"|Feature list 'a' has an unexpected number of values. "
r"Total values size: 5 is not consistent with output "
r"shape: \[\?,2\]"))
@test_util.with_forward_compatibility_horizons(None, [2019, 10, 31])
def testSequenceExampleListWithWrongShapeFails2(self):
# This exercises a different code path for FastParseSequenceExample than
# testSequenceExampleListWithWrongShapeFails (in that test, we can tell that
# the shape is bad based on the total number of values; in this test, we
# can't tell the shape is bad until we look at individual rows.)
original = sequence_example(
feature_lists=feature_lists({
"a": feature_list([int64_feature([2]),
int64_feature([2, 3, 4])]),
}))
serialized = original.SerializeToString()
self._testBoth(
{
"example_name": "in1",
"serialized": ops.convert_to_tensor(serialized),
"sequence_features": {
"a": parsing_ops.FixedLenSequenceFeature((2,), dtypes.int64)
}
},
expected_err=(errors_impl.OpError, r"Name: in1, Key: a, Index: 0."
r" Number of (int64 )?values != expected."
r" values size: 1 but output shape: \[2\]"))
@test_util.with_forward_compatibility_horizons(None, [2019, 10, 31])
def testSequenceExampleWithMissingFeatureListFails(self):
original = sequence_example(feature_lists=feature_lists({}))
@ -1923,6 +1965,7 @@ class ParseSequenceExampleTest(test.TestCase):
" feature_list_dense_missing_assumed_empty or"
" feature_list_dense_defaults?"))
@test_util.with_forward_compatibility_horizons(None, [2019, 10, 31])
def testSequenceExampleBatch(self):
first = sequence_example(
feature_lists=feature_lists({
@ -1935,14 +1978,21 @@ class ParseSequenceExampleTest(test.TestCase):
])
}))
second = sequence_example(
context=features({"c": float_feature([7])}),
feature_lists=feature_lists({
"a": feature_list([
int64_feature([21, 2, 11]),
])
]),
"b": feature_list([
int64_feature([5]),
]),
}))
serialized = [first.SerializeToString(), second.SerializeToString()]
expected_context_output = {
"c": np.array([-1, 7], dtype=np.float32),
}
expected_feature_list_output = {
"a":
np.array(
@ -1961,6 +2011,8 @@ class ParseSequenceExampleTest(test.TestCase):
]
],
dtype=np.int64),
"b":
np.array([[0], [5]], dtype=np.int64),
"d":
np.empty(shape=(2, 0, 5), dtype=np.float32), # allowed_missing
}
@ -1969,21 +2021,289 @@ class ParseSequenceExampleTest(test.TestCase):
{
"example_names": ops.convert_to_tensor(["in1", "in2"]),
"serialized": ops.convert_to_tensor(serialized),
"context_features": {
"c":
parsing_ops.FixedLenFeature(
(), dtypes.float32, default_value=-1),
},
"sequence_features": {
"a":
parsing_ops.FixedLenSequenceFeature((1, 3), dtypes.int64),
"b":
parsing_ops.FixedLenSequenceFeature(
(), dtypes.int64, allow_missing=True),
"d":
parsing_ops.FixedLenSequenceFeature(
(5,), dtypes.float32, allow_missing=True),
}
},
expected_context_values=expected_context_output,
expected_feat_list_values=expected_feature_list_output,
expected_length_values={
"a": [4, 1],
"b": [0, 1],
"d": [0, 0]
},
batch=True)
@test_util.with_forward_compatibility_horizons(None, [2019, 10, 31])
def testSerializedContainingRaggedFeatureWithNoPartitions(self):
original = [
sequence_example(
context=features({"a": float_feature([3, 4])}),
feature_lists=feature_lists({
"b": feature_list([float_feature([5]),
float_feature([3])]),
"c": feature_list([int64_feature([6, 7, 8, 9])])
})),
sequence_example(
context=features({"a": float_feature([9])}),
feature_lists=feature_lists({
"b": feature_list([]),
"c": feature_list([int64_feature([]),
int64_feature([1, 2, 3])])
})),
sequence_example(
feature_lists=feature_lists({
"b":
feature_list([
float_feature([1]),
float_feature([1, 2]),
float_feature([1, 2, 3])
])
})),
sequence_example(
context=features({"a": feature()}),
feature_lists=feature_lists({
"b": feature_list([feature()]),
"c": feature_list([int64_feature([3, 3, 3])])
}))
]
serialized = [m.SerializeToString() for m in original]
context_features = {"a": parsing_ops.RaggedFeature(dtype=dtypes.float32)}
sequence_features = {
"b":
parsing_ops.RaggedFeature(dtype=dtypes.float32),
"c":
parsing_ops.RaggedFeature(
dtype=dtypes.int64, row_splits_dtype=dtypes.int64)
}
expected_a = ragged_factory_ops.constant([[3, 4], [9], [], []],
dtype=dtypes.float32,
row_splits_dtype=dtypes.int32)
expected_b = ragged_factory_ops.constant(
[[[5], [3]], [], [[1], [1, 2], [1, 2, 3]], [[]]],
dtype=dtypes.float32,
row_splits_dtype=dtypes.int32)
expected_c = ragged_factory_ops.constant(
[[[6, 7, 8, 9]], [[], [1, 2, 3]], [], [[3, 3, 3]]],
dtype=dtypes.int64,
row_splits_dtype=dtypes.int64)
expected_context_output = dict(a=expected_a)
expected_feature_list_output = dict(b=expected_b, c=expected_c)
self._test(
{
"serialized": ops.convert_to_tensor(serialized),
"context_features": context_features,
"sequence_features": sequence_features,
},
expected_context_output,
expected_feature_list_output,
batch=True)
self._test(
{
"serialized": ops.convert_to_tensor(serialized)[0],
"context_features": context_features,
"sequence_features": sequence_features,
},
expected_context_values={"a": [3, 4]},
expected_feat_list_values={
"b": [[5], [3]],
"c": [[6, 7, 8, 9]]
},
batch=False)
# Test with a larger batch of examples.
batch_serialized = serialized * 64
batch_context_expected_out = {
"a": ragged_concat_ops.concat([expected_a] * 64, axis=0)
}
batch_feature_list_expected_out = {
"b": ragged_concat_ops.concat([expected_b] * 64, axis=0),
"c": ragged_concat_ops.concat([expected_c] * 64, axis=0)
}
self._test(
{
"serialized": ops.convert_to_tensor(batch_serialized),
"context_features": context_features,
"sequence_features": sequence_features,
},
batch_context_expected_out,
batch_feature_list_expected_out,
batch=True)
@test_util.with_forward_compatibility_horizons(None, [2019, 10, 31])
def testSerializedContainingNestedRaggedFeature(self):
"""Test RaggedFeatures with nested partitions."""
original = [
# rt shape: [(batch), 2, None, None]
sequence_example(
context=features({
# a[0] = [[[[1]], [[2, 3], [4]]], [[], [[5, 6, 7]]]]
"a_values": float_feature([1, 2, 3, 4, 5, 6, 7]),
"a_lengths_axis2": int64_feature([1, 2, 0, 1]),
"a_lengths_axis3": int64_feature([1, 2, 1, 3]),
"a_splits_axis3": int64_feature([0, 1, 3, 4, 7])
}),
feature_lists=feature_lists({
# b[0] = [[[1], [2, 3, 4]], [[2, 4], [6]]]
"b_values":
feature_list(
[float_feature([1, 2, 3, 4]),
float_feature([2, 4, 6])]),
"b_splits":
feature_list(
[int64_feature([0, 1, 4]),
int64_feature([0, 2, 3])]),
})),
sequence_example(
# a[1] = []
# b[1] = []
),
sequence_example(
context=features({
# a[2] = [[[[1, 2, 3], [4]], [[5], [6], [7, 8]]]]
"a_values": float_feature([1, 2, 3, 4, 5, 6, 7, 8]),
"a_lengths_axis2": int64_feature([2, 3]),
"a_lengths_axis3": int64_feature([3, 1, 1, 1, 2]),
"a_splits_axis3": int64_feature([0, 3, 4, 5, 6, 8])
}),
feature_lists=feature_lists({
# b[2] = [[[9], [8, 7, 6], [5]], [[4, 3, 2, 1]], [[0]]]
"b_values":
feature_list([
float_feature([9, 8, 7, 6, 5]),
float_feature([4, 3, 2, 1]),
float_feature([0])
]),
"b_splits":
feature_list([
int64_feature([0, 1, 4, 5]),
int64_feature([0, 4]),
int64_feature([0, 1])
])
}))
]
serialized = [m.SerializeToString() for m in original]
context_features = {
"a":
parsing_ops.RaggedFeature(
value_key="a_values",
partitions=[
parsing_ops.RaggedFeature.UniformRowLength(2),
parsing_ops.RaggedFeature.RowLengths("a_lengths_axis2"),
parsing_ops.RaggedFeature.RowSplits("a_splits_axis3"),
],
dtype=dtypes.float32,
row_splits_dtype=dtypes.int64,
)
}
sequence_features = {
"b":
parsing_ops.RaggedFeature(
value_key="b_values",
dtype=dtypes.float32,
partitions=[parsing_ops.RaggedFeature.RowSplits("b_splits")]),
"c":
parsing_ops.RaggedFeature(
value_key="b_values",
dtype=dtypes.float32,
partitions=[parsing_ops.RaggedFeature.UniformRowLength(1)]),
}
expected_context = {
"a":
ragged_factory_ops.constant(
[[[[[1]], [[2, 3], [4]]], [[], [[5, 6, 7]]]], [],
[[[[1, 2, 3], [4]], [[5], [6], [7, 8]]]]],
dtype=dtypes.float32,
row_splits_dtype=dtypes.int64)
}
expected_feature_list = {
"b":
ragged_factory_ops.constant(
[[[[1], [2, 3, 4]], [[2, 4], [6]]], [],
[[[9], [8, 7, 6], [5]], [[4, 3, 2, 1]], [[0]]]],
dtype=dtypes.float32,
row_splits_dtype=dtypes.int32),
"c":
ragged_factory_ops.constant(
[[[[1], [2], [3], [4]], [[2], [4], [6]]], [],
[[[9], [8], [7], [6], [5]], [[4], [3], [2], [1]], [[0]]]],
ragged_rank=2,
dtype=dtypes.float32,
row_splits_dtype=dtypes.int32),
}
self._test(
dict(
serialized=ops.convert_to_tensor(serialized),
context_features=context_features,
sequence_features=sequence_features),
expected_context,
expected_feature_list,
batch=True)
self._test(
dict(
serialized=ops.convert_to_tensor(serialized)[0],
context_features=context_features,
sequence_features=sequence_features),
{"a": expected_context["a"][0]}, {
"b": expected_feature_list["b"][0],
"c": expected_feature_list["c"][0]
},
batch=False)
@test_util.with_forward_compatibility_horizons(None, [2019, 10, 31])
def testSerializedContainingMisalignedNestedRaggedFeature(self):
"""FeatureList with 2 value tensors but only one splits tensor."""
original = sequence_example(
feature_lists=feature_lists({
"b_values":
feature_list(
[float_feature([1, 2, 3, 4]),
float_feature([2, 4, 6])]),
"b_splits":
feature_list([int64_feature([0, 1, 4])]),
}))
sequence_features = {
"b":
parsing_ops.RaggedFeature(
value_key="b_values",
dtype=dtypes.float32,
partitions=[parsing_ops.RaggedFeature.RowSplits("b_splits")],
validate=True)
}
self._testBoth(
dict(
serialized=ops.convert_to_tensor(original.SerializeToString()),
sequence_features=sequence_features),
expected_err=(
(errors_impl.OpError, ValueError),
# Message for batch=true:
"Feature b: values and partitions are not aligned"
# Message for batch=false in graph mode:
"|.* do not form a valid RaggedTensor"
# Message for batch=false in eager mode:
"|Dimensions 2 and 1 are not compatible"))
@test_util.run_all_in_graph_and_eager_modes
class DecodeRawTest(test.TestCase):

106
tensorflow/python/ops/parsing_config.py
View File

@ -26,6 +26,7 @@ from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.framework import tensor_shape
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import check_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import sparse_ops
from tensorflow.python.platform import tf_logging
@ -721,12 +722,25 @@ def _construct_tensors_for_composite_features(features, tensor_dict):
value_key = key if feature.value_key is None else feature.value_key
rt = tensor_dict[value_key]
if isinstance(rt, ragged_tensor.RaggedTensor):
# We processed a vector of serialized tf.Examples.
# We processed a batch of tf.Example or tf.SequenceExample, or single
# tf.SequenceExample.
if rt.ragged_rank > 1:
# We're processing a batch of SequenceExample, and we effectively have
# two batch dimensions. Cllapse those batch dimensions here, and
# restore them below (using outer_splits).
outer_splits = rt.row_splits
rt = rt.values
else:
outer_splits = None
for partition in reversed(feature.partitions):
rt = _add_batched_ragged_partition(rt, partition, tensor_dict,
feature.validate)
key, feature.validate,
outer_splits)
if outer_splits is not None:
rt = ragged_tensor.RaggedTensor.from_row_splits(
rt, outer_splits, validate=feature.validate)
else:
# We processed a single serialized tf.Example.
# We processed a single tf.Example.
for partition in reversed(feature.partitions):
rt = _add_ragged_partition(rt, partition, tensor_dict,
feature.row_splits_dtype, feature.validate)
@ -789,7 +803,8 @@ def _add_ragged_partition(values, partition, tensor_dict, row_splits_dtype,
raise ValueError("Unhandled partition type %r" % partition)
def _add_batched_ragged_partition(rt, partition, tensor_dict, validate):
def _add_batched_ragged_partition(rt, partition, tensor_dict, feature_key,
validate, outer_splits=None):
"""Adds a batched ragged partition tensor to a batched ragged tensor.
Args:
@ -799,7 +814,11 @@ def _add_batched_ragged_partition(rt, partition, tensor_dict, validate):
RaggedFeature.UniformRowLength, in which case there is no partition
tensor). The specified tensor must have shape [batch_size, ...].
tensor_dict: The dictionary mapping keys to tensors.
feature_key: The name of the feature being parsed (for error messages).
validate: Whether to validate that the values form a valid RaggedTensor.
outer_splits: If not None, then we have two batch dimensions, and this
is the row-splits for the collapsed batch dimension. Every partition
tensor must have an outer row_splits that matches this value.
Returns:
A new RaggedTensor where each batch item `rt[i]` has been partitioned
@ -823,40 +842,59 @@ def _add_batched_ragged_partition(rt, partition, tensor_dict, validate):
if partition_t.values.dtype != rt.row_splits.dtype:
partition_t = math_ops.cast(partition_t, rt.row_splits.dtype)
if isinstance(partition, (RaggedFeature.RowSplits, RaggedFeature.RowLimits)):
if isinstance(partition, RaggedFeature.RowSplits):
partition_t = partition_t[:, 1:]
adjusted_limits = partition_t.values + array_ops.repeat(
rt.row_starts(), partition_t.row_lengths())
return partition_t.with_values(
ragged_tensor.RaggedTensor.from_row_limits(
rt.values, adjusted_limits, validate=validate))
elif isinstance(partition, RaggedFeature.RowStarts):
adjusted_starts = partition_t.values + array_ops.repeat(
rt.row_starts(), partition_t.row_lengths())
return partition_t.with_values(
ragged_tensor.RaggedTensor.from_row_starts(
rt.values, adjusted_starts, validate=validate))
elif isinstance(partition, RaggedFeature.RowLengths):
return partition_t.with_values(
ragged_tensor.RaggedTensor.from_row_lengths(
rt.values, partition_t.values, validate=validate))
elif isinstance(partition, RaggedFeature.ValueRowIds):
nrows = math_ops.maximum( # number of rows in each batch item
ragged_math_ops.reduce_max(partition_t + 1, axis=1), 0)
adjusted_rowids = partition_t.values + array_ops.repeat(
math_ops.cumsum(nrows, exclusive=True), partition_t.row_lengths())
return ragged_tensor.RaggedTensor.from_row_lengths(
ragged_tensor.RaggedTensor.from_value_rowids(
rt.values, adjusted_rowids, validate=validate),
nrows,
validate=validate)
checks = []
if outer_splits is not None:
if validate:
checks.append(check_ops.assert_equal(
outer_splits, partition_t.row_splits,
message="Feature %s: values and partitions are not aligned"
% feature_key))
partition_t = partition_t.values
raise ValueError("Unhandled partition type %r" % partition)
with ops.control_dependencies(checks):
if isinstance(partition, (RaggedFeature.RowSplits,
RaggedFeature.RowLimits)):
if isinstance(partition, RaggedFeature.RowSplits):
partition_t = partition_t[:, 1:]
adjusted_limits = partition_t.values + array_ops.repeat(
rt.row_starts(), partition_t.row_lengths())
return partition_t.with_values(
ragged_tensor.RaggedTensor.from_row_limits(
rt.values, adjusted_limits, validate=validate))
elif isinstance(partition, RaggedFeature.RowStarts):
adjusted_starts = partition_t.values + array_ops.repeat(
rt.row_starts(), partition_t.row_lengths())
return partition_t.with_values(
ragged_tensor.RaggedTensor.from_row_starts(
rt.values, adjusted_starts, validate=validate))
elif isinstance(partition, RaggedFeature.RowLengths):
return partition_t.with_values(
ragged_tensor.RaggedTensor.from_row_lengths(
rt.values, partition_t.values, validate=validate))
elif isinstance(partition, RaggedFeature.ValueRowIds):
nrows = math_ops.maximum( # number of rows in each batch item
ragged_math_ops.reduce_max(partition_t + 1, axis=1), 0)
adjusted_rowids = partition_t.values + array_ops.repeat(
math_ops.cumsum(nrows, exclusive=True), partition_t.row_lengths())
return ragged_tensor.RaggedTensor.from_row_lengths(
ragged_tensor.RaggedTensor.from_value_rowids(
rt.values, adjusted_rowids, validate=validate),
nrows,
validate=validate)
raise ValueError("Unhandled partition type %r" % partition)
def _build_ragged_tensors(serialized_shape, ragged_values, ragged_row_splits):
def _build_ragged_tensors(serialized_shape,
ragged_values,
ragged_row_splits,
ragged_inner_splits=None):
"""Builds RaggedTensors from the outputs of a parse op."""
if ragged_inner_splits is not None:
ragged_values = [
ragged_tensor.RaggedTensor.from_row_splits(val, split, validate=False)
for (val, split) in zip(ragged_values, ragged_inner_splits)
]
if serialized_shape.ndims == 0:
return ragged_values
else:

260
tensorflow/python/ops/parsing_ops.py
View File

@ -470,7 +470,6 @@ def parse_single_example_v2_unoptimized(
[serialized, example_names]):
serialized = ops.convert_to_tensor(serialized, name="serialized")
serialized = _assert_scalar(serialized, "serialized")
serialized.set_shape([])
return parse_example_v2(serialized, features, example_names, name)
if example_names is None:
return parse_single_example_v2(serialized, features, name)
@ -557,16 +556,19 @@ def parse_sequence_example(serialized,
of `sequence_features` values may vary between `SequenceExample` protos,
and even between `feature_list` keys within the same `SequenceExample`.
`context_features` contains `VarLenFeature` and `FixedLenFeature` objects.
Each `VarLenFeature` is mapped to a `SparseTensor`, and each `FixedLenFeature`
is mapped to a `Tensor`, of the specified type, shape, and default value.
`context_features` contains `VarLenFeature`, `RaggedFeature`, and
`FixedLenFeature` objects. Each `VarLenFeature` is mapped to a
`SparseTensor`; each `RaggedFeature` is mapped to a `RaggedTensor`; and each
`FixedLenFeature` is mapped to a `Tensor`, of the specified type, shape, and
default value.
`sequence_features` contains `VarLenFeature` and `FixedLenSequenceFeature`
objects. Each `VarLenFeature` is mapped to a `SparseTensor`, and each
`FixedLenSequenceFeature` is mapped to a `Tensor`, each of the specified type.
The shape will be `(B,T,) + df.dense_shape` for `FixedLenSequenceFeature`
`df`, where `B` is the batch size, and `T` is the length of the associated
`FeatureList` in the `SequenceExample`. For instance,
`sequence_features` contains `VarLenFeature`, `RaggedFeature`, and
`FixedLenSequenceFeature` objects. Each `VarLenFeature` is mapped to a
`SparseTensor`; each `RaggedFeature` is mapped to a `RaggedTensor; and
each `FixedLenSequenceFeature` is mapped to a `Tensor`, each of the specified
type. The shape will be `(B,T,) + df.dense_shape` for
`FixedLenSequenceFeature` `df`, where `B` is the batch size, and `T` is the
length of the associated `FeatureList` in the `SequenceExample`. For instance,
`FixedLenSequenceFeature([])` yields a scalar 2-D `Tensor` of static shape
`[None, None]` and dynamic shape `[B, T]`, while
`FixedLenSequenceFeature([k])` (for `int k >= 1`) yields a 3-D matrix `Tensor`
@ -600,21 +602,21 @@ def parse_sequence_example(serialized,
serialized: A vector (1-D Tensor) of type string containing binary
serialized `SequenceExample` protos.
context_features: A `dict` mapping feature keys to `FixedLenFeature` or
`VarLenFeature` values. These features are associated with a
`SequenceExample` as a whole.
`VarLenFeature` or `RaggedFeature` values. These features are associated
with a `SequenceExample` as a whole.
sequence_features: A `dict` mapping feature keys to
`FixedLenSequenceFeature` or `VarLenFeature` values. These features are
associated with data within the `FeatureList` section of the
`SequenceExample` proto.
`FixedLenSequenceFeature` or `VarLenFeature` or `RaggedFeature` values.
These features are associated with data within the `FeatureList` section
of the `SequenceExample` proto.
example_names: A vector (1-D Tensor) of strings (optional), the name of the
serialized protos.
name: A name for this operation (optional).
Returns:
A tuple of three `dict`s, each mapping keys to `Tensor`s and
`SparseTensor`s. The first dict contains the context key/values,
the second dict contains the feature_list key/values, and the final dict
contains the lengths of any dense feature_list features.
A tuple of three `dict`s, each mapping keys to `Tensor`s,
`SparseTensor`s, and `RaggedTensor`. The first dict contains the context
key/values, the second dict contains the feature_list key/values, and the
final dict contains the lengths of any dense feature_list features.
Raises:
ValueError: if any feature is invalid.
@ -622,12 +624,26 @@ def parse_sequence_example(serialized,
if not (context_features or sequence_features):
raise ValueError("Missing features.")
context_params = _ParseOpParams.from_features(
context_features, [VarLenFeature, FixedLenFeature])
context_features, [VarLenFeature, FixedLenFeature, RaggedFeature])
feature_list_params = _ParseOpParams.from_features(
sequence_features, [VarLenFeature, FixedLenSequenceFeature])
sequence_features,
[VarLenFeature, FixedLenSequenceFeature, RaggedFeature])
return _parse_sequence_example_raw(serialized, example_names, context_params,
feature_list_params, name)
with ops.name_scope(name, "ParseSequenceExample",
[serialized, example_names]):
outputs = _parse_sequence_example_raw(serialized, example_names,
context_params, feature_list_params,
name)
context_output, feature_list_output, feature_list_lengths = outputs
if context_params.ragged_keys:
context_output = _construct_tensors_for_composite_features(
context_features, context_output)
if feature_list_params.ragged_keys:
feature_list_output = _construct_tensors_for_composite_features(
sequence_features, feature_list_output)
return context_output, feature_list_output, feature_list_lengths
def _parse_sequence_example_raw(serialized,
@ -649,9 +665,9 @@ def _parse_sequence_example_raw(serialized,
name: A name for this operation (optional).
Returns:
A tuple of three `dict`s, each mapping keys to `Tensor`s and
`SparseTensor`s. The first dict contains the context key/values,
the second dict contains the feature_list key/values, and the final dict
A tuple of three `dict`s, each mapping keys to `Tensor`s, `SparseTensor`s,
and `RaggedTensor`s. The first dict contains the context key/values, the
second dict contains the feature_list key/values, and the final dict
contains the lengths of any dense feature_list features.
Raises:
@ -670,33 +686,84 @@ def _parse_sequence_example_raw(serialized,
k)
feature_list_dense_missing_assumed_empty.append(k)
# pylint: disable=protected-access
outputs = gen_parsing_ops.parse_sequence_example(
serialized=serialized,
debug_name=debug_name,
Ncontext_sparse=len(context.sparse_keys),
Ncontext_dense=len(context.dense_keys),
Nfeature_list_sparse=len(feature_list.sparse_keys),
Nfeature_list_dense=len(feature_list.dense_keys),
context_dense_defaults=context.dense_defaults_vec,
context_sparse_keys=context.sparse_keys,
context_sparse_types=context.sparse_types,
context_dense_keys=context.dense_keys,
context_dense_shapes=context.dense_shapes_as_proto,
feature_list_sparse_keys=feature_list.sparse_keys,
feature_list_sparse_types=feature_list.sparse_types,
feature_list_dense_keys=feature_list.dense_keys,
feature_list_dense_types=feature_list.dense_types,
feature_list_dense_shapes=feature_list.dense_shapes,
feature_list_dense_missing_assumed_empty=(
feature_list_dense_missing_assumed_empty),
name=name)
# pylint: enable=protected-access
has_ragged = context.ragged_keys or feature_list.ragged_keys
if compat.forward_compatible(2019, 10, 26) or has_ragged:
serialized = ops.convert_to_tensor(serialized, name="serialized")
if has_ragged and serialized.shape.ndims is None:
raise ValueError("serialized must have statically-known rank to "
"parse ragged features.")
feature_list_dense_missing_assumed_empty_vector = [
key in feature_list_dense_missing_assumed_empty
for key in feature_list.dense_keys
]
outputs = gen_parsing_ops.parse_sequence_example_v2(
# Inputs
serialized=serialized,
debug_name=debug_name,
context_sparse_keys=context.sparse_keys,
context_dense_keys=context.dense_keys,
context_ragged_keys=context.ragged_keys,
feature_list_sparse_keys=feature_list.sparse_keys,
feature_list_dense_keys=feature_list.dense_keys,
feature_list_ragged_keys=feature_list.ragged_keys,
feature_list_dense_missing_assumed_empty=(
feature_list_dense_missing_assumed_empty_vector),
context_dense_defaults=context.dense_defaults_vec,
# Attrs
Ncontext_sparse=len(context.sparse_keys),
Nfeature_list_sparse=len(feature_list.sparse_keys),
Nfeature_list_dense=len(feature_list.dense_keys),
context_sparse_types=context.sparse_types,
context_ragged_value_types=context.ragged_value_types,
context_ragged_split_types=context.ragged_split_types,
feature_list_dense_types=feature_list.dense_types,
feature_list_sparse_types=feature_list.sparse_types,
feature_list_ragged_value_types=feature_list.ragged_value_types,
feature_list_ragged_split_types=feature_list.ragged_split_types,
context_dense_shapes=context.dense_shapes_as_proto,
feature_list_dense_shapes=feature_list.dense_shapes,
name=name)
(context_sparse_indices, context_sparse_values, context_sparse_shapes,
context_dense_values, context_ragged_values, context_ragged_row_splits,
feature_list_sparse_indices, feature_list_sparse_values,
feature_list_sparse_shapes, feature_list_dense_values,
feature_list_dense_lengths, feature_list_ragged_values,
feature_list_ragged_outer_splits,
feature_list_ragged_inner_splits) = outputs
# pylint: disable=protected-access
context_ragged_tensors = parsing_config._build_ragged_tensors(
serialized.shape, context_ragged_values, context_ragged_row_splits)
feature_list_ragged_tensors = parsing_config._build_ragged_tensors(
serialized.shape, feature_list_ragged_values,
feature_list_ragged_outer_splits, feature_list_ragged_inner_splits)
else:
outputs = gen_parsing_ops.parse_sequence_example(
serialized=serialized,
debug_name=debug_name,
Ncontext_sparse=len(context.sparse_keys),
Ncontext_dense=len(context.dense_keys),
Nfeature_list_sparse=len(feature_list.sparse_keys),
Nfeature_list_dense=len(feature_list.dense_keys),
context_dense_defaults=context.dense_defaults_vec,
context_sparse_keys=context.sparse_keys,
context_sparse_types=context.sparse_types,
context_dense_keys=context.dense_keys,
context_dense_shapes=context.dense_shapes_as_proto,
feature_list_sparse_keys=feature_list.sparse_keys,
feature_list_sparse_types=feature_list.sparse_types,
feature_list_dense_keys=feature_list.dense_keys,
feature_list_dense_types=feature_list.dense_types,
feature_list_dense_shapes=feature_list.dense_shapes,
feature_list_dense_missing_assumed_empty=(
feature_list_dense_missing_assumed_empty),
name=name)
(context_sparse_indices, context_sparse_values, context_sparse_shapes,
context_dense_values, feature_list_sparse_indices,
feature_list_sparse_values, feature_list_sparse_shapes,
feature_list_dense_values, feature_list_dense_lengths) = outputs
(context_sparse_indices, context_sparse_values, context_sparse_shapes,
context_dense_values, feature_list_sparse_indices,
feature_list_sparse_values, feature_list_sparse_shapes,
feature_list_dense_values, feature_list_dense_lengths) = outputs
context_ragged_tensors = []
feature_list_ragged_tensors = []
context_sparse_tensors = [
sparse_tensor.SparseTensor(ix, val, shape)
@ -713,19 +780,21 @@ def _parse_sequence_example_raw(serialized,
]
context_output = dict(
zip(context.sparse_keys + context.dense_keys,
context_sparse_tensors + context_dense_values))
zip(
context.sparse_keys + context.dense_keys + context.ragged_keys,
context_sparse_tensors + context_dense_values +
context_ragged_tensors))
feature_list_output = dict(
zip(feature_list.sparse_keys + feature_list.dense_keys,
feature_list_sparse_tensors + feature_list_dense_values))
zip(
feature_list.sparse_keys + feature_list.dense_keys +
feature_list.ragged_keys, feature_list_sparse_tensors +
feature_list_dense_values + feature_list_ragged_tensors))
feature_list_lengths = dict(
zip(feature_list.dense_keys, feature_list_dense_lengths))
return (context_output, feature_list_output, feature_list_lengths)
# TODO(sundberg): rewrite this method to call the batch version, which is more
# efficient especially for large inputs.
@tf_export("io.parse_single_sequence_example",
v1=["io.parse_single_sequence_example",
"parse_single_sequence_example"])
@ -755,17 +824,19 @@ def parse_single_sequence_example(
of `sequence_features` values may vary between `SequenceExample` protos,
and even between `feature_list` keys within the same `SequenceExample`.
`context_features` contains `VarLenFeature` and `FixedLenFeature` objects.
Each `VarLenFeature` is mapped to a `SparseTensor`, and each `FixedLenFeature`
`context_features` contains `VarLenFeature`, `RaggedFeature`, and
`FixedLenFeature` objects. Each `VarLenFeature` is mapped to a `SparseTensor`;
each `RaggedFeature` is mapped to a `RaggedTensor`; and each `FixedLenFeature`
is mapped to a `Tensor`, of the specified type, shape, and default value.
`sequence_features` contains `VarLenFeature` and `FixedLenSequenceFeature`
objects. Each `VarLenFeature` is mapped to a `SparseTensor`, and each
`sequence_features` contains `VarLenFeature`, `RaggedFeature`, and
`FixedLenSequenceFeature` objects. Each `VarLenFeature` is mapped to a
`SparseTensor`; each `RaggedFeature` is mapped to a `RaggedTensor`; and each
`FixedLenSequenceFeature` is mapped to a `Tensor`, each of the specified type.
The shape will be `(T,) + df.dense_shape` for `FixedLenSequenceFeature` `df`, where
`T` is the length of the associated `FeatureList` in the `SequenceExample`.
For instance, `FixedLenSequenceFeature([])` yields a scalar 1-D `Tensor` of
static shape `[None]` and dynamic shape `[T]`, while
The shape will be `(T,) + df.dense_shape` for `FixedLenSequenceFeature` `df`,
where `T` is the length of the associated `FeatureList` in the
`SequenceExample`. For instance, `FixedLenSequenceFeature([])` yields a scalar
1-D `Tensor` of static shape `[None]` and dynamic shape `[T]`, while
`FixedLenSequenceFeature([k])` (for `int k >= 1`) yields a 2-D matrix `Tensor`
of static shape `[None, k]` and dynamic shape `[T, k]`.
@ -790,20 +861,22 @@ def parse_single_sequence_example(
serialized: A scalar (0-D Tensor) of type string, a single binary
serialized `SequenceExample` proto.
context_features: A `dict` mapping feature keys to `FixedLenFeature` or
`VarLenFeature` values. These features are associated with a
`SequenceExample` as a whole.
`VarLenFeature` or `RaggedFeature` values. These features are associated
with a `SequenceExample` as a whole.
sequence_features: A `dict` mapping feature keys to
`FixedLenSequenceFeature` or `VarLenFeature` values. These features are
associated with data within the `FeatureList` section of the
`SequenceExample` proto.
`FixedLenSequenceFeature` or `VarLenFeature` or `RaggedFeature` values.
These features are associated with data within the `FeatureList` section
of the `SequenceExample` proto.
example_name: A scalar (0-D Tensor) of strings (optional), the name of
the serialized proto.
name: A name for this operation (optional).
Returns:
A tuple of two `dict`s, each mapping keys to `Tensor`s and `SparseTensor`s.
The first dict contains the context key/values.
The second dict contains the feature_list key/values.
A tuple of two `dict`s, each mapping keys to `Tensor`s and `SparseTensor`s
and `RaggedTensor`s.
* The first dict contains the context key/values.
* The second dict contains the feature_list key/values.
Raises:
ValueError: if any feature is invalid.
@ -812,13 +885,26 @@ def parse_single_sequence_example(
if not (context_features or sequence_features):
raise ValueError("Missing features.")
context_params = _ParseOpParams.from_features(
context_features, [VarLenFeature, FixedLenFeature])
context_features, [VarLenFeature, FixedLenFeature, RaggedFeature])
feature_list_params = _ParseOpParams.from_features(
sequence_features, [VarLenFeature, FixedLenSequenceFeature])
sequence_features,
[VarLenFeature, FixedLenSequenceFeature, RaggedFeature])
return _parse_single_sequence_example_raw(serialized, context_params,
feature_list_params, example_name,
name)
with ops.name_scope(name, "ParseSingleSequenceExample",
[serialized, example_name]):
context_output, feature_list_output = (
_parse_single_sequence_example_raw(serialized, context_params,
feature_list_params, example_name,
name))
if context_params.ragged_keys:
context_output = _construct_tensors_for_composite_features(
context_features, context_output)
if feature_list_params.ragged_keys:
feature_list_output = _construct_tensors_for_composite_features(
sequence_features, feature_list_output)
return context_output, feature_list_output
def _parse_single_sequence_example_raw(serialized,
@ -847,6 +933,14 @@ def _parse_single_sequence_example_raw(serialized,
Raises:
TypeError: if feature_list.dense_defaults is not either None or a dict.
"""
has_ragged = context.ragged_keys or feature_list.ragged_keys
if compat.forward_compatible(2019, 10, 26) or has_ragged:
with ops.name_scope(name, "ParseSingleExample", [serialized, debug_name]):
serialized = ops.convert_to_tensor(serialized, name="serialized")
serialized = _assert_scalar(serialized, "serialized")
return _parse_sequence_example_raw(serialized, debug_name, context,
feature_list, name)[:2]
if context.num_features + feature_list.num_features == 0:
raise ValueError("Must provide at least one feature key")
with ops.name_scope(name, "ParseSingleSequenceExample", [serialized]):
@ -1214,9 +1308,11 @@ def _assert_scalar(value, name):
math_ops.equal(array_ops.rank(value), 0),
["Input %s must be a scalar" % name],
name="%sIsScalar" % name.capitalize())
return control_flow_ops.with_dependencies([check],
value,
name="%sDependencies" % name)
result = control_flow_ops.with_dependencies([check],
value,
name="%sDependencies" % name)
result.set_shape([])
return result
elif value_rank == 0:
return value
else:

4
tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt
View File

@ -2556,6 +2556,10 @@ tf_module {
name: "ParseSequenceExample"
argspec: "args=[\'serialized\', \'debug_name\', \'context_dense_defaults\', \'feature_list_dense_missing_assumed_empty\', \'context_sparse_keys\', \'context_dense_keys\', \'feature_list_sparse_keys\', \'feature_list_dense_keys\', \'Ncontext_sparse\', \'Ncontext_dense\', \'Nfeature_list_sparse\', \'Nfeature_list_dense\', \'context_sparse_types\', \'feature_list_dense_types\', \'context_dense_shapes\', \'feature_list_sparse_types\', \'feature_list_dense_shapes\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'0\', \'0\', \'0\', \'[]\', \'[]\', \'[]\', \'[]\', \'[]\', \'None\'], "
}
member_method {
name: "ParseSequenceExampleV2"
argspec: "args=[\'serialized\', \'debug_name\', \'context_sparse_keys\', \'context_dense_keys\', \'context_ragged_keys\', \'feature_list_sparse_keys\', \'feature_list_dense_keys\', \'feature_list_ragged_keys\', \'feature_list_dense_missing_assumed_empty\', \'context_dense_defaults\', \'Ncontext_sparse\', \'context_sparse_types\', \'context_ragged_value_types\', \'context_ragged_split_types\', \'context_dense_shapes\', \'Nfeature_list_sparse\', \'Nfeature_list_dense\', \'feature_list_dense_types\', \'feature_list_sparse_types\', \'feature_list_ragged_value_types\', \'feature_list_ragged_split_types\', \'feature_list_dense_shapes\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'[]\', \'[]\', \'[]\', \'[]\', \'0\', \'0\', \'[]\', \'[]\', \'[]\', \'[]\', \'[]\', \'None\'], "
}
member_method {
name: "ParseSingleExample"
argspec: "args=[\'serialized\', \'dense_defaults\', \'num_sparse\', \'sparse_keys\', \'dense_keys\', \'sparse_types\', \'dense_shapes\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "

4
tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt
View File

@ -2556,6 +2556,10 @@ tf_module {
name: "ParseSequenceExample"
argspec: "args=[\'serialized\', \'debug_name\', \'context_dense_defaults\', \'feature_list_dense_missing_assumed_empty\', \'context_sparse_keys\', \'context_dense_keys\', \'feature_list_sparse_keys\', \'feature_list_dense_keys\', \'Ncontext_sparse\', \'Ncontext_dense\', \'Nfeature_list_sparse\', \'Nfeature_list_dense\', \'context_sparse_types\', \'feature_list_dense_types\', \'context_dense_shapes\', \'feature_list_sparse_types\', \'feature_list_dense_shapes\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'0\', \'0\', \'0\', \'[]\', \'[]\', \'[]\', \'[]\', \'[]\', \'None\'], "
}
member_method {
name: "ParseSequenceExampleV2"
argspec: "args=[\'serialized\', \'debug_name\', \'context_sparse_keys\', \'context_dense_keys\', \'context_ragged_keys\', \'feature_list_sparse_keys\', \'feature_list_dense_keys\', \'feature_list_ragged_keys\', \'feature_list_dense_missing_assumed_empty\', \'context_dense_defaults\', \'Ncontext_sparse\', \'context_sparse_types\', \'context_ragged_value_types\', \'context_ragged_split_types\', \'context_dense_shapes\', \'Nfeature_list_sparse\', \'Nfeature_list_dense\', \'feature_list_dense_types\', \'feature_list_sparse_types\', \'feature_list_ragged_value_types\', \'feature_list_ragged_split_types\', \'feature_list_dense_shapes\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'[]\', \'[]\', \'[]\', \'[]\', \'0\', \'0\', \'[]\', \'[]\', \'[]\', \'[]\', \'[]\', \'None\'], "
}
member_method {
name: "ParseSingleExample"
argspec: "args=[\'serialized\', \'dense_defaults\', \'num_sparse\', \'sparse_keys\', \'dense_keys\', \'sparse_types\', \'dense_shapes\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "