experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Clean up expired forward compatibility checks.

Browse Source 
PiperOrigin-RevId: 308455990
Change-Id: Icb38f1296a4c0326c3146e36e0e23a8dfaa37695
This commit is contained in:
Andrew Audibert 2020-04-25 19:14:41 -07:00 committed by TensorFlower Gardener
parent 2f55b904b5
commit 3b58a7de85
5 changed files with 34 additions and 84 deletions
tensorflow/python/data
experimental
kernel_tests/optimization
inject_prefetch_test.pymap_vectorization_test.pynoop_elimination_test.py
ops
parsing_ops.py
ops
readers.py

19
tensorflow/python/data/experimental/kernel_tests/optimization/inject_prefetch_test.py
View File

@ -19,7 +19,6 @@ from __future__ import print_function
from absl.testing import parameterized
from tensorflow.python.compat import compat
from tensorflow.python.data.experimental.ops import testing
from tensorflow.python.data.kernel_tests import test_base
from tensorflow.python.data.ops import dataset_ops
@ -37,11 +36,8 @@ class InjectPrefetchTest(test_base.DatasetTestBase, parameterized.TestCase):
@combinations.generate(test_base.default_test_combinations())
def testParallelMap(self):
dataset = dataset_ops.Dataset.range(100)
parallel_map = "ParallelMap"
if compat.forward_compatible(2020, 3, 6):
parallel_map = "ParallelMapV2"
dataset = dataset.apply(
testing.assert_next([parallel_map, "Prefetch", "FiniteTake"]))
testing.assert_next(["ParallelMapV2", "Prefetch", "FiniteTake"]))
dataset = dataset.map(
lambda x: x + 1, num_parallel_calls=dataset_ops.AUTOTUNE)
dataset = dataset.take(50)
@ -64,11 +60,8 @@ class InjectPrefetchTest(test_base.DatasetTestBase, parameterized.TestCase):
@combinations.generate(test_base.default_test_combinations())
def testParallelInterleave(self):
dataset = dataset_ops.Dataset.range(100)
parallel_interleave = "ParallelInterleaveV3"
if compat.forward_compatible(2020, 3, 6):
parallel_interleave = "ParallelInterleaveV4"
dataset = dataset.apply(
testing.assert_next([parallel_interleave, "Prefetch", "FiniteTake"]))
testing.assert_next(["ParallelInterleaveV4", "Prefetch", "FiniteTake"]))
dataset = dataset.interleave(
lambda x: dataset_ops.Dataset.from_tensors(x + 1),
num_parallel_calls=dataset_ops.AUTOTUNE)
@ -79,15 +72,9 @@ class InjectPrefetchTest(test_base.DatasetTestBase, parameterized.TestCase):
@combinations.generate(test_base.default_test_combinations())
def testChainedParallelDatasets(self):
dataset = dataset_ops.Dataset.range(100)
parallel_interleave = "ParallelInterleaveV3"
if compat.forward_compatible(2020, 3, 6):
parallel_interleave = "ParallelInterleaveV4"
parallel_map = "ParallelMap"
if compat.forward_compatible(2020, 3, 6):
parallel_map = "ParallelMapV2"
dataset = dataset.apply(
testing.assert_next([
parallel_map, "Prefetch", parallel_interleave, "Prefetch",
"ParallelMapV2", "Prefetch", "ParallelInterleaveV4", "Prefetch",
"MapAndBatch", "Prefetch", "FiniteTake"
]))
dataset = dataset.map(

5
tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
View File

@ -25,7 +25,6 @@ import numpy as np
from tensorflow.core.example import example_pb2
from tensorflow.core.example import feature_pb2
from tensorflow.python.compat import compat
from tensorflow.python.data.experimental.ops import batching
from tensorflow.python.data.experimental.ops import testing
from tensorflow.python.data.kernel_tests import test_base
@ -222,9 +221,7 @@ class MapVectorizationTest(test_base.DatasetTestBase, parameterized.TestCase):
"""
map_node_name = "Map"
if num_parallel_calls is not None:
map_node_name = "ParallelMap"
if compat.forward_compatible(2020, 3, 6):
map_node_name = "ParallelMapV2"
map_node_name = "ParallelMapV2"
def _make_dataset(node_names):
dataset = base_dataset.apply(testing.assert_next(node_names))

4
tensorflow/python/data/experimental/kernel_tests/optimization/noop_elimination_test.py
View File

@ -21,7 +21,6 @@ import functools
from absl.testing import parameterized
from tensorflow.python.compat import compat
from tensorflow.python.data.experimental.ops import testing
from tensorflow.python.data.kernel_tests import test_base
from tensorflow.python.data.ops import dataset_ops
@ -51,8 +50,7 @@ def _test_combinations():
ds = ds.map(lambda x: (x, x), num_parallel_calls=2) # Not eliminated
return ds.map(lambda x, y: (x, y)) # Eliminated
parallel_map_name = "ParallelMapV2" if compat.forward_compatible(
2020, 3, 6) else "ParallelMap"
parallel_map_name = "ParallelMapV2"
cases = [
("Skip0", lambda ds: ds.skip(0), None),

44
tensorflow/python/data/experimental/ops/parsing_ops.py
View File

@ -17,7 +17,6 @@ from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from tensorflow.python.compat import compat
from tensorflow.python.data.ops import dataset_ops
from tensorflow.python.data.util import structure
from tensorflow.python.framework import dtypes
@ -85,35 +84,20 @@ class _ParseExampleDataset(dataset_ops.UnaryDataset):
self._element_spec[key] = ragged_tensor.RaggedTensorSpec(
input_dataset_shape.concatenate([None]), value_type, 1, splits_type)
if deterministic is not None or compat.forward_compatible(2020, 3, 6):
variant_tensor = (
gen_experimental_dataset_ops.parse_example_dataset_v2(
self._input_dataset._variant_tensor, # pylint: disable=protected-access
self._num_parallel_calls,
self._dense_defaults,
self._sparse_keys,
self._dense_keys,
self._sparse_types,
self._dense_shapes,
deterministic=self._deterministic,
ragged_keys=self._ragged_keys,
ragged_value_types=self._ragged_value_types,
ragged_split_types=self._ragged_split_types,
**self._flat_structure))
else:
variant_tensor = (
gen_experimental_dataset_ops.parse_example_dataset(
self._input_dataset._variant_tensor, # pylint: disable=protected-access
self._num_parallel_calls,
self._dense_defaults,
self._sparse_keys,
self._dense_keys,
self._sparse_types,
self._dense_shapes,
ragged_keys=self._ragged_keys,
ragged_value_types=self._ragged_value_types,
ragged_split_types=self._ragged_split_types,
**self._flat_structure))
variant_tensor = (
gen_experimental_dataset_ops.parse_example_dataset_v2(
self._input_dataset._variant_tensor, # pylint: disable=protected-access
self._num_parallel_calls,
self._dense_defaults,
self._sparse_keys,
self._dense_keys,
self._sparse_types,
self._dense_shapes,
deterministic=self._deterministic,
ragged_keys=self._ragged_keys,
ragged_value_types=self._ragged_value_types,
ragged_split_types=self._ragged_split_types,
**self._flat_structure))
super(_ParseExampleDataset, self).__init__(input_dataset, variant_tensor)
@property

46
tensorflow/python/data/ops/readers.py
View File

@ -18,7 +18,6 @@ from __future__ import division
from __future__ import print_function
from tensorflow.python import tf2
from tensorflow.python.compat import compat
from tensorflow.python.data.ops import dataset_ops
from tensorflow.python.data.util import convert
from tensorflow.python.framework import dtypes
@ -249,9 +248,6 @@ class ParallelInterleaveDataset(dataset_ops.UnaryDataset):
cycle_length, dtype=dtypes.int64, name="cycle_length")
self._block_length = ops.convert_to_tensor(
block_length, dtype=dtypes.int64, name="block_length")
if sloppy is not None:
self._sloppy = ops.convert_to_tensor(
sloppy, dtype=dtypes.bool, name="sloppy")
self._buffer_output_elements = convert.optional_param_to_tensor(
"buffer_output_elements",
buffer_output_elements,
@ -260,34 +256,22 @@ class ParallelInterleaveDataset(dataset_ops.UnaryDataset):
"prefetch_input_elements",
prefetch_input_elements,
argument_default=2 * cycle_length)
if sloppy is None or compat.forward_compatible(2020, 3, 6):
if sloppy is None:
self._deterministic = "default"
elif sloppy:
self._deterministic = "false"
else:
self._deterministic = "true"
variant_tensor = ged_ops.legacy_parallel_interleave_dataset_v2(
self._input_dataset._variant_tensor, # pylint: disable=protected-access
self._map_func.function.captured_inputs,
self._cycle_length,
self._block_length,
self._buffer_output_elements,
self._prefetch_input_elements,
f=self._map_func.function,
deterministic=self._deterministic,
**self._flat_structure)
if sloppy is None:
self._deterministic = "default"
elif sloppy:
self._deterministic = "false"
else:
variant_tensor = ged_ops.parallel_interleave_dataset(
self._input_dataset._variant_tensor, # pylint: disable=protected-access
self._map_func.function.captured_inputs,
self._cycle_length,
self._block_length,
self._sloppy,
self._buffer_output_elements,
self._prefetch_input_elements,
f=self._map_func.function,
**self._flat_structure)
self._deterministic = "true"
variant_tensor = ged_ops.legacy_parallel_interleave_dataset_v2(
self._input_dataset._variant_tensor, # pylint: disable=protected-access
self._map_func.function.captured_inputs,
self._cycle_length,
self._block_length,
self._buffer_output_elements,
self._prefetch_input_elements,
f=self._map_func.function,
deterministic=self._deterministic,
**self._flat_structure)
super(ParallelInterleaveDataset, self).__init__(input_dataset,
variant_tensor)