Adding tf_export bindings for the tf.lookup API. Adds tf.lookup.StaticHashTable, tf.lookup.VocabularyTable, tf.lookup.experimental.DenseHashTable, tf.lookup.TextFileInitializer and tf.lookup.KeyValueTensorInitializer

PiperOrigin-RevId: 235990023
This commit is contained in:
Rohan Jain 2019-02-27 14:14:23 -08:00 committed by TensorFlower Gardener
parent 104098d46b
commit 6adc4f81d0
21 changed files with 839 additions and 385 deletions

View File

@ -30,6 +30,7 @@ from tensorflow.python.ops.lookup_ops import IdTableWithHashBuckets
from tensorflow.python.ops.lookup_ops import index_table_from_file
from tensorflow.python.ops.lookup_ops import index_to_string_table_from_file
from tensorflow.python.ops.lookup_ops import InitializableLookupTableBase
from tensorflow.python.ops.lookup_ops import InitializableLookupTableBaseV1
from tensorflow.python.ops.lookup_ops import KeyValueTensorInitializer
from tensorflow.python.ops.lookup_ops import LookupInterface
from tensorflow.python.ops.lookup_ops import StrongHashSpec
@ -284,7 +285,7 @@ def index_to_string(tensor, mapping, default_value="UNK", name=None):
return table.lookup(tensor)
class HashTable(InitializableLookupTableBase):
class HashTable(InitializableLookupTableBaseV1):
"""A generic hash table implementation.
Example usage:

File diff suppressed because it is too large Load Diff

View File

@ -174,15 +174,6 @@ class InitializableLookupTableBase(LookupInterface):
def _initialize(self):
return self._initializer.initialize(self)
@property
def initializer(self):
return self._init_op
@property
@deprecated("2018-12-15", "Use `initializer` instead.")
def init(self):
return self.initializer
@property
def default_value(self):
"""The default value of the table."""
@ -237,6 +228,14 @@ class InitializableLookupTableBase(LookupInterface):
return values
class InitializableLookupTableBaseV1(InitializableLookupTableBase):
@property
def initializer(self):
return self._init_op
@tf_export("lookup.StaticHashTable", v1=[])
class StaticHashTable(InitializableLookupTableBase):
"""A generic hash table implementation.
@ -311,8 +310,20 @@ class StaticHashTable(InitializableLookupTableBase):
return exported_keys, exported_values
@tf_export(v1=["lookup.StaticHashTable"])
class StaticHashTableV1(StaticHashTable):
@property
def initializer(self):
return self._init_op
# For backwards compatibility. This will be removed in TF 2.0.
HashTable = StaticHashTable
class HashTable(StaticHashTableV1):
@property
def init(self):
return self.initializer
class TableInitializerBase(trackable_base.Trackable):
@ -354,6 +365,7 @@ class TableInitializerBase(trackable_base.Trackable):
return shared_name
@tf_export("lookup.KeyValueTensorInitializer")
class KeyValueTensorInitializer(TableInitializerBase):
"""Table initializers given `keys` and `values` tensors."""
@ -412,6 +424,7 @@ class TextFileIndex(object):
LINE_NUMBER = -1
@tf_export("lookup.TextFileInitializer")
class TextFileInitializer(TableInitializerBase):
"""Table initializers from a text file.
@ -951,6 +964,7 @@ class IdTableWithHashBuckets(LookupInterface):
return ids
@tf_export("lookup.StaticVocabularyTable", v1=[])
class StaticVocabularyTable(LookupInterface):
"""String to Id table wrapper that assigns out-of-vocabulary keys to buckets.
@ -1063,18 +1077,6 @@ class StaticVocabularyTable(LookupInterface):
with ops.name_scope(None, "init"):
return control_flow_ops.no_op()
@property
def initializer(self):
if self._table is not None:
return self._table._init_op # pylint: disable=protected-access
with ops.name_scope(None, "init"):
return control_flow_ops.no_op()
@property
@deprecated("2018-12-15", "Use `initializer` instead.")
def init(self):
return self.initializer
@property
def resource_handle(self):
if self._table is not None:
@ -1136,6 +1138,17 @@ class StaticVocabularyTable(LookupInterface):
return ids
@tf_export(v1=["lookup.StaticVocabularyTable"])
class StaticVocabularyTableV1(StaticVocabularyTable):
@property
def initializer(self):
if self._table is not None:
return self._table._init_op # pylint: disable=protected-access
with ops.name_scope(None, "init"):
return control_flow_ops.no_op()
def index_table_from_file(vocabulary_file=None,
num_oov_buckets=0,
vocab_size=None,
@ -1244,7 +1257,7 @@ def index_table_from_file(vocabulary_file=None,
value_column_index=value_column_index,
delimiter=delimiter)
table = StaticHashTable(init, default_value)
table = StaticHashTableV1(init, default_value)
if num_oov_buckets:
table = IdTableWithHashBuckets(
table,
@ -1341,7 +1354,7 @@ def index_table_from_tensor(vocabulary_list,
table_keys.dtype.base_dtype,
dtypes.int64,
name="table_init")
table = StaticHashTable(init, default_value)
table = StaticHashTableV1(init, default_value)
if num_oov_buckets:
table = IdTableWithHashBuckets(
table,
@ -1438,7 +1451,7 @@ def index_to_string_table_from_file(vocabulary_file,
delimiter=delimiter)
# TODO(yleon): Use a more effienct structure.
return StaticHashTable(init, default_value)
return StaticHashTableV1(init, default_value)
def index_to_string_table_from_tensor(vocabulary_list,
@ -1499,7 +1512,7 @@ def index_to_string_table_from_tensor(vocabulary_list,
init = KeyValueTensorInitializer(
keys, vocabulary_list, dtypes.int64, dtypes.string, name="table_init")
# TODO(yleon): Use a more effienct structure.
return StaticHashTable(init, default_value)
return StaticHashTableV1(init, default_value)
class MutableHashTable(LookupInterface):
@ -1733,6 +1746,7 @@ class MutableHashTable(LookupInterface):
self.op.resource_handle, restored_tensors[0], restored_tensors[1])
@tf_export("lookup.experimental.DenseHashTable")
class DenseHashTable(LookupInterface):
"""A generic mutable hash table implementation using tensors as backing store.

View File

@ -30,6 +30,8 @@ TENSORFLOW_API_INIT_FILES = [
"lite/constants/__init__.py",
"lite/experimental/__init__.py",
"lite/experimental/nn/__init__.py",
"lookup/__init__.py",
"lookup/experimental/__init__.py",
"math/__init__.py",
"nest/__init__.py",
"nn/__init__.py",

View File

@ -37,6 +37,8 @@ TENSORFLOW_API_INIT_FILES_V1 = [
"lite/experimental/__init__.py",
"lite/experimental/nn/__init__.py",
"logging/__init__.py",
"lookup/__init__.py",
"lookup/experimental/__init__.py",
"losses/__init__.py",
"manip/__init__.py",
"math/__init__.py",

View File

@ -0,0 +1,23 @@
path: "tensorflow.lookup.KeyValueTensorInitializer"
tf_class {
is_instance: "<class \'tensorflow.python.ops.lookup_ops.KeyValueTensorInitializer\'>"
is_instance: "<class \'tensorflow.python.ops.lookup_ops.TableInitializerBase\'>"
is_instance: "<class \'tensorflow.python.training.tracking.base.Trackable\'>"
is_instance: "<type \'object\'>"
member {
name: "key_dtype"
mtype: "<type \'property\'>"
}
member {
name: "value_dtype"
mtype: "<type \'property\'>"
}
member_method {
name: "__init__"
argspec: "args=[\'self\', \'keys\', \'values\', \'key_dtype\', \'value_dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
}
member_method {
name: "initialize"
argspec: "args=[\'self\', \'table\'], varargs=None, keywords=None, defaults=None"
}
}

View File

@ -0,0 +1,50 @@
path: "tensorflow.lookup.StaticHashTable"
tf_class {
is_instance: "<class \'tensorflow.python.ops.lookup_ops.StaticHashTableV1\'>"
is_instance: "<class \'tensorflow.python.ops.lookup_ops.StaticHashTable\'>"
is_instance: "<class \'tensorflow.python.ops.lookup_ops.InitializableLookupTableBase\'>"
is_instance: "<class \'tensorflow.python.ops.lookup_ops.LookupInterface\'>"
is_instance: "<class \'tensorflow.python.training.tracking.tracking.TrackableResource\'>"
is_instance: "<class \'tensorflow.python.training.tracking.base.Trackable\'>"
is_instance: "<type \'object\'>"
member {
name: "default_value"
mtype: "<type \'property\'>"
}
member {
name: "initializer"
mtype: "<type \'property\'>"
}
member {
name: "key_dtype"
mtype: "<type \'property\'>"
}
member {
name: "name"
mtype: "<type \'property\'>"
}
member {
name: "resource_handle"
mtype: "<type \'property\'>"
}
member {
name: "value_dtype"
mtype: "<type \'property\'>"
}
member_method {
name: "__init__"
argspec: "args=[\'self\', \'initializer\', \'default_value\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
}
member_method {
name: "export"
argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
}
member_method {
name: "lookup"
argspec: "args=[\'self\', \'keys\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
}
member_method {
name: "size"
argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
}
}

View File

@ -0,0 +1,41 @@
path: "tensorflow.lookup.StaticVocabularyTable"
tf_class {
is_instance: "<class \'tensorflow.python.ops.lookup_ops.StaticVocabularyTableV1\'>"
is_instance: "<class \'tensorflow.python.ops.lookup_ops.StaticVocabularyTable\'>"
is_instance: "<class \'tensorflow.python.ops.lookup_ops.LookupInterface\'>"
is_instance: "<class \'tensorflow.python.training.tracking.tracking.TrackableResource\'>"
is_instance: "<class \'tensorflow.python.training.tracking.base.Trackable\'>"
is_instance: "<type \'object\'>"
member {
name: "initializer"
mtype: "<type \'property\'>"
}
member {
name: "key_dtype"
mtype: "<type \'property\'>"
}
member {
name: "name"
mtype: "<type \'property\'>"
}
member {
name: "resource_handle"
mtype: "<type \'property\'>"
}
member {
name: "value_dtype"
mtype: "<type \'property\'>"
}
member_method {
name: "__init__"
argspec: "args=[\'self\', \'initializer\', \'num_oov_buckets\', \'lookup_key_dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
}
member_method {
name: "lookup"
argspec: "args=[\'self\', \'keys\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
}
member_method {
name: "size"
argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
}
}

View File

@ -0,0 +1,23 @@
path: "tensorflow.lookup.TextFileInitializer"
tf_class {
is_instance: "<class \'tensorflow.python.ops.lookup_ops.TextFileInitializer\'>"
is_instance: "<class \'tensorflow.python.ops.lookup_ops.TableInitializerBase\'>"
is_instance: "<class \'tensorflow.python.training.tracking.base.Trackable\'>"
is_instance: "<type \'object\'>"
member {
name: "key_dtype"
mtype: "<type \'property\'>"
}
member {
name: "value_dtype"
mtype: "<type \'property\'>"
}
member_method {
name: "__init__"
argspec: "args=[\'self\', \'filename\', \'key_dtype\', \'key_index\', \'value_dtype\', \'value_index\', \'vocab_size\', \'delimiter\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'\\t\', \'None\'], "
}
member_method {
name: "initialize"
argspec: "args=[\'self\', \'table\'], varargs=None, keywords=None, defaults=None"
}
}

View File

@ -0,0 +1,56 @@
path: "tensorflow.lookup.experimental.DenseHashTable"
tf_class {
is_instance: "<class \'tensorflow.python.ops.lookup_ops.DenseHashTable\'>"
is_instance: "<class \'tensorflow.python.ops.lookup_ops.LookupInterface\'>"
is_instance: "<class \'tensorflow.python.training.tracking.tracking.TrackableResource\'>"
is_instance: "<class \'tensorflow.python.training.tracking.base.Trackable\'>"
is_instance: "<type \'object\'>"
member {
name: "key_dtype"
mtype: "<type \'property\'>"
}
member {
name: "name"
mtype: "<type \'property\'>"
}
member {
name: "resource_handle"
mtype: "<type \'property\'>"
}
member {
name: "value_dtype"
mtype: "<type \'property\'>"
}
member_method {
name: "__init__"
argspec: "args=[\'self\', \'key_dtype\', \'value_dtype\', \'default_value\', \'empty_key\', \'deleted_key\', \'initial_num_buckets\', \'name\', \'checkpoint\'], varargs=None, keywords=None, defaults=[\'None\', \'MutableDenseHashTable\', \'True\'], "
}
member_method {
name: "erase"
argspec: "args=[\'self\', \'keys\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
}
member_method {
name: "export"
argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
}
member_method {
name: "insert"
argspec: "args=[\'self\', \'keys\', \'values\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
}
member_method {
name: "insert_or_assign"
argspec: "args=[\'self\', \'keys\', \'values\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
}
member_method {
name: "lookup"
argspec: "args=[\'self\', \'keys\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
}
member_method {
name: "remove"
argspec: "args=[\'self\', \'keys\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
}
member_method {
name: "size"
argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
}
}

View File

@ -0,0 +1,7 @@
path: "tensorflow.lookup.experimental"
tf_module {
member {
name: "DenseHashTable"
mtype: "<type \'type\'>"
}
}

View File

@ -0,0 +1,23 @@
path: "tensorflow.lookup"
tf_module {
member {
name: "KeyValueTensorInitializer"
mtype: "<type \'type\'>"
}
member {
name: "StaticHashTable"
mtype: "<type \'type\'>"
}
member {
name: "StaticVocabularyTable"
mtype: "<type \'type\'>"
}
member {
name: "TextFileInitializer"
mtype: "<type \'type\'>"
}
member {
name: "experimental"
mtype: "<type \'module\'>"
}
}

View File

@ -464,6 +464,10 @@ tf_module {
name: "logging"
mtype: "<type \'module\'>"
}
member {
name: "lookup"
mtype: "<type \'module\'>"
}
member {
name: "losses"
mtype: "<type \'module\'>"

View File

@ -0,0 +1,23 @@
path: "tensorflow.lookup.KeyValueTensorInitializer"
tf_class {
is_instance: "<class \'tensorflow.python.ops.lookup_ops.KeyValueTensorInitializer\'>"
is_instance: "<class \'tensorflow.python.ops.lookup_ops.TableInitializerBase\'>"
is_instance: "<class \'tensorflow.python.training.tracking.base.Trackable\'>"
is_instance: "<type \'object\'>"
member {
name: "key_dtype"
mtype: "<type \'property\'>"
}
member {
name: "value_dtype"
mtype: "<type \'property\'>"
}
member_method {
name: "__init__"
argspec: "args=[\'self\', \'keys\', \'values\', \'key_dtype\', \'value_dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
}
member_method {
name: "initialize"
argspec: "args=[\'self\', \'table\'], varargs=None, keywords=None, defaults=None"
}
}

View File

@ -0,0 +1,45 @@
path: "tensorflow.lookup.StaticHashTable"
tf_class {
is_instance: "<class \'tensorflow.python.ops.lookup_ops.StaticHashTable\'>"
is_instance: "<class \'tensorflow.python.ops.lookup_ops.InitializableLookupTableBase\'>"
is_instance: "<class \'tensorflow.python.ops.lookup_ops.LookupInterface\'>"
is_instance: "<class \'tensorflow.python.training.tracking.tracking.TrackableResource\'>"
is_instance: "<class \'tensorflow.python.training.tracking.base.Trackable\'>"
is_instance: "<type \'object\'>"
member {
name: "default_value"
mtype: "<type \'property\'>"
}
member {
name: "key_dtype"
mtype: "<type \'property\'>"
}
member {
name: "name"
mtype: "<type \'property\'>"
}
member {
name: "resource_handle"
mtype: "<type \'property\'>"
}
member {
name: "value_dtype"
mtype: "<type \'property\'>"
}
member_method {
name: "__init__"
argspec: "args=[\'self\', \'initializer\', \'default_value\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
}
member_method {
name: "export"
argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
}
member_method {
name: "lookup"
argspec: "args=[\'self\', \'keys\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
}
member_method {
name: "size"
argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
}
}

View File

@ -0,0 +1,36 @@
path: "tensorflow.lookup.StaticVocabularyTable"
tf_class {
is_instance: "<class \'tensorflow.python.ops.lookup_ops.StaticVocabularyTable\'>"
is_instance: "<class \'tensorflow.python.ops.lookup_ops.LookupInterface\'>"
is_instance: "<class \'tensorflow.python.training.tracking.tracking.TrackableResource\'>"
is_instance: "<class \'tensorflow.python.training.tracking.base.Trackable\'>"
is_instance: "<type \'object\'>"
member {
name: "key_dtype"
mtype: "<type \'property\'>"
}
member {
name: "name"
mtype: "<type \'property\'>"
}
member {
name: "resource_handle"
mtype: "<type \'property\'>"
}
member {
name: "value_dtype"
mtype: "<type \'property\'>"
}
member_method {
name: "__init__"
argspec: "args=[\'self\', \'initializer\', \'num_oov_buckets\', \'lookup_key_dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
}
member_method {
name: "lookup"
argspec: "args=[\'self\', \'keys\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
}
member_method {
name: "size"
argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
}
}

View File

@ -0,0 +1,23 @@
path: "tensorflow.lookup.TextFileInitializer"
tf_class {
is_instance: "<class \'tensorflow.python.ops.lookup_ops.TextFileInitializer\'>"
is_instance: "<class \'tensorflow.python.ops.lookup_ops.TableInitializerBase\'>"
is_instance: "<class \'tensorflow.python.training.tracking.base.Trackable\'>"
is_instance: "<type \'object\'>"
member {
name: "key_dtype"
mtype: "<type \'property\'>"
}
member {
name: "value_dtype"
mtype: "<type \'property\'>"
}
member_method {
name: "__init__"
argspec: "args=[\'self\', \'filename\', \'key_dtype\', \'key_index\', \'value_dtype\', \'value_index\', \'vocab_size\', \'delimiter\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'\\t\', \'None\'], "
}
member_method {
name: "initialize"
argspec: "args=[\'self\', \'table\'], varargs=None, keywords=None, defaults=None"
}
}

View File

@ -0,0 +1,56 @@
path: "tensorflow.lookup.experimental.DenseHashTable"
tf_class {
is_instance: "<class \'tensorflow.python.ops.lookup_ops.DenseHashTable\'>"
is_instance: "<class \'tensorflow.python.ops.lookup_ops.LookupInterface\'>"
is_instance: "<class \'tensorflow.python.training.tracking.tracking.TrackableResource\'>"
is_instance: "<class \'tensorflow.python.training.tracking.base.Trackable\'>"
is_instance: "<type \'object\'>"
member {
name: "key_dtype"
mtype: "<type \'property\'>"
}
member {
name: "name"
mtype: "<type \'property\'>"
}
member {
name: "resource_handle"
mtype: "<type \'property\'>"
}
member {
name: "value_dtype"
mtype: "<type \'property\'>"
}
member_method {
name: "__init__"
argspec: "args=[\'self\', \'key_dtype\', \'value_dtype\', \'default_value\', \'empty_key\', \'deleted_key\', \'initial_num_buckets\', \'name\', \'checkpoint\'], varargs=None, keywords=None, defaults=[\'None\', \'MutableDenseHashTable\', \'True\'], "
}
member_method {
name: "erase"
argspec: "args=[\'self\', \'keys\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
}
member_method {
name: "export"
argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
}
member_method {
name: "insert"
argspec: "args=[\'self\', \'keys\', \'values\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
}
member_method {
name: "insert_or_assign"
argspec: "args=[\'self\', \'keys\', \'values\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
}
member_method {
name: "lookup"
argspec: "args=[\'self\', \'keys\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
}
member_method {
name: "remove"
argspec: "args=[\'self\', \'keys\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
}
member_method {
name: "size"
argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
}
}

View File

@ -0,0 +1,7 @@
path: "tensorflow.lookup.experimental"
tf_module {
member {
name: "DenseHashTable"
mtype: "<type \'type\'>"
}
}

View File

@ -0,0 +1,23 @@
path: "tensorflow.lookup"
tf_module {
member {
name: "KeyValueTensorInitializer"
mtype: "<type \'type\'>"
}
member {
name: "StaticHashTable"
mtype: "<type \'type\'>"
}
member {
name: "StaticVocabularyTable"
mtype: "<type \'type\'>"
}
member {
name: "TextFileInitializer"
mtype: "<type \'type\'>"
}
member {
name: "experimental"
mtype: "<type \'module\'>"
}
}

View File

@ -212,6 +212,10 @@ tf_module {
name: "lite"
mtype: "<type \'module\'>"
}
member {
name: "lookup"
mtype: "<type \'module\'>"
}
member {
name: "losses"
mtype: "<type \'module\'>"