Move hash table op kernels out of experimental directory
PiperOrigin-RevId: 307999605 Change-Id: Ib59a2979db8f59c84f0c5c3ce6b25f320cd0e917
This commit is contained in:
Jaesung Chung
TensorFlower Gardener
parent
35e3d0f075
commit
cf9d79b432
@ -125,50 +125,3 @@ cc_test(
|
||||
"@com_google_googletest//:gtest",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "hashtable_op_kernels",
|
||||
srcs = [
|
||||
"hashtable.cc",
|
||||
"hashtable_find.cc",
|
||||
"hashtable_import.cc",
|
||||
"hashtable_ops.cc",
|
||||
"hashtable_size.cc",
|
||||
],
|
||||
hdrs = [
|
||||
"hashtable_ops.h",
|
||||
],
|
||||
deps = [
|
||||
"//tensorflow/lite:framework",
|
||||
"//tensorflow/lite/c:common",
|
||||
"//tensorflow/lite/core/api",
|
||||
"//tensorflow/lite/experimental/resource",
|
||||
"//tensorflow/lite/kernels:kernel_util",
|
||||
"//tensorflow/lite/kernels:op_macros",
|
||||
"//tensorflow/lite/kernels/internal:tensor",
|
||||
"//tensorflow/lite/schema:schema_fbs",
|
||||
"@flatbuffers",
|
||||
],
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "hashtable_op_test",
|
||||
size = "small",
|
||||
srcs = [
|
||||
"hashtable_ops_test.cc",
|
||||
],
|
||||
deps = [
|
||||
":hashtable_op_kernels", # buildcleaner: keep
|
||||
"//tensorflow/lite:framework",
|
||||
"//tensorflow/lite/core/api",
|
||||
"//tensorflow/lite/experimental/resource",
|
||||
"//tensorflow/lite/kernels:test_main",
|
||||
"//tensorflow/lite/kernels:test_util",
|
||||
"//tensorflow/lite/kernels/internal:tensor",
|
||||
"//tensorflow/lite/testing:util",
|
||||
"@com_google_absl//absl/memory",
|
||||
"@com_google_absl//absl/strings",
|
||||
"@com_google_googletest//:gtest",
|
||||
"@flatbuffers",
|
||||
],
|
||||
)
|
||||
|
||||
@ -1,20 +0,0 @@
|
||||
/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
%{
|
||||
#include "tensorflow/lite/experimental/kernels/hashtable_ops.h"
|
||||
%}
|
||||
|
||||
%include "tensorflow/lite/experimental/kernels/hashtable_ops.h"
|
||||
@ -600,7 +600,7 @@ cc_library(
|
||||
":op_macros",
|
||||
"//tensorflow/lite:context",
|
||||
"//tensorflow/lite/c:common",
|
||||
"//tensorflow/lite/experimental/kernels:hashtable_op_kernels",
|
||||
"//tensorflow/lite/kernels/hashtable:hashtable_op_kernels",
|
||||
"//tensorflow/lite/kernels/internal:kernel_utils",
|
||||
"//tensorflow/lite/kernels/internal:tensor",
|
||||
"//third_party/fft2d:fft2d_headers",
|
||||
|
||||
54
tensorflow/lite/kernels/hashtable/BUILD
Normal file
54
tensorflow/lite/kernels/hashtable/BUILD
Normal file
@ -0,0 +1,54 @@
|
||||
load("//tensorflow/lite/micro:build_def.bzl", "cc_library")
|
||||
|
||||
package(
|
||||
default_visibility = [
|
||||
"//visibility:public",
|
||||
],
|
||||
licenses = ["notice"], # Apache 2.0
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "hashtable_op_kernels",
|
||||
srcs = [
|
||||
"hashtable.cc",
|
||||
"hashtable_find.cc",
|
||||
"hashtable_import.cc",
|
||||
"hashtable_ops.cc",
|
||||
"hashtable_size.cc",
|
||||
],
|
||||
hdrs = [
|
||||
"hashtable_ops.h",
|
||||
],
|
||||
deps = [
|
||||
"//tensorflow/lite:framework",
|
||||
"//tensorflow/lite/c:common",
|
||||
"//tensorflow/lite/core/api",
|
||||
"//tensorflow/lite/experimental/resource",
|
||||
"//tensorflow/lite/kernels:kernel_util",
|
||||
"//tensorflow/lite/kernels:op_macros",
|
||||
"//tensorflow/lite/kernels/internal:tensor",
|
||||
"//tensorflow/lite/schema:schema_fbs",
|
||||
"@flatbuffers",
|
||||
],
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "hashtable_op_test",
|
||||
size = "small",
|
||||
srcs = [
|
||||
"hashtable_ops_test.cc",
|
||||
],
|
||||
deps = [
|
||||
":hashtable_op_kernels", # buildcleaner: keep
|
||||
"//tensorflow/lite:framework",
|
||||
"//tensorflow/lite/core/api",
|
||||
"//tensorflow/lite/experimental/resource",
|
||||
"//tensorflow/lite/kernels:test_main",
|
||||
"//tensorflow/lite/kernels:test_util",
|
||||
"//tensorflow/lite/kernels/internal:tensor",
|
||||
"//tensorflow/lite/testing:util",
|
||||
"@com_google_absl//absl/memory",
|
||||
"@com_google_absl//absl/strings",
|
||||
"@flatbuffers",
|
||||
],
|
||||
)
|
||||
190
tensorflow/lite/kernels/hashtable/README.md
Normal file
190
tensorflow/lite/kernels/hashtable/README.md
Normal file
@ -0,0 +1,190 @@
|
||||
# How to use TF Lookup ops in TFLite
|
||||
|
||||
The objective of this file is to provide examples to demonstrate how to use TF
|
||||
Lookup ops in TFLite.
|
||||
|
||||
## Supported Tensorflow Lookup ops in TFLite
|
||||
|
||||
Here is the supported status of TensorFlow Lookup ops.
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<td><strong><em>TF Python lookup ops</em></strong>
|
||||
</td>
|
||||
<td colspan="5" ><strong><em>Supported status</em></strong>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td rowspan="2" >tf.lookup.StaticHashTable
|
||||
</td>
|
||||
<td rowspan="2" colspan="5" >Supported only with tensor initializers.
|
||||
<p>
|
||||
Supported mapping type: string → int64, int64 → string
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
</tr>
|
||||
<tr>
|
||||
<td rowspan="2" >tf.lookup.Hashtable
|
||||
</td>
|
||||
<td rowspan="2" colspan="5" >Supported only with tensor initializers.
|
||||
<p>
|
||||
Supported mapping type: string → int64, int64 → string
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
</tr>
|
||||
<tr>
|
||||
<td rowspan="2" >tf.lookup.index_to_string_table_from_tensor
|
||||
</td>
|
||||
<td rowspan="2" colspan="5" >Supported.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
</tr>
|
||||
<tr>
|
||||
<td rowspan="2" >tf.lookup.index_table_from_tensor
|
||||
</td>
|
||||
<td rowspan="2" colspan="5" >Supported natively when num_oov_bukcets=0 and dtype=dtypes.string.
|
||||
<p>
|
||||
For the oov concept, you will need a <a href="https://www.tensorflow.org/lite/guide/ops_select" title="Select TensorFlow operators to use in TensorFlow Lite">Flex delegate</a>.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>tf.lookup.StaticVocabularyTable
|
||||
</td>
|
||||
<td colspan="5" >Supported but you will need a <a href="https://www.tensorflow.org/lite/guide/ops_select" title="Select TensorFlow operators to use in TensorFlow Lite">Flex delegate</a>.
|
||||
<p>
|
||||
Use tf.index_table_from_tensor or tf.index_to_string_table_from_tensor instead if possible if you don’t want to use <a href="https://www.tensorflow.org/lite/guide/ops_select" title="Select TensorFlow operators to use in TensorFlow Lite">Flex delegate</a>.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>tf.lookup.experimental.DenseHashTable
|
||||
<p>
|
||||
tf.contrib.lookup.MutableHashTable
|
||||
<p>
|
||||
tf.contrib.lookup.MutableDenseHashTable
|
||||
</td>
|
||||
<td colspan="5" >Not supported yet.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>tf.lookup.IdTableWithHashBuckets
|
||||
</td>
|
||||
<td colspan="5" >Supported but you need a <a href="https://www.tensorflow.org/lite/guide/ops_select" title="Select TensorFlow operators to use in TensorFlow Lite">Flex delegate</a>.
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
|
||||
## Python Sample code
|
||||
|
||||
Here, you can find the Python sample code:
|
||||
|
||||
|
||||
|
||||
* Static hash table (string → int64)
|
||||
|
||||
```
|
||||
int64_values = tf.constant([1, 2, 3], dtype=tf.int64)
|
||||
string_values = tf.constant(['bar', 'foo', 'baz'], dtype=tf.string)
|
||||
|
||||
initializer = tf.lookup.KeyValueTensorInitializer(string_values, int64_values)
|
||||
table = tf.lookup.StaticHashTable(initializer, 4)
|
||||
|
||||
with tf.control_dependencies([tf.initializers.tables_initializer()]):
|
||||
input_string_tensor = tf.compat.v1.placeholder(tf.string, shape=[1])
|
||||
out_int64_tensor = table.lookup(input_string_tensor)
|
||||
```
|
||||
|
||||
* Static hash table, initialized from a file (string → int64)
|
||||
|
||||
```
|
||||
with open('/tmp/vocab.file', 'r') as f:
|
||||
words = f.read().splitlines()
|
||||
|
||||
string_values = tf.constant(words, dtype=tf.string)
|
||||
|
||||
initializer = tf.lookup.KeyValueTensorInitializer(string_values, int64_values)
|
||||
table = tf.lookup.StaticHashTable(initializer, 4)
|
||||
|
||||
with tf.control_dependencies([tf.initializers.tables_initializer()]):
|
||||
input_string_tensor = tf.placeholder(tf.string, shape=[1])
|
||||
out_int64_tensor = table.lookup(input_string_tensor)
|
||||
```
|
||||
|
||||
* Index table (string → int64)
|
||||
|
||||
```
|
||||
UNK_ID = -1
|
||||
vocab = tf.constant(["emerson", "lake", "palmer"])
|
||||
vocab_table = tf.lookup.index_table_from_tensor(vocab, default_value=UNK_ID)
|
||||
|
||||
input_tensor = tf.compat.v1.placeholder(tf.string, shape=[5])
|
||||
|
||||
with tf.control_dependencies([tf.initializers.tables_initializer()]):
|
||||
out_tensor = vocab_table.lookup(input_tensor)
|
||||
```
|
||||
|
||||
* Index table, initialized from a file (string → int64)
|
||||
|
||||
```
|
||||
with open('/tmp/vocab.file', 'r') as f:
|
||||
words = f.read().splitlines()
|
||||
|
||||
UNK_ID = -1
|
||||
vocab = tf.constant(words)
|
||||
vocab_table = tf.lookup.index_table_from_tensor(vocab, default_value=UNK_ID)
|
||||
|
||||
input_tensor = tf.compat.v1.placeholder(tf.string, shape=[5])
|
||||
|
||||
with tf.control_dependencies([tf.initializers.tables_initializer()]):
|
||||
out_tensor = vocab_table.lookup(input_tensor)
|
||||
```
|
||||
|
||||
* Index to string table (int64 → string)
|
||||
|
||||
```
|
||||
UNK_WORD = "unknown"
|
||||
vocab = tf.constant(["emerson", "lake", "palmer"])
|
||||
vocab_table = tf.lookup.index_to_string_table_from_tensor(vocab, default_value=UNK_WORD)
|
||||
|
||||
input_tensor = tf.compat.v1.placeholder(tf.int64, shape=[1])
|
||||
|
||||
with tf.control_dependencies([tf.initializers.tables_initializer()]):
|
||||
out_tensor = vocab_table.lookup(input_tensor)
|
||||
```
|
||||
|
||||
* Index to string table, initialized from a file (int64 → string)
|
||||
|
||||
```
|
||||
with open('/tmp/vocab.file', 'r') as f:
|
||||
words = f.read().splitlines()
|
||||
|
||||
UNK_WORD = "unknown"
|
||||
vocab = tf.constant(words)
|
||||
vocab_table = tf.lookup.index_to_string_table_from_tensor(vocab, default_value=UNK_WORD)
|
||||
|
||||
input_tensor = tf.compat.v1.placeholder(tf.int64, shape=[1])
|
||||
|
||||
with tf.control_dependencies([tf.initializers.tables_initializer()]):
|
||||
out_tensor = vocab_table.lookup(input_tensor)
|
||||
```
|
||||
|
||||
## How to Include Hashtable ops in your TFLite.
|
||||
|
||||
Currently, hashtable ops are not included in the builtin op set. You need to add
|
||||
hashtable ops manually by including the following dependency:
|
||||
|
||||
`"//tensorflow/lite/kernels/hashtable:hashtable_op_kernels"`
|
||||
|
||||
And then, your op resolver should add them like the following statements:
|
||||
|
||||
|
||||
```
|
||||
// Add hashtable op handlers.
|
||||
tflite::ops::custom::AddHashtableOps(&resolver);
|
||||
```
|
||||
@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#include "tensorflow/lite/experimental/kernels/hashtable_ops.h"
|
||||
#include "tensorflow/lite/kernels/hashtable/hashtable_ops.h"
|
||||
|
||||
namespace tflite {
|
||||
namespace ops {
|
||||
@ -13,8 +13,8 @@ See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_LITE_EXPERIMENTAL_KERNELS_HASHTABLE_OPS_H_
|
||||
#define TENSORFLOW_LITE_EXPERIMENTAL_KERNELS_HASHTABLE_OPS_H_
|
||||
#ifndef TENSORFLOW_LITE_KERNELS_HASHTABLE_HASHTABLE_OPS_H_
|
||||
#define TENSORFLOW_LITE_KERNELS_HASHTABLE_HASHTABLE_OPS_H_
|
||||
|
||||
#include "tensorflow/lite/mutable_op_resolver.h"
|
||||
|
||||
@ -33,4 +33,4 @@ extern "C" void AddHashtableOps(::tflite::MutableOpResolver* resolver);
|
||||
} // namespace ops
|
||||
} // namespace tflite
|
||||
|
||||
#endif // TENSORFLOW_LITE_EXPERIMENTAL_KERNELS_HASHTABLE_OPS_H_
|
||||
#endif // TENSORFLOW_LITE_KERNELS_HASHTABLE_HASHTABLE_OPS_H_
|
||||
@ -15,7 +15,6 @@ limitations under the License.
|
||||
#include <initializer_list>
|
||||
#include <vector>
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
#include "absl/memory/memory.h"
|
||||
#include "absl/strings/match.h"
|
||||
#include "flatbuffers/flexbuffers.h" // from @flatbuffers
|
||||
@ -222,10 +222,10 @@ cc_library(
|
||||
"@com_google_absl//absl/strings",
|
||||
"//tensorflow/lite:builtin_op_data",
|
||||
"//tensorflow/lite:framework",
|
||||
"//tensorflow/lite/experimental/kernels:hashtable_op_kernels",
|
||||
"//tensorflow/lite:string_util",
|
||||
"//tensorflow/lite/kernels:builtin_ops",
|
||||
"//tensorflow/lite/kernels:custom_ops",
|
||||
"//tensorflow/lite/kernels/hashtable:hashtable_op_kernels",
|
||||
"//tensorflow/lite/kernels:reference_ops",
|
||||
"//tensorflow/lite/tools/evaluation:utils",
|
||||
] + select({
|
||||
|
||||
@ -24,8 +24,8 @@ limitations under the License.
|
||||
#if !defined(__APPLE__)
|
||||
#include "tensorflow/lite/delegates/flex/delegate.h"
|
||||
#endif
|
||||
#include "tensorflow/lite/experimental/kernels/hashtable_ops.h"
|
||||
#include "tensorflow/lite/kernels/custom_ops_register.h"
|
||||
#include "tensorflow/lite/kernels/hashtable/hashtable_ops.h"
|
||||
#include "tensorflow/lite/kernels/register.h"
|
||||
#include "tensorflow/lite/kernels/register_ref.h"
|
||||
#include "tensorflow/lite/string_util.h"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user