Move hash table op kernels out of experimental directory

PiperOrigin-RevId: 307999605
Change-Id: Ib59a2979db8f59c84f0c5c3ce6b25f320cd0e917

tensorflow/lite/experimental/kernels/BUILD

@@ -125,50 +125,3 @@ cc_test(
         "@com_google_googletest//:gtest",
     ],
 )
-
-cc_library(
-    name = "hashtable_op_kernels",
-    srcs = [
-        "hashtable.cc",
-        "hashtable_find.cc",
-        "hashtable_import.cc",
-        "hashtable_ops.cc",
-        "hashtable_size.cc",
-    ],
-    hdrs = [
-        "hashtable_ops.h",
-    ],
-    deps = [
-        "//tensorflow/lite:framework",
-        "//tensorflow/lite/c:common",
-        "//tensorflow/lite/core/api",
-        "//tensorflow/lite/experimental/resource",
-        "//tensorflow/lite/kernels:kernel_util",
-        "//tensorflow/lite/kernels:op_macros",
-        "//tensorflow/lite/kernels/internal:tensor",
-        "//tensorflow/lite/schema:schema_fbs",
-        "@flatbuffers",
-    ],
-)
-
-cc_test(
-    name = "hashtable_op_test",
-    size = "small",
-    srcs = [
-        "hashtable_ops_test.cc",
-    ],
-    deps = [
-        ":hashtable_op_kernels",  # buildcleaner: keep
-        "//tensorflow/lite:framework",
-        "//tensorflow/lite/core/api",
-        "//tensorflow/lite/experimental/resource",
-        "//tensorflow/lite/kernels:test_main",
-        "//tensorflow/lite/kernels:test_util",
-        "//tensorflow/lite/kernels/internal:tensor",
-        "//tensorflow/lite/testing:util",
-        "@com_google_absl//absl/memory",
-        "@com_google_absl//absl/strings",
-        "@com_google_googletest//:gtest",
-        "@flatbuffers",
-    ],
-)

tensorflow/lite/experimental/kernels/hashtable_ops.i

@@ -1,20 +0,0 @@
-/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-%{
-#include "tensorflow/lite/experimental/kernels/hashtable_ops.h"
-%}
-
-%include "tensorflow/lite/experimental/kernels/hashtable_ops.h"

tensorflow/lite/kernels/BUILD

@@ -600,7 +600,7 @@ cc_library(
         ":op_macros",
         "//tensorflow/lite:context",
         "//tensorflow/lite/c:common",
-        "//tensorflow/lite/experimental/kernels:hashtable_op_kernels",
+        "//tensorflow/lite/kernels/hashtable:hashtable_op_kernels",
         "//tensorflow/lite/kernels/internal:kernel_utils",
         "//tensorflow/lite/kernels/internal:tensor",
         "//third_party/fft2d:fft2d_headers",

tensorflow/lite/kernels/hashtable/BUILD

@@ -0,0 +1,54 @@
+load("//tensorflow/lite/micro:build_def.bzl", "cc_library")
+
+package(
+    default_visibility = [
+        "//visibility:public",
+    ],
+    licenses = ["notice"],  # Apache 2.0
+)
+
+cc_library(
+    name = "hashtable_op_kernels",
+    srcs = [
+        "hashtable.cc",
+        "hashtable_find.cc",
+        "hashtable_import.cc",
+        "hashtable_ops.cc",
+        "hashtable_size.cc",
+    ],
+    hdrs = [
+        "hashtable_ops.h",
+    ],
+    deps = [
+        "//tensorflow/lite:framework",
+        "//tensorflow/lite/c:common",
+        "//tensorflow/lite/core/api",
+        "//tensorflow/lite/experimental/resource",
+        "//tensorflow/lite/kernels:kernel_util",
+        "//tensorflow/lite/kernels:op_macros",
+        "//tensorflow/lite/kernels/internal:tensor",
+        "//tensorflow/lite/schema:schema_fbs",
+        "@flatbuffers",
+    ],
+)
+
+cc_test(
+    name = "hashtable_op_test",
+    size = "small",
+    srcs = [
+        "hashtable_ops_test.cc",
+    ],
+    deps = [
+        ":hashtable_op_kernels",  # buildcleaner: keep
+        "//tensorflow/lite:framework",
+        "//tensorflow/lite/core/api",
+        "//tensorflow/lite/experimental/resource",
+        "//tensorflow/lite/kernels:test_main",
+        "//tensorflow/lite/kernels:test_util",
+        "//tensorflow/lite/kernels/internal:tensor",
+        "//tensorflow/lite/testing:util",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/strings",
+        "@flatbuffers",
+    ],
+)

tensorflow/lite/kernels/hashtable/README.md

@@ -0,0 +1,190 @@
+# How to use TF Lookup ops in TFLite
+
+The objective of this file is to provide examples to demonstrate how to use TF
+Lookup ops in TFLite.
+
+## Supported Tensorflow Lookup ops in TFLite
+
+Here is the supported status of TensorFlow Lookup ops.
+
+<table>
+  <tr>
+   <td><strong><em>TF Python lookup ops</em></strong>
+   </td>
+   <td colspan="5" ><strong><em>Supported status</em></strong>
+   </td>
+  </tr>
+  <tr>
+   <td rowspan="2" >tf.lookup.StaticHashTable
+   </td>
+   <td rowspan="2" colspan="5" >Supported only with tensor initializers.
+<p>
+Supported mapping type: string → int64, int64 → string
+   </td>
+  </tr>
+  <tr>
+  </tr>
+  <tr>
+   <td rowspan="2" >tf.lookup.Hashtable
+   </td>
+   <td rowspan="2" colspan="5" >Supported only with tensor initializers.
+<p>
+Supported mapping type: string → int64, int64 → string
+   </td>
+  </tr>
+  <tr>
+  </tr>
+  <tr>
+   <td rowspan="2" >tf.lookup.index_to_string_table_from_tensor
+   </td>
+   <td rowspan="2" colspan="5" >Supported.
+   </td>
+  </tr>
+  <tr>
+  </tr>
+  <tr>
+   <td rowspan="2" >tf.lookup.index_table_from_tensor
+   </td>
+   <td rowspan="2" colspan="5" >Supported natively when num_oov_bukcets=0 and dtype=dtypes.string.
+<p>
+For the oov concept, you will need a <a href="https://www.tensorflow.org/lite/guide/ops_select" title="Select TensorFlow operators to use in TensorFlow Lite">Flex delegate</a>.
+   </td>
+  </tr>
+  <tr>
+  </tr>
+  <tr>
+   <td>tf.lookup.StaticVocabularyTable
+   </td>
+   <td colspan="5" >Supported but you will need a <a href="https://www.tensorflow.org/lite/guide/ops_select" title="Select TensorFlow operators to use in TensorFlow Lite">Flex delegate</a>.
+<p>
+Use tf.index_table_from_tensor or tf.index_to_string_table_from_tensor instead if possible if you don’t want to use <a href="https://www.tensorflow.org/lite/guide/ops_select" title="Select TensorFlow operators to use in TensorFlow Lite">Flex delegate</a>.
+   </td>
+  </tr>
+  <tr>
+   <td>tf.lookup.experimental.DenseHashTable
+<p>
+tf.contrib.lookup.MutableHashTable
+<p>
+tf.contrib.lookup.MutableDenseHashTable
+   </td>
+   <td colspan="5" >Not supported yet.
+   </td>
+  </tr>
+  <tr>
+   <td>tf.lookup.IdTableWithHashBuckets
+   </td>
+   <td colspan="5" >Supported but you need a <a href="https://www.tensorflow.org/lite/guide/ops_select" title="Select TensorFlow operators to use in TensorFlow Lite">Flex delegate</a>.
+   </td>
+  </tr>
+</table>
+
+
+
+## Python Sample code
+
+Here, you can find the Python sample code:
+
+
+
+*   Static hash table (string → int64)
+
+```
+int64_values = tf.constant([1, 2, 3], dtype=tf.int64)
+string_values = tf.constant(['bar', 'foo', 'baz'], dtype=tf.string)
+
+initializer = tf.lookup.KeyValueTensorInitializer(string_values, int64_values)
+table = tf.lookup.StaticHashTable(initializer, 4)
+
+with tf.control_dependencies([tf.initializers.tables_initializer()]):
+  input_string_tensor = tf.compat.v1.placeholder(tf.string, shape=[1])
+  out_int64_tensor = table.lookup(input_string_tensor)
+```
+
+*   Static hash table, initialized from a file (string → int64)
+
+```
+with open('/tmp/vocab.file', 'r') as f:
+  words = f.read().splitlines()
+
+string_values = tf.constant(words, dtype=tf.string)
+
+initializer = tf.lookup.KeyValueTensorInitializer(string_values, int64_values)
+table = tf.lookup.StaticHashTable(initializer, 4)
+
+with tf.control_dependencies([tf.initializers.tables_initializer()]):
+  input_string_tensor = tf.placeholder(tf.string, shape=[1])
+  out_int64_tensor = table.lookup(input_string_tensor)
+```
+
+*   Index table (string → int64)
+
+```
+UNK_ID = -1
+vocab = tf.constant(["emerson", "lake", "palmer"])
+vocab_table = tf.lookup.index_table_from_tensor(vocab, default_value=UNK_ID)
+
+input_tensor = tf.compat.v1.placeholder(tf.string, shape=[5])
+
+with tf.control_dependencies([tf.initializers.tables_initializer()]):
+  out_tensor = vocab_table.lookup(input_tensor)
+```
+
+*   Index table, initialized from a file (string → int64)
+
+```
+with open('/tmp/vocab.file', 'r') as f:
+  words = f.read().splitlines()
+
+UNK_ID = -1
+vocab = tf.constant(words)
+vocab_table = tf.lookup.index_table_from_tensor(vocab, default_value=UNK_ID)
+
+input_tensor = tf.compat.v1.placeholder(tf.string, shape=[5])
+
+with tf.control_dependencies([tf.initializers.tables_initializer()]):
+  out_tensor = vocab_table.lookup(input_tensor)
+```
+
+*   Index to string table (int64 → string)
+
+```
+UNK_WORD = "unknown"
+vocab = tf.constant(["emerson", "lake", "palmer"])
+vocab_table = tf.lookup.index_to_string_table_from_tensor(vocab, default_value=UNK_WORD)
+
+input_tensor = tf.compat.v1.placeholder(tf.int64, shape=[1])
+
+with tf.control_dependencies([tf.initializers.tables_initializer()]):
+  out_tensor = vocab_table.lookup(input_tensor)
+```
+
+*   Index to string table, initialized from a file (int64 → string)
+
+```
+with open('/tmp/vocab.file', 'r') as f:
+  words = f.read().splitlines()
+
+UNK_WORD = "unknown"
+vocab = tf.constant(words)
+vocab_table = tf.lookup.index_to_string_table_from_tensor(vocab, default_value=UNK_WORD)
+
+input_tensor = tf.compat.v1.placeholder(tf.int64, shape=[1])
+
+with tf.control_dependencies([tf.initializers.tables_initializer()]):
+  out_tensor = vocab_table.lookup(input_tensor)
+```
+
+## How to Include Hashtable ops in your TFLite.
+
+Currently, hashtable ops are not included in the builtin op set. You need to add
+hashtable ops manually by including the following dependency:
+
+`"//tensorflow/lite/kernels/hashtable:hashtable_op_kernels"`
+
+And then, your op resolver should add them like the following statements:
+
+
+```
+  // Add hashtable op handlers.
+  tflite::ops::custom::AddHashtableOps(&resolver);
+```

tensorflow/lite/kernels/hashtable/hashtable_ops.cc

@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/lite/experimental/kernels/hashtable_ops.h"
+#include "tensorflow/lite/kernels/hashtable/hashtable_ops.h"
 
 namespace tflite {
 namespace ops {

tensorflow/lite/kernels/hashtable/hashtable_ops.h

@@ -13,8 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_LITE_EXPERIMENTAL_KERNELS_HASHTABLE_OPS_H_
+#ifndef TENSORFLOW_LITE_KERNELS_HASHTABLE_HASHTABLE_OPS_H_
-#define TENSORFLOW_LITE_EXPERIMENTAL_KERNELS_HASHTABLE_OPS_H_
+#define TENSORFLOW_LITE_KERNELS_HASHTABLE_HASHTABLE_OPS_H_
 
 #include "tensorflow/lite/mutable_op_resolver.h"
 
@@ -33,4 +33,4 @@ extern "C" void AddHashtableOps(::tflite::MutableOpResolver* resolver);
 }  // namespace ops
 }  // namespace tflite
 
-#endif  // TENSORFLOW_LITE_EXPERIMENTAL_KERNELS_HASHTABLE_OPS_H_
+#endif  // TENSORFLOW_LITE_KERNELS_HASHTABLE_HASHTABLE_OPS_H_

tensorflow/lite/kernels/hashtable/hashtable_ops_test.cc

@@ -15,7 +15,6 @@ limitations under the License.
 #include <initializer_list>
 #include <vector>
 
-#include <gtest/gtest.h>
 #include "absl/memory/memory.h"
 #include "absl/strings/match.h"
 #include "flatbuffers/flexbuffers.h"  // from @flatbuffers

tensorflow/lite/testing/BUILD

@@ -222,10 +222,10 @@ cc_library(
         "@com_google_absl//absl/strings",
         "//tensorflow/lite:builtin_op_data",
         "//tensorflow/lite:framework",
-        "//tensorflow/lite/experimental/kernels:hashtable_op_kernels",
         "//tensorflow/lite:string_util",
         "//tensorflow/lite/kernels:builtin_ops",
         "//tensorflow/lite/kernels:custom_ops",
+        "//tensorflow/lite/kernels/hashtable:hashtable_op_kernels",
         "//tensorflow/lite/kernels:reference_ops",
         "//tensorflow/lite/tools/evaluation:utils",
     ] + select({

tensorflow/lite/testing/tflite_driver.cc

@@ -24,8 +24,8 @@ limitations under the License.
 #if !defined(__APPLE__)
 #include "tensorflow/lite/delegates/flex/delegate.h"
 #endif
-#include "tensorflow/lite/experimental/kernels/hashtable_ops.h"
 #include "tensorflow/lite/kernels/custom_ops_register.h"
+#include "tensorflow/lite/kernels/hashtable/hashtable_ops.h"
 #include "tensorflow/lite/kernels/register.h"
 #include "tensorflow/lite/kernels/register_ref.h"
 #include "tensorflow/lite/string_util.h"