From b3b2c766b49384ec947056a9744844e42fe7cbb7 Mon Sep 17 00:00:00 2001 From: Koan-Sin Tan Date: Thu, 14 May 2020 09:54:18 +0800 Subject: [PATCH 1/5] [tflite] Java binding for fp16 in NNAPI delegate add Java binding to use allow_fp16 in NNAPI delegate --- .../tensorflow/lite/nnapi/NnApiDelegate.java | 18 ++++++++++++++++-- .../java/src/main/native/nnapi_delegate_jni.cc | 7 ++++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/tensorflow/lite/delegates/nnapi/java/src/main/java/org/tensorflow/lite/nnapi/NnApiDelegate.java b/tensorflow/lite/delegates/nnapi/java/src/main/java/org/tensorflow/lite/nnapi/NnApiDelegate.java index 105c02dadba..257902e45a1 100644 --- a/tensorflow/lite/delegates/nnapi/java/src/main/java/org/tensorflow/lite/nnapi/NnApiDelegate.java +++ b/tensorflow/lite/delegates/nnapi/java/src/main/java/org/tensorflow/lite/nnapi/NnApiDelegate.java @@ -118,12 +118,24 @@ public class NnApiDelegate implements Delegate, AutoCloseable { return this; } + /** + * Enable or disable to allow fp32 computation to be run in fp16 in NNAPI. + * See https://source.android.com/devices/neural-networks#android-9 + * + *

Only effective on Android 9 (API level 28) and above. + */ + public Options setAllowFp16(boolean enable) { + this.allowFp16 = enable; + return this; + } + private int executionPreference = EXECUTION_PREFERENCE_UNDEFINED; private String acceleratorName = null; private String cacheDir = null; private String modelToken = null; private Integer maxDelegatedPartitions = null; private Boolean useNnapiCpu = null; + private Boolean allowFp16 = null; } public NnApiDelegate(Options options) { @@ -139,7 +151,8 @@ public class NnApiDelegate implements Delegate, AutoCloseable { /*overrideDisallowCpu=*/ options.useNnapiCpu != null, /*disallowCpuValue=*/ options.useNnapiCpu != null ? !options.useNnapiCpu.booleanValue() - : false); + : false, + options.allowFp16 != null ? options.allowFp16 : false); } public NnApiDelegate() { @@ -204,7 +217,8 @@ public class NnApiDelegate implements Delegate, AutoCloseable { String modelToken, int maxDelegatedPartitions, boolean overrideDisallowCpu, - boolean disallowCpuValue); + boolean disallowCpuValue, + boolean allowFp16); private static native void deleteDelegate(long delegateHandle); diff --git a/tensorflow/lite/delegates/nnapi/java/src/main/native/nnapi_delegate_jni.cc b/tensorflow/lite/delegates/nnapi/java/src/main/native/nnapi_delegate_jni.cc index 6b5171ddfef..df2c4030e6c 100644 --- a/tensorflow/lite/delegates/nnapi/java/src/main/native/nnapi_delegate_jni.cc +++ b/tensorflow/lite/delegates/nnapi/java/src/main/native/nnapi_delegate_jni.cc @@ -27,7 +27,8 @@ JNIEXPORT jlong JNICALL Java_org_tensorflow_lite_nnapi_NnApiDelegate_createDelegate( JNIEnv* env, jclass clazz, jint preference, jstring accelerator_name, jstring cache_dir, jstring model_token, jint max_delegated_partitions, - jboolean override_disallow_cpu, jboolean disallow_cpu_value) { + jboolean override_disallow_cpu, jboolean disallow_cpu_value, + jboolean allow_fp16) { StatefulNnApiDelegate::Options options = StatefulNnApiDelegate::Options(); options.execution_preference = (StatefulNnApiDelegate::Options::ExecutionPreference)preference; @@ -49,6 +50,10 @@ Java_org_tensorflow_lite_nnapi_NnApiDelegate_createDelegate( options.disallow_nnapi_cpu = disallow_cpu_value; } + if (allow_fp16) { + options.allow_fp16 = allow_fp16; + } + auto delegate = new StatefulNnApiDelegate(options); if (options.accelerator_name) { From 38f02efb53fb3ceb1001bacf8662cf0488e341f2 Mon Sep 17 00:00:00 2001 From: Koan-Sin Tan Date: Fri, 15 May 2020 06:47:52 +0800 Subject: [PATCH 2/5] add @deprecated to Interpreter.Options.setAllowFp16PrecisionForFP32 --- .../java/src/main/java/org/tensorflow/lite/Interpreter.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tensorflow/lite/java/src/main/java/org/tensorflow/lite/Interpreter.java b/tensorflow/lite/java/src/main/java/org/tensorflow/lite/Interpreter.java index 5625ef98bb6..0faf3c008f4 100644 --- a/tensorflow/lite/java/src/main/java/org/tensorflow/lite/Interpreter.java +++ b/tensorflow/lite/java/src/main/java/org/tensorflow/lite/Interpreter.java @@ -102,7 +102,9 @@ public final class Interpreter implements AutoCloseable { * Sets whether to allow float16 precision for FP32 calculation when possible. Defaults to false * (disallow). * - *

WARNING: This is an experimental API and subject to change. + * @deprecated Prefer using + * {@link org.tensorflow.lite.nnapi.NnApiDelegate.Options#setAllowFp16(boolean enable)}. + * */ public Options setAllowFp16PrecisionForFp32(boolean allow) { this.allowFp16PrecisionForFp32 = allow; From 94be4e6db5982e116eea0e1a33b257586b99dcab Mon Sep 17 00:00:00 2001 From: Koan-Sin Tan Date: Wed, 27 May 2020 12:58:59 +0800 Subject: [PATCH 3/5] add @Deprecated to setAllowFp16PrecisionForFp32() --- .../lite/java/src/main/java/org/tensorflow/lite/Interpreter.java | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/lite/java/src/main/java/org/tensorflow/lite/Interpreter.java b/tensorflow/lite/java/src/main/java/org/tensorflow/lite/Interpreter.java index 0faf3c008f4..7b220d348cf 100644 --- a/tensorflow/lite/java/src/main/java/org/tensorflow/lite/Interpreter.java +++ b/tensorflow/lite/java/src/main/java/org/tensorflow/lite/Interpreter.java @@ -106,6 +106,7 @@ public final class Interpreter implements AutoCloseable { * {@link org.tensorflow.lite.nnapi.NnApiDelegate.Options#setAllowFp16(boolean enable)}. * */ + @Deprecated public Options setAllowFp16PrecisionForFp32(boolean allow) { this.allowFp16PrecisionForFp32 = allow; return this; From 4fb63ecd646062d110db9bbb64598d1c7dac00f1 Mon Sep 17 00:00:00 2001 From: Koan-Sin Tan Date: Mon, 1 Jun 2020 09:07:00 +0800 Subject: [PATCH 4/5] make //tensorflow/lite/java:InterpreterTest build surpress the two "drepcation" warnings --- .../java/src/test/java/org/tensorflow/lite/InterpreterTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow/lite/java/src/test/java/org/tensorflow/lite/InterpreterTest.java b/tensorflow/lite/java/src/test/java/org/tensorflow/lite/InterpreterTest.java index 6b6799eaad9..3daa9fe0766 100644 --- a/tensorflow/lite/java/src/test/java/org/tensorflow/lite/InterpreterTest.java +++ b/tensorflow/lite/java/src/test/java/org/tensorflow/lite/InterpreterTest.java @@ -65,6 +65,7 @@ public final class InterpreterTest { } @Test + @SuppressWarnings("deprecation") public void testInterpreterWithOptions() throws Exception { Interpreter interpreter = new Interpreter( @@ -390,6 +391,7 @@ public final class InterpreterTest { } @Test + @SuppressWarnings("deprecation") public void testTurnOnNNAPI() throws Exception { Interpreter interpreter = new Interpreter( From 777b6efed4f5f937a1817f58b4e09d63648017d5 Mon Sep 17 00:00:00 2001 From: Koan-Sin Tan Date: Mon, 1 Jun 2020 09:52:05 +0800 Subject: [PATCH 5/5] add allowFp16() test to NnApiDelegateTest --- .../lite/nnapi/NnApiDelegateTest.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tensorflow/lite/java/src/test/java/org/tensorflow/lite/nnapi/NnApiDelegateTest.java b/tensorflow/lite/java/src/test/java/org/tensorflow/lite/nnapi/NnApiDelegateTest.java index e3742dab9a3..45d66e24d35 100644 --- a/tensorflow/lite/java/src/test/java/org/tensorflow/lite/nnapi/NnApiDelegateTest.java +++ b/tensorflow/lite/java/src/test/java/org/tensorflow/lite/nnapi/NnApiDelegateTest.java @@ -56,6 +56,26 @@ public final class NnApiDelegateTest { } } + @Test + public void testInterpreterWithNnApiAllowFp16() throws Exception { + Interpreter.Options options = new Interpreter.Options(); + NnApiDelegate.Options nnApiOptions = new NnApiDelegate.Options(); + nnApiOptions.setAllowFp16(true); + + try (NnApiDelegate delegate = new NnApiDelegate(nnApiOptions); + Interpreter interpreter = new Interpreter(MODEL_BUFFER, options.addDelegate(delegate))) { + float[] oneD = {1.23f, 6.54f, 7.81f}; + float[][] twoD = {oneD, oneD, oneD, oneD, oneD, oneD, oneD, oneD}; + float[][][] threeD = {twoD, twoD, twoD, twoD, twoD, twoD, twoD, twoD}; + float[][][][] fourD = {threeD, threeD}; + float[][][][] parsedOutputs = new float[2][8][8][3]; + interpreter.run(fourD, parsedOutputs); + float[] outputOneD = parsedOutputs[0][0][0]; + float[] expected = {3.69f, 19.62f, 23.43f}; + assertThat(outputOneD).usingTolerance(0.1f).containsExactly(expected).inOrder(); + } + } + @Test public void testGetNnApiErrnoReturnsZeroIfNoNnapiCallFailed() throws Exception { Interpreter.Options options = new Interpreter.Options();