Nanosecond latency measurement for OVIC.

PiperOrigin-RevId: 308144385
Change-Id: I19c5942789110ad1ab92130c0d9914e78dfed1c0

tensorflow/lite/java/ovic/demo/app/OvicBenchmarkerActivity.java

@@ -45,6 +45,7 @@ public class OvicBenchmarkerActivity extends Activity {
 
   /** Name of the task-dependent data files stored in Assets. */
   private static String labelPath = null;
+
   private static String testImagePath = null;
   private static String modelPath = null;
   /**
@@ -91,7 +92,7 @@ public class OvicBenchmarkerActivity extends Activity {
       labelPath = "labels.txt";
       testImagePath = "test_image_224.jpg";
       modelPath = "quantized_model.lite";
-    } else {  // Benchmarking detection.
+    } else { // Benchmarking detection.
       benchmarker = new OvicDetectorBenchmarker(WALL_TIME);
       labelPath = "coco_labels.txt";
       testImagePath = "test_image_224.jpg";
@@ -145,6 +146,7 @@ public class OvicBenchmarkerActivity extends Activity {
   public void detectPressed(View view) throws IOException {
     benchmarkSession(false);
   }
+
   public void classifyPressed(View view) throws IOException {
     benchmarkSession(true);
   }
@@ -194,7 +196,7 @@ public class OvicBenchmarkerActivity extends Activity {
             displayText
                 + modelPath
                 + ": Average latency="
-                + df2.format(benchmarker.getTotalRunTime() / testIter)
+                + df2.format(benchmarker.getTotalRuntimeNano() * 1.0e-6 / testIter)
                 + "ms after "
                 + testIter
                 + " runs.");
@@ -204,12 +206,15 @@ public class OvicBenchmarkerActivity extends Activity {
     }
   }
 
+  // TODO(b/153429929) Remove with resolution of issue (see below).
+  @SuppressWarnings("RuntimeExec")
   private static void setProcessorAffinity(int mask) throws IOException {
     int myPid = Process.myPid();
     Log.i(TAG, String.format("Setting processor affinity to 0x%02x", mask));
 
     String command = String.format("taskset -a -p %x %d", mask, myPid);
     try {
+      // TODO(b/153429929) This is deprecated, but updating is not safe while verification is hard.
       Runtime.getRuntime().exec(command).waitFor();
     } catch (InterruptedException e) {
       throw new IOException("Interrupted: " + e);

tensorflow/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicBenchmarker.java

@@ -43,6 +43,7 @@ public abstract class OvicBenchmarker {
 
   /** Dimensions of inputs. */
   protected static final int DIM_BATCH_SIZE = 1;
+
   protected static final int DIM_PIXEL_SIZE = 3;
   protected int imgHeight = 224;
   protected int imgWidth = 224;
@@ -53,38 +54,38 @@ public abstract class OvicBenchmarker {
   /** A ByteBuffer to hold image data, to be feed into classifier as inputs. */
   protected ByteBuffer imgData = null;
 
-  /** Total runtime in ms. */
-  protected double totalRuntime = 0.0;
+  /** Total runtime in ns. */
+  protected double totalRuntimeNano = 0.0;
   /** Total allowed runtime in ms. */
-  protected double wallTime = 20000 * 30.0;
+  protected double wallTimeNano = 20000 * 30 * 1.0e6;
   /** Record whether benchmark has started (used to skip the first image). */
   protected boolean benchmarkStarted = false;
 
   /**
    * Initializes an {@link OvicBenchmarker}
    *
-   * @param wallTime: a double number specifying the total amount of time to benchmark.
+   * @param wallTimeNano: a double number specifying the total amount of time to benchmark.
    */
-  public OvicBenchmarker(double wallTime) {
+  public OvicBenchmarker(double wallTimeNano) {
     benchmarkStarted = false;
-    totalRuntime = 0.0;
-    this.wallTime = wallTime;
+    totalRuntimeNano = 0.0;
+    this.wallTimeNano = wallTimeNano;
   }
 
   /** Return the cumulative latency of all runs so far. */
-  public double getTotalRunTime() {
-    return totalRuntime;
+  public double getTotalRuntimeNano() {
+    return totalRuntimeNano;
   }
 
   /** Check whether the benchmarker should stop. */
   public Boolean shouldStop() {
-    if (totalRuntime >= wallTime) {
+    if (totalRuntimeNano >= wallTimeNano) {
       Log.e(
           TAG,
-          "Total runtime "
-              + Double.toString(totalRuntime)
-              + " exceeded walltime "
-              + Double.toString(wallTime));
+          "Total runtime (ms) "
+              + (totalRuntimeNano * 1.0e-6)
+              + " exceeded wall-time "
+              + (wallTimeNano * 1.0e-6));
       return true;
     }
     return false;
@@ -120,9 +121,9 @@ public abstract class OvicBenchmarker {
   public abstract String getLastResultString();
 
   /**
-   * Loads input buffer from intValues into ByteBuffer for the interpreter.
-   * Input buffer must be loaded in intValues and output will be placed in imgData.
-  */
+   * Loads input buffer from intValues into ByteBuffer for the interpreter. Input buffer must be
+   * loaded in intValues and output will be placed in imgData.
+   */
   protected void loadsInputToByteBuffer() {
     if (imgData == null || intValues == null) {
       throw new RuntimeException("Benchmarker is not yet ready to test.");

tensorflow/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicClassificationResult.java

@@ -21,34 +21,39 @@ public class OvicClassificationResult {
 
   /** Top K classes and probabilities. */
   public final ArrayList<String> topKClasses;
+
   public final ArrayList<Float> topKProbs;
   public final ArrayList<Integer> topKIndices;
 
   /** Latency (ms). */
-  public Long latency;
+  public Long latencyMilli;
+
+  /** Latency (ns). */
+  public Long latencyNano;
 
   OvicClassificationResult() {
     topKClasses = new ArrayList<>();
     topKProbs = new ArrayList<>();
     topKIndices = new ArrayList<>();
-    latency = -1L;
+    latencyMilli = -1L;
+    latencyNano = -1L;
   }
 
   @Override
   public String toString() {
-    String textToShow = latency + "ms";
+    String textToShow = latencyMilli + "ms";
+    textToShow += "\n" + latencyNano + "ns";
     for (int k = 0; k < topKProbs.size(); ++k) {
       textToShow +=
           "\nPrediction ["
               + k
               + "] = Class "
-              + Integer.toString(topKIndices.get(k))
+              + topKIndices.get(k)
               + " ("
               + topKClasses.get(k)
               + ") : "
-              + Float.toString(topKProbs.get(k));
+              + topKProbs.get(k);
     }
     return textToShow;
   }
-
 }

tensorflow/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicClassifier.java

@@ -14,13 +14,14 @@ limitations under the License.
 ==============================================================================*/
 package org.tensorflow.ovic;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
+
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.nio.ByteBuffer;
 import java.nio.MappedByteBuffer;
-import java.nio.charset.StandardCharsets;
 import java.util.AbstractMap;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -44,7 +45,7 @@ public class OvicClassifier {
   private Interpreter tflite;
 
   /** Labels corresponding to the output of the vision model. */
-  private List<String> labelList;
+  private final List<String> labelList;
 
   /** An array to hold inference results, to be feed into Tensorflow Lite as outputs. */
   private byte[][] inferenceOutputArray = null;
@@ -56,19 +57,18 @@ public class OvicClassifier {
   /** Whether the model runs as float or quantized. */
   private Boolean outputIsFloat = null;
 
-  private PriorityQueue<Map.Entry<Integer, Float>> sortedLabels =
+  private final PriorityQueue<Map.Entry<Integer, Float>> sortedLabels =
       new PriorityQueue<>(
           RESULTS_TO_SHOW,
           new Comparator<Map.Entry<Integer, Float>>() {
             @Override
             public int compare(Map.Entry<Integer, Float> o1, Map.Entry<Integer, Float> o2) {
-              return (o1.getValue()).compareTo(o2.getValue());
+              return o1.getValue().compareTo(o2.getValue());
             }
           });
 
   /** Initializes an {@code OvicClassifier}. */
-  public OvicClassifier(InputStream labelInputStream, MappedByteBuffer model)
-      throws IOException, RuntimeException {
+  public OvicClassifier(InputStream labelInputStream, MappedByteBuffer model) throws IOException {
     if (model == null) {
       throw new RuntimeException("Input model is empty.");
     }
@@ -80,12 +80,12 @@ public class OvicClassifier {
       throw new RuntimeException("The model's input dimensions must be 4 (BWHC).");
     }
     if (inputDims[0] != 1) {
-      throw new RuntimeException("The model must have a batch size of 1, got "
-          + inputDims[0] + " instead.");
+      throw new IllegalStateException(
+          "The model must have a batch size of 1, got " + inputDims[0] + " instead.");
     }
     if (inputDims[3] != 3) {
-      throw new RuntimeException("The model must have three color channels, got "
-          + inputDims[3] + " instead.");
+      throw new IllegalStateException(
+          "The model must have three color channels, got " + inputDims[3] + " instead.");
     }
     int minSide = Math.min(inputDims[1], inputDims[2]);
     int maxSide = Math.max(inputDims[1], inputDims[2]);
@@ -93,12 +93,15 @@ public class OvicClassifier {
       throw new RuntimeException("The model's resolution must be between (0, 1000].");
     }
     String outputDataType = TestHelper.getOutputDataType(tflite, 0);
-    if (outputDataType.equals("float")) {
-      outputIsFloat = true;
-    } else if (outputDataType.equals("byte")) {
-      outputIsFloat = false;
-    } else {
-      throw new RuntimeException("Cannot process output type: " + outputDataType);
+    switch (outputDataType) {
+      case "float":
+        outputIsFloat = true;
+        break;
+      case "byte":
+        outputIsFloat = false;
+        break;
+      default:
+        throw new IllegalStateException("Cannot process output type: " + outputDataType);
     }
     inferenceOutputArray = new byte[1][labelList.size()];
     labelProbArray = new float[1][labelList.size()];
@@ -123,7 +126,8 @@ public class OvicClassifier {
       }
     }
     OvicClassificationResult iterResult = computeTopKLabels();
-    iterResult.latency = getLastNativeInferenceLatencyMilliseconds();
+    iterResult.latencyMilli = getLastNativeInferenceLatencyMilliseconds();
+    iterResult.latencyNano = getLastNativeInferenceLatencyNanoseconds();
     return iterResult;
   }
 
@@ -154,6 +158,18 @@ public class OvicClassifier {
     return (latency == null) ? null : (Long) (latency / 1000000);
   }
 
+  /*
+   * Get native inference latency of last image classification run.
+   *  @throws RuntimeException if model is uninitialized.
+   */
+  public Long getLastNativeInferenceLatencyNanoseconds() {
+    if (tflite == null) {
+      throw new IllegalStateException(
+          TAG + ": ImageNet classifier has not been initialized; Failed.");
+    }
+    return tflite.getLastNativeInferenceDurationNanoseconds();
+  }
+
   /** Closes tflite to release resources. */
   public void close() {
     tflite.close();
@@ -162,9 +178,9 @@ public class OvicClassifier {
 
   /** Reads label list from Assets. */
   private static List<String> loadLabelList(InputStream labelInputStream) throws IOException {
-    List<String> labelList = new ArrayList<String>();
+    List<String> labelList = new ArrayList<>();
     try (BufferedReader reader =
-        new BufferedReader(new InputStreamReader(labelInputStream, StandardCharsets.UTF_8))) {
+        new BufferedReader(new InputStreamReader(labelInputStream, UTF_8))) {
       String line;
       while ((line = reader.readLine()) != null) {
         labelList.add(line);

tensorflow/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicClassifierBenchmarker.java

@@ -88,16 +88,17 @@ public final class OvicClassifierBenchmarker extends OvicBenchmarker {
       Log.e(TAG, e.getMessage());
       Log.e(TAG, "Failed to classify image.");
     }
-    if (iterResult == null || iterResult.latency == null) {
+    if (iterResult == null || iterResult.latencyMilli == null || iterResult.latencyNano == null) {
       throw new RuntimeException("Classification result or timing is invalid.");
     }
-    Log.d(TAG, "Native inference latency: " + iterResult.latency);
+    Log.d(TAG, "Native inference latency (ms): " + iterResult.latencyMilli);
+    Log.d(TAG, "Native inference latency (ns): " + iterResult.latencyNano);
     Log.i(TAG, iterResult.toString());
 
     if (!benchmarkStarted) {  // Skip the first image to discount warming-up time.
       benchmarkStarted = true;
     } else {
-      totalRuntime += ((double) iterResult.latency);
+      totalRuntimeNano += ((double) iterResult.latencyNano);
     }
     return true;
   }

tensorflow/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicDetectionResult.java

@@ -22,7 +22,9 @@ public class OvicDetectionResult {
   // Top K classes and probabilities.
   public final ArrayList<BoundingBox> detections;
   // Latency (ms).
-  public Long latency = -1L;
+  public Long latencyMilli = -1L;
+  // Latency (ns).
+  public Long latencyNano = -1L;
   // id of the image.
   public int id = -1;
   // Number of valid detections (separately maintained, maybe different from detections.size()).
@@ -37,9 +39,10 @@ public class OvicDetectionResult {
     }
   }
 
-  public void resetTo(Long latency, int id) {
+  public void resetTo(Long latencyMilli, Long latencyNano, int id) {
     count = 0;
-    this.latency = latency;
+    this.latencyMilli = latencyMilli;
+    this.latencyNano = latencyNano;
     this.id = id;
   }
 
@@ -64,7 +67,8 @@ public class OvicDetectionResult {
 
   @Override
   public String toString() {
-    String textToShow = latency + "ms";
+    String textToShow = latencyMilli + "ms";
+    textToShow += "\n" + latencyNano + "ns";
     int k = 0;
     for (BoundingBox box : detections) {
       textToShow +=

tensorflow/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicDetector.java

@@ -14,13 +14,14 @@ limitations under the License.
 ==============================================================================*/
 package org.tensorflow.ovic;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
+
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.nio.ByteBuffer;
 import java.nio.MappedByteBuffer;
-import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -105,7 +106,7 @@ public class OvicDetector implements AutoCloseable {
   private static List<String> loadLabelList(InputStream labelInputStream) throws IOException {
     List<String> labelList = new ArrayList<>();
     try (BufferedReader reader =
-        new BufferedReader(new InputStreamReader(labelInputStream, StandardCharsets.UTF_8))) {
+        new BufferedReader(new InputStreamReader(labelInputStream, UTF_8))) {
       String line;
       while ((line = reader.readLine()) != null) {
         labelList.add(line);
@@ -131,10 +132,11 @@ public class OvicDetector implements AutoCloseable {
     Object[] inputArray = {imgData};
     tflite.runForMultipleInputsOutputs(inputArray, outputMap);
 
-    Long latency = getLastNativeInferenceLatencyMilliseconds();
+    Long latencyMilli = getLastNativeInferenceLatencyMilliseconds();
+    Long latencyNano = getLastNativeInferenceLatencyNanoseconds();
 
     // Update the results.
-    result.resetTo(latency, imageId);
+    result.resetTo(latencyMilli, latencyNano, imageId);
     for (int i = 0; i < NUM_RESULTS; i++) {
       // The model returns normalized coordinates [start_y, start_x, end_y, end_x].
       // The boxes expect pixel coordinates [x1, y1, x2, y2].
@@ -154,7 +156,7 @@ public class OvicDetector implements AutoCloseable {
   /*
    * Get native inference latency of last image detection run.
    *  @throws RuntimeException if model is uninitialized.
-   *  @return The inference latency in millisecond.
+   *  @return The inference latency in milliseconds.
    */
   public Long getLastNativeInferenceLatencyMilliseconds() {
     if (tflite == null) {
@@ -164,6 +166,19 @@ public class OvicDetector implements AutoCloseable {
     return (latency == null) ? null : (Long) (latency / 1000000);
   }
 
+  /*
+   * Get native inference latency of last image detection run.
+   *  @throws RuntimeException if model is uninitialized.
+   *  @return The inference latency in nanoseconds.
+   */
+  public Long getLastNativeInferenceLatencyNanoseconds() {
+    if (tflite == null) {
+      throw new IllegalStateException(
+          TAG + ": ImageNet classifier has not been initialized; Failed.");
+    }
+    return tflite.getLastNativeInferenceDurationNanoseconds();
+  }
+
   public int[] getInputDims() {
     return inputDims;
   }

tensorflow/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicDetectorBenchmarker.java

@@ -98,7 +98,7 @@ public final class OvicDetectorBenchmarker extends OvicBenchmarker {
     if (!benchmarkStarted) { // Skip the first image to discount warming-up time.
       benchmarkStarted = true;
     } else {
-      totalRuntime += ((double) detector.result.latency);
+      totalRuntimeNano += ((double) detector.result.latencyNano);
     }
     return true;  // Indicating that result is ready.
   }

tensorflow/lite/java/ovic/src/test/java/org/tensorflow/ovic/OvicClassifierTest.java

@@ -116,7 +116,7 @@ public final class OvicClassifierTest {
   public void ovicClassifier_latencyNotNull() throws Exception {
     classifier = new OvicClassifier(labelsInputStream, floatModel);
     testResult = classifier.classifyByteBuffer(testImage);
-    assertThat(testResult.latency).isNotNull();
+    assertThat(testResult.latencyNano).isNotNull();
   }
 
   @Test