From 9533f3f841f4d80305a2a31cbad9c927d2c76386 Mon Sep 17 00:00:00 2001 From: Alex Stark Date: Thu, 23 Apr 2020 15:40:57 -0700 Subject: [PATCH] Nanosecond latency measurement for OVIC. PiperOrigin-RevId: 308144385 Change-Id: I19c5942789110ad1ab92130c0d9914e78dfed1c0 --- .../demo/app/OvicBenchmarkerActivity.java | 9 +++- .../org/tensorflow/ovic/OvicBenchmarker.java | 35 ++++++------ .../ovic/OvicClassificationResult.java | 17 +++--- .../org/tensorflow/ovic/OvicClassifier.java | 54 ++++++++++++------- .../ovic/OvicClassifierBenchmarker.java | 7 +-- .../tensorflow/ovic/OvicDetectionResult.java | 12 +++-- .../org/tensorflow/ovic/OvicDetector.java | 25 +++++++-- .../ovic/OvicDetectorBenchmarker.java | 2 +- .../tensorflow/ovic/OvicClassifierTest.java | 2 +- 9 files changed, 105 insertions(+), 58 deletions(-) diff --git a/tensorflow/lite/java/ovic/demo/app/OvicBenchmarkerActivity.java b/tensorflow/lite/java/ovic/demo/app/OvicBenchmarkerActivity.java index 144530390ff..76e0cedad5f 100644 --- a/tensorflow/lite/java/ovic/demo/app/OvicBenchmarkerActivity.java +++ b/tensorflow/lite/java/ovic/demo/app/OvicBenchmarkerActivity.java @@ -45,6 +45,7 @@ public class OvicBenchmarkerActivity extends Activity { /** Name of the task-dependent data files stored in Assets. */ private static String labelPath = null; + private static String testImagePath = null; private static String modelPath = null; /** @@ -91,7 +92,7 @@ public class OvicBenchmarkerActivity extends Activity { labelPath = "labels.txt"; testImagePath = "test_image_224.jpg"; modelPath = "quantized_model.lite"; - } else { // Benchmarking detection. + } else { // Benchmarking detection. benchmarker = new OvicDetectorBenchmarker(WALL_TIME); labelPath = "coco_labels.txt"; testImagePath = "test_image_224.jpg"; @@ -145,6 +146,7 @@ public class OvicBenchmarkerActivity extends Activity { public void detectPressed(View view) throws IOException { benchmarkSession(false); } + public void classifyPressed(View view) throws IOException { benchmarkSession(true); } @@ -194,7 +196,7 @@ public class OvicBenchmarkerActivity extends Activity { displayText + modelPath + ": Average latency=" - + df2.format(benchmarker.getTotalRunTime() / testIter) + + df2.format(benchmarker.getTotalRuntimeNano() * 1.0e-6 / testIter) + "ms after " + testIter + " runs."); @@ -204,12 +206,15 @@ public class OvicBenchmarkerActivity extends Activity { } } + // TODO(b/153429929) Remove with resolution of issue (see below). + @SuppressWarnings("RuntimeExec") private static void setProcessorAffinity(int mask) throws IOException { int myPid = Process.myPid(); Log.i(TAG, String.format("Setting processor affinity to 0x%02x", mask)); String command = String.format("taskset -a -p %x %d", mask, myPid); try { + // TODO(b/153429929) This is deprecated, but updating is not safe while verification is hard. Runtime.getRuntime().exec(command).waitFor(); } catch (InterruptedException e) { throw new IOException("Interrupted: " + e); diff --git a/tensorflow/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicBenchmarker.java b/tensorflow/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicBenchmarker.java index 32bdd5a97a7..49cf21debc5 100644 --- a/tensorflow/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicBenchmarker.java +++ b/tensorflow/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicBenchmarker.java @@ -43,6 +43,7 @@ public abstract class OvicBenchmarker { /** Dimensions of inputs. */ protected static final int DIM_BATCH_SIZE = 1; + protected static final int DIM_PIXEL_SIZE = 3; protected int imgHeight = 224; protected int imgWidth = 224; @@ -53,38 +54,38 @@ public abstract class OvicBenchmarker { /** A ByteBuffer to hold image data, to be feed into classifier as inputs. */ protected ByteBuffer imgData = null; - /** Total runtime in ms. */ - protected double totalRuntime = 0.0; + /** Total runtime in ns. */ + protected double totalRuntimeNano = 0.0; /** Total allowed runtime in ms. */ - protected double wallTime = 20000 * 30.0; + protected double wallTimeNano = 20000 * 30 * 1.0e6; /** Record whether benchmark has started (used to skip the first image). */ protected boolean benchmarkStarted = false; /** * Initializes an {@link OvicBenchmarker} * - * @param wallTime: a double number specifying the total amount of time to benchmark. + * @param wallTimeNano: a double number specifying the total amount of time to benchmark. */ - public OvicBenchmarker(double wallTime) { + public OvicBenchmarker(double wallTimeNano) { benchmarkStarted = false; - totalRuntime = 0.0; - this.wallTime = wallTime; + totalRuntimeNano = 0.0; + this.wallTimeNano = wallTimeNano; } /** Return the cumulative latency of all runs so far. */ - public double getTotalRunTime() { - return totalRuntime; + public double getTotalRuntimeNano() { + return totalRuntimeNano; } /** Check whether the benchmarker should stop. */ public Boolean shouldStop() { - if (totalRuntime >= wallTime) { + if (totalRuntimeNano >= wallTimeNano) { Log.e( TAG, - "Total runtime " - + Double.toString(totalRuntime) - + " exceeded walltime " - + Double.toString(wallTime)); + "Total runtime (ms) " + + (totalRuntimeNano * 1.0e-6) + + " exceeded wall-time " + + (wallTimeNano * 1.0e-6)); return true; } return false; @@ -120,9 +121,9 @@ public abstract class OvicBenchmarker { public abstract String getLastResultString(); /** - * Loads input buffer from intValues into ByteBuffer for the interpreter. - * Input buffer must be loaded in intValues and output will be placed in imgData. - */ + * Loads input buffer from intValues into ByteBuffer for the interpreter. Input buffer must be + * loaded in intValues and output will be placed in imgData. + */ protected void loadsInputToByteBuffer() { if (imgData == null || intValues == null) { throw new RuntimeException("Benchmarker is not yet ready to test."); diff --git a/tensorflow/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicClassificationResult.java b/tensorflow/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicClassificationResult.java index 5ab804e6ee2..9aad3719167 100644 --- a/tensorflow/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicClassificationResult.java +++ b/tensorflow/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicClassificationResult.java @@ -21,34 +21,39 @@ public class OvicClassificationResult { /** Top K classes and probabilities. */ public final ArrayList topKClasses; + public final ArrayList topKProbs; public final ArrayList topKIndices; /** Latency (ms). */ - public Long latency; + public Long latencyMilli; + + /** Latency (ns). */ + public Long latencyNano; OvicClassificationResult() { topKClasses = new ArrayList<>(); topKProbs = new ArrayList<>(); topKIndices = new ArrayList<>(); - latency = -1L; + latencyMilli = -1L; + latencyNano = -1L; } @Override public String toString() { - String textToShow = latency + "ms"; + String textToShow = latencyMilli + "ms"; + textToShow += "\n" + latencyNano + "ns"; for (int k = 0; k < topKProbs.size(); ++k) { textToShow += "\nPrediction [" + k + "] = Class " - + Integer.toString(topKIndices.get(k)) + + topKIndices.get(k) + " (" + topKClasses.get(k) + ") : " - + Float.toString(topKProbs.get(k)); + + topKProbs.get(k); } return textToShow; } - } diff --git a/tensorflow/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicClassifier.java b/tensorflow/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicClassifier.java index d8a54c1f3bc..e27272a5d41 100644 --- a/tensorflow/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicClassifier.java +++ b/tensorflow/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicClassifier.java @@ -14,13 +14,14 @@ limitations under the License. ==============================================================================*/ package org.tensorflow.ovic; +import static java.nio.charset.StandardCharsets.UTF_8; + import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.nio.ByteBuffer; import java.nio.MappedByteBuffer; -import java.nio.charset.StandardCharsets; import java.util.AbstractMap; import java.util.ArrayList; import java.util.Collections; @@ -44,7 +45,7 @@ public class OvicClassifier { private Interpreter tflite; /** Labels corresponding to the output of the vision model. */ - private List labelList; + private final List labelList; /** An array to hold inference results, to be feed into Tensorflow Lite as outputs. */ private byte[][] inferenceOutputArray = null; @@ -56,19 +57,18 @@ public class OvicClassifier { /** Whether the model runs as float or quantized. */ private Boolean outputIsFloat = null; - private PriorityQueue> sortedLabels = + private final PriorityQueue> sortedLabels = new PriorityQueue<>( RESULTS_TO_SHOW, new Comparator>() { @Override public int compare(Map.Entry o1, Map.Entry o2) { - return (o1.getValue()).compareTo(o2.getValue()); + return o1.getValue().compareTo(o2.getValue()); } }); /** Initializes an {@code OvicClassifier}. */ - public OvicClassifier(InputStream labelInputStream, MappedByteBuffer model) - throws IOException, RuntimeException { + public OvicClassifier(InputStream labelInputStream, MappedByteBuffer model) throws IOException { if (model == null) { throw new RuntimeException("Input model is empty."); } @@ -80,12 +80,12 @@ public class OvicClassifier { throw new RuntimeException("The model's input dimensions must be 4 (BWHC)."); } if (inputDims[0] != 1) { - throw new RuntimeException("The model must have a batch size of 1, got " - + inputDims[0] + " instead."); + throw new IllegalStateException( + "The model must have a batch size of 1, got " + inputDims[0] + " instead."); } if (inputDims[3] != 3) { - throw new RuntimeException("The model must have three color channels, got " - + inputDims[3] + " instead."); + throw new IllegalStateException( + "The model must have three color channels, got " + inputDims[3] + " instead."); } int minSide = Math.min(inputDims[1], inputDims[2]); int maxSide = Math.max(inputDims[1], inputDims[2]); @@ -93,12 +93,15 @@ public class OvicClassifier { throw new RuntimeException("The model's resolution must be between (0, 1000]."); } String outputDataType = TestHelper.getOutputDataType(tflite, 0); - if (outputDataType.equals("float")) { - outputIsFloat = true; - } else if (outputDataType.equals("byte")) { - outputIsFloat = false; - } else { - throw new RuntimeException("Cannot process output type: " + outputDataType); + switch (outputDataType) { + case "float": + outputIsFloat = true; + break; + case "byte": + outputIsFloat = false; + break; + default: + throw new IllegalStateException("Cannot process output type: " + outputDataType); } inferenceOutputArray = new byte[1][labelList.size()]; labelProbArray = new float[1][labelList.size()]; @@ -123,7 +126,8 @@ public class OvicClassifier { } } OvicClassificationResult iterResult = computeTopKLabels(); - iterResult.latency = getLastNativeInferenceLatencyMilliseconds(); + iterResult.latencyMilli = getLastNativeInferenceLatencyMilliseconds(); + iterResult.latencyNano = getLastNativeInferenceLatencyNanoseconds(); return iterResult; } @@ -154,6 +158,18 @@ public class OvicClassifier { return (latency == null) ? null : (Long) (latency / 1000000); } + /* + * Get native inference latency of last image classification run. + * @throws RuntimeException if model is uninitialized. + */ + public Long getLastNativeInferenceLatencyNanoseconds() { + if (tflite == null) { + throw new IllegalStateException( + TAG + ": ImageNet classifier has not been initialized; Failed."); + } + return tflite.getLastNativeInferenceDurationNanoseconds(); + } + /** Closes tflite to release resources. */ public void close() { tflite.close(); @@ -162,9 +178,9 @@ public class OvicClassifier { /** Reads label list from Assets. */ private static List loadLabelList(InputStream labelInputStream) throws IOException { - List labelList = new ArrayList(); + List labelList = new ArrayList<>(); try (BufferedReader reader = - new BufferedReader(new InputStreamReader(labelInputStream, StandardCharsets.UTF_8))) { + new BufferedReader(new InputStreamReader(labelInputStream, UTF_8))) { String line; while ((line = reader.readLine()) != null) { labelList.add(line); diff --git a/tensorflow/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicClassifierBenchmarker.java b/tensorflow/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicClassifierBenchmarker.java index b35b8ff2c34..8eafd7a25cf 100644 --- a/tensorflow/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicClassifierBenchmarker.java +++ b/tensorflow/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicClassifierBenchmarker.java @@ -88,16 +88,17 @@ public final class OvicClassifierBenchmarker extends OvicBenchmarker { Log.e(TAG, e.getMessage()); Log.e(TAG, "Failed to classify image."); } - if (iterResult == null || iterResult.latency == null) { + if (iterResult == null || iterResult.latencyMilli == null || iterResult.latencyNano == null) { throw new RuntimeException("Classification result or timing is invalid."); } - Log.d(TAG, "Native inference latency: " + iterResult.latency); + Log.d(TAG, "Native inference latency (ms): " + iterResult.latencyMilli); + Log.d(TAG, "Native inference latency (ns): " + iterResult.latencyNano); Log.i(TAG, iterResult.toString()); if (!benchmarkStarted) { // Skip the first image to discount warming-up time. benchmarkStarted = true; } else { - totalRuntime += ((double) iterResult.latency); + totalRuntimeNano += ((double) iterResult.latencyNano); } return true; } diff --git a/tensorflow/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicDetectionResult.java b/tensorflow/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicDetectionResult.java index cf2902a5cb2..15e62c5a22f 100644 --- a/tensorflow/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicDetectionResult.java +++ b/tensorflow/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicDetectionResult.java @@ -22,7 +22,9 @@ public class OvicDetectionResult { // Top K classes and probabilities. public final ArrayList detections; // Latency (ms). - public Long latency = -1L; + public Long latencyMilli = -1L; + // Latency (ns). + public Long latencyNano = -1L; // id of the image. public int id = -1; // Number of valid detections (separately maintained, maybe different from detections.size()). @@ -37,9 +39,10 @@ public class OvicDetectionResult { } } - public void resetTo(Long latency, int id) { + public void resetTo(Long latencyMilli, Long latencyNano, int id) { count = 0; - this.latency = latency; + this.latencyMilli = latencyMilli; + this.latencyNano = latencyNano; this.id = id; } @@ -64,7 +67,8 @@ public class OvicDetectionResult { @Override public String toString() { - String textToShow = latency + "ms"; + String textToShow = latencyMilli + "ms"; + textToShow += "\n" + latencyNano + "ns"; int k = 0; for (BoundingBox box : detections) { textToShow += diff --git a/tensorflow/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicDetector.java b/tensorflow/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicDetector.java index 84c9816d2b1..c43eb131180 100644 --- a/tensorflow/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicDetector.java +++ b/tensorflow/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicDetector.java @@ -14,13 +14,14 @@ limitations under the License. ==============================================================================*/ package org.tensorflow.ovic; +import static java.nio.charset.StandardCharsets.UTF_8; + import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.nio.ByteBuffer; import java.nio.MappedByteBuffer; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -105,7 +106,7 @@ public class OvicDetector implements AutoCloseable { private static List loadLabelList(InputStream labelInputStream) throws IOException { List labelList = new ArrayList<>(); try (BufferedReader reader = - new BufferedReader(new InputStreamReader(labelInputStream, StandardCharsets.UTF_8))) { + new BufferedReader(new InputStreamReader(labelInputStream, UTF_8))) { String line; while ((line = reader.readLine()) != null) { labelList.add(line); @@ -131,10 +132,11 @@ public class OvicDetector implements AutoCloseable { Object[] inputArray = {imgData}; tflite.runForMultipleInputsOutputs(inputArray, outputMap); - Long latency = getLastNativeInferenceLatencyMilliseconds(); + Long latencyMilli = getLastNativeInferenceLatencyMilliseconds(); + Long latencyNano = getLastNativeInferenceLatencyNanoseconds(); // Update the results. - result.resetTo(latency, imageId); + result.resetTo(latencyMilli, latencyNano, imageId); for (int i = 0; i < NUM_RESULTS; i++) { // The model returns normalized coordinates [start_y, start_x, end_y, end_x]. // The boxes expect pixel coordinates [x1, y1, x2, y2]. @@ -154,7 +156,7 @@ public class OvicDetector implements AutoCloseable { /* * Get native inference latency of last image detection run. * @throws RuntimeException if model is uninitialized. - * @return The inference latency in millisecond. + * @return The inference latency in milliseconds. */ public Long getLastNativeInferenceLatencyMilliseconds() { if (tflite == null) { @@ -164,6 +166,19 @@ public class OvicDetector implements AutoCloseable { return (latency == null) ? null : (Long) (latency / 1000000); } + /* + * Get native inference latency of last image detection run. + * @throws RuntimeException if model is uninitialized. + * @return The inference latency in nanoseconds. + */ + public Long getLastNativeInferenceLatencyNanoseconds() { + if (tflite == null) { + throw new IllegalStateException( + TAG + ": ImageNet classifier has not been initialized; Failed."); + } + return tflite.getLastNativeInferenceDurationNanoseconds(); + } + public int[] getInputDims() { return inputDims; } diff --git a/tensorflow/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicDetectorBenchmarker.java b/tensorflow/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicDetectorBenchmarker.java index 15a4c988123..0c03269c27a 100644 --- a/tensorflow/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicDetectorBenchmarker.java +++ b/tensorflow/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicDetectorBenchmarker.java @@ -98,7 +98,7 @@ public final class OvicDetectorBenchmarker extends OvicBenchmarker { if (!benchmarkStarted) { // Skip the first image to discount warming-up time. benchmarkStarted = true; } else { - totalRuntime += ((double) detector.result.latency); + totalRuntimeNano += ((double) detector.result.latencyNano); } return true; // Indicating that result is ready. } diff --git a/tensorflow/lite/java/ovic/src/test/java/org/tensorflow/ovic/OvicClassifierTest.java b/tensorflow/lite/java/ovic/src/test/java/org/tensorflow/ovic/OvicClassifierTest.java index 37d716583fc..7ded4df9e07 100644 --- a/tensorflow/lite/java/ovic/src/test/java/org/tensorflow/ovic/OvicClassifierTest.java +++ b/tensorflow/lite/java/ovic/src/test/java/org/tensorflow/ovic/OvicClassifierTest.java @@ -116,7 +116,7 @@ public final class OvicClassifierTest { public void ovicClassifier_latencyNotNull() throws Exception { classifier = new OvicClassifier(labelsInputStream, floatModel); testResult = classifier.classifyByteBuffer(testImage); - assertThat(testResult.latency).isNotNull(); + assertThat(testResult.latencyNano).isNotNull(); } @Test