Integrate the benchmarks of Keras.io examples to MLCompass for tracking.

PiperOrigin-RevId: 348675646
Change-Id: If8ed588b1a69a54f60535e9547b6834f40ca7c8b

tensorflow/python/keras/benchmarks/BUILD

@@ -130,6 +130,7 @@ cuda_py_test(
     tags = COMMON_TAGS,
     deps = [
         ":benchmark_util",
+        ":profiler_lib",
         "//tensorflow:tensorflow_py",
     ],
 )
@@ -141,6 +142,7 @@ cuda_py_test(
     tags = COMMON_TAGS,
     deps = [
         ":benchmark_util",
+        ":profiler_lib",
         "//tensorflow:tensorflow_py",
     ],
 )
@@ -152,6 +154,7 @@ cuda_py_test(
     tags = COMMON_TAGS,
     deps = [
         ":benchmark_util",
+        ":profiler_lib",
         "//tensorflow:tensorflow_py",
         "//third_party/py/numpy",
     ],
@@ -164,6 +167,7 @@ cuda_py_test(
     tags = COMMON_TAGS,
     deps = [
         ":benchmark_util",
+        ":profiler_lib",
         "//tensorflow:tensorflow_py",
     ],
 )
@@ -175,6 +179,7 @@ cuda_py_test(
     tags = COMMON_TAGS,
     deps = [
         ":benchmark_util",
+        ":profiler_lib",
         "//tensorflow:tensorflow_py",
     ],
 )
@@ -186,6 +191,7 @@ cuda_py_test(
     tags = COMMON_TAGS,
     deps = [
         ":benchmark_util",
+        ":profiler_lib",
         "//tensorflow:tensorflow_py",
         "//third_party/py/numpy",
     ],
@@ -198,6 +204,7 @@ cuda_py_test(
     tags = COMMON_TAGS,
     deps = [
         ":benchmark_util",
+        ":profiler_lib",
         "//tensorflow:tensorflow_py",
     ],
 )
@@ -208,7 +215,9 @@ cuda_py_test(
     python_version = "PY3",
     tags = COMMON_TAGS,
     deps = [
+        ":benchmark_util",
         ":distribution_util",
+        ":profiler_lib",
         "//tensorflow:tensorflow_py",
     ],
 )

tensorflow/python/keras/benchmarks/benchmark_util.py

@@ -64,6 +64,16 @@ def generate_benchmark_params_cpu_gpu(*params_list):
   return benchmark_params
 
 
+def get_keras_examples_metadata(keras_model,
+                                batch_size,
+                                impl='.keras.cfit_graph'):
+  return {
+      'model_name': 'keras_examples',
+      'implementation': keras_model + impl,
+      'parameters': 'bs_' + str(batch_size),
+  }
+
+
 class TimerCallBack(tf.keras.callbacks.Callback):
   """Callback for logging time in each epoch or batch."""
 

tensorflow/python/keras/benchmarks/keras_examples_benchmarks/antirectifier_benchmark_test.py

@@ -65,6 +65,9 @@ class AntirectifierBenchmark(tf.test.Benchmark):
         loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
         metrics=["sparse_categorical_accuracy"])
 
+    metadata = benchmark_util.get_keras_examples_metadata(
+        "antirectifier", batch_size)
+    extras.update(metadata)
     self.report_benchmark(wall_time=wall_time, metrics=metrics, extras=extras)
 
   def benchmark_antirectifier_bs_256(self):
@@ -79,6 +82,9 @@ class AntirectifierBenchmark(tf.test.Benchmark):
         loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
         metrics=["sparse_categorical_accuracy"])
 
+    metadata = benchmark_util.get_keras_examples_metadata(
+        "antirectifier", batch_size)
+    extras.update(metadata)
     self.report_benchmark(wall_time=wall_time, metrics=metrics, extras=extras)
 
   def benchmark_antirectifier_bs_512(self):
@@ -93,6 +99,9 @@ class AntirectifierBenchmark(tf.test.Benchmark):
         loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
         metrics=["sparse_categorical_accuracy"])
 
+    metadata = benchmark_util.get_keras_examples_metadata(
+        "antirectifier", batch_size)
+    extras.update(metadata)
     self.report_benchmark(wall_time=wall_time, metrics=metrics, extras=extras)
 
   def benchmark_antirectifier_bs_512_gpu_2(self):
@@ -112,6 +121,9 @@ class AntirectifierBenchmark(tf.test.Benchmark):
         loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
         metrics=["sparse_categorical_accuracy"])
 
+    metadata = benchmark_util.get_keras_examples_metadata(
+        "antirectifier", batch_size)
+    extras.update(metadata)
     self.report_benchmark(wall_time=wall_time, metrics=metrics, extras=extras)
 
 

tensorflow/python/keras/benchmarks/keras_examples_benchmarks/bidirectional_lstm_benchmark_test.py

@@ -67,6 +67,9 @@ class BidirectionalLSTMBenchmark(tf.test.Benchmark):
         loss='binary_crossentropy',
         metrics=['accuracy'])
 
+    metadata = benchmark_util.get_keras_examples_metadata(
+        'bidirectional_lstm', batch_size)
+    extras.update(metadata)
     self.report_benchmark(wall_time=wall_time, metrics=metrics, extras=extras)
 
   def benchmark_bidirect_lstm_imdb_bs_256(self):
@@ -81,6 +84,9 @@ class BidirectionalLSTMBenchmark(tf.test.Benchmark):
         loss='binary_crossentropy',
         metrics=['accuracy'])
 
+    metadata = benchmark_util.get_keras_examples_metadata(
+        'bidirectional_lstm', batch_size)
+    extras.update(metadata)
     self.report_benchmark(wall_time=wall_time, metrics=metrics, extras=extras)
 
   def benchmark_bidirect_lstm_imdb_bs_512(self):
@@ -95,6 +101,9 @@ class BidirectionalLSTMBenchmark(tf.test.Benchmark):
         loss='binary_crossentropy',
         metrics=['accuracy'])
 
+    metadata = benchmark_util.get_keras_examples_metadata(
+        'bidirectional_lstm', batch_size)
+    extras.update(metadata)
     self.report_benchmark(wall_time=wall_time, metrics=metrics, extras=extras)
 
   def benchmark_bidirect_lstm_imdb_bs_512_gpu_2(self):
@@ -114,6 +123,9 @@ class BidirectionalLSTMBenchmark(tf.test.Benchmark):
         loss='binary_crossentropy',
         metrics=['accuracy'])
 
+    metadata = benchmark_util.get_keras_examples_metadata(
+        'bidirectional_lstm', batch_size)
+    extras.update(metadata)
     self.report_benchmark(wall_time=wall_time, metrics=metrics, extras=extras)
 
 

tensorflow/python/keras/benchmarks/keras_examples_benchmarks/cifar10_cnn_benchmark_test.py

@@ -82,6 +82,8 @@ class Cifar10CNNBenchmark(tf.test.Benchmark):
         loss='categorical_crossentropy',
         metrics=['accuracy'])
 
+    metadata = benchmark_util.get_keras_examples_metadata('cnn', batch_size)
+    extras.update(metadata)
     self.report_benchmark(wall_time=wall_time, metrics=metrics, extras=extras)
 
   def benchmark_cnn_cifar10_bs_512(self):
@@ -97,6 +99,8 @@ class Cifar10CNNBenchmark(tf.test.Benchmark):
         loss='categorical_crossentropy',
         metrics=['accuracy'])
 
+    metadata = benchmark_util.get_keras_examples_metadata('cnn', batch_size)
+    extras.update(metadata)
     self.report_benchmark(wall_time=wall_time, metrics=metrics, extras=extras)
 
   def benchmark_cnn_cifar10_bs_1024(self):
@@ -112,6 +116,8 @@ class Cifar10CNNBenchmark(tf.test.Benchmark):
         loss='categorical_crossentropy',
         metrics=['accuracy'])
 
+    metadata = benchmark_util.get_keras_examples_metadata('cnn', batch_size)
+    extras.update(metadata)
     self.report_benchmark(wall_time=wall_time, metrics=metrics, extras=extras)
 
   def benchmark_cnn_cifar10_bs_1024_gpu_2(self):
@@ -132,6 +138,8 @@ class Cifar10CNNBenchmark(tf.test.Benchmark):
         loss='categorical_crossentropy',
         metrics=['accuracy'])
 
+    metadata = benchmark_util.get_keras_examples_metadata('cnn', batch_size)
+    extras.update(metadata)
     self.report_benchmark(wall_time=wall_time, metrics=metrics, extras=extras)
 
 

tensorflow/python/keras/benchmarks/keras_examples_benchmarks/mnist_conv_benchmark_test.py

@@ -73,6 +73,8 @@ class ConvMnistBenchmark(tf.test.Benchmark):
         loss='categorical_crossentropy',
         metrics=['accuracy'])
 
+    metadata = benchmark_util.get_keras_examples_metadata('conv', batch_size)
+    extras.update(metadata)
     self.report_benchmark(wall_time=wall_time, metrics=metrics, extras=extras)
 
   def benchmark_conv_mnist_bs_256(self):
@@ -88,6 +90,8 @@ class ConvMnistBenchmark(tf.test.Benchmark):
         loss='categorical_crossentropy',
         metrics=['accuracy'])
 
+    metadata = benchmark_util.get_keras_examples_metadata('conv', batch_size)
+    extras.update(metadata)
     self.report_benchmark(wall_time=wall_time, metrics=metrics, extras=extras)
 
   def benchmark_conv_mnist_bs_512(self):
@@ -103,6 +107,8 @@ class ConvMnistBenchmark(tf.test.Benchmark):
         loss='categorical_crossentropy',
         metrics=['accuracy'])
 
+    metadata = benchmark_util.get_keras_examples_metadata('conv', batch_size)
+    extras.update(metadata)
     self.report_benchmark(wall_time=wall_time, metrics=metrics, extras=extras)
 
   def benchmark_conv_mnist_bs_512_gpu_2(self):
@@ -123,6 +129,8 @@ class ConvMnistBenchmark(tf.test.Benchmark):
         loss='categorical_crossentropy',
         metrics=['accuracy'])
 
+    metadata = benchmark_util.get_keras_examples_metadata('conv', batch_size)
+    extras.update(metadata)
     self.report_benchmark(wall_time=wall_time, metrics=metrics, extras=extras)
 
 

tensorflow/python/keras/benchmarks/keras_examples_benchmarks/mnist_conv_custom_training_benchmark_test.py

@@ -22,6 +22,7 @@ import numpy as np
 
 import tensorflow as tf
 
+from tensorflow.python.keras.benchmarks import benchmark_util
 from tensorflow.python.keras.benchmarks import distribution_util
 
 
@@ -281,7 +282,10 @@ class CustomMnistBenchmark(tf.test.Benchmark):
     metrics, wall_time = self.measure_performance(model, train_dataset, loss_fn,
                                                   optimizer, batch_size,
                                                   run_iters, self.epochs)
-    self.report_benchmark(iters=run_iters, wall_time=wall_time, metrics=metrics)
+    extras = benchmark_util.get_keras_examples_metadata('conv', batch_size,
+                                                        '.keras.ctl_graph')
+    self.report_benchmark(
+        iters=run_iters, wall_time=wall_time, metrics=metrics, extras=extras)
 
   def benchmark_custom_training_mnist_bs_256(self):
     """Measure performance with batch_size=256 and run_iters=5."""
@@ -300,7 +304,10 @@ class CustomMnistBenchmark(tf.test.Benchmark):
     metrics, wall_time = self.measure_performance(model, train_dataset, loss_fn,
                                                   optimizer, batch_size,
                                                   run_iters, self.epochs)
-    self.report_benchmark(iters=run_iters, wall_time=wall_time, metrics=metrics)
+    extras = benchmark_util.get_keras_examples_metadata('conv', batch_size,
+                                                        '.keras.ctl_graph')
+    self.report_benchmark(
+        iters=run_iters, wall_time=wall_time, metrics=metrics, extras=extras)
 
   def benchmark_custom_training_mnist_bs_512(self):
     """Measure performance with batch_size=512 and run_iters=10."""
@@ -319,7 +326,10 @@ class CustomMnistBenchmark(tf.test.Benchmark):
     metrics, wall_time = self.measure_performance(model, train_dataset, loss_fn,
                                                   optimizer, batch_size,
                                                   run_iters, self.epochs)
-    self.report_benchmark(iters=run_iters, wall_time=wall_time, metrics=metrics)
+    extras = benchmark_util.get_keras_examples_metadata('conv', batch_size,
+                                                        '.keras.ctl_graph')
+    self.report_benchmark(
+        iters=run_iters, wall_time=wall_time, metrics=metrics, extras=extras)
 
   def benchmark_custom_training_mnist_bs_512_gpu_2(self):
     """Measure performance with batch_size=512, run_iters=10, gpu=2 and
@@ -353,7 +363,10 @@ class CustomMnistBenchmark(tf.test.Benchmark):
                                                   optimizer, batch_size,
                                                   run_iters, self.epochs,
                                                   strategy)
-    self.report_benchmark(iters=run_iters, wall_time=wall_time, metrics=metrics)
+    extras = benchmark_util.get_keras_examples_metadata('conv', batch_size,
+                                                        '.keras.ctl_graph')
+    self.report_benchmark(
+        iters=run_iters, wall_time=wall_time, metrics=metrics, extras=extras)
 
 
 if __name__ == '__main__':

tensorflow/python/keras/benchmarks/keras_examples_benchmarks/mnist_hierarchical_rnn_benchmark_test.py

@@ -73,6 +73,9 @@ class HierarchicalRNNBenchmark(tf.test.Benchmark):
         loss='categorical_crossentropy',
         metrics=['accuracy'])
 
+    metadata = benchmark_util.get_keras_examples_metadata(
+        'hierarchical_rnn', batch_size)
+    extras.update(metadata)
     self.report_benchmark(wall_time=wall_time, metrics=metrics, extras=extras)
 
   def benchmark_hrnn_mnist_bs_512(self):
@@ -87,6 +90,9 @@ class HierarchicalRNNBenchmark(tf.test.Benchmark):
         loss='categorical_crossentropy',
         metrics=['accuracy'])
 
+    metadata = benchmark_util.get_keras_examples_metadata(
+        'hierarchical_rnn', batch_size)
+    extras.update(metadata)
     self.report_benchmark(wall_time=wall_time, metrics=metrics, extras=extras)
 
   def benchmark_hrnn_mnist_bs_1024(self):
@@ -101,6 +107,9 @@ class HierarchicalRNNBenchmark(tf.test.Benchmark):
         loss='categorical_crossentropy',
         metrics=['accuracy'])
 
+    metadata = benchmark_util.get_keras_examples_metadata(
+        'hierarchical_rnn', batch_size)
+    extras.update(metadata)
     self.report_benchmark(wall_time=wall_time, metrics=metrics, extras=extras)
 
   def benchmark_hrnn_mnist_bs_1024_gpu_2(self):
@@ -120,6 +129,9 @@ class HierarchicalRNNBenchmark(tf.test.Benchmark):
         loss='categorical_crossentropy',
         metrics=['accuracy'])
 
+    metadata = benchmark_util.get_keras_examples_metadata(
+        'hierarchical_rnn', batch_size)
+    extras.update(metadata)
     self.report_benchmark(wall_time=wall_time, metrics=metrics, extras=extras)
 
 

tensorflow/python/keras/benchmarks/keras_examples_benchmarks/mnist_irnn_benchmark_test.py

@@ -73,6 +73,8 @@ class IRNNMnistBenchmark(tf.test.Benchmark):
         loss='categorical_crossentropy',
         metrics=['accuracy'])
 
+    metadata = benchmark_util.get_keras_examples_metadata('irnn', batch_size)
+    extras.update(metadata)
     self.report_benchmark(wall_time=wall_time, metrics=metrics, extras=extras)
 
   def benchmark_irnn_mnist_bs_512(self):
@@ -87,6 +89,8 @@ class IRNNMnistBenchmark(tf.test.Benchmark):
         loss='categorical_crossentropy',
         metrics=['accuracy'])
 
+    metadata = benchmark_util.get_keras_examples_metadata('irnn', batch_size)
+    extras.update(metadata)
     self.report_benchmark(wall_time=wall_time, metrics=metrics, extras=extras)
 
   def benchmark_irnn_mnist_bs_1024(self):
@@ -101,6 +105,8 @@ class IRNNMnistBenchmark(tf.test.Benchmark):
         loss='categorical_crossentropy',
         metrics=['accuracy'])
 
+    metadata = benchmark_util.get_keras_examples_metadata('irnn', batch_size)
+    extras.update(metadata)
     self.report_benchmark(wall_time=wall_time, metrics=metrics, extras=extras)
 
   def benchmark_irnn_mnist_bs_1024_gpu_2(self):
@@ -120,6 +126,8 @@ class IRNNMnistBenchmark(tf.test.Benchmark):
         loss='categorical_crossentropy',
         metrics=['accuracy'])
 
+    metadata = benchmark_util.get_keras_examples_metadata('irnn', batch_size)
+    extras.update(metadata)
     self.report_benchmark(wall_time=wall_time, metrics=metrics, extras=extras)
 
 

tensorflow/python/keras/benchmarks/keras_examples_benchmarks/reuters_mlp_benchmark_test.py

@@ -73,6 +73,8 @@ class MLPReutersBenchmark(tf.test.Benchmark):
         loss='categorical_crossentropy',
         metrics=['accuracy'])
 
+    metadata = benchmark_util.get_keras_examples_metadata('mlp', batch_size)
+    extras.update(metadata)
     self.report_benchmark(wall_time=wall_time, metrics=metrics, extras=extras)
 
   def benchmark_mlp_reuters_bs_256(self):
@@ -88,6 +90,8 @@ class MLPReutersBenchmark(tf.test.Benchmark):
         loss='categorical_crossentropy',
         metrics=['accuracy'])
 
+    metadata = benchmark_util.get_keras_examples_metadata('mlp', batch_size)
+    extras.update(metadata)
     self.report_benchmark(wall_time=wall_time, metrics=metrics, extras=extras)
 
   def benchmark_mlp_reuters_bs_512(self):
@@ -103,6 +107,8 @@ class MLPReutersBenchmark(tf.test.Benchmark):
         loss='categorical_crossentropy',
         metrics=['accuracy'])
 
+    metadata = benchmark_util.get_keras_examples_metadata('mlp', batch_size)
+    extras.update(metadata)
     self.report_benchmark(wall_time=wall_time, metrics=metrics, extras=extras)
 
   def benchmark_mlp_reuters_bs_512_gpu_2(self):
@@ -123,6 +129,8 @@ class MLPReutersBenchmark(tf.test.Benchmark):
         loss='categorical_crossentropy',
         metrics=['accuracy'])
 
+    metadata = benchmark_util.get_keras_examples_metadata('mlp', batch_size)
+    extras.update(metadata)
     self.report_benchmark(wall_time=wall_time, metrics=metrics, extras=extras)
 
 

tensorflow/python/keras/benchmarks/keras_examples_benchmarks/text_classification_transformer_benchmark_test.py

@@ -77,6 +77,9 @@ class TextWithTransformerBenchmark(tf.test.Benchmark):
         loss='sparse_categorical_crossentropy',
         metrics=['accuracy'])
 
+    metadata = benchmark_util.get_keras_examples_metadata(
+        'transformer', batch_size)
+    extras.update(metadata)
     self.report_benchmark(wall_time=wall_time, metrics=metrics, extras=extras)
 
   def benchmark_text_classification_bs_256(self):
@@ -91,6 +94,9 @@ class TextWithTransformerBenchmark(tf.test.Benchmark):
         loss='sparse_categorical_crossentropy',
         metrics=['accuracy'])
 
+    metadata = benchmark_util.get_keras_examples_metadata(
+        'transformer', batch_size)
+    extras.update(metadata)
     self.report_benchmark(wall_time=wall_time, metrics=metrics, extras=extras)
 
   def benchmark_text_classification_bs_512(self):
@@ -105,6 +111,9 @@ class TextWithTransformerBenchmark(tf.test.Benchmark):
         loss='sparse_categorical_crossentropy',
         metrics=['accuracy'])
 
+    metadata = benchmark_util.get_keras_examples_metadata(
+        'transformer', batch_size)
+    extras.update(metadata)
     self.report_benchmark(wall_time=wall_time, metrics=metrics, extras=extras)
 
   def benchmark_text_classification_bs_512_gpu_2(self):
@@ -124,6 +133,9 @@ class TextWithTransformerBenchmark(tf.test.Benchmark):
         loss='sparse_categorical_crossentropy',
         metrics=['accuracy'])
 
+    metadata = benchmark_util.get_keras_examples_metadata(
+        'transformer', batch_size)
+    extras.update(metadata)
     self.report_benchmark(wall_time=wall_time, metrics=metrics, extras=extras)