experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[ParseExample] Reduce variance in benchmark.

Browse Source 
This change switches the various ParseExample benchmarks to run using tf.data's single-threaded executor. This eliminates noise from thread scheduling, which is often larger than the actual execution time for some of the benchmarks.

PiperOrigin-RevId: 297471781
Change-Id: I1043679ae080692b7a04a3b72223f55ab2200678
This commit is contained in:
Derek Murray 2020-02-26 16:46:58 -08:00 committed by TensorFlower Gardener
parent cddf574279
commit 92e4dfdd1e
2 changed files with 32 additions and 21 deletions
tensorflow/core/kernels
BUILDexample_parsing_ops_test.cc

1
tensorflow/core/kernels/BUILD
View File

@ -1901,6 +1901,7 @@ tf_cc_test(
"//tensorflow/core:test",
"//tensorflow/core:test_main",
"//tensorflow/core:testlib",
"//tensorflow/core/kernels/data:single_threaded_executor",
"@com_google_absl//absl/base",
],
)

52
tensorflow/core/kernels/example_parsing_ops_test.cc
View File

@ -24,6 +24,7 @@ limitations under the License.
#include "tensorflow/core/framework/tensor_shape.h"
#include "tensorflow/core/framework/tensor_types.h"
#include "tensorflow/core/framework/types.pb.h"
#include "tensorflow/core/graph/algorithm.h"
#include "tensorflow/core/graph/graph.h"
#include "tensorflow/core/graph/node_builder.h"
#include "tensorflow/core/lib/core/status_test_util.h"
@ -196,6 +197,7 @@ static Graph* ParseExample(int batch_size, int num_keys, int feature_size) {
.Attr("dense_shapes", dense_shapes)
.Finalize(g, &ret));
FixupSourceAndSinkEdges(g);
return g;
}
@ -272,6 +274,7 @@ static Graph* ParseExampleV2(int batch_size, int num_keys, int feature_size) {
.Attr("dense_shapes", dense_shapes)
.Finalize(g, &ret));
FixupSourceAndSinkEdges(g);
return g;
}
@ -322,6 +325,7 @@ static Graph* ParseSingleExample(int num_keys, int feature_size) {
.Attr("dense_shapes", dense_shapes)
.Finalize(g, &ret));
FixupSourceAndSinkEdges(g);
return g;
}
@ -344,13 +348,15 @@ typedef BenchmarkOptions<ExampleStore<FloatFiller>, kRagged> RaggedFloat;
// B == batch_size, K == num_keys. F == feature_size.
// K must be one of 10, 100, 1000
#define BM_ParseExample(TYPE, B, K, F) \
static void BM_ParseExample##_##TYPE##_##B##_##K##_##F(int iters) { \
int64 items_per_iter = static_cast<int64>(B) * K * F; \
testing::UseRealTime(); \
testing::ItemsProcessed(static_cast<int64>(iters) * items_per_iter); \
test::Benchmark("cpu", ParseExample<TYPE>(B, K, F)).Run(iters); \
} \
#define BM_ParseExample(TYPE, B, K, F) \
static void BM_ParseExample##_##TYPE##_##B##_##K##_##F(int iters) { \
int64 items_per_iter = static_cast<int64>(B) * K * F; \
testing::UseRealTime(); \
testing::ItemsProcessed(static_cast<int64>(iters) * items_per_iter); \
test::Benchmark("cpu", ParseExample<TYPE>(B, K, F), nullptr, nullptr, \
nullptr, "SINGLE_THREADED_EXECUTOR") \
.Run(iters); \
} \
BENCHMARK(BM_ParseExample##_##TYPE##_##B##_##K##_##F);
#define BM_AllParseExample(Type) \
@ -378,13 +384,15 @@ BM_AllParseExample(VarLenDenseFloat);
// B == batch_size, K == num_keys. F == feature_size.
// K must be one of 10, 100, 1000
// B=0 indicates that a scalar input should be used (instead of a vector).
#define BM_ParseExampleV2(TYPE, B, K, F) \
static void BM_ParseExampleV2##_##TYPE##_##B##_##K##_##F(int iters) { \
int64 items_per_iter = static_cast<int64>(std::max(B, 1)) * K * F; \
testing::UseRealTime(); \
testing::ItemsProcessed(static_cast<int64>(iters) * items_per_iter); \
test::Benchmark("cpu", ParseExampleV2<TYPE>(B, K, F)).Run(iters); \
} \
#define BM_ParseExampleV2(TYPE, B, K, F) \
static void BM_ParseExampleV2##_##TYPE##_##B##_##K##_##F(int iters) { \
int64 items_per_iter = static_cast<int64>(std::max(B, 1)) * K * F; \
testing::UseRealTime(); \
testing::ItemsProcessed(static_cast<int64>(iters) * items_per_iter); \
test::Benchmark("cpu", ParseExampleV2<TYPE>(B, K, F), nullptr, nullptr, \
nullptr, "SINGLE_THREADED_EXECUTOR") \
.Run(iters); \
} \
BENCHMARK(BM_ParseExampleV2##_##TYPE##_##B##_##K##_##F);
#define BM_AllParseExampleV2(Type) \
@ -428,13 +436,15 @@ BM_AllParseExampleV2(RaggedFloat);
// K == num_keys. F == feature_size.
// K must be one of 10, 100, 1000
#define BM_ParseSingleExample(TYPE, K, F) \
static void BM_ParseSingleExample##_##TYPE##_1_##K##_##F(int iters) { \
int64 items_per_iter = K * F; \
testing::UseRealTime(); \
testing::ItemsProcessed(static_cast<int64>(iters) * items_per_iter); \
test::Benchmark("cpu", ParseSingleExample<TYPE>(K, F)).Run(iters); \
} \
#define BM_ParseSingleExample(TYPE, K, F) \
static void BM_ParseSingleExample##_##TYPE##_1_##K##_##F(int iters) { \
int64 items_per_iter = K * F; \
testing::UseRealTime(); \
testing::ItemsProcessed(static_cast<int64>(iters) * items_per_iter); \
test::Benchmark("cpu", ParseSingleExample<TYPE>(K, F), nullptr, nullptr, \
nullptr, "SINGLE_THREADED_EXECUTOR") \
.Run(iters); \
} \
BENCHMARK(BM_ParseSingleExample##_##TYPE##_1_##K##_##F);
#define BM_AllParseSingleExample(Type) \