experiments
/
STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #38124 from tfeher:trt_opconv_test_unified_memory 

PiperOrigin-RevId: 306753772
Change-Id: Id87c8340ef01e44c700f273972e9e6f7ee1b8cbb
This commit is contained in:
TensorFlower Gardener 2020-04-15 17:37:23 -07:00
parent 2835c023c0 81d68f14d0
commit 4735df8ff4
1 changed files with 106 additions and 131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

237
tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
View File

@ -19,6 +19,9 @@ limitations under the License.
#include <unordered_map> #include <unordered_map>
#include <vector> #include <vector>
#if GOOGLE_CUDA
#if GOOGLE_TENSORRT
#include <gmock/gmock.h> #include <gmock/gmock.h>
#include <gtest/gtest.h> #include <gtest/gtest.h>
#include "absl/strings/match.h" #include "absl/strings/match.h"
@ -26,6 +29,8 @@ limitations under the License.
#include "absl/strings/str_cat.h" #include "absl/strings/str_cat.h"
#include "absl/strings/string_view.h" #include "absl/strings/string_view.h"
#include "absl/types/span.h" #include "absl/types/span.h"
#include "third_party/gpus/cuda/include/cuda.h"
#include "third_party/gpus/cuda/include/cuda_runtime_api.h"
#include "tensorflow/cc/framework/ops.h" #include "tensorflow/cc/framework/ops.h"
#include "tensorflow/cc/framework/scope.h" #include "tensorflow/cc/framework/scope.h"
#include "tensorflow/cc/ops/nn_ops_internal.h" #include "tensorflow/cc/ops/nn_ops_internal.h"
@ -33,6 +38,8 @@ limitations under the License.
#include "tensorflow/compiler/tf2tensorrt/convert/utils.h" #include "tensorflow/compiler/tf2tensorrt/convert/utils.h"
#include "tensorflow/compiler/tf2tensorrt/utils/trt_engine_utils.h" #include "tensorflow/compiler/tf2tensorrt/utils/trt_engine_utils.h"
#include "tensorflow/compiler/tf2tensorrt/utils/trt_logger.h" #include "tensorflow/compiler/tf2tensorrt/utils/trt_logger.h"
#include "tensorflow/core/common_runtime/gpu/gpu_managed_allocator.h"
#include "tensorflow/core/framework/allocator.h"
#include "tensorflow/core/framework/node_def.pb.h" // NOLINT #include "tensorflow/core/framework/node_def.pb.h" // NOLINT
#include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/framework/tensor.pb.h" // NOLINT #include "tensorflow/core/framework/tensor.pb.h" // NOLINT
@ -48,11 +55,6 @@ limitations under the License.
#include "tensorflow/core/platform/test.h" #include "tensorflow/core/platform/test.h"
#include "tensorflow/core/protobuf/config.pb.h" // NOLINT #include "tensorflow/core/protobuf/config.pb.h" // NOLINT
#include "tensorflow/core/public/session.h" #include "tensorflow/core/public/session.h"
#if GOOGLE_CUDA
#if GOOGLE_TENSORRT
#include "third_party/gpus/cuda/include/cuda.h"
#include "third_party/gpus/cuda/include/cuda_runtime_api.h"
#include "third_party/tensorrt/NvInfer.h" #include "third_party/tensorrt/NvInfer.h"
namespace tensorflow { namespace tensorflow {
@ -1215,12 +1217,6 @@ TEST_F(ConvertGraphDefToEngineTest, IdentityGraph) {
TF_EXPECT_OK(RunConvertGraphDefToEngine(&s)); TF_EXPECT_OK(RunConvertGraphDefToEngine(&s));
} }
template <typename T>
Tensor ConstructTensor(int data_size, const T& value = T()) {
std::vector<T> values(data_size, value);
return test::AsTensor<T>(values);
}
template <typename T> template <typename T>
inline absl::Span<const T> GetSpanForData(const InputOutputData& data) { inline absl::Span<const T> GetSpanForData(const InputOutputData& data) {
const auto& tensor_map = data.tensor.flat<T>(); const auto& tensor_map = data.tensor.flat<T>();
@ -1231,7 +1227,8 @@ inline absl::Span<const T> GetSpanForData(const InputOutputData& data) {
// Converter. // Converter.
class OpConverterTest : public ::testing::Test { class OpConverterTest : public ::testing::Test {
public: public:
OpConverterTest() : scope_(Scope::NewRootScope()) { OpConverterTest()
: scope_(Scope::NewRootScope()), allocator_(new GpuManagedAllocator()) {
QCHECK_EQ(0, cudaStreamCreate(&stream_)); QCHECK_EQ(0, cudaStreamCreate(&stream_));
Reset(); Reset();
} }
@ -1258,6 +1255,32 @@ class OpConverterTest : public ::testing::Test {
scope_ = Scope::NewRootScope(); scope_ = Scope::NewRootScope();
} }
// Constructs a flat tensor with 'vals' in Unified Memory.
template <typename T>
Tensor AsTensor(gtl::ArraySlice<T> vals) { // non-absl ok
Tensor ret(allocator_.get(), DataTypeToEnum<T>::value,
{static_cast<int64>(vals.size())});
std::copy_n(vals.data(), vals.size(), ret.flat<T>().data());
return ret;
}
// Constructs a tensor of "shape" with values "vals" in Unified Memory.
template <typename T>
Tensor AsTensor(gtl::ArraySlice<T> vals, // non-absl ok
const TensorShape& shape) {
Tensor ret(allocator_.get(), DataTypeToEnum<T>::value,
{static_cast<int64>(vals.size())});
CHECK(ret.CopyFrom(AsTensor(vals), shape));
return ret;
}
// Constructs a flat tensor in Unified Memory.
template <typename T>
Tensor ConstructTensor(int data_size, const T& value = T()) {
std::vector<T> values(data_size, value);
return AsTensor<T>(values);
}
void CheckDataTypeMatches(const DataVec& datas) { void CheckDataTypeMatches(const DataVec& datas) {
for (const auto& data : datas) { for (const auto& data : datas) {
const int input_index = engine_->getBindingIndex(data.name.c_str()); const int input_index = engine_->getBindingIndex(data.name.c_str());
@ -1313,51 +1336,10 @@ class OpConverterTest : public ::testing::Test {
buffers, converter_->use_implicit_batch(), buffers, converter_->use_implicit_batch(),
batch_size, nullptr, output_data)); batch_size, nullptr, output_data));
// Allocate buffers on GPU and copy data there. This is necessary because
// the test tensors are allocated in host memory, so the pointers that
// SetTrtEngin(In|Out)puts placed into buffers[] cannot be used on the GPU.
// We allocate the GPU buffers, copy the data there, and overwrite the
// addresses in the buffers array.
//
// TODO(tfeher): This step can be avoided if we allocate the Tensors in
// unified memory.
for (const auto& data : input_data) {
const int input_index = engine_->getBindingIndex(data.name.c_str());
ASSERT_NE(-1, input_index);
ASSERT_EQ(0, cudaMalloc(&buffers[input_index], data.TotalBytes()));
ASSERT_EQ(0, cudaMemcpyAsync(buffers[input_index], data.Buffer(),
data.TotalBytes(), cudaMemcpyHostToDevice,
stream_));
}
struct SizeAndIndex {
SizeAndIndex(int in_size, int in_index)
: size(in_size), index(in_index) {}
int size;
int index;
};
std::vector<SizeAndIndex> output_infos;
for (const auto& data : *output_data) {
const int output_index = engine_->getBindingIndex(data.name.c_str());
ASSERT_NE(-1, output_index);
output_infos.emplace_back(data.TotalBytes(), output_index);
ASSERT_EQ(0, cudaMalloc(&buffers[output_index], data.TotalBytes()));
}
// Execute the TRT engine. // Execute the TRT engine.
TF_ASSERT_OK(TrtEnqueue(execution_context.get(), buffers, stream_, TF_ASSERT_OK(TrtEnqueue(execution_context.get(), buffers, stream_,
converter_->use_implicit_batch(), batch_size)); converter_->use_implicit_batch(), batch_size));
for (int i = 0; i < output_infos.size(); ++i) {
const auto& output_info = output_infos[i];
ASSERT_EQ(0, cudaMemcpyAsync(output_data->at(i).Buffer(),
buffers[output_info.index], output_info.size,
cudaMemcpyDeviceToHost, stream_));
}
cudaStreamSynchronize(stream_); cudaStreamSynchronize(stream_);
for (int i = 0; i < num_bindings; ++i) {
ASSERT_EQ(0, cudaFree(buffers[i]));
}
} }
bool HasStaticShape(const nvinfer1::Dims& dims) const { bool HasStaticShape(const nvinfer1::Dims& dims) const {
@ -1395,7 +1377,7 @@ class OpConverterTest : public ::testing::Test {
// Add weights for validation. // Add weights for validation.
TensorShape shape; TensorShape shape;
TF_EXPECT_OK(TensorShapeUtils::MakeShape(dims, &shape)); TF_EXPECT_OK(TensorShapeUtils::MakeShape(dims, &shape));
Tensor t = test::AsTensor<T>(values, shape); Tensor t = AsTensor<T>(values, shape);
node_inputs_[name] = ops::Const(scope_.WithOpName(name), t); node_inputs_[name] = ops::Const(scope_.WithOpName(name), t);
// Add weights for conversion. // Add weights for conversion.
@ -1488,6 +1470,7 @@ class OpConverterTest : public ::testing::Test {
// GraphProperties. // GraphProperties.
Scope scope_; Scope scope_;
std::unordered_map<string, Output> node_inputs_; std::unordered_map<string, Output> node_inputs_;
std::unique_ptr<Allocator> allocator_;
}; };
template <typename T> template <typename T>
@ -1564,13 +1547,13 @@ void TestConvertConst(OpConverterTest* test) {
reset_and_test(t, true, {1}, {12}); reset_and_test(t, true, {1}, {12});
} }
{ {
Tensor t = test::AsTensor<InputCType>({1, 2}); Tensor t = test->AsTensor<InputCType>({1, 2});
reset_and_test(t, false, {2}, {1, 2}); reset_and_test(t, false, {2}, {1, 2});
reset_and_test(t, true, {2}, {1, 2}); reset_and_test(t, true, {2}, {1, 2});
} }
{ {
Tensor t = Tensor t =
test::AsTensor<InputCType>({1, 2, 3, 4, 5, 6}, TensorShape({2, 3})); test->AsTensor<InputCType>({1, 2, 3, 4, 5, 6}, TensorShape({2, 3}));
reset_and_test(t, false, {2, 3}, {1, 2, 3, 4, 5, 6}); reset_and_test(t, false, {2, 3}, {1, 2, 3, 4, 5, 6});
reset_and_test(t, true, {2, 3}, {1, 2, 3, 4, 5, 6}); reset_and_test(t, true, {2, 3}, {1, 2, 3, 4, 5, 6});
} }
@ -1578,7 +1561,7 @@ void TestConvertConst(OpConverterTest* test) {
// Set all tensor elements to the same value. Such tensors are encoded // Set all tensor elements to the same value. Such tensors are encoded
// using a single element list in tensor proto. // using a single element list in tensor proto.
Tensor t = Tensor t =
test::AsTensor<InputCType>({1, 1, 1, 1, 1, 1}, TensorShape({2, 3})); test->AsTensor<InputCType>({1, 1, 1, 1, 1, 1}, TensorShape({2, 3}));
reset_and_test(t, false, {2, 3}, {1, 1, 1, 1, 1, 1}); reset_and_test(t, false, {2, 3}, {1, 1, 1, 1, 1, 1});
reset_and_test(t, true, {2, 3}, {1, 1, 1, 1, 1, 1}); reset_and_test(t, true, {2, 3}, {1, 1, 1, 1, 1, 1});
} }
@ -1586,7 +1569,7 @@ void TestConvertConst(OpConverterTest* test) {
// Set trailing tensor elements to the same value. Such tensors are // Set trailing tensor elements to the same value. Such tensors are
// encoded by truncating all equal elements except the first one. // encoded by truncating all equal elements except the first one.
Tensor t = Tensor t =
test::AsTensor<InputCType>({2, 2, 1, 1, 1, 1}, TensorShape({2, 3})); test->AsTensor<InputCType>({2, 2, 1, 1, 1, 1}, TensorShape({2, 3}));
reset_and_test(t, false, {2, 3}, {2, 2, 1, 1, 1, 1}); reset_and_test(t, false, {2, 3}, {2, 2, 1, 1, 1, 1});
reset_and_test(t, true, {2, 3}, {2, 2, 1, 1, 1, 1}); reset_and_test(t, true, {2, 3}, {2, 2, 1, 1, 1, 1});
} }
@ -1601,10 +1584,9 @@ TEST_F(OpConverterTest, ConvertConst) {
} }
{ {
Reset(); Reset();
Tensor tensor = Tensor tensor = AsTensor<int64>({1, std::numeric_limits<int64>::max(), 1, 1,
test::AsTensor<int64>({1, std::numeric_limits<int64>::max(), 1, 1, 1, 1, std::numeric_limits<int64>::lowest()},
std::numeric_limits<int64>::lowest()}, TensorShape({2, 3}));
TensorShape({2, 3}));
NodeDef node_def; NodeDef node_def;
node_def.set_name("my_const"); node_def.set_name("my_const");
node_def.set_op("Const"); node_def.set_op("Const");
@ -1673,8 +1655,7 @@ TEST_F(OpConverterTest, ConvertTranspose) {
ASSERT_TRUE(output.is_tensor()); ASSERT_TRUE(output.is_tensor());
ExpectTrtDimsEqualsArray({3, 1, 2}, output.tensor()->getDimensions()); ExpectTrtDimsEqualsArray({3, 1, 2}, output.tensor()->getDimensions());
const DataVec input_data{ const DataVec input_data{{"input", AsTensor<float>({1, 2, 3, 4, 5, 6})}};
{"input", test::AsTensor<float>({1, 2, 3, 4, 5, 6})}};
DataVec output_data{{"my_transpose", ConstructTensor<float>(6)}}; DataVec output_data{{"my_transpose", ConstructTensor<float>(6)}};
BuildAndRun(input_data, &output_data); BuildAndRun(input_data, &output_data);
EXPECT_THAT(GetSpanForData<float>(output_data[0]), EXPECT_THAT(GetSpanForData<float>(output_data[0]),
@ -1772,7 +1753,7 @@ TEST_F(OpConverterTest, ConvertReshape) {
std::vector<float> input_vec(TrtTensorDimsNumElements(actual_output_dims) * std::vector<float> input_vec(TrtTensorDimsNumElements(actual_output_dims) *
batch_size); batch_size);
std::iota(input_vec.begin(), input_vec.end(), 1); std::iota(input_vec.begin(), input_vec.end(), 1);
const DataVec input_data{{"input", test::AsTensor<float>(input_vec)}}; const DataVec input_data{{"input", AsTensor<float>(input_vec)}};
DataVec output_data{ DataVec output_data{
{"my_reshape", ConstructTensor<float>(input_vec.size())}}; {"my_reshape", ConstructTensor<float>(input_vec.size())}};
BuildAndRun(input_data, &output_data, TrtPrecisionMode::FP32, batch_size); BuildAndRun(input_data, &output_data, TrtPrecisionMode::FP32, batch_size);
@ -1828,8 +1809,8 @@ void TestMatMulHelper(
ASSERT_TRUE(output.is_tensor()); ASSERT_TRUE(output.is_tensor());
ExpectTrtDimsEqualsArray({2}, output.tensor()->getDimensions()); ExpectTrtDimsEqualsArray({2}, output.tensor()->getDimensions());
const DataVec input_data{{"input", test::AsTensor<float>({0, 1})}}; const DataVec input_data{{"input", test->AsTensor<float>({0, 1})}};
DataVec output_data{{"my_matmul", ConstructTensor<float>(2)}}; DataVec output_data{{"my_matmul", test->ConstructTensor<float>(2)}};
test->BuildAndRun(input_data, &output_data); test->BuildAndRun(input_data, &output_data);
if (transpose_b) { if (transpose_b) {
EXPECT_THAT(GetSpanForData<float>(output_data[0]), ElementsAre(1, 3)); EXPECT_THAT(GetSpanForData<float>(output_data[0]), ElementsAre(1, 3));
@ -1855,8 +1836,8 @@ void TestMatMulHelper(
TF_EXPECT_OK(test->GetTensorOrWeights("my_matmul", &output)); TF_EXPECT_OK(test->GetTensorOrWeights("my_matmul", &output));
ASSERT_TRUE(output.is_tensor()); ASSERT_TRUE(output.is_tensor());
ExpectTrtDimsEqualsArray({2}, output.tensor()->getDimensions()); ExpectTrtDimsEqualsArray({2}, output.tensor()->getDimensions());
const DataVec input_data{{"input", test::AsTensor<float>({0, 1})}}; const DataVec input_data{{"input", test->AsTensor<float>({0, 1})}};
DataVec output_data{{"my_matmul", ConstructTensor<float>(2)}}; DataVec output_data{{"my_matmul", test->ConstructTensor<float>(2)}};
test->BuildAndRun(input_data, &output_data); test->BuildAndRun(input_data, &output_data);
if (transpose_b) { if (transpose_b) {
EXPECT_THAT(GetSpanForData<float>(output_data[0]), ElementsAre(1, 3)); EXPECT_THAT(GetSpanForData<float>(output_data[0]), ElementsAre(1, 3));
@ -2004,7 +1985,7 @@ TEST_F(OpConverterTest, ConvertBatchMatMul) {
TF_EXPECT_OK(GetTensorOrWeights("my_matmul", &output)); TF_EXPECT_OK(GetTensorOrWeights("my_matmul", &output));
ASSERT_TRUE(output.is_tensor()); ASSERT_TRUE(output.is_tensor());
ExpectTrtDimsEqualsArray({2, 2}, output.tensor()->getDimensions()); ExpectTrtDimsEqualsArray({2, 2}, output.tensor()->getDimensions());
const DataVec input_data{{"input", test::AsTensor<float>({0, 1, 2, 3})}}; const DataVec input_data{{"input", AsTensor<float>({0, 1, 2, 3})}};
DataVec output_data{{"my_matmul", ConstructTensor<float>(4)}}; DataVec output_data{{"my_matmul", ConstructTensor<float>(4)}};
BuildAndRun(input_data, &output_data); BuildAndRun(input_data, &output_data);
if (!transpose_a && !transpose_b) { if (!transpose_a && !transpose_b) {
@ -2077,8 +2058,9 @@ void TestConvertBiasAdd(OpConverterTest* test) {
num_input); num_input);
const DataVec input_data{ const DataVec input_data{
{"input", ConstructTensor<CType>(num_input, CType(0))}}; {"input", test->ConstructTensor<CType>(num_input, CType(0))}};
DataVec output_data{{"my_biasadd", ConstructTensor<CType>(num_input)}}; DataVec output_data{
{"my_biasadd", test->ConstructTensor<CType>(num_input)}};
test->BuildAndRun(input_data, &output_data); test->BuildAndRun(input_data, &output_data);
if (trt_input_rank == 1) { if (trt_input_rank == 1) {
if (data_format == "NHWC") { if (data_format == "NHWC") {
@ -2147,14 +2129,14 @@ void TestBinaryOp(OpConverterTest* test, bool operand_1_is_tensor,
if (operand_1_is_tensor) { if (operand_1_is_tensor) {
input_data.push_back( input_data.push_back(
{"input1", {"input1",
test::AsTensor<CType>({CType(3), CType(6), CType(3), CType(6)})}); test->AsTensor<CType>({CType(3), CType(6), CType(3), CType(6)})});
} }
if (operand_2_is_tensor) { if (operand_2_is_tensor) {
input_data.push_back( input_data.push_back(
{"input2", {"input2",
test::AsTensor<CType>({CType(2), CType(3), CType(2), CType(3)})}); test->AsTensor<CType>({CType(2), CType(3), CType(2), CType(3)})});
} }
DataVec output_data{{"my_binary", ConstructTensor<CType>(8)}}; DataVec output_data{{"my_binary", test->ConstructTensor<CType>(8)}};
// Check output dims. // Check output dims.
TRT_TensorOrWeights output; TRT_TensorOrWeights output;
TF_EXPECT_OK(test->GetTensorOrWeights("my_binary", &output)); TF_EXPECT_OK(test->GetTensorOrWeights("my_binary", &output));
@ -2287,7 +2269,7 @@ void TestAddN(OpConverterTest* test) {
for (const auto name : {"inp1", "inp2", "inp3"}) { for (const auto name : {"inp1", "inp2", "inp3"}) {
test->AddTestTensor(name, /*dims=*/{1, 2}, /*batch_size=*/2, test->AddTestTensor(name, /*dims=*/{1, 2}, /*batch_size=*/2,
TfDataTypeToTrt(dtype)); TfDataTypeToTrt(dtype));
input_data.push_back({name, test::AsTensor<CType>({CType(1), CType(2), input_data.push_back({name, test->AsTensor<CType>({CType(1), CType(2),
CType(3), CType(4)})}); CType(3), CType(4)})});
} }
const NodeDef node_def = GetAddNNodeDef({"inp1", "inp2", "inp3"}, dtype); const NodeDef node_def = GetAddNNodeDef({"inp1", "inp2", "inp3"}, dtype);
@ -2298,7 +2280,7 @@ void TestAddN(OpConverterTest* test) {
ASSERT_TRUE(output.is_tensor()); ASSERT_TRUE(output.is_tensor());
ExpectTrtDimsEqualsArray({1, 2}, output.tensor()->getDimensions()); ExpectTrtDimsEqualsArray({1, 2}, output.tensor()->getDimensions());
DataVec output_data{{"my_addn", ConstructTensor<CType>(4)}}; DataVec output_data{{"my_addn", test->ConstructTensor<CType>(4)}};
test->BuildAndRun( test->BuildAndRun(
input_data, &output_data, input_data, &output_data,
dtype == DT_HALF ? TrtPrecisionMode::FP16 : TrtPrecisionMode::FP32, dtype == DT_HALF ? TrtPrecisionMode::FP16 : TrtPrecisionMode::FP32,
@ -2313,7 +2295,7 @@ void TestAddN(OpConverterTest* test) {
for (const auto name : {"inp1", "inp2"}) { for (const auto name : {"inp1", "inp2"}) {
test->AddTestTensor(name, /*dims=*/{1, 2}, /*batch_size=*/1, test->AddTestTensor(name, /*dims=*/{1, 2}, /*batch_size=*/1,
TfDataTypeToTrt(dtype)); TfDataTypeToTrt(dtype));
input_data.push_back({name, test::AsTensor<CType>({CType(1), CType(2)})}); input_data.push_back({name, test->AsTensor<CType>({CType(1), CType(2)})});
} }
test->AddTestWeights("inp3", /*dims=*/{1, 1, 2}, test->AddTestWeights("inp3", /*dims=*/{1, 1, 2},
/*values=*/std::vector<CType>{CType(3), CType(4)}); /*values=*/std::vector<CType>{CType(3), CType(4)});
@ -2325,7 +2307,7 @@ void TestAddN(OpConverterTest* test) {
ASSERT_TRUE(output.is_tensor()); ASSERT_TRUE(output.is_tensor());
ExpectTrtDimsEqualsArray({1, 2}, output.tensor()->getDimensions()); ExpectTrtDimsEqualsArray({1, 2}, output.tensor()->getDimensions());
DataVec output_data{{"my_addn", ConstructTensor<CType>(2)}}; DataVec output_data{{"my_addn", test->ConstructTensor<CType>(2)}};
test->BuildAndRun( test->BuildAndRun(
input_data, &output_data, input_data, &output_data,
dtype == DT_HALF ? TrtPrecisionMode::FP16 : TrtPrecisionMode::FP32); dtype == DT_HALF ? TrtPrecisionMode::FP16 : TrtPrecisionMode::FP32);
@ -2491,10 +2473,10 @@ void TestConvertSquare(OpConverterTest* test) {
inputs[i] = value; inputs[i] = value;
expected_outputs[i] = value * value; expected_outputs[i] = value * value;
} }
const DataVec input_data{{"input", test::AsTensor<CType>(inputs)}}; const DataVec input_data{{"input", test->AsTensor<CType>(inputs)}};
// Engine outputs are converted to FP16 automatically if we set FP16 mode in // Engine outputs are converted to FP16 automatically if we set FP16 mode in
// the builder. // the builder.
DataVec output_data{{"my_square", ConstructTensor<CType>(num_inputs)}}; DataVec output_data{{"my_square", test->ConstructTensor<CType>(num_inputs)}};
test->BuildAndRun( test->BuildAndRun(
input_data, &output_data, input_data, &output_data,
dtype == DT_HALF ? TrtPrecisionMode::FP16 : TrtPrecisionMode::FP32); dtype == DT_HALF ? TrtPrecisionMode::FP16 : TrtPrecisionMode::FP32);
@ -2607,9 +2589,8 @@ TEST_F(OpConverterTest, ConvertCombinedNMS) {
{"my_nms:2", ConstructTensor<float>(2)}, {"my_nms:2", ConstructTensor<float>(2)},
{"my_nms:3", ConstructTensor<int32>(1)}, {"my_nms:3", ConstructTensor<int32>(1)},
}; };
const DataVec input_data{ const DataVec input_data{{"boxes", AsTensor<float>({0, 0, 0.3, 0.4})},
{"boxes", test::AsTensor<float>({0, 0, 0.3, 0.4})}, {"scores", AsTensor<float>({0.4, 0.7, 0.3})}};
{"scores", test::AsTensor<float>({0.4, 0.7, 0.3})}};
BuildAndRun(input_data, &output_data); BuildAndRun(input_data, &output_data);
EXPECT_THAT(GetSpanForData<float>(output_data[0]), EXPECT_THAT(GetSpanForData<float>(output_data[0]),
ElementsAre(0, 0, 0.3, 0.4, 0, 0, 0.3, 0.4)); ElementsAre(0, 0, 0.3, 0.4, 0, 0, 0.3, 0.4));
@ -2734,7 +2715,7 @@ TEST_F(OpConverterTest, ConvertActivation) {
// std::exp in Softplus will overflow for input > 88 // std::exp in Softplus will overflow for input > 88
const std::vector<float> input = {-100, -2, -1, 0, 1, 88}; const std::vector<float> input = {-100, -2, -1, 0, 1, 88};
const DataVec input_data{{"input", test::AsTensor<float>(input)}}; const DataVec input_data{{"input", AsTensor<float>(input)}};
DataVec output_data{{"my_act", ConstructTensor<float>(6)}}; DataVec output_data{{"my_act", ConstructTensor<float>(6)}};
BuildAndRun(input_data, &output_data); BuildAndRun(input_data, &output_data);
for (int i = 0; i < input.size(); i++) { for (int i = 0; i < input.size(); i++) {
@ -2839,8 +2820,7 @@ TEST_F(OpConverterTest, ConvertExpandDims) {
ExpectTrtDimsEqualsArray(ok_params[i].expected_output_dims, ExpectTrtDimsEqualsArray(ok_params[i].expected_output_dims,
output.tensor()->getDimensions()); output.tensor()->getDimensions());
const DataVec input_data{ const DataVec input_data{{"input", AsTensor<float>({1, 2, 3, 4, 5, 6})}};
{"input", test::AsTensor<float>({1, 2, 3, 4, 5, 6})}};
DataVec output_data{{"my_expanddims", ConstructTensor<float>(6)}}; DataVec output_data{{"my_expanddims", ConstructTensor<float>(6)}};
BuildAndRun(input_data, &output_data); BuildAndRun(input_data, &output_data);
EXPECT_THAT(GetSpanForData<float>(output_data[0]), EXPECT_THAT(GetSpanForData<float>(output_data[0]),
@ -2961,8 +2941,7 @@ TEST_F(OpConverterTest, ConvertSqueeze) {
ExpectTrtDimsEqualsArray(ok_params[i].expected_output_dims, ExpectTrtDimsEqualsArray(ok_params[i].expected_output_dims,
output.tensor()->getDimensions()); output.tensor()->getDimensions());
const DataVec input_data{ const DataVec input_data{{"input", AsTensor<float>({1, 2, 3, 4, 5, 6})}};
{"input", test::AsTensor<float>({1, 2, 3, 4, 5, 6})}};
DataVec output_data{{"my_squeeze", ConstructTensor<float>(6)}}; DataVec output_data{{"my_squeeze", ConstructTensor<float>(6)}};
BuildAndRun(input_data, &output_data); BuildAndRun(input_data, &output_data);
EXPECT_THAT(GetSpanForData<float>(output_data[0]), EXPECT_THAT(GetSpanForData<float>(output_data[0]),
@ -3565,7 +3544,7 @@ TEST_F(OpConverterTest, ConvertStridedSlice) {
ExpectTrtDimsEqualsArray(ok_params[i].expected_output_dims, ExpectTrtDimsEqualsArray(ok_params[i].expected_output_dims,
output.tensor()->getDimensions()); output.tensor()->getDimensions());
const DataVec input_data{{"input", test::AsTensor<float>(ok_input)}}; const DataVec input_data{{"input", AsTensor<float>(ok_input)}};
DataVec output_data{ DataVec output_data{
{"my_strided_slice", {"my_strided_slice",
ConstructTensor<float>(ok_params[i].expected_output.size())}}; ConstructTensor<float>(ok_params[i].expected_output.size())}};
@ -3706,8 +3685,7 @@ TEST_F(OpConverterTest, ConvertSlice) {
ExpectTrtDimsEqualsArray(ok_params[i].expected_output_dims, ExpectTrtDimsEqualsArray(ok_params[i].expected_output_dims,
output.tensor()->getDimensions()); output.tensor()->getDimensions());
const DataVec input_data{ const DataVec input_data{{"input", AsTensor<float>({1, 2, 3, 4, 5, 6})}};
{"input", test::AsTensor<float>({1, 2, 3, 4, 5, 6})}};
DataVec output_data{{"my_slice", ConstructTensor<float>( DataVec output_data{{"my_slice", ConstructTensor<float>(
ok_params[i].expected_output.size())}}; ok_params[i].expected_output.size())}};
BuildAndRun(input_data, &output_data); BuildAndRun(input_data, &output_data);
@ -3991,8 +3969,7 @@ TEST_F(OpConverterTest, ConvertConv2D) {
ExpectTrtDimsEqualsArray(ok_params[i].expected_output_dims, ExpectTrtDimsEqualsArray(ok_params[i].expected_output_dims,
output.tensor()->getDimensions()); output.tensor()->getDimensions());
const DataVec input_data{ const DataVec input_data{{"input", AsTensor<float>(ok_params[i].input)}};
{"input", test::AsTensor<float>(ok_params[i].input)}};
DataVec output_data{ DataVec output_data{
{"my_conv2d", {"my_conv2d",
ConstructTensor<float>(ok_params[i].expected_output.size())}}; ConstructTensor<float>(ok_params[i].expected_output.size())}};
@ -4323,8 +4300,7 @@ TEST_F(OpConverterTest, ConvertConv3D) {
ExpectTrtDimsEqualsArray(ok_params[i].expected_output_dims, ExpectTrtDimsEqualsArray(ok_params[i].expected_output_dims,
output.tensor()->getDimensions()); output.tensor()->getDimensions());
const DataVec input_data{ const DataVec input_data{{"input", AsTensor<float>(ok_params[i].input)}};
{"input", test::AsTensor<float>(ok_params[i].input)}};
DataVec output_data{ DataVec output_data{
{"my_conv3d", {"my_conv3d",
ConstructTensor<float>(ok_params[i].expected_output.size())}}; ConstructTensor<float>(ok_params[i].expected_output.size())}};
@ -4511,8 +4487,7 @@ TEST_F(OpConverterTest, ConvertPool3D) {
ExpectTrtDimsEqualsArray(ok_params[i].expected_output_dims, ExpectTrtDimsEqualsArray(ok_params[i].expected_output_dims,
output.tensor()->getDimensions()); output.tensor()->getDimensions());
const DataVec input_data{ const DataVec input_data{{"input", AsTensor<float>(ok_params[i].input)}};
{"input", test::AsTensor<float>(ok_params[i].input)}};
DataVec output_data{ DataVec output_data{
{expected_node_name, {expected_node_name,
ConstructTensor<float>(ok_params[i].expected_output.size())}}; ConstructTensor<float>(ok_params[i].expected_output.size())}};
@ -4558,7 +4533,7 @@ TEST_F(OpConverterTest, ConvertTopK) {
} }
const DataVec input_data{ const DataVec input_data{
{"input", test::AsTensor<float>({-9, 3, 5, 1, 6, -5, 7, 1, 0, -1})}}; {"input", AsTensor<float>({-9, 3, 5, 1, 6, -5, 7, 1, 0, -1})}};
DataVec output_data{{"my_topk", ConstructTensor<float>(4)}, DataVec output_data{{"my_topk", ConstructTensor<float>(4)},
{"my_topk:1", ConstructTensor<int32>(4)}}; {"my_topk:1", ConstructTensor<int32>(4)}};
BuildAndRun(input_data, &output_data); BuildAndRun(input_data, &output_data);
@ -4741,13 +4716,13 @@ void TestConvertGather(OpConverterTest* test) {
DataVec input_data; DataVec input_data;
if (ok_params[i].params_is_tensor) { if (ok_params[i].params_is_tensor) {
input_data = {{"params", test::AsTensor<CType>(params_input)}, input_data = {{"params", test->AsTensor<CType>(params_input)},
{"indices", test::AsTensor<int32>(ok_params[i].indices)}}; {"indices", test->AsTensor<int32>(ok_params[i].indices)}};
} else { } else {
input_data = {{"indices", test::AsTensor<int32>(ok_params[i].indices)}}; input_data = {{"indices", test->AsTensor<int32>(ok_params[i].indices)}};
} }
DataVec output_data{ DataVec output_data{
{"my_gather", ConstructTensor<CType>(expected_output.size())}}; {"my_gather", test->ConstructTensor<CType>(expected_output.size())}};
test->BuildAndRun( test->BuildAndRun(
input_data, &output_data, input_data, &output_data,
dtype == DT_HALF ? TrtPrecisionMode::FP16 : TrtPrecisionMode::FP32, dtype == DT_HALF ? TrtPrecisionMode::FP16 : TrtPrecisionMode::FP32,
@ -4973,7 +4948,7 @@ TEST_F(OpConverterTest, ConvertUnary) {
ExpectTrtDimsEqualsArray({1, 2, 3}, output.tensor()->getDimensions()); ExpectTrtDimsEqualsArray({1, 2, 3}, output.tensor()->getDimensions());
const std::vector<float> input = {-0.9f, 0.6f, 0.0f, -3.5f, 100.0f, 2.9f}; const std::vector<float> input = {-0.9f, 0.6f, 0.0f, -3.5f, 100.0f, 2.9f};
const DataVec input_data{{"input", test::AsTensor<float>(input)}}; const DataVec input_data{{"input", AsTensor<float>(input)}};
DataVec output_data{{"my_unary", ConstructTensor<float>(6)}}; DataVec output_data{{"my_unary", ConstructTensor<float>(6)}};
BuildAndRun(input_data, &output_data); BuildAndRun(input_data, &output_data);
for (int i = 0; i < input.size(); ++i) { for (int i = 0; i < input.size(); ++i) {
@ -5079,11 +5054,11 @@ void TestConvertConcat(OpConverterTest* test) {
for (int j = 0; j < num_inputs; ++j) { for (int j = 0; j < num_inputs; ++j) {
input_data.push_back( input_data.push_back(
{StrCat("values_", j), {StrCat("values_", j),
test::AsTensor<CType>(ok_params[i].input_values[j])}); test->AsTensor<CType>(ok_params[i].input_values[j])});
} }
DataVec output_data{ DataVec output_data{
{"my_concat", {"my_concat",
ConstructTensor<CType>(ok_params[i].expected_output.size())}}; test->ConstructTensor<CType>(ok_params[i].expected_output.size())}};
test->BuildAndRun( test->BuildAndRun(
input_data, &output_data, input_data, &output_data,
dtype == DT_HALF ? TrtPrecisionMode::FP16 : TrtPrecisionMode::FP32); dtype == DT_HALF ? TrtPrecisionMode::FP16 : TrtPrecisionMode::FP32);
@ -5244,13 +5219,13 @@ void TestConvertSplit(OpConverterTest* test) {
outputs[j].tensor()->getDimensions()); outputs[j].tensor()->getDimensions());
// Create buffer to store output. // Create buffer to store output.
output_data.push_back( output_data.push_back(
{name, {name, test->ConstructTensor<CType>(
ConstructTensor<CType>(ok_params[i].expected_outputs[j].size())}); ok_params[i].expected_outputs[j].size())});
} }
// Verify output values are correct. // Verify output values are correct.
const DataVec input_data{ const DataVec input_data{
{"value", test::AsTensor<CType>(ok_params[i].value)}}; {"value", test->AsTensor<CType>(ok_params[i].value)}};
test->BuildAndRun( test->BuildAndRun(
input_data, &output_data, input_data, &output_data,
dtype == DT_HALF ? TrtPrecisionMode::FP16 : TrtPrecisionMode::FP32); dtype == DT_HALF ? TrtPrecisionMode::FP16 : TrtPrecisionMode::FP32);
@ -5423,13 +5398,13 @@ void TestConvertUnpack(OpConverterTest* test) {
outputs[j].tensor()->getDimensions()); outputs[j].tensor()->getDimensions());
// Create buffer to store output. // Create buffer to store output.
output_data.push_back( output_data.push_back(
{name, {name, test->ConstructTensor<CType>(
ConstructTensor<CType>(ok_params[i].expected_outputs[j].size())}); ok_params[i].expected_outputs[j].size())});
} }
// Verify output values are correct. // Verify output values are correct.
const DataVec input_data{ const DataVec input_data{
{"value", test::AsTensor<CType>(ok_params[i].value)}}; {"value", test->AsTensor<CType>(ok_params[i].value)}};
test->BuildAndRun( test->BuildAndRun(
input_data, &output_data, input_data, &output_data,
dtype == DT_HALF ? TrtPrecisionMode::FP16 : TrtPrecisionMode::FP32); dtype == DT_HALF ? TrtPrecisionMode::FP16 : TrtPrecisionMode::FP32);
@ -5597,10 +5572,10 @@ void TestConvertPack(OpConverterTest* test) {
DataVec input_data; DataVec input_data;
for (int j = 0; j < num_inputs; ++j) { for (int j = 0; j < num_inputs; ++j) {
input_data.push_back({StrCat("values_", j), input_data.push_back({StrCat("values_", j),
test::AsTensor<CType>(params[i].input_values[j])}); test->AsTensor<CType>(params[i].input_values[j])});
} }
DataVec output_data{ DataVec output_data{{"my_pack", test->ConstructTensor<CType>(
{"my_pack", ConstructTensor<CType>(params[i].expected_output.size())}}; params[i].expected_output.size())}};
test->BuildAndRun( test->BuildAndRun(
input_data, &output_data, input_data, &output_data,
dtype == DT_HALF ? TrtPrecisionMode::FP16 : TrtPrecisionMode::FP32); dtype == DT_HALF ? TrtPrecisionMode::FP16 : TrtPrecisionMode::FP32);
@ -5747,10 +5722,10 @@ void TestConvertArgMinMax(OpConverterTest* test) {
output.tensor()->getDimensions()); output.tensor()->getDimensions());
// Create input data for tensors. // Create input data for tensors.
const DataVec input_data{ const DataVec input_data{
{"input", test::AsTensor<CType>(params[i].input_value)}}; {"input", test->AsTensor<CType>(params[i].input_value)}};
DataVec output_data{ DataVec output_data{
{"my_arg", {"my_arg", test->ConstructTensor<int32>(
ConstructTensor<int32>(params[i].expected_argmax_output.size())}}; params[i].expected_argmax_output.size())}};
test->BuildAndRun( test->BuildAndRun(
input_data, &output_data, input_data, &output_data,
dtype == DT_HALF ? TrtPrecisionMode::FP16 : TrtPrecisionMode::FP32); dtype == DT_HALF ? TrtPrecisionMode::FP16 : TrtPrecisionMode::FP32);
@ -5849,8 +5824,8 @@ void TestConvertDepthSpaceShuffle(
ExpectTrtDimsEqualsArray(params[i].expected_output_dims, ExpectTrtDimsEqualsArray(params[i].expected_output_dims,
output.tensor()->getDimensions()); output.tensor()->getDimensions());
DataVec input_data{{"input", test::AsTensor<CType>(params[i].input_value)}}; DataVec input_data{{"input", test->AsTensor<CType>(params[i].input_value)}};
DataVec output_data{{"my_shuffle", ConstructTensor<CType>( DataVec output_data{{"my_shuffle", test->ConstructTensor<CType>(
params[i].expected_output.size())}}; params[i].expected_output.size())}};
test->BuildAndRun( test->BuildAndRun(
input_data, &output_data, input_data, &output_data,
@ -6127,9 +6102,9 @@ void TestConvertClipByValue(OpConverterTest* test) {
EXPECT_TRUE(output.is_tensor()); EXPECT_TRUE(output.is_tensor());
ExpectTrtDimsEqualsArray(params[i].dims, output.tensor()->getDimensions()); ExpectTrtDimsEqualsArray(params[i].dims, output.tensor()->getDimensions());
DataVec input_data{{"t", test::AsTensor<CType>(params[i].input_value)}}; DataVec input_data{{"t", test->AsTensor<CType>(params[i].input_value)}};
DataVec output_data{ DataVec output_data{{"my_clip", test->ConstructTensor<CType>(
{"my_clip", ConstructTensor<CType>(params[i].expected_output.size())}}; params[i].expected_output.size())}};
test->BuildAndRun( test->BuildAndRun(
input_data, &output_data, input_data, &output_data,
dtype == DT_HALF ? TrtPrecisionMode::FP16 : TrtPrecisionMode::FP32); dtype == DT_HALF ? TrtPrecisionMode::FP16 : TrtPrecisionMode::FP32);
@ -6235,11 +6210,11 @@ void TestConvertSquaredDifference(OpConverterTest* test) {
ExpectTrtDimsEqualsArray(params[i].expected_output_dims, ExpectTrtDimsEqualsArray(params[i].expected_output_dims,
output.tensor()->getDimensions()); output.tensor()->getDimensions());
DataVec input_data{{"x", test::AsTensor<CType>(params[i].value_x)}, DataVec input_data{{"x", test->AsTensor<CType>(params[i].value_x)},
{"y", test::AsTensor<CType>(params[i].value_y)}}; {"y", test->AsTensor<CType>(params[i].value_y)}};
DataVec output_data{ DataVec output_data{
{"my_squared_diff", {"my_squared_diff",
ConstructTensor<CType>(params[i].expected_output.size())}}; test->ConstructTensor<CType>(params[i].expected_output.size())}};
test->BuildAndRun( test->BuildAndRun(
input_data, &output_data, input_data, &output_data,
dtype == DT_HALF ? TrtPrecisionMode::FP16 : TrtPrecisionMode::FP32); dtype == DT_HALF ? TrtPrecisionMode::FP16 : TrtPrecisionMode::FP32);
@ -6342,9 +6317,9 @@ void TestConvertResize(OpConverterTest* test) {
// Create input data for tensors. // Create input data for tensors.
const DataVec input_data{ const DataVec input_data{
{"input", test::AsTensor<CType>(params[i].input_values)}}; {"input", test->AsTensor<CType>(params[i].input_values)}};
DataVec output_data{ DataVec output_data{
{"my_resize", ConstructTensor<CType>( {"my_resize", test->ConstructTensor<CType>(
params[i].expected_nearest_output_values.size())}}; params[i].expected_nearest_output_values.size())}};
test->BuildAndRun( test->BuildAndRun(
@ -6444,10 +6419,10 @@ void TestConvertPad(OpConverterTest* test) {
// Create input data for tensors. // Create input data for tensors.
const DataVec input_data{ const DataVec input_data{
{"input", test::AsTensor<CType>(params[i].input_values)}}; {"input", test->AsTensor<CType>(params[i].input_values)}};
DataVec output_data{ DataVec output_data{
{"my_pad", {"my_pad", test->ConstructTensor<CType>(
ConstructTensor<CType>(params[i].expected_output_values.size())}}; params[i].expected_output_values.size())}};
test->BuildAndRun( test->BuildAndRun(
input_data, &output_data, input_data, &output_data,