experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #46214 from ddavis-2015:LeakyRelu-pr5

Browse Source 
PiperOrigin-RevId: 361249658
Change-Id: I4f53043186901ec35a25839ada26b3184160430b
This commit is contained in:
TensorFlower Gardener 2021-03-05 16:42:03 -08:00
commit d2a9a946f2
7 changed files with 279 additions and 164 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
tensorflow/lite/micro/all_ops_resolver.cc
View File

@ -41,6 +41,7 @@ AllOpsResolver::AllOpsResolver() {
AddGreaterEqual();
AddHardSwish();
AddL2Normalization();
AddLeakyRelu();
AddLess();
AddLessEqual();
AddLog();

16
tensorflow/lite/micro/kernels/BUILD
View File

@ -273,6 +273,7 @@ cc_library(
"fill.cc",
"floor.cc",
"l2norm.cc",
"leaky_relu.cc",
"logical.cc",
"logistic.cc",
"maximum_minimum.cc",
@ -671,6 +672,21 @@ cc_test(
],
)
cc_test(
name = "leaky_relu_test",
srcs = [
"leaky_relu_test.cc",
],
deps = [
":kernel_runner",
"//tensorflow/lite/c:common",
"//tensorflow/lite/micro:debug_log",
"//tensorflow/lite/micro:op_resolvers",
"//tensorflow/lite/micro:test_helpers",
"//tensorflow/lite/micro/testing:micro_test",
],
)
cc_test(
name = "logical_test",
srcs = [

187
tensorflow/lite/micro/kernels/leaky_relu.cc
View File

@ -23,122 +23,131 @@ limitations under the License.
#include "tensorflow/lite/micro/kernels/kernel_util.h"
namespace tflite {
namespace ops {
namespace micro {
namespace activations {
namespace {
// OLD-TODO(b/142762739): We should figure out a multi-threading plan for most
// of the activation ops below.
// Input/output tensor index.
constexpr int kInputTensor = 0;
constexpr int kOutputTensor = 0;
struct OpData {};
struct LeakyReluOpData : public OpData {
int32_t output_multiplier_alpha = 0;
int32_t output_shift_alpha = 0;
int32_t output_multiplier_identity = 0;
int32_t output_shift_identity = 0;
struct LeakyReluOpData {
// quantization parameters
int32_t output_multiplier_alpha;
int32_t output_shift_alpha;
int32_t output_multiplier_identity;
int32_t output_shift_identity;
int32_t input_zero_point;
int32_t output_zero_point;
};
template <typename T>
void QuantizeLeakyRelu(const TfLiteTensor* input, TfLiteTensor* output,
const LeakyReluOpData* data) {
LeakyReluParams op_params;
void QuantizeLeakyRelu(const LeakyReluOpData& data,
const TfLiteEvalTensor* input,
TfLiteEvalTensor* output) {
LeakyReluParams op_params = {};
op_params.input_offset = input->params.zero_point;
op_params.output_offset = output->params.zero_point;
op_params.output_multiplier_alpha = data->output_multiplier_alpha;
op_params.output_shift_alpha = data->output_shift_alpha;
op_params.output_multiplier_identity = data->output_multiplier_identity;
op_params.output_shift_identity = data->output_shift_identity;
reference_ops::QuantizeLeakyRelu(
op_params, GetTensorShape(input), GetTensorData<T>(input),
GetTensorShape(output), GetTensorData<T>(output));
op_params.input_offset = data.input_zero_point;
op_params.output_offset = data.output_zero_point;
op_params.output_multiplier_alpha = data.output_multiplier_alpha;
op_params.output_shift_alpha = data.output_shift_alpha;
op_params.output_multiplier_identity = data.output_multiplier_identity;
op_params.output_shift_identity = data.output_shift_identity;
reference_ops::QuantizeLeakyRelu(op_params,
tflite::micro::GetTensorShape(input),
tflite::micro::GetTensorData<T>(input),
tflite::micro::GetTensorShape(output),
tflite::micro::GetTensorData<T>(output));
}
} // namespace
void* LeakyReluInit(TfLiteContext* context, const char* buffer, size_t length) {
return nullptr;
}
TfLiteStatus LeakyReluPrepare(TfLiteContext* context, TfLiteNode* node) {
TfLiteStatus CalculateOpData(TfLiteContext* context, TfLiteNode* node) {
TF_LITE_ENSURE_EQ(context, NumInputs(node), 1);
TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1);
const TfLiteTensor* input;
TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, 0, &input));
TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, kInputTensor, &input));
TfLiteTensor* output;
TF_LITE_ENSURE_OK(context, GetOutputSafe(context, node, 0, &output));
TF_LITE_ENSURE_OK(context,
GetOutputSafe(context, node, kOutputTensor, &output));
TF_LITE_ENSURE_TYPES_EQ(context, input->type, output->type);
LeakyReluOpData* data = reinterpret_cast<LeakyReluOpData*>(node->user_data);
if (output->type == kTfLiteUInt8 || output->type == kTfLiteInt8 ||
output->type == kTfLiteInt16) {
if (output->type == kTfLiteInt8) {
LeakyReluOpData* data = static_cast<LeakyReluOpData*>(node->user_data);
const auto* params =
reinterpret_cast<TfLiteLeakyReluParams*>(node->builtin_data);
static_cast<TfLiteLeakyReluParams*>(node->builtin_data);
double alpha_multiplier =
input->params.scale * params->alpha / output->params.scale;
data->input_zero_point = input->params.zero_point;
data->output_zero_point = output->params.zero_point;
int output_shift_alpha;
double alpha_multiplier = static_cast<double>(
input->params.scale * params->alpha / output->params.scale);
QuantizeMultiplier(alpha_multiplier, &data->output_multiplier_alpha,
&data->output_shift_alpha);
double identity_multiplier = input->params.scale / output->params.scale;
&output_shift_alpha);
data->output_shift_alpha = static_cast<int32_t>(output_shift_alpha);
int output_shift_identity;
double identity_multiplier =
static_cast<double>(input->params.scale / output->params.scale);
QuantizeMultiplier(identity_multiplier, &data->output_multiplier_identity,
&data->output_shift_identity);
&output_shift_identity);
data->output_shift_identity = static_cast<int32_t>(output_shift_identity);
}
if (input->type == kTfLiteInt16 && output->type == kTfLiteInt16) {
TF_LITE_ENSURE_EQ(context, input->params.zero_point, 0);
TF_LITE_ENSURE_EQ(context, output->params.zero_point, 0);
return kTfLiteOk;
}
void* LeakyReluInit(TfLiteContext* context, const char* buffer, size_t length) {
TFLITE_DCHECK(context->AllocatePersistentBuffer != nullptr);
return context->AllocatePersistentBuffer(context, sizeof(LeakyReluOpData));
}
TfLiteStatus LeakyReluPrepare(TfLiteContext* context, TfLiteNode* node) {
return CalculateOpData(context, node);
}
TfLiteStatus LeakyReluEval(TfLiteContext* context, TfLiteNode* node) {
const TfLiteEvalTensor* input =
tflite::micro::GetEvalInput(context, node, kInputTensor);
TfLiteEvalTensor* output =
tflite::micro::GetEvalOutput(context, node, kOutputTensor);
const LeakyReluOpData& data = *static_cast<LeakyReluOpData*>(node->user_data);
switch (input->type) {
case kTfLiteFloat32: {
LeakyReluParams op_params = {};
const auto* params =
static_cast<TfLiteLeakyReluParams*>(node->builtin_data);
op_params.alpha = params->alpha;
reference_ops::LeakyRelu(op_params, tflite::micro::GetTensorShape(input),
tflite::micro::GetTensorData<float>(input),
tflite::micro::GetTensorShape(output),
tflite::micro::GetTensorData<float>(output));
return kTfLiteOk;
} break;
case kTfLiteInt8: {
QuantizeLeakyRelu<int8_t>(data, input, output);
return kTfLiteOk;
} break;
default:
TF_LITE_KERNEL_LOG(
context, "Only float32, int8 are supported by LEAKY_RELU, got %s.",
TfLiteTypeGetName(input->type));
return kTfLiteError;
}
return kTfLiteError;
}
TfLiteStatus LeakyReluEval(TfLiteContext* context, TfLiteNode* node) {
const TfLiteTensor* input;
TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, 0, &input));
TfLiteTensor* output;
TF_LITE_ENSURE_OK(context, GetOutputSafe(context, node, 0, &output));
const auto* params =
reinterpret_cast<TfLiteLeakyReluParams*>(node->builtin_data);
const LeakyReluOpData* data =
reinterpret_cast<LeakyReluOpData*>(node->user_data);
} // namespace
LeakyReluParams op_params;
switch (input->type) {
case kTfLiteFloat32: {
op_params.alpha = params->alpha;
reference_ops::LeakyRelu(
op_params, GetTensorShape(input), GetTensorData<float>(input),
GetTensorShape(output), GetTensorData<float>(output));
return kTfLiteOk;
} break;
case kTfLiteUInt8: {
QuantizeLeakyRelu<uint8_t>(input, output, data);
return kTfLiteOk;
} break;
case kTfLiteInt8: {
QuantizeLeakyRelu<int8_t>(input, output, data);
return kTfLiteOk;
} break;
case kTfLiteInt16: {
QuantizeLeakyRelu<int16_t>(input, output, data);
return kTfLiteOk;
} break;
default:
TF_LITE_KERNEL_LOG(
context,
"Only float32, int8, int16 and uint8 is supported currently, got %s.",
TfLiteTypeGetName(input->type));
return kTfLiteError;
}
TfLiteRegistration Register_LEAKY_RELU() {
return {/*init=*/LeakyReluInit,
/*free=*/nullptr,
/*prepare=*/LeakyReluPrepare,
/*invoke=*/LeakyReluEval,
/*profiling_string=*/nullptr,
/*builtin_code=*/0,
/*custom_name=*/nullptr,
/*version=*/0};
}
} // namespace activations
TfLiteRegistration* Register_LEAKY_RELU() { return nullptr; }
} // namespace micro
} // namespace ops
} // namespace tflite

230
tensorflow/lite/micro/kernels/leaky_relu_test.cc
View File

@ -26,6 +26,88 @@ namespace tflite {
namespace testing {
namespace {
// min/max are used to compute scale, zero-point, compare tolerance
template <typename T>
struct TestLeakyReluParams {
// general parameters
float alpha; // alpha multiplier
// quantization parameters
float data_min; // input and output data minimum value
float data_max; // input and output data maximum value
T* input_data; // quantized input storage
T* output_data; // quantized output storage
float tolerance; // output vs expected value tolerance
};
void ExecuteLeakyReluTest(const float alpha, const int tensors_count,
TfLiteTensor* tensors) {
TfLiteLeakyReluParams builtin_data = {};
builtin_data.alpha = alpha;
constexpr int kInputArrayData[] = {1, 0};
TfLiteIntArray* inputs_array = IntArrayFromInts(kInputArrayData);
constexpr int kOutputArrayData[] = {1, 1};
TfLiteIntArray* outputs_array = IntArrayFromInts(kOutputArrayData);
const TfLiteRegistration registration = tflite::Register_LEAKY_RELU();
micro::KernelRunner runner(registration, tensors, tensors_count, inputs_array,
outputs_array, static_cast<void*>(&builtin_data));
TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, runner.InitAndPrepare());
TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, runner.Invoke());
}
template <typename T>
void TestLeakyRelu(const TestLeakyReluParams<T>& params,
const int* input_dims_data, const T* input_data,
const int* expected_dims, const T* expected_data,
T* output_data) {
TfLiteIntArray* input_dims = IntArrayFromInts(input_dims_data);
TfLiteIntArray* output_dims = IntArrayFromInts(expected_dims);
const int output_count = ElementCount(*output_dims);
TfLiteTensor tensors[] = {
CreateTensor(input_data, input_dims),
CreateTensor(output_data, output_dims),
};
constexpr int tensors_count = std::extent<decltype(tensors)>::value;
ExecuteLeakyReluTest(params.alpha, tensors_count, tensors);
for (int i = 0; i < output_count; i++) {
TF_LITE_MICRO_EXPECT_EQ(expected_data[i], output_data[i]);
}
}
template <typename T>
void TestLeakyReluQuantized(const TestLeakyReluParams<T>& params,
const int* input_dims_data, const float* input_data,
const int* expected_dims,
const float* expected_data, float* output_data) {
TfLiteIntArray* input_dims = IntArrayFromInts(input_dims_data);
TfLiteIntArray* output_dims = IntArrayFromInts(expected_dims);
const int output_count = ElementCount(*output_dims);
const float scale = ScaleFromMinMax<T>(params.data_min, params.data_max);
const int zero_point =
ZeroPointFromMinMax<T>(params.data_min, params.data_max);
TfLiteTensor tensors[] = {
CreateQuantizedTensor(input_data, params.input_data, input_dims, scale,
zero_point),
CreateQuantizedTensor(params.output_data, output_dims, scale, zero_point),
};
constexpr int kTensorsCount = std::extent<decltype(tensors)>::value;
ExecuteLeakyReluTest(params.alpha, kTensorsCount, tensors);
Dequantize(params.output_data, output_count, scale, zero_point, output_data);
const float kTolerance = params.tolerance;
for (int i = 0; i < output_count; i++) {
TF_LITE_MICRO_EXPECT_NEAR(expected_data[i], output_data[i], kTolerance);
}
}
// Our fixed-point math function implementations have roughly 12 bits of
// accuracy, when specialized to 16-bit fixed-point arithmetic.
// That is purely an implementation compromise, it would have been possible
@ -43,92 +125,90 @@ namespace {
// is 2, our representable values are often diluted by a factor of 2, whence
// the factor of 2 below.
const float kQuantizedTolerance = 2 * (1. / 256);
const float kQuantizedToleranceInt16 = 2 * (1. / 4096);
template <TensorType tensor_type, typename integer_dtype>
template <typename integer_dtype>
void QuantizedActivationsOpTestLeakyRelu() {
const float kMin = -1;
const float kMax =
std::numeric_limits<integer_dtype>::max() /
static_cast<float>(std::numeric_limits<integer_dtype>::max() + 1);
#ifdef notdef
QuantizedActivationsOpModel m(
/*input=*/{tensor_type, {5, 5}, 5 * kMin, 5 * kMax}, 0.1);
m.SetInput<integer_dtype>({
constexpr int kDims[] = {2, 5, 5};
constexpr float kInput[] = {
-5.0f, -4.6f, -4.2f, -3.8f, -3.4f, // Row 1
-3.0f, -2.6f, -2.2f, -1.8f, -1.4f, // Row 2
-1.0f, -0.6f, -0.2f, 0.2f, 0.6f, // Row 3
1.0f, 1.4f, 1.8f, 2.2f, 2.6f, // Row 4
3.0f, 3.4f, 3.8f, 4.2f, 4.6f, // Row 5
});
};
constexpr float kExpect[] = {
-0.50f, -0.46f, -0.42f, -0.38f, -0.34f, // Row 1
-0.30f, -0.26f, -0.22f, -0.18f, -0.14f, // Row 2
-0.10f, -0.06f, -0.02f, 0.20f, 0.60f, // Row 3
1.00f, 1.40f, 1.80f, 2.20f, 2.60f, // Row 4
3.00f, 3.40f, 3.80f, 4.20f, 4.60f, // Row 5
};
constexpr int kOutputCount = std::extent<decltype(kExpect)>::value;
float output_data[kOutputCount];
float kTestQuantizedTolerance = tensor_type == TensorType_INT16
? kQuantizedToleranceInt16
: kQuantizedTolerance * 5;
// setup quantization storage and parameters
integer_dtype q_output_data[kOutputCount];
integer_dtype q_input_data[kOutputCount];
constexpr float kMin = -1;
constexpr float kMax =
std::numeric_limits<integer_dtype>::max() /
static_cast<float>(std::numeric_limits<integer_dtype>::max() + 1);
TestLeakyReluParams<integer_dtype> params = {};
params.alpha = 0.1f;
params.data_min = 5 * kMin;
params.data_max = 5 * kMax;
params.input_data = q_input_data;
params.output_data = q_output_data;
params.tolerance = kQuantizedTolerance * 5;
EXPECT_THAT(m.GetDequantizedOutput<integer_dtype>(),
ElementsAreArray(ArrayFloatNear(
{
-0.50f, -0.46f, -0.42f, -0.38f, -0.34f, // Row 1
-0.30f, -0.26f, -0.22f, -0.18f, -0.14f, // Row 2
-0.10f, -0.06f, -0.02f, 0.20f, 0.60f, // Row 3
1.00f, 1.40f, 1.80f, 2.20f, 2.60f, // Row 4
3.00f, 3.40f, 3.80f, 4.20f, 4.60f, // Row 5
},
kTestQuantizedTolerance)));
#endif // notdef
TestLeakyReluQuantized(params, kDims, kInput, kDims, kExpect, output_data);
}
TF_LITE_MICRO_TESTS_BEGIN
TF_LITE_MICRO_TEST(QuantizedActivationsOpTestLeakyReluUint8) {
const float kMin = -1;
const float kMax = 127.f / 128.f;
#ifdef notdef
QuantizedActivationsOpModel m(
/*input=*/{TensorType_UINT8, {2, 3}, 8 * kMin, 8 * kMax}, 0.5);
m.SetInput<uint8_t>({
0.0f, 1.0f, 3.0f, // Row 1
1.0f, -1.0f, -2.0f, // Row 2
});
EXPECT_THAT(m.GetDequantizedOutput<uint8_t>(),
ElementsAreArray(ArrayFloatNear(
{
0.0f, 1.0f, 3.0f, // Row 1
1.0f, -0.5f, -1.0f, // Row 2
},
kQuantizedTolerance * 8)));
#endif // notdef
}
TF_LITE_MICRO_TEST(QuantizedActivationsOpTestLeakyReluInt8) {
QuantizedActivationsOpTestLeakyRelu<TensorType_INT8, int8_t>();
}
TF_LITE_MICRO_TEST(QuantizedActivationsOpTestLeakyReluInt16) {
QuantizedActivationsOpTestLeakyRelu<TensorType_INT16, int16_t>();
}
TF_LITE_MICRO_TEST(FloatActivationsOpTestLeakyRelu) {
#ifdef notdef
LeakyReluOpModel m({TensorType_FLOAT32, {2, 3}}, 0.5f);
m.SetInput({
0.0f, 1.0f, 3.0f, // Row 1
1.0f, -1.0f, -2.0f, // Row 2
});
m.Invoke();
EXPECT_THAT(m.GetOutput(), ElementsAreArray({
0.0f, 1.0f, 3.0f, // Row 1
1.0f, -0.5f, -1.0f, // Row 2
}));
#endif // notdef
}
TF_LITE_MICRO_TESTS_END
} // namespace
} // namespace testing
} // namespace tflite
TF_LITE_MICRO_TESTS_BEGIN
TF_LITE_MICRO_TEST(QuantizedActivationsOpTestLeakyReluInt8_1) {
constexpr int kDims[] = {2, 2, 3};
constexpr float kInput[] = {0.0f, 1.0f, 3.0f, 1.0f, -1.0f, -2.0f};
constexpr float kExpect[] = {0.0f, 1.0f, 3.0f, 1.0f, -0.5f, -1.0f};
constexpr int kOutputCount = std::extent<decltype(kExpect)>::value;
float output_data[kOutputCount];
// setup quantization storage and parameters
int8_t q_output_data[kOutputCount];
int8_t q_input_data[kOutputCount];
constexpr float kMin = -1;
constexpr float kMax = 127.f / 128.f;
tflite::testing::TestLeakyReluParams<int8_t> params = {};
params.alpha = 0.5f;
params.data_min = 8 * kMin;
params.data_max = 8 * kMax;
params.input_data = q_input_data;
params.output_data = q_output_data;
params.tolerance = tflite::testing::kQuantizedTolerance * 8;
tflite::testing::TestLeakyReluQuantized(params, kDims, kInput, kDims, kExpect,
output_data);
}
TF_LITE_MICRO_TEST(QuantizedActivationsOpTestLeakyReluInt8_2) {
tflite::testing::QuantizedActivationsOpTestLeakyRelu<int8_t>();
}
TF_LITE_MICRO_TEST(FloatActivationsOpTestLeakyRelu) {
constexpr int kDims[] = {2, 2, 3};
constexpr float kInput[] = {0.0f, 1.0f, 3.0f, 1.0f, -1.0f, -2.0f};
constexpr float kExpect[] = {0.0f, 1.0f, 3.0f, 1.0f, -0.5f, -1.0f};
constexpr int kOutputCount = std::extent<decltype(kExpect)>::value;
float output_data[kOutputCount];
tflite::testing::TestLeakyReluParams<float> params = {};
params.alpha = 0.5f;
tflite::testing::TestLeakyRelu(params, kDims, kInput, kDims, kExpect,
output_data);
}
TF_LITE_MICRO_TESTS_END

1
tensorflow/lite/micro/kernels/micro_ops.h
View File

@ -37,6 +37,7 @@ TfLiteRegistration Register_CONV_2D();
TfLiteRegistration Register_DEPTHWISE_CONV_2D();
TfLiteRegistration Register_ELU();
TfLiteRegistration Register_EXP();
TfLiteRegistration Register_LEAKY_RELU();
TfLiteRegistration Register_FILL();
TfLiteRegistration Register_QUANTIZE();
TfLiteRegistration Register_SHAPE();

5
tensorflow/lite/micro/micro_mutable_op_resolver.h
View File

@ -243,6 +243,11 @@ class MicroMutableOpResolver : public MicroOpResolver {
ParseL2Normalization);
}
TfLiteStatus AddLeakyRelu() {
return AddBuiltin(BuiltinOperator_LEAKY_RELU, tflite::Register_LEAKY_RELU(),
ParseLeakyRelu);
}
TfLiteStatus AddLess() {
return AddBuiltin(BuiltinOperator_LESS, tflite::ops::micro::Register_LESS(),
ParseLess);

3
tensorflow/lite/micro/tools/make/Makefile
View File

@ -277,6 +277,7 @@ tensorflow/lite/micro/kernels/floor_test.cc \
tensorflow/lite/micro/kernels/fully_connected_test.cc \
tensorflow/lite/micro/kernels/hard_swish_test.cc \
tensorflow/lite/micro/kernels/l2norm_test.cc \
tensorflow/lite/micro/kernels/leaky_relu_test.cc \
tensorflow/lite/micro/kernels/logical_test.cc \
tensorflow/lite/micro/kernels/logistic_test.cc \
tensorflow/lite/micro/kernels/maximum_minimum_test.cc \
@ -337,6 +338,7 @@ tensorflow/lite/micro/kernels/hard_swish.cc \
tensorflow/lite/micro/kernels/kernel_runner.cc \
tensorflow/lite/micro/kernels/kernel_util.cc \
tensorflow/lite/micro/kernels/l2norm.cc \
tensorflow/lite/micro/kernels/leaky_relu.cc \
tensorflow/lite/micro/kernels/logical.cc \
tensorflow/lite/micro/kernels/logistic.cc \
tensorflow/lite/micro/kernels/maximum_minimum.cc \
@ -432,6 +434,7 @@ tensorflow/lite/kernels/internal/reference/integer_ops/pooling.h \
tensorflow/lite/kernels/internal/reference/integer_ops/tanh.h \
tensorflow/lite/kernels/internal/reference/integer_ops/transpose_conv.h \
tensorflow/lite/kernels/internal/reference/l2normalization.h \
tensorflow/lite/kernels/internal/reference/leaky_relu.h \
tensorflow/lite/kernels/internal/reference/maximum_minimum.h \
tensorflow/lite/kernels/internal/reference/mul.h \
tensorflow/lite/kernels/internal/reference/neg.h \