From 333c4ff5569a7d9a5b5b3ab8fda638588d008fba Mon Sep 17 00:00:00 2001 From: Jian Li Date: Mon, 15 Oct 2018 10:59:37 -0700 Subject: [PATCH] Add speech frontend. PiperOrigin-RevId: 217171519 --- .../lite/experimental/microfrontend/lib/BUILD | 188 +++++++++++++++ .../experimental/microfrontend/lib/README | 9 + .../experimental/microfrontend/lib/bits.h | 102 ++++++++ .../lite/experimental/microfrontend/lib/fft.c | 54 +++++ .../lite/experimental/microfrontend/lib/fft.h | 50 ++++ .../experimental/microfrontend/lib/fft_io.c | 33 +++ .../experimental/microfrontend/lib/fft_io.h | 34 +++ .../microfrontend/lib/fft_test.cc | 49 ++++ .../experimental/microfrontend/lib/fft_util.c | 71 ++++++ .../experimental/microfrontend/lib/fft_util.h | 34 +++ .../microfrontend/lib/filterbank.c | 134 +++++++++++ .../microfrontend/lib/filterbank.h | 63 +++++ .../microfrontend/lib/filterbank_io.c | 66 +++++ .../microfrontend/lib/filterbank_io.h | 35 +++ .../microfrontend/lib/filterbank_test.cc | 194 +++++++++++++++ .../microfrontend/lib/filterbank_util.c | 225 ++++++++++++++++++ .../microfrontend/lib/filterbank_util.h | 50 ++++ .../experimental/microfrontend/lib/frontend.c | 72 ++++++ .../experimental/microfrontend/lib/frontend.h | 64 +++++ .../microfrontend/lib/frontend_io.c | 69 ++++++ .../microfrontend/lib/frontend_io.h | 31 +++ .../microfrontend/lib/frontend_main.c | 70 ++++++ .../lib/frontend_memmap_generator.c | 47 ++++ .../microfrontend/lib/frontend_memmap_main.c | 58 +++++ .../microfrontend/lib/frontend_test.cc | 120 ++++++++++ .../microfrontend/lib/frontend_util.c | 87 +++++++ .../microfrontend/lib/frontend_util.h | 52 ++++ .../experimental/microfrontend/lib/log_lut.c | 30 +++ .../experimental/microfrontend/lib/log_lut.h | 40 ++++ .../microfrontend/lib/log_scale.c | 83 +++++++ .../microfrontend/lib/log_scale.h | 39 +++ .../microfrontend/lib/log_scale_io.c | 21 ++ .../microfrontend/lib/log_scale_io.h | 33 +++ .../microfrontend/lib/log_scale_test.cc | 58 +++++ .../microfrontend/lib/log_scale_util.c | 27 +++ .../microfrontend/lib/log_scale_util.h | 45 ++++ .../microfrontend/lib/noise_reduction.c | 51 ++++ .../microfrontend/lib/noise_reduction.h | 46 ++++ .../microfrontend/lib/noise_reduction_io.c | 34 +++ .../microfrontend/lib/noise_reduction_io.h | 36 +++ .../microfrontend/lib/noise_reduction_test.cc | 70 ++++++ .../microfrontend/lib/noise_reduction_util.c | 45 ++++ .../microfrontend/lib/noise_reduction_util.h | 50 ++++ .../microfrontend/lib/pcan_gain_control.c | 56 +++++ .../microfrontend/lib/pcan_gain_control.h | 46 ++++ .../lib/pcan_gain_control_test.cc | 59 +++++ .../lib/pcan_gain_control_util.c | 90 +++++++ .../lib/pcan_gain_control_util.h | 57 +++++ .../experimental/microfrontend/lib/window.c | 70 ++++++ .../experimental/microfrontend/lib/window.h | 49 ++++ .../microfrontend/lib/window_io.c | 42 ++++ .../microfrontend/lib/window_io.h | 34 +++ .../microfrontend/lib/window_test.cc | 157 ++++++++++++ .../microfrontend/lib/window_util.c | 71 ++++++ .../microfrontend/lib/window_util.h | 45 ++++ tensorflow/workspace.bzl | 2 + third_party/kissfft/BUILD | 1 + third_party/kissfft/BUILD.bazel | 23 ++ third_party/kissfft/workspace.bzl | 15 ++ 59 files changed, 3586 insertions(+) create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/BUILD create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/README create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/bits.h create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/fft.c create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/fft.h create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/fft_io.c create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/fft_io.h create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/fft_test.cc create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/fft_util.c create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/fft_util.h create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/filterbank.c create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/filterbank.h create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/filterbank_io.c create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/filterbank_io.h create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/filterbank_test.cc create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/filterbank_util.c create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/filterbank_util.h create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/frontend.c create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/frontend.h create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/frontend_io.c create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/frontend_io.h create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/frontend_main.c create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/frontend_memmap_generator.c create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/frontend_memmap_main.c create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/frontend_test.cc create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/frontend_util.c create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/frontend_util.h create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/log_lut.c create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/log_lut.h create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/log_scale.c create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/log_scale.h create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/log_scale_io.c create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/log_scale_io.h create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/log_scale_test.cc create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/log_scale_util.c create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/log_scale_util.h create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/noise_reduction.c create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/noise_reduction.h create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/noise_reduction_io.c create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/noise_reduction_io.h create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/noise_reduction_test.cc create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/noise_reduction_util.c create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/noise_reduction_util.h create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/pcan_gain_control.c create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/pcan_gain_control.h create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/pcan_gain_control_test.cc create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/pcan_gain_control_util.c create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/pcan_gain_control_util.h create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/window.c create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/window.h create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/window_io.c create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/window_io.h create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/window_test.cc create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/window_util.c create mode 100644 tensorflow/contrib/lite/experimental/microfrontend/lib/window_util.h create mode 100644 third_party/kissfft/BUILD create mode 100644 third_party/kissfft/BUILD.bazel create mode 100644 third_party/kissfft/workspace.bzl diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/BUILD b/tensorflow/contrib/lite/experimental/microfrontend/lib/BUILD new file mode 100644 index 00000000000..3fd4b9fe82f --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/BUILD @@ -0,0 +1,188 @@ +# Library for generating feature vectors from audio data + +package( + default_visibility = ["//visibility:private"], +) + +licenses(["notice"]) # Apache 2.0 + +cc_library( + name = "bits", + hdrs = ["bits.h"], +) + +cc_library( + name = "fft", + srcs = [ + "fft.c", + "fft_util.c", + ], + hdrs = [ + "fft.h", + "fft_util.h", + ], + deps = ["@kissfft//:kiss_fftr_16"], +) + +cc_library( + name = "filterbank", + srcs = [ + "filterbank.c", + "filterbank_util.c", + ], + hdrs = [ + "filterbank.h", + "filterbank_util.h", + ], + deps = [ + ":bits", + ":fft", + ], +) + +cc_library( + name = "frontend", + srcs = [ + "frontend.c", + "frontend_util.c", + ], + hdrs = [ + "frontend.h", + "frontend_util.h", + ], + deps = [ + ":bits", + ":fft", + ":filterbank", + ":log_scale", + ":noise_reduction", + ":pcan_gain_control", + ":window", + ], +) + +cc_library( + name = "log_scale", + srcs = [ + "log_lut.c", + "log_scale.c", + "log_scale_util.c", + ], + hdrs = [ + "log_lut.h", + "log_scale.h", + "log_scale_util.h", + ], + deps = [ + ":bits", + ], +) + +cc_library( + name = "noise_reduction", + srcs = [ + "noise_reduction.c", + "noise_reduction_util.c", + ], + hdrs = [ + "noise_reduction.h", + "noise_reduction_util.h", + ], +) + +cc_library( + name = "pcan_gain_control", + srcs = [ + "pcan_gain_control.c", + "pcan_gain_control_util.c", + ], + hdrs = [ + "pcan_gain_control.h", + "pcan_gain_control_util.h", + ], + deps = [ + ":bits", + ], +) + +cc_library( + name = "window", + srcs = [ + "window.c", + "window_util.c", + ], + hdrs = [ + "window.h", + "window_util.h", + ], +) + +cc_test( + name = "fft_test", + size = "small", + srcs = ["fft_test.cc"], + deps = [ + ":fft", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "filterbank_test", + size = "small", + srcs = ["filterbank_test.cc"], + deps = [ + ":filterbank", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "frontend_test", + size = "small", + srcs = ["frontend_test.cc"], + deps = [ + ":frontend", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "log_scale_test", + size = "small", + srcs = ["log_scale_test.cc"], + deps = [ + ":log_scale", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "noise_reduction_test", + size = "small", + srcs = ["noise_reduction_test.cc"], + deps = [ + ":noise_reduction", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "pcan_gain_control_test", + size = "small", + srcs = ["pcan_gain_control_test.cc"], + deps = [ + ":pcan_gain_control", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "window_test", + size = "small", + srcs = ["window_test.cc"], + deps = [ + ":window", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/README b/tensorflow/contrib/lite/experimental/microfrontend/lib/README new file mode 100644 index 00000000000..731d88c5bda --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/README @@ -0,0 +1,9 @@ +The binary frontend_main shows sample usage of the frontend, printing out +coefficients when it has processed enough data. + +The binary frontend_memmap_main shows a sample usage of how to avoid all the +init code in your runtime, by first running "frontend_generate_memmap" to +create a header/source file that uses a baked in frontend state. This command +could be automated as part of your build process, or you can just use the output +directly. + diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/bits.h b/tensorflow/contrib/lite/experimental/microfrontend/lib/bits.h new file mode 100644 index 00000000000..f81bc2b023e --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/bits.h @@ -0,0 +1,102 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_BITS_H_ +#define TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_BITS_H_ + +#ifdef __cplusplus +#include + +extern "C" { +#endif + +static inline int CountLeadingZeros32Slow(uint64_t n) { + int zeroes = 28; + if (n >> 16) zeroes -= 16, n >>= 16; + if (n >> 8) zeroes -= 8, n >>= 8; + if (n >> 4) zeroes -= 4, n >>= 4; + return "\4\3\2\2\1\1\1\1\0\0\0\0\0\0\0"[n] + zeroes; +} + +static inline int CountLeadingZeros32(uint32_t n) { +#if defined(_MSC_VER) + unsigned long result = 0; // NOLINT(runtime/int) + if (_BitScanReverse(&result, n)) { + return 31 - result; + } + return 32; +#elif defined(__GNUC__) + + // Handle 0 as a special case because __builtin_clz(0) is undefined. + if (n == 0) { + return 32; + } + return __builtin_clz(n); +#else + return CountLeadingZeros32Slow(n); +#endif +} + +static inline int MostSignificantBit32(uint32_t n) { + return 32 - CountLeadingZeros32(n); +} + +static inline int CountLeadingZeros64Slow(uint64_t n) { + int zeroes = 60; + if (n >> 32) zeroes -= 32, n >>= 32; + if (n >> 16) zeroes -= 16, n >>= 16; + if (n >> 8) zeroes -= 8, n >>= 8; + if (n >> 4) zeroes -= 4, n >>= 4; + return "\4\3\2\2\1\1\1\1\0\0\0\0\0\0\0"[n] + zeroes; +} + +static inline int CountLeadingZeros64(uint64_t n) { +#if defined(_MSC_VER) && defined(_M_X64) + // MSVC does not have __buitin_clzll. Use _BitScanReverse64. + unsigned long result = 0; // NOLINT(runtime/int) + if (_BitScanReverse64(&result, n)) { + return 63 - result; + } + return 64; +#elif defined(_MSC_VER) + // MSVC does not have __buitin_clzll. Compose two calls to _BitScanReverse + unsigned long result = 0; // NOLINT(runtime/int) + if ((n >> 32) && _BitScanReverse(&result, n >> 32)) { + return 31 - result; + } + if (_BitScanReverse(&result, n)) { + return 63 - result; + } + return 64; +#elif defined(__GNUC__) + + // Handle 0 as a special case because __builtin_clzll(0) is undefined. + if (n == 0) { + return 64; + } + return __builtin_clzll(n); +#else + return CountLeadingZeros64Slow(n); +#endif +} + +static inline int MostSignificantBit64(uint64_t n) { + return 64 - CountLeadingZeros64(n); +} + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_BITS_H_ diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/fft.c b/tensorflow/contrib/lite/experimental/microfrontend/lib/fft.c new file mode 100644 index 00000000000..1ecbb30b514 --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/fft.c @@ -0,0 +1,54 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/fft.h" + +#include + +#define FIXED_POINT 16 +#include "kiss_fft.h" +// Internal test dependency placeholder1 +// Internal test dependency placeholder2 +#include "tools/kiss_fftr.h" +// Internal test dependency placeholder3 + +void FftCompute(struct FftState* state, const int16_t* input, + int input_scale_shift) { + const size_t input_size = state->input_size; + const size_t fft_size = state->fft_size; + + int16_t* fft_input = state->input; + // First, scale the input by the given shift. + int i; + for (i = 0; i < input_size; ++i) { + *fft_input++ = (*input++) << input_scale_shift; + } + // Zero out whatever else remains in the top part of the input. + for (; i < fft_size; ++i) { + *fft_input++ = 0; + } + + // Apply the FFT. + kiss_fftr((const kiss_fftr_cfg)state->scratch, state->input, + (kiss_fft_cpx*)state->output); +} + +void FftInit(struct FftState* state) { + // All the initialization is done in FftPopulateState() +} + +void FftReset(struct FftState* state) { + memset(state->input, 0, state->fft_size * sizeof(*state->input)); + memset(state->output, 0, (state->fft_size / 2 + 1) * sizeof(*state->output)); +} diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/fft.h b/tensorflow/contrib/lite/experimental/microfrontend/lib/fft.h new file mode 100644 index 00000000000..e7644bf2a70 --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/fft.h @@ -0,0 +1,50 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FFT_H_ +#define TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FFT_H_ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct complex_int16_t { + int16_t real; + int16_t imag; +}; + +struct FftState { + int16_t* input; + struct complex_int16_t* output; + size_t fft_size; + size_t input_size; + void* scratch; + size_t scratch_size; +}; + +void FftCompute(struct FftState* state, const int16_t* input, + int input_scale_shift); + +void FftInit(struct FftState* state); + +void FftReset(struct FftState* state); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FFT_H_ diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/fft_io.c b/tensorflow/contrib/lite/experimental/microfrontend/lib/fft_io.c new file mode 100644 index 00000000000..cc1ce209d85 --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/fft_io.c @@ -0,0 +1,33 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/fft_io.h" + +void FftWriteMemmapPreamble(FILE* fp, const struct FftState* state) { + fprintf(fp, "static int16_t fft_input[%zu];\n", state->fft_size); + fprintf(fp, "static struct complex_int16_t fft_output[%zu];\n", + state->fft_size / 2 + 1); + fprintf(fp, "static char fft_scratch[%zu];\n", state->scratch_size); + fprintf(fp, "\n"); +} + +void FftWriteMemmap(FILE* fp, const struct FftState* state, + const char* variable) { + fprintf(fp, "%s->input = fft_input;\n", variable); + fprintf(fp, "%s->output = fft_output;\n", variable); + fprintf(fp, "%s->fft_size = %zu;\n", variable, state->fft_size); + fprintf(fp, "%s->input_size = %zu;\n", variable, state->input_size); + fprintf(fp, "%s->scratch = fft_scratch;\n", variable); + fprintf(fp, "%s->scratch_size = %zu;\n", variable, state->scratch_size); +} diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/fft_io.h b/tensorflow/contrib/lite/experimental/microfrontend/lib/fft_io.h new file mode 100644 index 00000000000..4d10c3a92af --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/fft_io.h @@ -0,0 +1,34 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FFT_IO_H_ +#define TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FFT_IO_H_ + +#include + +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/fft.h" + +#ifdef __cplusplus +extern "C" { +#endif + +void FftWriteMemmapPreamble(FILE* fp, const struct FftState* state); +void FftWriteMemmap(FILE* fp, const struct FftState* state, + const char* variable); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FFT_IO_H_ diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/fft_test.cc b/tensorflow/contrib/lite/experimental/microfrontend/lib/fft_test.cc new file mode 100644 index 00000000000..b8684a0b5c0 --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/fft_test.cc @@ -0,0 +1,49 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/fft.h" +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/fft_util.h" + +#include +#include + +namespace { + +const int16_t kFakeWindow[] = { + 0, 1151, 0, -5944, 0, 13311, 0, -21448, 0, 28327, 0, -32256, 0, 32255, + 0, -28328, 0, 21447, 0, -13312, 0, 5943, 0, -1152, 0}; +const int kScaleShift = 0; + +TEST(FftTest, CheckOutputValues) { + struct FftState state; + ASSERT_TRUE( + FftPopulateState(&state, sizeof(kFakeWindow) / sizeof(kFakeWindow[0]))); + + FftInit(&state); + FftCompute(&state, kFakeWindow, kScaleShift); + + const struct complex_int16_t expected[] = { + {0, 0}, {-10, 9}, {-20, 0}, {-9, -10}, {0, 25}, {-119, 119}, + {-887, 0}, {3000, 3000}, {0, -6401}, {-3000, 3000}, {886, 0}, {118, 119}, + {0, 25}, {9, -10}, {19, 0}, {9, 9}, {0, 0}}; + ASSERT_EQ(state.fft_size / 2 + 1, sizeof(expected) / sizeof(expected[0])); + for (int i = 0; i <= state.fft_size / 2; ++i) { + EXPECT_EQ(state.output[i].real, expected[i].real); + EXPECT_EQ(state.output[i].imag, expected[i].imag); + } + + FftFreeStateContents(&state); +} + +} // namespace diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/fft_util.c b/tensorflow/contrib/lite/experimental/microfrontend/lib/fft_util.c new file mode 100644 index 00000000000..55494422f37 --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/fft_util.c @@ -0,0 +1,71 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/fft_util.h" + +#include + +#define FIXED_POINT 16 +#include "kiss_fft.h" +#include "tools/kiss_fftr.h" + +int FftPopulateState(struct FftState* state, size_t input_size) { + state->input_size = input_size; + state->fft_size = 1; + while (state->fft_size < state->input_size) { + state->fft_size <<= 1; + } + + state->input = malloc(state->fft_size * sizeof(*state->input)); + if (state->input == NULL) { + fprintf(stderr, "Failed to alloc fft input buffer\n"); + return 0; + } + + state->output = + malloc((state->fft_size / 2 + 1) * sizeof(*state->output) * 2); + if (state->output == NULL) { + fprintf(stderr, "Failed to alloc fft output buffer\n"); + return 0; + } + + // Ask kissfft how much memory it wants. + size_t scratch_size = 0; + kiss_fftr_cfg kfft_cfg = + kiss_fftr_alloc(state->fft_size, 0, NULL, &scratch_size); + if (kfft_cfg != NULL) { + fprintf(stderr, "Kiss memory sizing failed.\n"); + return 0; + } + state->scratch = malloc(scratch_size); + if (state->scratch == NULL) { + fprintf(stderr, "Failed to alloc fft scratch buffer\n"); + return 0; + } + state->scratch_size = scratch_size; + // Let kissfft configure the scratch space we just allocated + kfft_cfg = kiss_fftr_alloc(state->fft_size, 0, state->scratch, &scratch_size); + if (kfft_cfg != state->scratch) { + fprintf(stderr, "Kiss memory preallocation strategy failed.\n"); + return 0; + } + return 1; +} + +void FftFreeStateContents(struct FftState* state) { + free(state->input); + free(state->output); + free(state->scratch); +} + diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/fft_util.h b/tensorflow/contrib/lite/experimental/microfrontend/lib/fft_util.h new file mode 100644 index 00000000000..4935e87fc1a --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/fft_util.h @@ -0,0 +1,34 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FFT_UTIL_H_ +#define TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FFT_UTIL_H_ + +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/fft.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Prepares and FFT for the given input size. +int FftPopulateState(struct FftState* state, size_t input_size); + +// Frees any allocated buffers. +void FftFreeStateContents(struct FftState* state); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FFT_UTIL_H_ diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/filterbank.c b/tensorflow/contrib/lite/experimental/microfrontend/lib/filterbank.c new file mode 100644 index 00000000000..944eb1a7379 --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/filterbank.c @@ -0,0 +1,134 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/filterbank.h" + +#include + +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/bits.h" + +void FilterbankConvertFftComplexToEnergy(struct FilterbankState* state, + struct complex_int16_t* fft_output, + int32_t* energy) { + const int end_index = state->end_index; + int i; + energy += state->start_index; + fft_output += state->start_index; + for (i = state->start_index; i < end_index; ++i) { + const int32_t real = fft_output->real; + const int32_t imag = fft_output->imag; + fft_output++; + const uint32_t mag_squared = (real * real) + (imag * imag); + *energy++ = mag_squared; + } +} + +void FilterbankAccumulateChannels(struct FilterbankState* state, + const int32_t* energy) { + uint64_t* work = state->work; + uint64_t weight_accumulator = 0; + uint64_t unweight_accumulator = 0; + + const int16_t* channel_frequency_starts = state->channel_frequency_starts; + const int16_t* channel_weight_starts = state->channel_weight_starts; + const int16_t* channel_widths = state->channel_widths; + + int num_channels_plus_1 = state->num_channels + 1; + int i; + for (i = 0; i < num_channels_plus_1; ++i) { + const int32_t* magnitudes = energy + *channel_frequency_starts++; + const int16_t* weights = state->weights + *channel_weight_starts; + const int16_t* unweights = state->unweights + *channel_weight_starts++; + const int width = *channel_widths++; + int j; + for (j = 0; j < width; ++j) { + weight_accumulator += *weights++ * ((uint64_t) *magnitudes); + unweight_accumulator += *unweights++ * ((uint64_t) *magnitudes); + ++magnitudes; + } + *work++ = weight_accumulator; + weight_accumulator = unweight_accumulator; + unweight_accumulator = 0; + } +} + +static uint16_t Sqrt32(uint32_t num) { + if (num == 0) { + return 0; + } + uint32_t res = 0; + int max_bit_number = 32 - MostSignificantBit32(num); + max_bit_number |= 1; + uint32_t bit = 1U << (31 - max_bit_number); + int iterations = (31 - max_bit_number) / 2 + 1; + while (iterations--) { + if (num >= res + bit) { + num -= res + bit; + res = (res >> 1U) + bit; + } else { + res >>= 1U; + } + bit >>= 2U; + } + // Do rounding - if we have the bits. + if (num > res && res != 0xFFFF) { + ++res; + } + return res; +} + +static uint32_t Sqrt64(uint64_t num) { + // Take a shortcut and just use 32 bit operations if the upper word is all + // clear. This will cause a slight off by one issue for numbers close to 2^32, + // but it probably isn't going to matter (and gives us a big performance win). + if ((num >> 32) == 0) { + return Sqrt32((uint32_t) num); + } + uint64_t res = 0; + int max_bit_number = 64 - MostSignificantBit64(num); + max_bit_number |= 1; + uint64_t bit = 1ULL << (63 - max_bit_number); + int iterations = (63 - max_bit_number) / 2 + 1; + while (iterations--) { + if (num >= res + bit) { + num -= res + bit; + res = (res >> 1U) + bit; + } else { + res >>= 1U; + } + bit >>= 2U; + } + // Do rounding - if we have the bits. + if (num > res && res != 0xFFFFFFFFLL) { + ++res; + } + return res; +} + +uint32_t* FilterbankSqrt(struct FilterbankState* state, int scale_down_shift) { + const int num_channels = state->num_channels; + const int64_t* work = state->work + 1; + // Reuse the work buffer since we're fine clobbering it at this point to hold + // the output. + uint32_t* output = (uint32_t*) state->work; + int i; + for (i = 0; i < num_channels; ++i) { + *output++ = Sqrt64(*work++) >> scale_down_shift; + } + return (uint32_t*) state->work; +} + +void FilterbankReset(struct FilterbankState* state) { + memset(state->work, 0, (state->num_channels + 1) * sizeof(*state->work)); +} diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/filterbank.h b/tensorflow/contrib/lite/experimental/microfrontend/lib/filterbank.h new file mode 100644 index 00000000000..0dd9c3fa651 --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/filterbank.h @@ -0,0 +1,63 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FILTERBANK_H_ +#define TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FILTERBANK_H_ + +#include +#include + +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/fft.h" + +#define kFilterbankBits 12 + +#ifdef __cplusplus +extern "C" { +#endif + +struct FilterbankState { + int num_channels; + int start_index; + int end_index; + int16_t* channel_frequency_starts; + int16_t* channel_weight_starts; + int16_t* channel_widths; + int16_t* weights; + int16_t* unweights; + uint64_t* work; +}; + +// Converts the relevant complex values of an FFT output into energy (the +// square magnitude). +void FilterbankConvertFftComplexToEnergy(struct FilterbankState* state, + struct complex_int16_t* fft_output, + int32_t* energy); + +// Computes the mel-scale filterbank on the given energy array. Output is cached +// internally - to fetch it, you need to call FilterbankSqrt. +void FilterbankAccumulateChannels(struct FilterbankState* state, + const int32_t* energy); + +// Applies an integer square root to the 64 bit intermediate values of the +// filterbank, and returns a pointer to them. Memory will be invalidated the +// next time FilterbankAccumulateChannels is called. +uint32_t* FilterbankSqrt(struct FilterbankState* state, int scale_down_shift); + +void FilterbankReset(struct FilterbankState* state); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FILTERBANK_H_ diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/filterbank_io.c b/tensorflow/contrib/lite/experimental/microfrontend/lib/filterbank_io.c new file mode 100644 index 00000000000..672ddd530f8 --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/filterbank_io.c @@ -0,0 +1,66 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/filterbank_io.h" + +static void PrintArray(FILE* fp, const char* name, const int16_t* values, + size_t size) { + fprintf(fp, "static int16_t filterbank_%s[] = {", name); + for (int i = 0; i < size; ++i) { + fprintf(fp, "%d", values[i]); + if (i < size - 1) { + fprintf(fp, ", "); + } + } + fprintf(fp, "};\n"); +} + +void FilterbankWriteMemmapPreamble(FILE* fp, + const struct FilterbankState* state) { + const int num_channels_plus_1 = state->num_channels + 1; + + PrintArray(fp, "channel_frequency_starts", state->channel_frequency_starts, + num_channels_plus_1); + PrintArray(fp, "channel_weight_starts", state->channel_weight_starts, + num_channels_plus_1); + PrintArray(fp, "channel_widths", state->channel_widths, num_channels_plus_1); + int num_weights = 0; + int i; + for (i = 0; i < num_channels_plus_1; ++i) { + num_weights += state->channel_widths[i]; + } + PrintArray(fp, "weights", state->weights, num_weights); + PrintArray(fp, "unweights", state->unweights, num_weights); + + fprintf(fp, "static uint64_t filterbank_work[%d];\n", num_channels_plus_1); + fprintf(fp, "\n"); +} + +void FilterbankWriteMemmap(FILE* fp, const struct FilterbankState* state, + const char* variable) { + fprintf(fp, "%s->num_channels = %d;\n", variable, state->num_channels); + fprintf(fp, "%s->start_index = %d;\n", variable, state->start_index); + fprintf(fp, "%s->end_index = %d;\n", variable, state->end_index); + + fprintf( + fp, + "%s->channel_frequency_starts = filterbank_channel_frequency_starts;\n", + variable); + fprintf(fp, "%s->channel_weight_starts = filterbank_channel_weight_starts;\n", + variable); + fprintf(fp, "%s->channel_widths = filterbank_channel_widths;\n", variable); + fprintf(fp, "%s->weights = filterbank_weights;\n", variable); + fprintf(fp, "%s->unweights = filterbank_unweights;\n", variable); + fprintf(fp, "%s->work = filterbank_work;\n", variable); +} diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/filterbank_io.h b/tensorflow/contrib/lite/experimental/microfrontend/lib/filterbank_io.h new file mode 100644 index 00000000000..1ddc314df22 --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/filterbank_io.h @@ -0,0 +1,35 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FILTERBANK_IO_H_ +#define TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FILTERBANK_IO_H_ + +#include + +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/filterbank.h" + +#ifdef __cplusplus +extern "C" { +#endif + +void FilterbankWriteMemmapPreamble(FILE* fp, + const struct FilterbankState* state); +void FilterbankWriteMemmap(FILE* fp, const struct FilterbankState* state, + const char* variable); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FILTERBANK_IO_H_ diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/filterbank_test.cc b/tensorflow/contrib/lite/experimental/microfrontend/lib/filterbank_test.cc new file mode 100644 index 00000000000..88d8de4b8f0 --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/filterbank_test.cc @@ -0,0 +1,194 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/filterbank.h" +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/filterbank_util.h" + +#include + +#include +#include + +namespace { + +const int kSampleRate = 1000; +const int kSpectrumSize = 17; +const int kStartIndex = 1; +const int kEndIndex = 15; +const int32_t kEnergy[] = {-1, 181, 400, 181, 625, 28322, + 786769, 18000000, 40972801, 18000000, 784996, 28085, + 625, 181, 361, -1, -1}; +const uint64_t kWork[] = {1835887, 61162970173, 258694800000}; +const int kScaleShift = 0; + +// Test filterbank generation using scaled-down defaults. +class FilterbankTest : public ::testing::Test { + protected: + FilterbankTest() { + config_.num_channels = 2; + config_.lower_band_limit = 8.0; + config_.upper_band_limit = 450.0; + } + + struct FilterbankConfig config_; +}; + +TEST_F(FilterbankTest, CheckStartIndex) { + struct FilterbankState state; + ASSERT_TRUE( + FilterbankPopulateState(&config_, &state, kSampleRate, kSpectrumSize)); + + EXPECT_EQ(state.start_index, kStartIndex); + + FilterbankFreeStateContents(&state); +} + +TEST_F(FilterbankTest, CheckEndIndex) { + struct FilterbankState state; + ASSERT_TRUE( + FilterbankPopulateState(&config_, &state, kSampleRate, kSpectrumSize)); + + EXPECT_EQ(state.end_index, kEndIndex); + + FilterbankFreeStateContents(&state); +} + +TEST_F(FilterbankTest, CheckChannelFrequencyStarts) { + struct FilterbankState state; + ASSERT_TRUE( + FilterbankPopulateState(&config_, &state, kSampleRate, kSpectrumSize)); + + const int16_t expected[] = {0, 4, 8}; + ASSERT_EQ(state.num_channels + 1, sizeof(expected) / sizeof(expected[0])); + for (int i = 0; i <= state.num_channels; ++i) { + EXPECT_EQ(state.channel_frequency_starts[i], expected[i]); + } + + FilterbankFreeStateContents(&state); +} + +TEST_F(FilterbankTest, CheckChannelWeightStarts) { + struct FilterbankState state; + ASSERT_TRUE( + FilterbankPopulateState(&config_, &state, kSampleRate, kSpectrumSize)); + + const int16_t expected[] = {0, 8, 16}; + ASSERT_EQ(state.num_channels + 1, sizeof(expected) / sizeof(expected[0])); + for (int i = 0; i <= state.num_channels; ++i) { + EXPECT_EQ(state.channel_weight_starts[i], expected[i]); + } + + FilterbankFreeStateContents(&state); +} + +TEST_F(FilterbankTest, CheckChannelWidths) { + struct FilterbankState state; + ASSERT_TRUE( + FilterbankPopulateState(&config_, &state, kSampleRate, kSpectrumSize)); + + const int16_t expected[] = {8, 8, 8}; + ASSERT_EQ(state.num_channels + 1, sizeof(expected) / sizeof(expected[0])); + for (int i = 0; i <= state.num_channels; ++i) { + EXPECT_EQ(state.channel_widths[i], expected[i]); + } + + FilterbankFreeStateContents(&state); +} + +TEST_F(FilterbankTest, CheckWeights) { + struct FilterbankState state; + ASSERT_TRUE( + FilterbankPopulateState(&config_, &state, kSampleRate, kSpectrumSize)); + + const int16_t expected[] = {0, 3277, 2217, 1200, 222, 0, 0, 0, + 0, 3376, 2468, 1591, 744, 0, 0, 0, + 0, 4020, 3226, 2456, 1708, 983, 277, 0}; + ASSERT_EQ(state.channel_weight_starts[state.num_channels] + + state.channel_widths[state.num_channels], + sizeof(expected) / sizeof(expected[0])); + for (int i = 0; i < sizeof(expected) / sizeof(expected[0]); ++i) { + EXPECT_EQ(state.weights[i], expected[i]); + } + + FilterbankFreeStateContents(&state); +} + +TEST_F(FilterbankTest, CheckUnweights) { + struct FilterbankState state; + ASSERT_TRUE( + FilterbankPopulateState(&config_, &state, kSampleRate, kSpectrumSize)); + + const int16_t expected[] = {0, 819, 1879, 2896, 3874, 0, 0, 0, + 0, 720, 1628, 2505, 3352, 0, 0, 0, + 0, 76, 870, 1640, 2388, 3113, 3819, 0}; + ASSERT_EQ(state.channel_weight_starts[state.num_channels] + + state.channel_widths[state.num_channels], + sizeof(expected) / sizeof(expected[0])); + for (int i = 0; i < sizeof(expected) / sizeof(expected[0]); ++i) { + EXPECT_EQ(state.unweights[i], expected[i]); + } + + FilterbankFreeStateContents(&state); +} + +TEST_F(FilterbankTest, CheckConvertFftComplexToEnergy) { + struct FilterbankState state; + state.start_index = kStartIndex; + state.end_index = kEndIndex; + + struct complex_int16_t fake_fft[] = { + {0, 0}, {-10, 9}, {-20, 0}, {-9, -10}, {0, 25}, {-119, 119}, + {-887, 0}, {3000, 3000}, {0, -6401}, {-3000, 3000}, {886, 0}, {118, 119}, + {0, 25}, {9, -10}, {19, 0}, {9, 9}, {0, 0}}; + int32_t* energy = reinterpret_cast(fake_fft); + FilterbankConvertFftComplexToEnergy(&state, fake_fft, energy); + + for (int i = state.start_index; i < state.end_index; ++i) { + EXPECT_EQ(energy[i], kEnergy[i]); + } +} + +TEST_F(FilterbankTest, CheckAccumulateChannels) { + struct FilterbankState state; + ASSERT_TRUE( + FilterbankPopulateState(&config_, &state, kSampleRate, kSpectrumSize)); + + FilterbankAccumulateChannels(&state, kEnergy); + + ASSERT_EQ(state.num_channels + 1, sizeof(kWork) / sizeof(kWork[0])); + for (int i = 0; i <= state.num_channels; ++i) { + EXPECT_EQ(state.work[i], kWork[i]); + } + + FilterbankFreeStateContents(&state); +} + +TEST_F(FilterbankTest, CheckSqrt) { + struct FilterbankState state; + ASSERT_TRUE( + FilterbankPopulateState(&config_, &state, kSampleRate, kSpectrumSize)); + std::memcpy(state.work, kWork, sizeof(kWork)); + + uint32_t* scaled_filterbank = FilterbankSqrt(&state, kScaleShift); + + const uint32_t expected[] = {247311, 508620}; + ASSERT_EQ(state.num_channels, sizeof(expected) / sizeof(expected[0])); + for (int i = 0; i < state.num_channels; ++i) { + EXPECT_EQ(scaled_filterbank[i], expected[i]); + } + + FilterbankFreeStateContents(&state); +} + +} // namespace diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/filterbank_util.c b/tensorflow/contrib/lite/experimental/microfrontend/lib/filterbank_util.c new file mode 100644 index 00000000000..53b5e450734 --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/filterbank_util.c @@ -0,0 +1,225 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/filterbank_util.h" + +#include +#include +#include + +#define kFilterbankIndexAlignment 4 +#define kFilterbankChannelBlockSize 4 + +void FilterbankFillConfigWithDefaults(struct FilterbankConfig* config) { + config->num_channels = 32; + config->lower_band_limit = 125.0f; + config->upper_band_limit = 7500.0f; + config->output_scale_shift = 7; +} + +static float FreqToMel(float freq) { + return 1127.0 * log(1.0 + (freq / 700.0)); +} + +static void CalculateCenterFrequencies(const int num_channels, + const float lower_frequency_limit, + const float upper_frequency_limit, + float* center_frequencies) { + assert(lower_frequency_limit >= 0.0f); + assert(upper_frequency_limit > lower_frequency_limit); + + const float mel_low = FreqToMel(lower_frequency_limit); + const float mel_hi = FreqToMel(upper_frequency_limit); + const float mel_span = mel_hi - mel_low; + const float mel_spacing = mel_span / ((float) num_channels); + int i; + for (i = 0; i < num_channels; ++i) { + center_frequencies[i] = mel_low + (mel_spacing * (i + 1)); + } +} + +static void QuantizeFilterbankWeights(const float float_weight, + int16_t* weight, int16_t* unweight) { + *weight = floor(float_weight * (1 << kFilterbankBits) + 0.5); + *unweight = floor((1.0 - float_weight) * (1 << kFilterbankBits) + 0.5); +} + +int FilterbankPopulateState(const struct FilterbankConfig* config, + struct FilterbankState* state, + int sample_rate, int spectrum_size) { + state->num_channels = config->num_channels; + const int num_channels_plus_1 = config->num_channels + 1; + + // How should we align things to index counts given the byte alignment? + const int index_alignment = + (kFilterbankIndexAlignment < sizeof(int16_t) + ? 1 + : kFilterbankIndexAlignment / sizeof(int16_t)); + + state->channel_frequency_starts = + malloc(num_channels_plus_1 * sizeof(*state->channel_frequency_starts)); + state->channel_weight_starts = + malloc(num_channels_plus_1 * sizeof(*state->channel_weight_starts)); + state->channel_widths = + malloc(num_channels_plus_1 * sizeof(*state->channel_widths)); + state->work = malloc(num_channels_plus_1 * sizeof(*state->work)); + + float* center_mel_freqs = + malloc(num_channels_plus_1 * sizeof(*center_mel_freqs)); + int16_t* actual_channel_starts = + malloc(num_channels_plus_1 * sizeof(*actual_channel_starts)); + int16_t* actual_channel_widths = + malloc(num_channels_plus_1 * sizeof(*actual_channel_widths)); + + if (state->channel_frequency_starts == NULL || + state->channel_weight_starts == NULL || + state->channel_widths == NULL || + center_mel_freqs == NULL || + actual_channel_starts == NULL || + actual_channel_widths == NULL) { + free(center_mel_freqs); + free(actual_channel_starts); + free(actual_channel_widths); + fprintf(stderr, "Failed to allocate channel buffers\n"); + return 0; + } + + CalculateCenterFrequencies(num_channels_plus_1, config->lower_band_limit, + config->upper_band_limit, center_mel_freqs); + + // Always exclude DC. + const float hz_per_sbin = 0.5 * sample_rate / ((float) spectrum_size - 1); + state->start_index = 1.5 + config->lower_band_limit / hz_per_sbin; + state->end_index = 0; // Initialized to zero here, but actually set below. + + // For each channel, we need to figure out what frequencies belong to it, and + // how much padding we need to add so that we can efficiently multiply the + // weights and unweights for accumulation. To simplify the multiplication + // logic, all channels will have some multiplication to do (even if there are + // no frequencies that accumulate to that channel) - they will be directed to + // a set of zero weights. + int chan_freq_index_start = state->start_index; + int weight_index_start = 0; + int needs_zeros = 0; + + int chan; + for (chan = 0; chan < num_channels_plus_1; ++chan) { + // Keep jumping frequencies until we overshoot the bound on this channel. + int freq_index = chan_freq_index_start; + while (FreqToMel((freq_index) * hz_per_sbin) <= center_mel_freqs[chan]) { + ++freq_index; + } + + const int width = freq_index - chan_freq_index_start; + actual_channel_starts[chan] = chan_freq_index_start; + actual_channel_widths[chan] = width; + + if (width == 0) { + // This channel doesn't actually get anything from the frequencies, it's + // always zero. We need then to insert some 'zero' weights into the + // output, and just redirect this channel to do a single multiplication at + // this point. For simplicity, the zeros are placed at the beginning of + // the weights arrays, so we have to go and update all the other + // weight_starts to reflect this shift (but only once). + state->channel_frequency_starts[chan] = 0; + state->channel_weight_starts[chan] = 0; + state->channel_widths[chan] = kFilterbankChannelBlockSize; + if (!needs_zeros) { + needs_zeros = 1; + int j; + for (j = 0; j < chan; ++j) { + state->channel_weight_starts[j] += kFilterbankChannelBlockSize; + } + weight_index_start += kFilterbankChannelBlockSize; + } + } else { + // How far back do we need to go to ensure that we have the proper + // alignment? + const int aligned_start = + (chan_freq_index_start / index_alignment) * index_alignment; + const int aligned_width = + (chan_freq_index_start - aligned_start + width); + const int padded_width = + (((aligned_width - 1) / kFilterbankChannelBlockSize) + 1) * + kFilterbankChannelBlockSize; + + state->channel_frequency_starts[chan] = aligned_start; + state->channel_weight_starts[chan] = weight_index_start; + state->channel_widths[chan] = padded_width; + weight_index_start += padded_width; + } + chan_freq_index_start = freq_index; + } + + // Allocate the two arrays to store the weights - weight_index_start contains + // the index of what would be the next set of weights that we would need to + // add, so that's how many weights we need to allocate. + state->weights = calloc(weight_index_start, sizeof(*state->weights)); + state->unweights = calloc(weight_index_start, sizeof(*state->unweights)); + + // If the alloc failed, we also need to nuke the arrays. + if (state->weights == NULL || state->unweights == NULL) { + free(center_mel_freqs); + free(actual_channel_starts); + free(actual_channel_widths); + fprintf(stderr, "Failed to allocate weights or unweights\n"); + return 0; + } + + // Next pass, compute all the weights. Since everything has been memset to + // zero, we only need to fill in the weights that correspond to some frequency + // for a channel. + const float mel_low = FreqToMel(config->lower_band_limit); + for (chan = 0; chan < num_channels_plus_1; ++chan) { + int frequency = actual_channel_starts[chan]; + const int num_frequencies = actual_channel_widths[chan]; + const int frequency_offset = + frequency - state->channel_frequency_starts[chan]; + const int weight_start = state->channel_weight_starts[chan]; + const float denom_val = (chan == 0) ? mel_low : center_mel_freqs[chan - 1]; + + int j; + for (j = 0; j < num_frequencies; ++j, ++frequency) { + const float weight = + (center_mel_freqs[chan] - FreqToMel(frequency * hz_per_sbin)) / + (center_mel_freqs[chan] - denom_val); + + // Make the float into an integer for the weights (and unweights). + const int weight_index = weight_start + frequency_offset + j; + QuantizeFilterbankWeights(weight, state->weights + weight_index, + state->unweights + weight_index); + } + if (frequency > state->end_index) { + state->end_index = frequency; + } + } + + free(center_mel_freqs); + free(actual_channel_starts); + free(actual_channel_widths); + if (state->end_index >= spectrum_size) { + fprintf(stderr, "Filterbank end_index is above spectrum size.\n"); + return 0; + } + return 1; +} + +void FilterbankFreeStateContents(struct FilterbankState* state) { + free(state->channel_frequency_starts); + free(state->channel_weight_starts); + free(state->channel_widths); + free(state->weights); + free(state->unweights); + free(state->work); +} diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/filterbank_util.h b/tensorflow/contrib/lite/experimental/microfrontend/lib/filterbank_util.h new file mode 100644 index 00000000000..9ec9bc93028 --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/filterbank_util.h @@ -0,0 +1,50 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FILTERBANK_UTIL_H_ +#define TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FILTERBANK_UTIL_H_ + +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/filterbank.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct FilterbankConfig { + // number of frequency channel buckets for filterbank + int num_channels; + // maximum frequency to include + float upper_band_limit; + // minimum frequency to include + float lower_band_limit; + // unused + int output_scale_shift; +}; + +// Fills the frontendConfig with "sane" defaults. +void FilterbankFillConfigWithDefaults(struct FilterbankConfig* config); + +// Allocates any buffers. +int FilterbankPopulateState(const struct FilterbankConfig* config, + struct FilterbankState* state, int sample_rate, + int spectrum_size); + +// Frees any allocated buffers. +void FilterbankFreeStateContents(struct FilterbankState* state); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FILTERBANK_UTIL_H_ diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/frontend.c b/tensorflow/contrib/lite/experimental/microfrontend/lib/frontend.c new file mode 100644 index 00000000000..de7a60b56fd --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/frontend.c @@ -0,0 +1,72 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/frontend.h" + +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/bits.h" + +struct FrontendOutput FrontendProcessSamples(struct FrontendState* state, + const int16_t* samples, + size_t num_samples, + size_t* num_samples_read) { + struct FrontendOutput output; + output.values = NULL; + output.size = 0; + + // Try to apply the window - if it fails, return and wait for more data. + if (!WindowProcessSamples(&state->window, samples, num_samples, + num_samples_read)) { + return output; + } + + // Apply the FFT to the window's output (and scale it so that the fixed point + // FFT can have as much resolution as possible). + int input_shift = + 15 - MostSignificantBit32(state->window.max_abs_output_value); + FftCompute(&state->fft, state->window.output, input_shift); + + // We can re-ruse the fft's output buffer to hold the energy. + int32_t* energy = (int32_t*) state->fft.output; + + FilterbankConvertFftComplexToEnergy(&state->filterbank, state->fft.output, + energy); + + FilterbankAccumulateChannels(&state->filterbank, energy); + uint32_t* scaled_filterbank = FilterbankSqrt(&state->filterbank, input_shift); + + // Apply noise reduction. + NoiseReductionApply(&state->noise_reduction, scaled_filterbank); + + if (state->pcan_gain_control.enable_pcan) { + PcanGainControlApply(&state->pcan_gain_control, scaled_filterbank); + } + + // Apply the log and scale. + int correction_bits = + MostSignificantBit32(state->fft.fft_size) - 1 - (kFilterbankBits / 2); + uint16_t* logged_filterbank = + LogScaleApply(&state->log_scale, scaled_filterbank, + state->filterbank.num_channels, correction_bits); + + output.size = state->filterbank.num_channels; + output.values = logged_filterbank; + return output; +} + +void FrontendReset(struct FrontendState* state) { + WindowReset(&state->window); + FftReset(&state->fft); + FilterbankReset(&state->filterbank); + NoiseReductionReset(&state->noise_reduction); +} diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/frontend.h b/tensorflow/contrib/lite/experimental/microfrontend/lib/frontend.h new file mode 100644 index 00000000000..71ae81024cb --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/frontend.h @@ -0,0 +1,64 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FRONTEND_H_ +#define TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FRONTEND_H_ + +#include +#include + +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/fft.h" +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/filterbank.h" +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/log_scale.h" +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/noise_reduction.h" +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/pcan_gain_control.h" +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/window.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct FrontendState { + struct WindowState window; + struct FftState fft; + struct FilterbankState filterbank; + struct NoiseReductionState noise_reduction; + struct PcanGainControlState pcan_gain_control; + struct LogScaleState log_scale; +}; + +struct FrontendOutput { + const uint16_t* values; + size_t size; +}; + +// Main entry point to processing frontend samples. Updates num_samples_read to +// contain the number of samples that have been consumed from the input array. +// Returns a struct containing the generated output. If not enough samples were +// added to generate a feature vector, the returned size will be 0 and the +// values pointer will be NULL. Note that the output pointer will be invalidated +// as soon as FrontendProcessSamples is called again, so copy the contents +// elsewhere if you need to use them later. +struct FrontendOutput FrontendProcessSamples(struct FrontendState* state, + const int16_t* samples, + size_t num_samples, + size_t* num_samples_read); + +void FrontendReset(struct FrontendState* state); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FRONTEND_H_ diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/frontend_io.c b/tensorflow/contrib/lite/experimental/microfrontend/lib/frontend_io.c new file mode 100644 index 00000000000..40bcf247497 --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/frontend_io.c @@ -0,0 +1,69 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/frontend_io.h" + +#include + +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/fft_io.h" +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/filterbank_io.h" +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/log_scale_io.h" +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/noise_reduction_io.h" +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/window_io.h" + +int WriteFrontendStateMemmap(const char* header, const char* source, + const struct FrontendState* state) { + // Write a header that just has our init function. + FILE* fp = fopen(header, "w"); + if (!fp) { + fprintf(stderr, "Failed to open header '%s' for write\n", header); + return 0; + } + fprintf(fp, "#ifndef FRONTEND_STATE_MEMMAP_H_\n"); + fprintf(fp, "#define FRONTEND_STATE_MEMMAP_H_\n"); + fprintf(fp, "\n"); + fprintf(fp, "#include \"frontend.h\"\n"); + fprintf(fp, "\n"); + fprintf(fp, "struct FrontendState* GetFrontendStateMemmap();\n"); + fprintf(fp, "\n"); + fprintf(fp, "#endif // FRONTEND_STATE_MEMMAP_H_\n"); + fclose(fp); + + // Write out the source file that actually has everything in it. + fp = fopen(source, "w"); + if (!fp) { + fprintf(stderr, "Failed to open source '%s' for write\n", source); + return 0; + } + fprintf(fp, "#include \"%s\"\n", header); + fprintf(fp, "\n"); + WindowWriteMemmapPreamble(fp, &state->window); + FftWriteMemmapPreamble(fp, &state->fft); + FilterbankWriteMemmapPreamble(fp, &state->filterbank); + NoiseReductionWriteMemmapPreamble(fp, &state->noise_reduction); + fprintf(fp, "static struct FrontendState state;\n"); + fprintf(fp, "struct FrontendState* GetFrontendStateMemmap() {\n"); + WindowWriteMemmap(fp, &state->window, " (&state.window)"); + FftWriteMemmap(fp, &state->fft, " (&state.fft)"); + FilterbankWriteMemmap(fp, &state->filterbank, " (&state.filterbank)"); + NoiseReductionWriteMemmap(fp, &state->noise_reduction, + " (&state.noise_reduction)"); + LogScaleWriteMemmap(fp, &state->log_scale, " (&state.log_scale)"); + fprintf(fp, " FftInit(&state.fft);\n"); + fprintf(fp, " FrontendReset(&state);\n"); + fprintf(fp, " return &state;\n"); + fprintf(fp, "}\n"); + fclose(fp); + return 1; +} diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/frontend_io.h b/tensorflow/contrib/lite/experimental/microfrontend/lib/frontend_io.h new file mode 100644 index 00000000000..4f45577caea --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/frontend_io.h @@ -0,0 +1,31 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FRONTEND_IO_H_ +#define TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FRONTEND_IO_H_ + +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/frontend.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int WriteFrontendStateMemmap(const char* header, const char* source, + const struct FrontendState* state); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FRONTEND_IO_H_ diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/frontend_main.c b/tensorflow/contrib/lite/experimental/microfrontend/lib/frontend_main.c new file mode 100644 index 00000000000..46caebeec90 --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/frontend_main.c @@ -0,0 +1,70 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include + +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/frontend.h" +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/frontend_util.h" + +int main(int argc, char** argv) { + struct FrontendConfig frontend_config; + FrontendFillConfigWithDefaults(&frontend_config); + + char* filename = argv[1]; + int sample_rate = 16000; + + struct FrontendState frontend_state; + if (!FrontendPopulateState(&frontend_config, &frontend_state, sample_rate)) { + fprintf(stderr, "Failed to populate frontend state\n"); + FrontendFreeStateContents(&frontend_state); + return 1; + } + + + FILE* fp = fopen(filename, "r"); + if (fp == NULL) { + fprintf(stderr, "Failed to open %s for read\n", filename); + return 1; + } + fseek(fp, 0L, SEEK_END); + size_t audio_file_size = ftell(fp) / sizeof(int16_t); + fseek(fp, 0L, SEEK_SET); + int16_t* audio_data = malloc(audio_file_size * sizeof(int16_t)); + int16_t* original_audio_data = audio_data; + if (audio_file_size != + fread(audio_data, sizeof(int16_t), audio_file_size, fp)) { + fprintf(stderr, "Failed to read in all audio data\n"); + return 1; + } + + while (audio_file_size > 0) { + size_t num_samples_read; + struct FrontendOutput output = FrontendProcessSamples( + &frontend_state, audio_data, audio_file_size, &num_samples_read); + audio_data += num_samples_read; + audio_file_size -= num_samples_read; + + if (output.values != NULL) { + int i; + for (i = 0; i < output.size; ++i) { + printf("%d ", output.values[i]); + } + printf("\n"); + } + } + + FrontendFreeStateContents(&frontend_state); + free(original_audio_data); + return 0; +} diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/frontend_memmap_generator.c b/tensorflow/contrib/lite/experimental/microfrontend/lib/frontend_memmap_generator.c new file mode 100644 index 00000000000..a4c59b0ccca --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/frontend_memmap_generator.c @@ -0,0 +1,47 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include + +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/frontend.h" +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/frontend_util.h" +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/frontend_io.h" + +int main(int argc, char** argv) { + if (argc != 3) { + fprintf(stderr, + "%s requires exactly two parameters - the names of the header and " + "source files to save\n"); + return 1; + } + struct FrontendConfig frontend_config; + FrontendFillConfigWithDefaults(&frontend_config); + + int sample_rate = 16000; + struct FrontendState frontend_state; + if (!FrontendPopulateState(&frontend_config, &frontend_state, sample_rate)) { + fprintf(stderr, "Failed to populate frontend state\n"); + FrontendFreeStateContents(&frontend_state); + return 1; + } + + if (!WriteFrontendStateMemmap(argv[1], argv[2], &frontend_state)) { + fprintf(stderr, "Failed to write memmap\n"); + FrontendFreeStateContents(&frontend_state); + return 1; + } + + FrontendFreeStateContents(&frontend_state); + return 0; +} diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/frontend_memmap_main.c b/tensorflow/contrib/lite/experimental/microfrontend/lib/frontend_memmap_main.c new file mode 100644 index 00000000000..a4264922b94 --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/frontend_memmap_main.c @@ -0,0 +1,58 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include + +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/frontend.h" +#include "memmap.h" + +int main(int argc, char** argv) { + struct FrontendState* frontend_state = GetFrontendStateMemmap(); + + char* filename = argv[1]; + FILE* fp = fopen(filename, "r"); + if (fp == NULL) { + fprintf(stderr, "Failed to open %s for read\n", filename); + return 1; + } + fseek(fp, 0L, SEEK_END); + size_t audio_file_size = ftell(fp) / sizeof(int16_t); + fseek(fp, 0L, SEEK_SET); + int16_t* audio_data = malloc(audio_file_size * sizeof(int16_t)); + int16_t* original_audio_data = audio_data; + if (audio_file_size != + fread(audio_data, sizeof(int16_t), audio_file_size, fp)) { + fprintf(stderr, "Failed to read in all audio data\n"); + return 1; + } + + while (audio_file_size > 0) { + size_t num_samples_read; + struct FrontendOutput output = FrontendProcessSamples( + frontend_state, audio_data, audio_file_size, &num_samples_read); + audio_data += num_samples_read; + audio_file_size -= num_samples_read; + + if (output.values != NULL) { + int i; + for (i = 0; i < output.size; ++i) { + printf("%d ", output.values[i]); + } + printf("\n"); + } + } + + free(original_audio_data); + return 0; +} diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/frontend_test.cc b/tensorflow/contrib/lite/experimental/microfrontend/lib/frontend_test.cc new file mode 100644 index 00000000000..f06e2565c28 --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/frontend_test.cc @@ -0,0 +1,120 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/frontend.h" +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/frontend_util.h" + +#include +#include + +namespace { + +const int kSampleRate = 1000; +const int kWindowSamples = 25; +const int kStepSamples = 10; +const int16_t kFakeAudioData[] = { + 0, 32767, 0, -32768, 0, 32767, 0, -32768, 0, 32767, 0, -32768, + 0, 32767, 0, -32768, 0, 32767, 0, -32768, 0, 32767, 0, -32768, + 0, 32767, 0, -32768, 0, 32767, 0, -32768, 0, 32767, 0, -32768}; + +// Test end-to-end frontend behaviors. +class FrontendTest : public ::testing::Test { + protected: + FrontendTest() { + config_.window.size_ms = 25; + config_.window.step_size_ms = 10; + config_.noise_reduction.smoothing_bits = 10; + config_.filterbank.num_channels = 2; + config_.filterbank.lower_band_limit = 8.0; + config_.filterbank.upper_band_limit = 450.0; + config_.noise_reduction.smoothing_bits = 10; + config_.noise_reduction.even_smoothing = 0.025; + config_.noise_reduction.odd_smoothing = 0.06; + config_.noise_reduction.min_signal_remaining = 0.05; + config_.pcan_gain_control.enable_pcan = true; + config_.pcan_gain_control.strength = 0.95; + config_.pcan_gain_control.offset = 80.0; + config_.pcan_gain_control.gain_bits = 21; + config_.log_scale.enable_log = true; + config_.log_scale.scale_shift = 6; + } + + struct FrontendConfig config_; +}; + +TEST_F(FrontendTest, CheckOutputValues) { + struct FrontendState state; + ASSERT_TRUE(FrontendPopulateState(&config_, &state, kSampleRate)); + size_t num_samples_read; + + struct FrontendOutput output = FrontendProcessSamples( + &state, kFakeAudioData, + sizeof(kFakeAudioData) / sizeof(kFakeAudioData[0]), &num_samples_read); + + const uint16_t expected[] = {479, 425}; + ASSERT_EQ(output.size, sizeof(expected) / sizeof(expected[0])); + for (int i = 0; i < output.size; ++i) { + EXPECT_EQ(output.values[i], expected[i]); + } + + FrontendFreeStateContents(&state); +} + +TEST_F(FrontendTest, CheckConsecutiveWindow) { + struct FrontendState state; + ASSERT_TRUE(FrontendPopulateState(&config_, &state, kSampleRate)); + size_t num_samples_read; + + FrontendProcessSamples(&state, kFakeAudioData, + sizeof(kFakeAudioData) / sizeof(kFakeAudioData[0]), + &num_samples_read); + struct FrontendOutput output = FrontendProcessSamples( + &state, kFakeAudioData + kWindowSamples, + sizeof(kFakeAudioData) / sizeof(kFakeAudioData[0]) - kWindowSamples, + &num_samples_read); + + const int16_t expected[] = {436, 378}; + ASSERT_EQ(output.size, sizeof(expected) / sizeof(expected[0])); + for (int i = 0; i < output.size; ++i) { + EXPECT_EQ(output.values[i], expected[i]); + } + + FrontendFreeStateContents(&state); +} + +TEST_F(FrontendTest, CheckNotEnoughSamples) { + struct FrontendState state; + ASSERT_TRUE(FrontendPopulateState(&config_, &state, kSampleRate)); + size_t num_samples_read; + + FrontendProcessSamples(&state, kFakeAudioData, + sizeof(kFakeAudioData) / sizeof(kFakeAudioData[0]), + &num_samples_read); + FrontendProcessSamples( + &state, kFakeAudioData + kWindowSamples, + sizeof(kFakeAudioData) / sizeof(kFakeAudioData[0]) - kWindowSamples, + &num_samples_read); + struct FrontendOutput output = FrontendProcessSamples( + &state, kFakeAudioData + kWindowSamples + kStepSamples, + sizeof(kFakeAudioData) / sizeof(kFakeAudioData[0]) - kWindowSamples - + kStepSamples, + &num_samples_read); + + EXPECT_EQ(output.size, 0); + EXPECT_EQ(output.values, nullptr); + + FrontendFreeStateContents(&state); +} + +} // namespace diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/frontend_util.c b/tensorflow/contrib/lite/experimental/microfrontend/lib/frontend_util.c new file mode 100644 index 00000000000..ae2d9ae6c4c --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/frontend_util.c @@ -0,0 +1,87 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/frontend_util.h" + +#include +#include + +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/bits.h" + +void FrontendFillConfigWithDefaults(struct FrontendConfig* config) { + WindowFillConfigWithDefaults(&config->window); + FilterbankFillConfigWithDefaults(&config->filterbank); + NoiseReductionFillConfigWithDefaults(&config->noise_reduction); + PcanGainControlFillConfigWithDefaults(&config->pcan_gain_control); + LogScaleFillConfigWithDefaults(&config->log_scale); +} + +int FrontendPopulateState(const struct FrontendConfig* config, + struct FrontendState* state, int sample_rate) { + memset(state, 0, sizeof(*state)); + + if (!WindowPopulateState(&config->window, &state->window, sample_rate)) { + fprintf(stderr, "Failed to populate window state\n"); + return 0; + } + + if (!FftPopulateState(&state->fft, state->window.size)) { + fprintf(stderr, "Failed to populate fft state\n"); + return 0; + } + FftInit(&state->fft); + + if (!FilterbankPopulateState(&config->filterbank, &state->filterbank, + sample_rate, state->fft.fft_size / 2 + 1)) { + fprintf(stderr, "Failed to populate filterbank state\n"); + return 0; + } + + if (!NoiseReductionPopulateState(&config->noise_reduction, + &state->noise_reduction, + state->filterbank.num_channels)) { + fprintf(stderr, "Failed to populate noise reduction state\n"); + return 0; + } + + int input_correction_bits = + MostSignificantBit32(state->fft.fft_size) - 1 - (kFilterbankBits / 2); + if (!PcanGainControlPopulateState(&config->pcan_gain_control, + &state->pcan_gain_control, + state->noise_reduction.estimate, + state->filterbank.num_channels, + state->noise_reduction.smoothing_bits, + input_correction_bits)) { + fprintf(stderr, "Failed to populate pcan gain control state\n"); + return 0; + } + + if (!LogScalePopulateState(&config->log_scale, &state->log_scale)) { + fprintf(stderr, "Failed to populate log scale state\n"); + return 0; + } + + FrontendReset(state); + + // All good, return a true value. + return 1; +} + +void FrontendFreeStateContents(struct FrontendState* state) { + WindowFreeStateContents(&state->window); + FftFreeStateContents(&state->fft); + FilterbankFreeStateContents(&state->filterbank); + NoiseReductionFreeStateContents(&state->noise_reduction); + PcanGainControlFreeStateContents(&state->pcan_gain_control); +} diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/frontend_util.h b/tensorflow/contrib/lite/experimental/microfrontend/lib/frontend_util.h new file mode 100644 index 00000000000..a958b610eae --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/frontend_util.h @@ -0,0 +1,52 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FRONTEND_UTIL_H_ +#define TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FRONTEND_UTIL_H_ + +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/fft_util.h" +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/filterbank_util.h" +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/frontend.h" +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/log_scale_util.h" +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/noise_reduction_util.h" +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/pcan_gain_control_util.h" +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/window_util.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct FrontendConfig { + struct WindowConfig window; + struct FilterbankConfig filterbank; + struct NoiseReductionConfig noise_reduction; + struct PcanGainControlConfig pcan_gain_control; + struct LogScaleConfig log_scale; +}; + +// Fills the frontendConfig with "sane" defaults. +void FrontendFillConfigWithDefaults(struct FrontendConfig* config); + +// Allocates any buffers. +int FrontendPopulateState(const struct FrontendConfig* config, + struct FrontendState* state, int sample_rate); + +// Frees any allocated buffers. +void FrontendFreeStateContents(struct FrontendState* state); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FRONTEND_UTIL_H_ diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/log_lut.c b/tensorflow/contrib/lite/experimental/microfrontend/lib/log_lut.c new file mode 100644 index 00000000000..f8d32102336 --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/log_lut.c @@ -0,0 +1,30 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/log_lut.h" +const uint16_t kLogLut[] +#ifndef _MSC_VER + __attribute__((aligned(4))) +#endif // _MSV_VER + = {0, 224, 442, 654, 861, 1063, 1259, 1450, 1636, 1817, 1992, 2163, + 2329, 2490, 2646, 2797, 2944, 3087, 3224, 3358, 3487, 3611, 3732, 3848, + 3960, 4068, 4172, 4272, 4368, 4460, 4549, 4633, 4714, 4791, 4864, 4934, + 5001, 5063, 5123, 5178, 5231, 5280, 5326, 5368, 5408, 5444, 5477, 5507, + 5533, 5557, 5578, 5595, 5610, 5622, 5631, 5637, 5640, 5641, 5638, 5633, + 5626, 5615, 5602, 5586, 5568, 5547, 5524, 5498, 5470, 5439, 5406, 5370, + 5332, 5291, 5249, 5203, 5156, 5106, 5054, 5000, 4944, 4885, 4825, 4762, + 4697, 4630, 4561, 4490, 4416, 4341, 4264, 4184, 4103, 4020, 3935, 3848, + 3759, 3668, 3575, 3481, 3384, 3286, 3186, 3084, 2981, 2875, 2768, 2659, + 2549, 2437, 2323, 2207, 2090, 1971, 1851, 1729, 1605, 1480, 1353, 1224, + 1094, 963, 830, 695, 559, 421, 282, 142, 0, 0}; diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/log_lut.h b/tensorflow/contrib/lite/experimental/microfrontend/lib/log_lut.h new file mode 100644 index 00000000000..53dd1fa4052 --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/log_lut.h @@ -0,0 +1,40 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_LOG_LUT_H_ +#define TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_LOG_LUT_H_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +// Number of segments in the log lookup table. The table will be kLogSegments+1 +// in length (with some padding). +#define kLogSegments 128 +#define kLogSegmentsLog2 7 + +// Scale used by lookup table. +#define kLogScale 65536 +#define kLogScaleLog2 16 +#define kLogCoeff 45426 + +extern const uint16_t kLogLut[]; + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_LOG_LUT_H_ diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/log_scale.c b/tensorflow/contrib/lite/experimental/microfrontend/lib/log_scale.c new file mode 100644 index 00000000000..4b124618715 --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/log_scale.c @@ -0,0 +1,83 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/log_scale.h" + +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/bits.h" +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/log_lut.h" + +#define kuint16max 0x0000FFFF + +// The following functions implement integer logarithms of various sizes. The +// approximation is calculated according to method described in +// www.inti.gob.ar/electronicaeinformatica/instrumentacion/utic/ +// publicaciones/SPL2007/Log10-spl07.pdf +// It first calculates log2 of the input and then converts it to natural +// logarithm. + +static uint32_t Log2FractionPart(const uint32_t x, const uint32_t log2x) { + // Part 1 + int32_t frac = x - (1LL << log2x); + if (log2x < kLogScaleLog2) { + frac <<= kLogScaleLog2 - log2x; + } else { + frac >>= log2x - kLogScaleLog2; + } + // Part 2 + const uint32_t base_seg = frac >> (kLogScaleLog2 - kLogSegmentsLog2); + const uint32_t seg_unit = + (((uint32_t) 1) << kLogScaleLog2) >> kLogSegmentsLog2; + + const int32_t c0 = kLogLut[base_seg]; + const int32_t c1 = kLogLut[base_seg + 1]; + const int32_t seg_base = seg_unit * base_seg; + const int32_t rel_pos = ((c1 - c0) * (frac - seg_base)) >> kLogScaleLog2; + return frac + c0 + rel_pos; +} + +static uint32_t Log(const uint32_t x, const uint32_t scale_shift) { + const uint32_t integer = MostSignificantBit32(x) - 1; + const uint32_t fraction = Log2FractionPart(x, integer); + const uint32_t log2 = (integer << kLogScaleLog2) + fraction; + const uint32_t round = kLogScale / 2; + const uint32_t loge = + (((uint64_t) kLogCoeff) * log2 + round) >> kLogScaleLog2; + // Finally scale to our output scale + const uint32_t loge_scaled = ((loge << scale_shift) + round) >> kLogScaleLog2; + return loge_scaled; +} + +uint16_t* LogScaleApply(struct LogScaleState* state, uint32_t* signal, + int signal_size, int correction_bits) { + const int scale_shift = state->scale_shift; + uint16_t* output = (uint16_t*) signal; + uint16_t* ret = output; + for (int i = 0; i < signal_size; ++i) { + uint32_t value = *signal++; + if (state->enable_log) { + if (correction_bits < 0) { + value >>= -correction_bits; + } else { + value <<= correction_bits; + } + if (value > 1) { + value = Log(value, scale_shift); + } else { + value = 0; + } + } + *output++ = (value < kuint16max) ? value : kuint16max; + } + return ret; +} diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/log_scale.h b/tensorflow/contrib/lite/experimental/microfrontend/lib/log_scale.h new file mode 100644 index 00000000000..8fd60999330 --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/log_scale.h @@ -0,0 +1,39 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_LOG_SCALE_H_ +#define TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_LOG_SCALE_H_ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct LogScaleState { + int enable_log; + int scale_shift; +}; + +// Applies a fixed point logarithm to the signal and converts it to 16 bit. Note +// that the signal array will be modified. +uint16_t* LogScaleApply(struct LogScaleState* state, uint32_t* signal, + int signal_size, int correction_bits); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_LOG_SCALE_H_ diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/log_scale_io.c b/tensorflow/contrib/lite/experimental/microfrontend/lib/log_scale_io.c new file mode 100644 index 00000000000..f59cde951ca --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/log_scale_io.c @@ -0,0 +1,21 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/log_scale_io.h" + +void LogScaleWriteMemmap(FILE* fp, const struct LogScaleState* state, + const char* variable) { + fprintf(fp, "%s->enable_log = %d;\n", variable, state->enable_log); + fprintf(fp, "%s->scale_shift = %d;\n", variable, state->scale_shift); +} diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/log_scale_io.h b/tensorflow/contrib/lite/experimental/microfrontend/lib/log_scale_io.h new file mode 100644 index 00000000000..5444303b244 --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/log_scale_io.h @@ -0,0 +1,33 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_LOG_SCALE_IO_H_ +#define TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_LOG_SCALE_IO_H_ + +#include + +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/log_scale.h" + +#ifdef __cplusplus +extern "C" { +#endif + +void LogScaleWriteMemmap(FILE* fp, const struct LogScaleState* state, + const char* variable); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_LOG_SCALE_IO_H_ diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/log_scale_test.cc b/tensorflow/contrib/lite/experimental/microfrontend/lib/log_scale_test.cc new file mode 100644 index 00000000000..312d7ea7406 --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/log_scale_test.cc @@ -0,0 +1,58 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/log_scale.h" +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/log_scale_util.h" + +#include +#include + +namespace { + +const int kScaleShift = 6; +const int kCorrectionBits = -1; + +TEST(LogScaleTest, CheckOutputValues) { + struct LogScaleState state; + state.enable_log = true; + state.scale_shift = kScaleShift; + + uint32_t fake_signal[] = {3578, 1533}; + uint16_t* output = LogScaleApply(&state, fake_signal, + sizeof(fake_signal) / sizeof(fake_signal[0]), + kCorrectionBits); + + const uint16_t expected[] = {479, 425}; + for (int i = 0; i < sizeof(expected) / sizeof(expected[0]); ++i) { + EXPECT_EQ(output[i], expected[i]); + } +} + +TEST(LogScaleTest, CheckOutputValuesNoLog) { + struct LogScaleState state; + state.enable_log = false; + state.scale_shift = kScaleShift; + + uint32_t fake_signal[] = {85964, 45998}; + uint16_t* output = LogScaleApply(&state, fake_signal, + sizeof(fake_signal) / sizeof(fake_signal[0]), + kCorrectionBits); + + const uint16_t expected[] = {65535, 45998}; + for (int i = 0; i < sizeof(expected) / sizeof(expected[0]); ++i) { + EXPECT_EQ(output[i], expected[i]); + } +} + +} // namespace diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/log_scale_util.c b/tensorflow/contrib/lite/experimental/microfrontend/lib/log_scale_util.c new file mode 100644 index 00000000000..8a025fbf72d --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/log_scale_util.c @@ -0,0 +1,27 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/log_scale_util.h" + +void LogScaleFillConfigWithDefaults(struct LogScaleConfig* config) { + config->enable_log = 1; + config->scale_shift = 6; +} + +int LogScalePopulateState(const struct LogScaleConfig* config, + struct LogScaleState* state) { + state->enable_log = config->enable_log; + state->scale_shift = config->scale_shift; + return 1; +} diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/log_scale_util.h b/tensorflow/contrib/lite/experimental/microfrontend/lib/log_scale_util.h new file mode 100644 index 00000000000..33b21f30b10 --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/log_scale_util.h @@ -0,0 +1,45 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_LOG_SCALE_UTIL_H_ +#define TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_LOG_SCALE_UTIL_H_ + +#include +#include + +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/log_scale.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct LogScaleConfig { + // set to false (0) to disable this module + int enable_log; + // scale results by 2^(scale_shift) + int scale_shift; +}; + +// Populates the LogScaleConfig with "sane" default values. +void LogScaleFillConfigWithDefaults(struct LogScaleConfig* config); + +// Allocates any buffers. +int LogScalePopulateState(const struct LogScaleConfig* config, + struct LogScaleState* state); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_LOG_SCALE_UTIL_H_ diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/noise_reduction.c b/tensorflow/contrib/lite/experimental/microfrontend/lib/noise_reduction.c new file mode 100644 index 00000000000..92f8b58d74f --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/noise_reduction.c @@ -0,0 +1,51 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/noise_reduction.h" + +#include + +void NoiseReductionApply(struct NoiseReductionState* state, uint32_t* signal) { + int i; + for (i = 0; i < state->num_channels; ++i) { + const uint32_t smoothing = + ((i & 1) == 0) ? state->even_smoothing : state->odd_smoothing; + const uint32_t one_minus_smoothing = (1 << kNoiseReductionBits) - smoothing; + + // Update the estimate of the noise. + const uint32_t signal_scaled_up = signal[i] << state->smoothing_bits; + uint32_t estimate = + (((uint64_t) signal_scaled_up * smoothing) + + ((uint64_t) state->estimate[i] * one_minus_smoothing)) >> + kNoiseReductionBits; + state->estimate[i] = estimate; + + // Make sure that we can't get a negative value for the signal - estimate. + if (estimate > signal_scaled_up) { + estimate = signal_scaled_up; + } + + const uint32_t floor = + ((uint64_t) signal[i] * state->min_signal_remaining) >> + kNoiseReductionBits; + const uint32_t subtracted = (signal_scaled_up - estimate) >> + state->smoothing_bits; + const uint32_t output = subtracted > floor ? subtracted : floor; + signal[i] = output; + } +} + +void NoiseReductionReset(struct NoiseReductionState* state) { + memset(state->estimate, 0, sizeof(*state->estimate) * state->num_channels); +} diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/noise_reduction.h b/tensorflow/contrib/lite/experimental/microfrontend/lib/noise_reduction.h new file mode 100644 index 00000000000..cc2cf2d9b74 --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/noise_reduction.h @@ -0,0 +1,46 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_NOISE_REDUCTION_H_ +#define TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_NOISE_REDUCTION_H_ + +#define kNoiseReductionBits 14 + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct NoiseReductionState { + int smoothing_bits; + uint16_t even_smoothing; + uint16_t odd_smoothing; + uint16_t min_signal_remaining; + int num_channels; + uint32_t* estimate; +}; + +// Removes stationary noise from each channel of the signal using a low pass +// filter. +void NoiseReductionApply(struct NoiseReductionState* state, uint32_t* signal); + +void NoiseReductionReset(struct NoiseReductionState* state); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_NOISE_REDUCTION_H_ diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/noise_reduction_io.c b/tensorflow/contrib/lite/experimental/microfrontend/lib/noise_reduction_io.c new file mode 100644 index 00000000000..1cba410436a --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/noise_reduction_io.c @@ -0,0 +1,34 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/noise_reduction_io.h" + +void NoiseReductionWriteMemmapPreamble( + FILE* fp, const struct NoiseReductionState* state) { + fprintf(fp, "static uint32_t noise_reduction_estimate[%zu];\n", + state->num_channels); + fprintf(fp, "\n"); +} + +void NoiseReductionWriteMemmap(FILE* fp, + const struct NoiseReductionState* state, + const char* variable) { + fprintf(fp, "%s->even_smoothing = %d;\n", variable, state->even_smoothing); + fprintf(fp, "%s->odd_smoothing = %d;\n", variable, state->odd_smoothing); + fprintf(fp, "%s->min_signal_remaining = %d;\n", variable, + state->min_signal_remaining); + fprintf(fp, "%s->num_channels = %d;\n", variable, state->num_channels); + + fprintf(fp, "%s->estimate = noise_reduction_estimate;\n", variable); +} diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/noise_reduction_io.h b/tensorflow/contrib/lite/experimental/microfrontend/lib/noise_reduction_io.h new file mode 100644 index 00000000000..afeedfce99d --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/noise_reduction_io.h @@ -0,0 +1,36 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_NOISE_REDUCTION_IO_H_ +#define TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_NOISE_REDUCTION_IO_H_ + +#include + +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/noise_reduction.h" + +#ifdef __cplusplus +extern "C" { +#endif + +void NoiseReductionWriteMemmapPreamble(FILE* fp, + const struct NoiseReductionState* state); +void NoiseReductionWriteMemmap(FILE* fp, + const struct NoiseReductionState* state, + const char* variable); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_NOISE_REDUCTION_IO_H_ diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/noise_reduction_test.cc b/tensorflow/contrib/lite/experimental/microfrontend/lib/noise_reduction_test.cc new file mode 100644 index 00000000000..f4cf486227a --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/noise_reduction_test.cc @@ -0,0 +1,70 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/noise_reduction.h" +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/noise_reduction_util.h" + +#include +#include + +namespace { + +const int kNumChannels = 2; + +// Test noise reduction using default config values. +class NoiseReductionTest : public ::testing::Test { + protected: + NoiseReductionTest() { + config_.smoothing_bits = 10; + config_.even_smoothing = 0.025; + config_.odd_smoothing = 0.06; + config_.min_signal_remaining = 0.05; + } + + struct NoiseReductionConfig config_; +}; + +TEST_F(NoiseReductionTest, TestNoiseReductionEstimate) { + struct NoiseReductionState state; + ASSERT_TRUE(NoiseReductionPopulateState(&config_, &state, kNumChannels)); + + uint32_t signal[] = {247311, 508620}; + NoiseReductionApply(&state, signal); + + const uint32_t expected[] = {6321887, 31248341}; + ASSERT_EQ(state.num_channels, sizeof(expected) / sizeof(expected[0])); + for (int i = 0; i < state.num_channels; ++i) { + EXPECT_EQ(state.estimate[i], expected[i]); + } + + NoiseReductionFreeStateContents(&state); +} + +TEST_F(NoiseReductionTest, TestNoiseReduction) { + struct NoiseReductionState state; + ASSERT_TRUE(NoiseReductionPopulateState(&config_, &state, kNumChannels)); + + uint32_t signal[] = {247311, 508620}; + NoiseReductionApply(&state, signal); + + const uint32_t expected[] = {241137, 478104}; + ASSERT_EQ(state.num_channels, sizeof(expected) / sizeof(expected[0])); + for (int i = 0; i < state.num_channels; ++i) { + EXPECT_EQ(signal[i], expected[i]); + } + + NoiseReductionFreeStateContents(&state); +} + +} // namespace diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/noise_reduction_util.c b/tensorflow/contrib/lite/experimental/microfrontend/lib/noise_reduction_util.c new file mode 100644 index 00000000000..46f475352e0 --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/noise_reduction_util.c @@ -0,0 +1,45 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/noise_reduction_util.h" + +#include + +void NoiseReductionFillConfigWithDefaults(struct NoiseReductionConfig* config) { + config->smoothing_bits = 10; + config->even_smoothing = 0.025; + config->odd_smoothing = 0.06; + config->min_signal_remaining = 0.05; +} + +int NoiseReductionPopulateState(const struct NoiseReductionConfig* config, + struct NoiseReductionState* state, + int num_channels) { + state->smoothing_bits = config->smoothing_bits; + state->odd_smoothing = config->odd_smoothing * (1 << kNoiseReductionBits); + state->even_smoothing = config->even_smoothing * (1 << kNoiseReductionBits); + state->min_signal_remaining = + config->min_signal_remaining * (1 << kNoiseReductionBits); + state->num_channels = num_channels; + state->estimate = calloc(state->num_channels, sizeof(*state->estimate)); + if (state->estimate == NULL) { + fprintf(stderr, "Failed to alloc estimate buffer\n"); + return 0; + } + return 1; +} + +void NoiseReductionFreeStateContents(struct NoiseReductionState* state) { + free(state->estimate); +} diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/noise_reduction_util.h b/tensorflow/contrib/lite/experimental/microfrontend/lib/noise_reduction_util.h new file mode 100644 index 00000000000..207b8a679da --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/noise_reduction_util.h @@ -0,0 +1,50 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_NOISE_REDUCTION_UTIL_H_ +#define TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_NOISE_REDUCTION_UTIL_H_ + +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/noise_reduction.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct NoiseReductionConfig { + // scale the signal up by 2^(smoothing_bits) before reduction + int smoothing_bits; + // smoothing coefficient for even-numbered channels + float even_smoothing; + // smoothing coefficient for odd-numbered channels + float odd_smoothing; + // fraction of signal to preserve (1.0 disables this module) + float min_signal_remaining; +}; + +// Populates the NoiseReductionConfig with "sane" default values. +void NoiseReductionFillConfigWithDefaults(struct NoiseReductionConfig* config); + +// Allocates any buffers. +int NoiseReductionPopulateState(const struct NoiseReductionConfig* config, + struct NoiseReductionState* state, + int num_channels); + +// Frees any allocated buffers. +void NoiseReductionFreeStateContents(struct NoiseReductionState* state); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_NOISE_REDUCTION_UTIL_H_ diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/pcan_gain_control.c b/tensorflow/contrib/lite/experimental/microfrontend/lib/pcan_gain_control.c new file mode 100644 index 00000000000..551d552e8f6 --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/pcan_gain_control.c @@ -0,0 +1,56 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/pcan_gain_control.h" + +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/bits.h" + +int16_t WideDynamicFunction(const uint32_t x, const int16_t* lut) { + if (x <= 2) { + return lut[x]; + } + + const int16_t interval = MostSignificantBit32(x); + lut += 4 * interval - 6; + + const int16_t frac = ((interval < 11) + ? (x << (11 - interval)) + : (x >> (interval - 11)) + ) & 0x3FF; + + int32_t result = ((int32_t) lut[2] * frac) >> 5; + result += ((int32_t) lut[1]) << 5; + result *= frac; + result = (result + (1 << 14)) >> 15; + result += lut[0]; + return (int16_t) result; +} + +uint32_t PcanShrink(const uint32_t x) { + if (x < (2 << kPcanSnrBits)) { + return (x * x) >> (2 + 2 * kPcanSnrBits - kPcanOutputBits); + } else { + return (x >> (kPcanSnrBits - kPcanOutputBits)) - (1 << kPcanOutputBits); + } +} + +void PcanGainControlApply(struct PcanGainControlState* state, + uint32_t* signal) { + for (int i = 0; i < state->num_channels; ++i) { + const uint32_t gain = WideDynamicFunction(state->noise_estimate[i], + state->gain_lut); + const uint32_t snr = ((uint64_t) signal[i] * gain) >> state->snr_shift; + signal[i] = PcanShrink(snr); + } +} diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/pcan_gain_control.h b/tensorflow/contrib/lite/experimental/microfrontend/lib/pcan_gain_control.h new file mode 100644 index 00000000000..cab74f49dbe --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/pcan_gain_control.h @@ -0,0 +1,46 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_PCAN_GAIN_CONTROL_H_ +#define TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_PCAN_GAIN_CONTROL_H_ + +#include +#include + +#define kPcanSnrBits 12 +#define kPcanOutputBits 6 + +#ifdef __cplusplus +extern "C" { +#endif + +struct PcanGainControlState { + int enable_pcan; + uint32_t* noise_estimate; + int num_channels; + int16_t* gain_lut; + int32_t snr_shift; +}; + +int16_t WideDynamicFunction(const uint32_t x, const int16_t* lut); + +uint32_t PcanShrink(const uint32_t x); + +void PcanGainControlApply(struct PcanGainControlState* state, uint32_t* signal); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_PCAN_GAIN_CONTROL_H_ diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/pcan_gain_control_test.cc b/tensorflow/contrib/lite/experimental/microfrontend/lib/pcan_gain_control_test.cc new file mode 100644 index 00000000000..bbc36d6eac7 --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/pcan_gain_control_test.cc @@ -0,0 +1,59 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/pcan_gain_control.h" +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/pcan_gain_control_util.h" + +#include +#include + +namespace { + +const int kNumChannels = 2; +const int kSmoothingBits = 10; +const int kCorrectionBits = -1; + +// Test pcan auto gain control using default config values. +class PcanGainControlTest : public ::testing::Test { + protected: + PcanGainControlTest() { + config_.enable_pcan = 1; + config_.strength = 0.95; + config_.offset = 80.0; + config_.gain_bits = 21; + } + + struct PcanGainControlConfig config_; +}; + +TEST_F(PcanGainControlTest, TestPcanGainControl) { + uint32_t estimate[] = {6321887, 31248341}; + struct PcanGainControlState state; + ASSERT_TRUE(PcanGainControlPopulateState(&config_, &state, estimate, + kNumChannels, kSmoothingBits, + kCorrectionBits)); + + uint32_t signal[] = {241137, 478104}; + PcanGainControlApply(&state, signal); + + const uint32_t expected[] = {3578, 1533}; + ASSERT_EQ(state.num_channels, sizeof(expected) / sizeof(expected[0])); + for (int i = 0; i < state.num_channels; ++i) { + EXPECT_EQ(signal[i], expected[i]); + } + + PcanGainControlFreeStateContents(&state); +} + +} // namespace diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/pcan_gain_control_util.c b/tensorflow/contrib/lite/experimental/microfrontend/lib/pcan_gain_control_util.c new file mode 100644 index 00000000000..4226b390bc1 --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/pcan_gain_control_util.c @@ -0,0 +1,90 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/pcan_gain_control_util.h" + +#include +#include + +#define kint16max 0x00007FFF + +void PcanGainControlFillConfigWithDefaults( + struct PcanGainControlConfig* config) { + config->enable_pcan = 0; + config->strength = 0.95; + config->offset = 80.0; + config->gain_bits = 21; +} + +int16_t PcanGainLookupFunction(const struct PcanGainControlConfig* config, + int32_t input_bits, uint32_t x) { + const float x_as_float = ((float) x) / ((uint32_t) 1 << input_bits); + const float gain_as_float = ((uint32_t) 1 << config->gain_bits) * + powf(x_as_float + config->offset, -config->strength); + + if (gain_as_float > kint16max) { + return kint16max; + } + return (int16_t) (gain_as_float + 0.5f); +} + +int PcanGainControlPopulateState(const struct PcanGainControlConfig* config, + struct PcanGainControlState* state, + uint32_t* noise_estimate, + const int num_channels, + const uint16_t smoothing_bits, + const int32_t input_correction_bits) { + state->enable_pcan = config->enable_pcan; + if (!state->enable_pcan) { + return 1; + } + state->noise_estimate = noise_estimate; + state->num_channels = num_channels; + state->gain_lut = malloc(kWideDynamicFunctionLUTSize * sizeof(int16_t)); + if (state->gain_lut == NULL) { + fprintf(stderr, "Failed to allocate gain LUT\n"); + return 0; + } + state->snr_shift = config->gain_bits - input_correction_bits - kPcanSnrBits; + + const int32_t input_bits = smoothing_bits - input_correction_bits; + state->gain_lut[0] = PcanGainLookupFunction(config, input_bits, 0); + state->gain_lut[1] = PcanGainLookupFunction(config, input_bits, 1); + state->gain_lut -= 6; + for (int interval = 2; interval <= kWideDynamicFunctionBits; ++interval) { + const uint32_t x0 = (uint32_t) 1 << (interval - 1); + const uint32_t x1 = x0 + (x0 >> 1); + const uint32_t x2 = (interval == kWideDynamicFunctionBits) + ? x0 + (x0 - 1) : 2 * x0; + + const int16_t y0 = PcanGainLookupFunction(config, input_bits, x0); + const int16_t y1 = PcanGainLookupFunction(config, input_bits, x1); + const int16_t y2 = PcanGainLookupFunction(config, input_bits, x2); + + const int32_t diff1 = (int32_t) y1 - y0; + const int32_t diff2 = (int32_t) y2 - y0; + const int32_t a1 = 4 * diff1 - diff2; + const int32_t a2 = diff2 - a1; + + state->gain_lut[4 * interval] = y0; + state->gain_lut[4 * interval + 1] = (int16_t) a1; + state->gain_lut[4 * interval + 2] = (int16_t) a2; + } + state->gain_lut += 6; + return 1; +} + +void PcanGainControlFreeStateContents(struct PcanGainControlState* state) { + free(state->gain_lut); +} diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/pcan_gain_control_util.h b/tensorflow/contrib/lite/experimental/microfrontend/lib/pcan_gain_control_util.h new file mode 100644 index 00000000000..79c0b1da693 --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/pcan_gain_control_util.h @@ -0,0 +1,57 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_PCAN_GAIN_CONTROL_UTIL_H_ +#define TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_PCAN_GAIN_CONTROL_UTIL_H_ + +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/pcan_gain_control.h" + +#define kWideDynamicFunctionBits 32 +#define kWideDynamicFunctionLUTSize (4 * kWideDynamicFunctionBits - 3) + +#ifdef __cplusplus +extern "C" { +#endif + +struct PcanGainControlConfig { + // set to false (0) to disable this module + int enable_pcan; + // gain normalization exponent (0.0 disables, 1.0 full strength) + float strength; + // positive value added in the normalization denominator + float offset; + // number of fractional bits in the gain + int gain_bits; +}; + +void PcanGainControlFillConfigWithDefaults( + struct PcanGainControlConfig* config); + +int16_t PcanGainLookupFunction(const struct PcanGainControlConfig* config, + int32_t input_bits, uint32_t x); + +int PcanGainControlPopulateState(const struct PcanGainControlConfig* config, + struct PcanGainControlState* state, + uint32_t* noise_estimate, + const int num_channels, + const uint16_t smoothing_bits, + const int32_t input_correction_bits); + +void PcanGainControlFreeStateContents(struct PcanGainControlState* state); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_PCAN_GAIN_CONTROL_UTIL_H_ diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/window.c b/tensorflow/contrib/lite/experimental/microfrontend/lib/window.c new file mode 100644 index 00000000000..0fdc040a7a5 --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/window.c @@ -0,0 +1,70 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/window.h" + +#include + +int WindowProcessSamples(struct WindowState* state, const int16_t* samples, + size_t num_samples, size_t* num_samples_read) { + const int size = state->size; + + // Copy samples from the samples buffer over to our local input. + size_t max_samples_to_copy = state->size - state->input_used; + if (max_samples_to_copy > num_samples) { + max_samples_to_copy = num_samples; + } + memcpy(state->input + state->input_used, samples, + max_samples_to_copy * sizeof(*samples)); + *num_samples_read = max_samples_to_copy; + state->input_used += max_samples_to_copy; + + if (state->input_used < state->size) { + // We don't have enough samples to compute a window. + return 0; + } + + // Apply the window to the input. + const int16_t* coefficients = state->coefficients; + const int16_t* input = state->input; + int16_t* output = state->output; + int i; + int16_t max_abs_output_value = 0; + for (i = 0; i < size; ++i) { + int16_t new_value = + (((int32_t) *input++) * *coefficients++) >> kFrontendWindowBits; + *output++ = new_value; + if (new_value < 0) { + new_value = -new_value; + } + if (new_value > max_abs_output_value) { + max_abs_output_value = new_value; + } + } + // Shuffle the input down by the step size, and update how much we have used. + memmove(state->input, state->input + state->step, + sizeof(*state->input) * (state->size - state->step)); + state->input_used -= state->step; + state->max_abs_output_value = max_abs_output_value; + + // Indicate that the output buffer is valid for the next stage. + return 1; +} + +void WindowReset(struct WindowState* state) { + memset(state->input, 0, state->size * sizeof(*state->input)); + memset(state->output, 0, state->size * sizeof(*state->output)); + state->input_used = 0; + state->max_abs_output_value = 0; +} diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/window.h b/tensorflow/contrib/lite/experimental/microfrontend/lib/window.h new file mode 100644 index 00000000000..90291e5c723 --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/window.h @@ -0,0 +1,49 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_WINDOW_H_ +#define TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_WINDOW_H_ + +#include +#include + +#define kFrontendWindowBits 12 + +#ifdef __cplusplus +extern "C" { +#endif + +struct WindowState { + size_t size; + int16_t* coefficients; + size_t step; + + int16_t* input; + size_t input_used; + int16_t* output; + int16_t max_abs_output_value; +}; + +// Applies a window to the samples coming in, stepping forward at the given +// rate. +int WindowProcessSamples(struct WindowState* state, const int16_t* samples, + size_t num_samples, size_t* num_samples_read); + +void WindowReset(struct WindowState* state); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_WINDOW_H_ diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/window_io.c b/tensorflow/contrib/lite/experimental/microfrontend/lib/window_io.c new file mode 100644 index 00000000000..f1fee7c1eda --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/window_io.c @@ -0,0 +1,42 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/window_io.h" + +void WindowWriteMemmapPreamble(FILE* fp, const struct WindowState* state) { + fprintf(fp, "static int16_t window_coefficients[] = {\n"); + for (int i = 0; i < state->size; ++i) { + fprintf(fp, "%d", state->coefficients[i]); + if (i < state->size - 1) { + fprintf(fp, ", "); + } + } + fprintf(fp, "};\n"); + fprintf(fp, "static int16_t window_input[%zu];\n", state->size); + fprintf(fp, "static int16_t window_output[%zu];\n", state->size); + fprintf(fp, "\n"); +} + +void WindowWriteMemmap(FILE* fp, const struct WindowState* state, + const char* variable) { + fprintf(fp, "%s->size = %zu;\n", variable, state->size); + fprintf(fp, "%s->coefficients = window_coefficients;\n", variable); + fprintf(fp, "%s->step = %zu;\n", variable, state->step); + + fprintf(fp, "%s->input = window_input;\n", variable); + fprintf(fp, "%s->input_used = %zu;\n", variable, state->input_used); + fprintf(fp, "%s->output = window_output;\n", variable); + fprintf(fp, "%s->max_abs_output_value = %d;\n", variable, + state->max_abs_output_value); +} diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/window_io.h b/tensorflow/contrib/lite/experimental/microfrontend/lib/window_io.h new file mode 100644 index 00000000000..2bab9064c1f --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/window_io.h @@ -0,0 +1,34 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_WINDOW_IO_H_ +#define TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_WINDOW_IO_H_ + +#include + +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/window.h" + +#ifdef __cplusplus +extern "C" { +#endif + +void WindowWriteMemmapPreamble(FILE* fp, const struct WindowState* state); +void WindowWriteMemmap(FILE* fp, const struct WindowState* state, + const char* variable); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_WINDOW_IO_H_ diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/window_test.cc b/tensorflow/contrib/lite/experimental/microfrontend/lib/window_test.cc new file mode 100644 index 00000000000..a6c0879faa8 --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/window_test.cc @@ -0,0 +1,157 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/window.h" +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/window_util.h" + +#include +#include + +namespace { + +const int kSampleRate = 1000; +const int kWindowSamples = 25; +const int kStepSamples = 10; +const int16_t kFakeAudioData[] = { + 0, 32767, 0, -32768, 0, 32767, 0, -32768, 0, 32767, 0, -32768, + 0, 32767, 0, -32768, 0, 32767, 0, -32768, 0, 32767, 0, -32768, + 0, 32767, 0, -32768, 0, 32767, 0, -32768, 0, 32767, 0, -32768}; + +// Test window function behaviors using default config values. +class WindowTest : public ::testing::Test { + protected: + WindowTest() { + config_.size_ms = 25; + config_.step_size_ms = 10; + } + + struct WindowConfig config_; +}; + +TEST_F(WindowTest, CheckCoefficients) { + struct WindowState state; + ASSERT_TRUE(WindowPopulateState(&config_, &state, kSampleRate)); + + const int16_t expected[] = {16, 144, 391, 743, 1176, 1664, 2177, + 2681, 3145, 3541, 3843, 4032, 4096, 4032, + 3843, 3541, 3145, 2681, 2177, 1664, 1176, + 743, 391, 144, 16}; + ASSERT_EQ(state.size, sizeof(expected) / sizeof(expected[0])); + for (int i = 0; i < state.size; ++i) { + EXPECT_EQ(state.coefficients[i], expected[i]); + } + + WindowFreeStateContents(&state); +} + +TEST_F(WindowTest, CheckResidualInput) { + struct WindowState state; + ASSERT_TRUE(WindowPopulateState(&config_, &state, kSampleRate)); + size_t num_samples_read; + + ASSERT_TRUE(WindowProcessSamples( + &state, kFakeAudioData, + sizeof(kFakeAudioData) / sizeof(kFakeAudioData[0]), &num_samples_read)); + + for (int i = kStepSamples; i < kWindowSamples; ++i) { + EXPECT_EQ(state.input[i - kStepSamples], kFakeAudioData[i]); + } + + WindowFreeStateContents(&state); +} + +TEST_F(WindowTest, CheckOutputValues) { + struct WindowState state; + ASSERT_TRUE(WindowPopulateState(&config_, &state, kSampleRate)); + size_t num_samples_read; + + ASSERT_TRUE(WindowProcessSamples( + &state, kFakeAudioData, + sizeof(kFakeAudioData) / sizeof(kFakeAudioData[0]), &num_samples_read)); + + const int16_t expected[] = { + 0, 1151, 0, -5944, 0, 13311, 0, -21448, 0, 28327, 0, -32256, 0, 32255, + 0, -28328, 0, 21447, 0, -13312, 0, 5943, 0, -1152, 0}; + ASSERT_EQ(state.size, sizeof(expected) / sizeof(expected[0])); + for (int i = 0; i < state.size; ++i) { + EXPECT_EQ(state.output[i], expected[i]); + } + + WindowFreeStateContents(&state); +} + +TEST_F(WindowTest, CheckMaxAbsValue) { + struct WindowState state; + ASSERT_TRUE(WindowPopulateState(&config_, &state, kSampleRate)); + size_t num_samples_read; + + ASSERT_TRUE(WindowProcessSamples( + &state, kFakeAudioData, + sizeof(kFakeAudioData) / sizeof(kFakeAudioData[0]), &num_samples_read)); + + EXPECT_EQ(state.max_abs_output_value, 32256); + + WindowFreeStateContents(&state); +} + +TEST_F(WindowTest, CheckConsecutiveWindow) { + struct WindowState state; + ASSERT_TRUE(WindowPopulateState(&config_, &state, kSampleRate)); + size_t num_samples_read; + + ASSERT_TRUE(WindowProcessSamples( + &state, kFakeAudioData, + sizeof(kFakeAudioData) / sizeof(kFakeAudioData[0]), &num_samples_read)); + ASSERT_TRUE(WindowProcessSamples( + &state, kFakeAudioData + kWindowSamples, + sizeof(kFakeAudioData) / sizeof(kFakeAudioData[0]) - kWindowSamples, + &num_samples_read)); + + const int16_t expected[] = { + 0, -1152, 0, 5943, 0, -13312, 0, 21447, 0, -28328, 0, 32255, 0, -32256, + 0, 28327, 0, -21448, 0, 13311, 0, -5944, 0, 1151, 0}; + ASSERT_EQ(state.size, sizeof(expected) / sizeof(expected[0])); + for (int i = 0; i < state.size; ++i) { + EXPECT_EQ(state.output[i], expected[i]); + } + + WindowFreeStateContents(&state); +} + +TEST_F(WindowTest, CheckNotEnoughSamples) { + struct WindowState state; + ASSERT_TRUE(WindowPopulateState(&config_, &state, kSampleRate)); + size_t num_samples_read; + + ASSERT_TRUE(WindowProcessSamples( + &state, kFakeAudioData, + sizeof(kFakeAudioData) / sizeof(kFakeAudioData[0]), &num_samples_read)); + ASSERT_TRUE(WindowProcessSamples( + &state, kFakeAudioData + kWindowSamples, + sizeof(kFakeAudioData) / sizeof(kFakeAudioData[0]) - kWindowSamples, + &num_samples_read)); + ASSERT_FALSE(WindowProcessSamples( + &state, kFakeAudioData + kWindowSamples + kStepSamples, + sizeof(kFakeAudioData) / sizeof(kFakeAudioData[0]) - kWindowSamples - + kStepSamples, + &num_samples_read)); + + EXPECT_EQ( + state.input_used, + sizeof(kFakeAudioData) / sizeof(kFakeAudioData[0]) - 2 * kStepSamples); + + WindowFreeStateContents(&state); +} + +} // namespace diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/window_util.c b/tensorflow/contrib/lite/experimental/microfrontend/lib/window_util.c new file mode 100644 index 00000000000..3adde0fb0a6 --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/window_util.c @@ -0,0 +1,71 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/window_util.h" + +#include +#include +#include +#include + +void WindowFillConfigWithDefaults(struct WindowConfig* config) { + config->size_ms = 25; + config->step_size_ms = 10; +} + +int WindowPopulateState(const struct WindowConfig* config, + struct WindowState* state, int sample_rate) { + state->size = config->size_ms * sample_rate / 1000; + state->step = config->step_size_ms * sample_rate / 1000; + + state->coefficients = malloc( + state->size * sizeof(*state->coefficients)); + if (state->coefficients == NULL) { + fprintf(stderr, "Failed to allocate window coefficients\n"); + return 0; + } + + // Populate the window values. + const float arg = M_PI * 2.0 / ((float) state->size); + int i; + for (i = 0; i < state->size; ++i) { + float float_value = 0.5 - (0.5 * cos(arg * (i + 0.5))); + // Scale it to fixed point and round it. + state->coefficients[i] = + floor(float_value * (1 << kFrontendWindowBits) + 0.5); + } + + state->input_used = 0; + state->input = malloc( + state->size * sizeof(*state->input)); + if (state->input == NULL) { + fprintf(stderr, "Failed to allocate window input\n"); + return 0; + } + + state->output = malloc( + state->size * sizeof(*state->output)); + if (state->output == NULL) { + fprintf(stderr, "Failed to allocate window output\n"); + return 0; + } + + return 1; +} + +void WindowFreeStateContents(struct WindowState* state) { + free(state->coefficients); + free(state->input); + free(state->output); +} diff --git a/tensorflow/contrib/lite/experimental/microfrontend/lib/window_util.h b/tensorflow/contrib/lite/experimental/microfrontend/lib/window_util.h new file mode 100644 index 00000000000..52dc8f38cc8 --- /dev/null +++ b/tensorflow/contrib/lite/experimental/microfrontend/lib/window_util.h @@ -0,0 +1,45 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_WINDOW_UTIL_H_ +#define TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_WINDOW_UTIL_H_ + +#include "tensorflow/contrib/lite/experimental/microfrontend/lib/window.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct WindowConfig { + // length of window frame in milliseconds + size_t size_ms; + // length of step for next frame in milliseconds + size_t step_size_ms; +}; + +// Populates the WindowConfig with "sane" default values. +void WindowFillConfigWithDefaults(struct WindowConfig* config); + +// Allocates any buffers. +int WindowPopulateState(const struct WindowConfig* config, + struct WindowState* state, int sample_rate); + +// Frees any allocated buffers. +void WindowFreeStateContents(struct WindowState* state); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_WINDOW_UTIL_H_ diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index dc2118694a0..81e6676a975 100755 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -26,6 +26,7 @@ load("//third_party/highwayhash:workspace.bzl", highwayhash = "repo") load("//third_party/icu:workspace.bzl", icu = "repo") load("//third_party/jpeg:workspace.bzl", jpeg = "repo") load("//third_party/nasm:workspace.bzl", nasm = "repo") +load("//third_party/kissfft:workspace.bzl", kissfft = "repo") def initialize_third_party(): """ Load third party repositories. See above load() statements. """ @@ -33,6 +34,7 @@ def initialize_third_party(): flatbuffers() highwayhash() icu() + kissfft() jpeg() nasm() diff --git a/third_party/kissfft/BUILD b/third_party/kissfft/BUILD new file mode 100644 index 00000000000..82bab3ffd96 --- /dev/null +++ b/third_party/kissfft/BUILD @@ -0,0 +1 @@ +# This empty BUILD file is required to make Bazel treat this directory as a package. diff --git a/third_party/kissfft/BUILD.bazel b/third_party/kissfft/BUILD.bazel new file mode 100644 index 00000000000..a57cb6ebda4 --- /dev/null +++ b/third_party/kissfft/BUILD.bazel @@ -0,0 +1,23 @@ +package( + default_visibility = ["//visibility:public"], +) + +licenses(["notice"]) # Apache 2.0 + +exports_files(["LICENSE"]) + +cc_library( + name = "kiss_fftr_16", + srcs = [ + "kiss_fft.c", + "tools/kiss_fftr.c", + ], + hdrs = [ + "_kiss_fft_guts.h", + "kiss_fft.h", + "tools/kiss_fftr.h", + ], + copts = [ + "-DFIXED_POINT=16", + ], +) diff --git a/third_party/kissfft/workspace.bzl b/third_party/kissfft/workspace.bzl new file mode 100644 index 00000000000..1754eb1a90f --- /dev/null +++ b/third_party/kissfft/workspace.bzl @@ -0,0 +1,15 @@ +"""Loads the kissfft library, used by TF Lite.""" + +load("//third_party:repo.bzl", "third_party_http_archive") + +def repo(): + third_party_http_archive( + name = "kissfft", + strip_prefix = "kissfft-cddf3833fdf24fa84b79be37efdcd348cae0e39c", + sha256 = "7ba83a3da1636350472e501e3e6c3418df72466990530ea273c05fa7e3dd8635", + urls = [ + "https://mirror.bazel.build/github.com/mborgerding/kissfft/archive/cddf3833fdf24fa84b79be37efdcd348cae0e39c.tar.gz", + "https://github.com/mborgerding/kissfft/archive/cddf3833fdf24fa84b79be37efdcd348cae0e39c.tar.gz", + ], + build_file = "//third_party/kissfft:BUILD.bazel", + )