87 lines
2.7 KiB
C++
87 lines
2.7 KiB
C++
#include "lm/interpolate/normalize.hh"
|
|
|
|
#include "lm/interpolate/interpolate_info.hh"
|
|
#include "lm/interpolate/merge_probabilities.hh"
|
|
#include "lm/common/ngram_stream.hh"
|
|
#include "util/stream/chain.hh"
|
|
#include "util/stream/multi_stream.hh"
|
|
|
|
#define BOOST_TEST_MODULE NormalizeTest
|
|
#include <boost/test/unit_test.hpp>
|
|
|
|
namespace lm { namespace interpolate { namespace {
|
|
|
|
// log without backoff
|
|
const float kInputs[] = {-0.3, 1.2, -9.8, 4.0, -7.0, 0.0};
|
|
|
|
class WriteInput {
|
|
public:
|
|
WriteInput() {}
|
|
void Run(const util::stream::ChainPosition &to) {
|
|
util::stream::Stream out(to);
|
|
for (WordIndex i = 0; i < sizeof(kInputs) / sizeof(float); ++i, ++out) {
|
|
memcpy(out.Get(), &i, sizeof(WordIndex));
|
|
memcpy((uint8_t*)out.Get() + sizeof(WordIndex), &kInputs[i], sizeof(float));
|
|
}
|
|
out.Poison();
|
|
}
|
|
};
|
|
|
|
void CheckOutput(const util::stream::ChainPosition &from) {
|
|
NGramStream<float> in(from);
|
|
float sum = 0.0;
|
|
for (WordIndex i = 0; i < sizeof(kInputs) / sizeof(float) - 1 /* <s> at the end */; ++i) {
|
|
sum += pow(10.0, kInputs[i]);
|
|
}
|
|
sum = log10(sum);
|
|
BOOST_REQUIRE(in);
|
|
BOOST_CHECK_CLOSE(kInputs[0] - sum, in->Value(), 0.0001);
|
|
BOOST_REQUIRE(++in);
|
|
BOOST_CHECK_CLOSE(kInputs[1] - sum, in->Value(), 0.0001);
|
|
BOOST_REQUIRE(++in);
|
|
BOOST_CHECK_CLOSE(kInputs[2] - sum, in->Value(), 0.0001);
|
|
BOOST_REQUIRE(++in);
|
|
BOOST_CHECK_CLOSE(kInputs[3] - sum, in->Value(), 0.0001);
|
|
BOOST_REQUIRE(++in);
|
|
BOOST_CHECK_CLOSE(kInputs[4] - sum, in->Value(), 0.0001);
|
|
BOOST_REQUIRE(++in);
|
|
BOOST_CHECK_CLOSE(kInputs[5] - sum, in->Value(), 0.0001);
|
|
BOOST_CHECK(!++in);
|
|
}
|
|
|
|
BOOST_AUTO_TEST_CASE(Unigrams) {
|
|
InterpolateInfo info;
|
|
info.lambdas.push_back(2.0);
|
|
info.lambdas.push_back(-0.1);
|
|
info.orders.push_back(1);
|
|
info.orders.push_back(1);
|
|
|
|
BOOST_CHECK_EQUAL(0, MakeEncoder(info, 1).EncodedLength());
|
|
|
|
// No backoffs.
|
|
util::stream::Chains blank(0);
|
|
util::FixedArray<util::stream::ChainPositions> models_by_order(2);
|
|
models_by_order.push_back(blank);
|
|
models_by_order.push_back(blank);
|
|
|
|
util::stream::Chains merged_probabilities(1);
|
|
util::stream::Chains probabilities_out(1);
|
|
util::stream::Chains backoffs_out(0);
|
|
|
|
merged_probabilities.push_back(util::stream::ChainConfig(sizeof(WordIndex) + sizeof(float) + sizeof(float), 2, 24));
|
|
probabilities_out.push_back(util::stream::ChainConfig(sizeof(WordIndex) + sizeof(float), 2, 100));
|
|
|
|
merged_probabilities[0] >> WriteInput();
|
|
Normalize(info, models_by_order, merged_probabilities, probabilities_out, backoffs_out);
|
|
|
|
util::stream::ChainPosition checker(probabilities_out[0].Add());
|
|
|
|
merged_probabilities >> util::stream::kRecycle;
|
|
probabilities_out >> util::stream::kRecycle;
|
|
|
|
CheckOutput(checker);
|
|
probabilities_out.Wait();
|
|
}
|
|
|
|
}}} // namespaces
|