STT/native_client/ctcdecode/scorer.cpp

#ifdef _MSC_VER
  #include <stdlib.h>
  #include <io.h>
  #include <windows.h> 

  #define R_OK    4       /* Read permission.  */
  #define W_OK    2       /* Write permission.  */ 
  #define F_OK    0       /* Existence.  */

  #define access _access

#else          /* _MSC_VER  */
  #include <unistd.h>
#endif

#include "scorer.h"
#include <iostream>
#include <fstream>

#include "lm/config.hh"
#include "lm/model.hh"
#include "lm/state.hh"
#include "util/string_piece.hh"

#include "decoder_utils.h"

static const int32_t MAGIC = 'TRIE';
static const int32_t FILE_VERSION = 6;

int
Scorer::init(const std::string& lm_path,
             const Alphabet& alphabet)
{
  set_alphabet(alphabet);
  return load_lm(lm_path);
}

int
Scorer::init(const std::string& lm_path,
             const std::string& alphabet_config_path)
{
  int err = alphabet_.init(alphabet_config_path.c_str());
  if (err != 0) {
    return err;
  }
  setup_char_map();
  return load_lm(lm_path);
}

void
Scorer::set_alphabet(const Alphabet& alphabet)
{
  alphabet_ = alphabet;
  setup_char_map();
}

void Scorer::setup_char_map()
{
  // (Re-)Initialize character map
  char_map_.clear();

  SPACE_ID_ = alphabet_.GetSpaceLabel();

  for (int i = 0; i < alphabet_.GetSize(); i++) {
    // The initial state of FST is state 0, hence the index of chars in
    // the FST should start from 1 to avoid the conflict with the initial
    // state, otherwise wrong decoding results would be given.
    char_map_[alphabet_.DecodeSingle(i)] = i + 1;
  }
}

int Scorer::load_lm(const std::string& lm_path)
{
  // Check if file is readable to avoid KenLM throwing an exception
  const char* filename = lm_path.c_str();
  if (access(filename, R_OK) != 0) {
    return DS_ERR_SCORER_UNREADABLE;
  }

  // Check if the file format is valid to avoid KenLM throwing an exception
  lm::ngram::ModelType model_type;
  if (!lm::ngram::RecognizeBinary(filename, model_type)) {
    return DS_ERR_SCORER_INVALID_LM;
  }

  // Load the LM
  lm::ngram::Config config;
  config.load_method = util::LoadMethod::LAZY;
  language_model_.reset(lm::ngram::LoadVirtual(filename, config));
  max_order_ = language_model_->Order();

  uint64_t package_size;
  {
    util::scoped_fd fd(util::OpenReadOrThrow(filename));
    package_size = util::SizeFile(fd.get());
  }
  uint64_t trie_offset = language_model_->GetEndOfSearchOffset();
  if (package_size <= trie_offset) {
    // File ends without a trie structure
    return DS_ERR_SCORER_NO_TRIE;
  }

  // Read metadata and trie from file
  std::ifstream fin(lm_path, std::ios::binary);
  fin.seekg(trie_offset);
  return load_trie(fin, lm_path);
}

int Scorer::load_trie(std::ifstream& fin, const std::string& file_path)
{
  int magic;
  fin.read(reinterpret_cast<char*>(&magic), sizeof(magic));
  if (magic != MAGIC) {
    std::cerr << "Error: Can't parse scorer file, invalid header. Try updating "
                 "your scorer file." << std::endl;
    return DS_ERR_SCORER_INVALID_TRIE;
  }

  int version;
  fin.read(reinterpret_cast<char*>(&version), sizeof(version));
  if (version != FILE_VERSION) {
    std::cerr << "Error: Scorer file version mismatch (" << version
              << " instead of expected " << FILE_VERSION
              << "). ";
    if (version < FILE_VERSION) {
      std::cerr << "Update your scorer file.";
    } else {
      std::cerr << "Downgrade your scorer file or update your version of DeepSpeech.";
    }
    std::cerr << std::endl;
    return DS_ERR_SCORER_VERSION_MISMATCH;
  }

  fin.read(reinterpret_cast<char*>(&is_utf8_mode_), sizeof(is_utf8_mode_));

  // Read hyperparameters from header
  double alpha, beta;
  fin.read(reinterpret_cast<char*>(&alpha), sizeof(alpha));
  fin.read(reinterpret_cast<char*>(&beta), sizeof(beta));
  reset_params(alpha, beta);

  fst::FstReadOptions opt;
  opt.mode = fst::FstReadOptions::MAP;
  opt.source = file_path;
  dictionary.reset(FstType::Read(fin, opt));
  return DS_ERR_OK;
}

bool Scorer::save_dictionary(const std::string& path, bool append_instead_of_overwrite)
{
  std::ios::openmode om;
  if (append_instead_of_overwrite) {
    om = std::ios::in|std::ios::out|std::ios::binary|std::ios::ate;
  } else {
    om = std::ios::out|std::ios::binary;
  }
  std::fstream fout(path, om);
  if (!fout ||fout.bad()) {
    std::cerr << "Error opening '" << path << "'" << std::endl;
    return false;
  }
  fout.write(reinterpret_cast<const char*>(&MAGIC), sizeof(MAGIC));
  if (fout.bad()) {
    std::cerr << "Error writing MAGIC '" << path << "'" << std::endl;
    return false;
  }
  fout.write(reinterpret_cast<const char*>(&FILE_VERSION), sizeof(FILE_VERSION));
  if (fout.bad()) {
    std::cerr << "Error writing FILE_VERSION '" << path << "'" << std::endl;
    return false;
  }
  fout.write(reinterpret_cast<const char*>(&is_utf8_mode_), sizeof(is_utf8_mode_));
  if (fout.bad()) {
    std::cerr << "Error writing is_utf8_mode '" << path << "'" << std::endl;
    return false;
  }
  fout.write(reinterpret_cast<const char*>(&alpha), sizeof(alpha));
  if (fout.bad()) {
    std::cerr << "Error writing alpha '" << path << "'" << std::endl;
    return false;
  }
  fout.write(reinterpret_cast<const char*>(&beta), sizeof(beta));
  if (fout.bad()) {
    std::cerr << "Error writing beta '" << path << "'" << std::endl;
    return false;
  }
  fst::FstWriteOptions opt;
  opt.align = true;
  opt.source = path;
  return dictionary->Write(fout, opt);
}

bool Scorer::is_scoring_boundary(PathTrie* prefix, size_t new_label)
{
  if (is_utf8_mode()) {
    if (prefix->character == -1) {
      return false;
    }
    unsigned char first_byte;
    int distance_to_boundary = prefix->distance_to_codepoint_boundary(&first_byte, alphabet_);
    int needed_bytes;
    if ((first_byte >> 3) == 0x1E) {
      needed_bytes = 4;
    } else if ((first_byte >> 4) == 0x0E) {
      needed_bytes = 3;
    } else if ((first_byte >> 5) == 0x06) {
      needed_bytes = 2;
    } else if ((first_byte >> 7) == 0x00) {
      needed_bytes = 1;
    } else {
      assert(false); // invalid byte sequence. should be unreachable, disallowed by vocabulary/trie
      return false;
    }
    return distance_to_boundary == needed_bytes;
  } else {
    return new_label == SPACE_ID_;
  }
}

double Scorer::get_log_cond_prob(const std::vector<std::string>& words,
                                 bool bos,
                                 bool eos)
{
  return get_log_cond_prob(words.begin(), words.end(), bos, eos);
}

double Scorer::get_log_cond_prob(const std::vector<std::string>::const_iterator& begin,
                                 const std::vector<std::string>::const_iterator& end,
                                 bool bos,
                                 bool eos)
{
  const auto& vocab = language_model_->BaseVocabulary();
  lm::ngram::State state_vec[2];
  lm::ngram::State *in_state = &state_vec[0];
  lm::ngram::State *out_state = &state_vec[1];

  if (bos) {
    language_model_->BeginSentenceWrite(in_state);
  } else {
    language_model_->NullContextWrite(in_state);
  }

  double cond_prob = 0.0;
  for (auto it = begin; it != end; ++it) {
    lm::WordIndex word_index = vocab.Index(*it);

    // encounter OOV
    if (word_index == lm::kUNK) {
      return OOV_SCORE;
    }

    cond_prob = language_model_->BaseScore(in_state, word_index, out_state);
    std::swap(in_state, out_state);
  }

  if (eos) {
    cond_prob = language_model_->BaseScore(in_state, vocab.EndSentence(), out_state);
  }

  // return loge prob
  return cond_prob/NUM_FLT_LOGE;
}

void Scorer::reset_params(float alpha, float beta)
{
  this->alpha = alpha;
  this->beta = beta;
}

std::vector<std::string> Scorer::split_labels_into_scored_units(const std::vector<unsigned int>& labels)
{
  if (labels.empty()) return {};

  std::string s = alphabet_.Decode(labels);
  std::vector<std::string> words;
  if (is_utf8_mode_) {
    words = split_into_codepoints(s);
  } else {
    words = split_str(s, " ");
  }
  return words;
}

std::vector<std::string> Scorer::make_ngram(PathTrie* prefix)
{
  std::vector<std::string> ngram;
  PathTrie* current_node = prefix;
  PathTrie* new_node = nullptr;

  for (int order = 0; order < max_order_; order++) {
    if (!current_node || current_node->character == -1) {
      break;
    }

    std::vector<unsigned int> prefix_vec;

    if (is_utf8_mode_) {
      new_node = current_node->get_prev_grapheme(prefix_vec, alphabet_);
    } else {
      new_node = current_node->get_prev_word(prefix_vec, alphabet_);
    }
    current_node = new_node->parent;

    // reconstruct word
    std::string word = alphabet_.Decode(prefix_vec);
    ngram.push_back(word);
  }
  std::reverse(ngram.begin(), ngram.end());
  return ngram;
}

void Scorer::fill_dictionary(const std::unordered_set<std::string>& vocabulary)
{
  // ConstFst is immutable, so we need to use a MutableFst to create the trie,
  // and then we convert to a ConstFst for the decoder and for storing on disk.
  fst::StdVectorFst dictionary;
  // For each unigram convert to ints and put in trie
  for (const auto& word : vocabulary) {
    if (word != START_TOKEN && word != UNK_TOKEN && word != END_TOKEN) {
      add_word_to_dictionary(word, char_map_, is_utf8_mode_, SPACE_ID_ + 1, &dictionary);
    }
  }

  /* Simplify FST

   * This gets rid of "epsilon" transitions in the FST.
   * These are transitions that don't require a string input to be taken.
   * Getting rid of them is necessary to make the FST deterministic, but
   * can greatly increase the size of the FST
   */
  fst::RmEpsilon(&dictionary);
  std::unique_ptr<fst::StdVectorFst> new_dict(new fst::StdVectorFst);

  /* This makes the FST deterministic, meaning for any string input there's
   * only one possible state the FST could be in.  It is assumed our
   * dictionary is deterministic when using it.
   * (lest we'd have to check for multiple transitions at each state)
   */
  fst::Determinize(dictionary, new_dict.get());

  /* Finds the simplest equivalent fst. This is unnecessary but decreases
   * memory usage of the dictionary
   */
  fst::Minimize(new_dict.get());

  // Now we convert the MutableFst to a ConstFst (Scorer::FstType) via its ctor
  std::unique_ptr<FstType> converted(new FstType(*new_dict));
  this->dictionary = std::move(converted);
}