experiments/STT
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Don't add special tokens to vocabulary

Browse Source
This commit is contained in:
Reuben Morais 2019-10-14 11:48:44 +02:00
parent 3015237e8d
commit c8802a38e7
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
native_client/ctcdecode/scorer.cpp
View File

@ -296,8 +296,10 @@ void Scorer::fill_dictionary(const std::vector<std::string>& vocabulary, bool ad
fst::StdVectorFst dictionary; fst::StdVectorFst dictionary;
// For each unigram convert to ints and put in trie // For each unigram convert to ints and put in trie
for (const auto& word : vocabulary) { for (const auto& word : vocabulary) {
if (word != START_TOKEN && word != UNK_TOKEN && word != END_TOKEN) {
add_word_to_dictionary(word, char_map_, add_space, SPACE_ID_ + 1, &dictionary); add_word_to_dictionary(word, char_map_, add_space, SPACE_ID_ + 1, &dictionary);
} }
}
/* Simplify FST /* Simplify FST