diff --git a/native_client/ctcdecode/scorer.cpp b/native_client/ctcdecode/scorer.cpp
index 49a6d794..c265b430 100644
--- a/native_client/ctcdecode/scorer.cpp
+++ b/native_client/ctcdecode/scorer.cpp
@@ -296,7 +296,9 @@ void Scorer::fill_dictionary(const std::vector<std::string>& vocabulary, bool ad
   fst::StdVectorFst dictionary;
   // For each unigram convert to ints and put in trie
   for (const auto& word : vocabulary) {
-    add_word_to_dictionary(word, char_map_, add_space, SPACE_ID_ + 1, &dictionary);
+    if (word != START_TOKEN && word != UNK_TOKEN && word != END_TOKEN) {
+      add_word_to_dictionary(word, char_map_, add_space, SPACE_ID_ + 1, &dictionary);
+    }
   }
 
   /* Simplify FST