diff --git a/native_client/client.cc b/native_client/client.cc
index 981b1bda..99af904e 100644
--- a/native_client/client.cc
+++ b/native_client/client.cc
@@ -302,8 +302,11 @@ WordsFromMetadata(Metadata* metadata)
     MetadataItem item = metadata->items[i];
 
     // Append character to word if it's not a space
-    if (strcmp(item.character, " ") != 0 
-        && strcmp(item.character, u8"　") != 0) {
+    if (strcmp(item.character, u8" ") != 0) {
+      // Log the start time of the new word
+      if (word.length() == 0) {
+        word_start_time = item.start_time;
+      }
       word.append(item.character);
     }
 
@@ -328,10 +331,6 @@ WordsFromMetadata(Metadata* metadata)
       // Reset
       word = "";
       word_start_time = 0;
-    } else {
-      if (word.length() == 1) {
-        word_start_time = item.start_time; // Log the start time of the new word
-      }
     }
   }
 
diff --git a/native_client/python/client.py b/native_client/python/client.py
index 71ad382c..91a63491 100644
--- a/native_client/python/client.py
+++ b/native_client/python/client.py
@@ -42,6 +42,10 @@ def words_from_metadata(metadata):
         item = metadata.items[i]
         # Append character to word if it's not a space
         if item.character != " ":
+            if len(word) == 0:
+                # Log the start time of the new word
+                word_start_time = item.start_time
+
             word = word + item.character
         # Word boundary is either a space or the last character in the array
         if item.character == " " or i == metadata.num_items - 1:
@@ -59,10 +63,6 @@ def words_from_metadata(metadata):
             # Reset
             word = ""
             word_start_time = 0
-        else:
-            if len(word) == 1:
-                # Log the start time of the new word
-                word_start_time = item.start_time
 
     return word_list