Computing audio hours at import

bin/import_cv2.py

@@ -27,6 +27,7 @@ from multiprocessing.dummy import Pool
 from multiprocessing import cpu_count
 from util.downloader import SIMPLE_BAR
 from util.text import Alphabet, validate_label
+from util.feeding import secs_to_hours
 
 
 FIELDNAMES = ['wav_filename', 'wav_filesize', 'transcript']
@@ -56,7 +57,7 @@ def _maybe_convert_set(input_tsv, audio_dir, label_filter, space_after_every_cha
             samples.append((row['path'], row['sentence']))
 
     # Keep track of how many samples are good vs. problematic
-    counter = {'all': 0, 'failed': 0, 'invalid_label': 0, 'too_short': 0, 'too_long': 0}
+    counter = {'all': 0, 'failed': 0, 'invalid_label': 0, 'too_short': 0, 'too_long': 0, 'total_time': 0}
     lock = RLock()
     num_samples = len(samples)
     rows = []
@@ -91,6 +92,7 @@ def _maybe_convert_set(input_tsv, audio_dir, label_filter, space_after_every_cha
                 # This one is good - keep it for the target CSV
                 rows.append((wav_filename, file_size, label))
             counter['all'] += 1
+            counter['total_time'] += frames
 
     print("Importing mp3 files...")
     pool = Pool(cpu_count())
@@ -121,6 +123,7 @@ def _maybe_convert_set(input_tsv, audio_dir, label_filter, space_after_every_cha
         print('Skipped %d samples that were too short to match the transcript.' % counter['too_short'])
     if counter['too_long'] > 0:
         print('Skipped %d samples that were longer than %d seconds.' % (counter['too_long'], MAX_SECS))
+    print('Final amount of imported audio: %s.' % secs_to_hours(counter['total_time'] / SAMPLE_RATE))
 
 
 def _maybe_convert_wav(mp3_filename, wav_filename):

bin/import_lingua_libre.py

@@ -27,6 +27,7 @@ from glob import glob
 
 from util.downloader import maybe_download
 from util.text import Alphabet, validate_label
+from util.feeding import secs_to_hours
 
 FIELDNAMES = ['wav_filename', 'wav_filesize', 'transcript']
 SAMPLE_RATE = 16000
@@ -76,7 +77,7 @@ def _maybe_convert_sets(target_dir, extracted_data):
         samples.append((record_file, os.path.splitext(os.path.basename(record_file))[0]))
 
     # Keep track of how many samples are good vs. problematic
-    counter = {'all': 0, 'failed': 0, 'invalid_label': 0, 'too_short': 0, 'too_long': 0}
+    counter = {'all': 0, 'failed': 0, 'invalid_label': 0, 'too_short': 0, 'too_long': 0, 'total_time': 0}
     lock = RLock()
     num_samples = len(samples)
     rows = []
@@ -109,6 +110,7 @@ def _maybe_convert_sets(target_dir, extracted_data):
                 # This one is good - keep it for the target CSV
                 rows.append((wav_filename, file_size, label))
             counter['all'] += 1
+            counter['total_time'] += frames
 
     print("Importing ogg files...")
     pool = Pool(cpu_count())
@@ -156,6 +158,7 @@ def _maybe_convert_sets(target_dir, extracted_data):
         print('Skipped %d samples that were too short to match the transcript.' % counter['too_short'])
     if counter['too_long'] > 0:
         print('Skipped %d samples that were longer than %d seconds.' % (counter['too_long'], MAX_SECS))
+    print('Final amount of imported audio: %s.' % secs_to_hours(counter['total_time'] / SAMPLE_RATE))
 
 def _maybe_convert_wav(ogg_filename, wav_filename):
     if not path.exists(wav_filename):

bin/import_ts.py

@@ -27,6 +27,7 @@ from os import path
 
 from util.downloader import maybe_download
 from util.text import validate_label
+from util.feeding import secs_to_hours
 
 FIELDNAMES = ['wav_filename', 'wav_filesize', 'transcript']
 SAMPLE_RATE = 16000
@@ -74,7 +75,7 @@ def _maybe_convert_sets(target_dir, extracted_data, english_compatible=False):
         ]
 
     # Keep track of how many samples are good vs. problematic
-    counter = {'all': 0, 'failed': 0, 'invalid_label': 0, 'too_short': 0, 'too_long': 0}
+    counter = {'all': 0, 'failed': 0, 'invalid_label': 0, 'too_short': 0, 'too_long': 0, 'total_time': 0}
     lock = RLock()
     num_samples = len(data)
     rows = []
@@ -109,6 +110,7 @@ def _maybe_convert_sets(target_dir, extracted_data, english_compatible=False):
                 # This one is good - keep it for the target CSV
                 rows.append((wav_filename, file_size, label))
             counter['all'] += 1
+            counter['total_time'] += frames
 
     print("Importing wav files...")
     pool = Pool(cpu_count())
@@ -157,6 +159,7 @@ def _maybe_convert_sets(target_dir, extracted_data, english_compatible=False):
         print('Skipped %d samples that were too short to match the transcript.' % counter['too_short'])
     if counter['too_long'] > 0:
         print('Skipped %d samples that were longer than %d seconds.' % (counter['too_long'], MAX_SECS))
+    print('Final amount of imported audio: %s.' % secs_to_hours(counter['total_time'] / SAMPLE_RATE))
 
 def _maybe_convert_wav(orig_filename, wav_filename):
     if not path.exists(wav_filename):

stats.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+
+from util.feeding import read_csvs, secs_to_hours
+
+def main():
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument("-csv", "--csv-files", help="Str. Filenames as a comma separated list", required=True)
+    parser.add_argument("--sample-rate", type=int, default=16000, required=False, help="Audio sample rate")
+    parser.add_argument("--channels", type=int, default=1, required=False, help="Audio channels")
+    parser.add_argument("--bits-per-sample", type=int, default=16, required=False, help="Audio bits per sample")
+    args = parser.parse_args()
+    in_files = [os.path.abspath(i) for i in args.csv_files.split(",")]
+
+    csv_dataframe = read_csvs(in_files)
+    total_bytes = csv_dataframe['wav_filesize'].sum()
+    total_files = len(csv_dataframe.index)
+
+    bytes_without_headers = total_bytes - 44 * total_files
+
+    total_time = bytes_without_headers / (args.sample_rate * args.channels * args.bits_per_sample / 8)
+
+    print('total_bytes', total_bytes)
+    print('total_files', total_files)
+    print('bytes_without_headers', bytes_without_headers)
+    print('total_time', secs_to_hours(total_time))
+
+if __name__ == '__main__':
+    main()

util/feeding.py

@@ -8,6 +8,7 @@ from functools import partial
 import numpy as np
 import pandas
 import tensorflow as tf
+import datetime
 
 from tensorflow.contrib.framework.python.ops import audio_ops as contrib_audio
 
@@ -98,3 +99,8 @@ def create_dataset(csvs, batch_size, cache_path=''):
                               .prefetch(num_gpus))
 
     return dataset
+
+def secs_to_hours(secs):
+    hours, remainder = divmod(secs, 3600)
+    minutes, seconds = divmod(remainder, 60)
+    return '%d:%02d:%02d' % (hours, minutes, seconds)