diff --git a/bin/import_cv2.py b/bin/import_cv2.py index 5af4fb7f..61b56554 100755 --- a/bin/import_cv2.py +++ b/bin/import_cv2.py @@ -27,6 +27,7 @@ from multiprocessing.dummy import Pool from multiprocessing import cpu_count from util.downloader import SIMPLE_BAR from util.text import Alphabet, validate_label +from util.feeding import secs_to_hours FIELDNAMES = ['wav_filename', 'wav_filesize', 'transcript'] @@ -56,7 +57,7 @@ def _maybe_convert_set(input_tsv, audio_dir, label_filter, space_after_every_cha samples.append((row['path'], row['sentence'])) # Keep track of how many samples are good vs. problematic - counter = {'all': 0, 'failed': 0, 'invalid_label': 0, 'too_short': 0, 'too_long': 0} + counter = {'all': 0, 'failed': 0, 'invalid_label': 0, 'too_short': 0, 'too_long': 0, 'total_time': 0} lock = RLock() num_samples = len(samples) rows = [] @@ -91,6 +92,7 @@ def _maybe_convert_set(input_tsv, audio_dir, label_filter, space_after_every_cha # This one is good - keep it for the target CSV rows.append((wav_filename, file_size, label)) counter['all'] += 1 + counter['total_time'] += frames print("Importing mp3 files...") pool = Pool(cpu_count()) @@ -121,6 +123,7 @@ def _maybe_convert_set(input_tsv, audio_dir, label_filter, space_after_every_cha print('Skipped %d samples that were too short to match the transcript.' % counter['too_short']) if counter['too_long'] > 0: print('Skipped %d samples that were longer than %d seconds.' % (counter['too_long'], MAX_SECS)) + print('Final amount of imported audio: %s.' % secs_to_hours(counter['total_time'] / SAMPLE_RATE)) def _maybe_convert_wav(mp3_filename, wav_filename): diff --git a/bin/import_lingua_libre.py b/bin/import_lingua_libre.py index 94c9f204..62c2b7d6 100644 --- a/bin/import_lingua_libre.py +++ b/bin/import_lingua_libre.py @@ -27,6 +27,7 @@ from glob import glob from util.downloader import maybe_download from util.text import Alphabet, validate_label +from util.feeding import secs_to_hours FIELDNAMES = ['wav_filename', 'wav_filesize', 'transcript'] SAMPLE_RATE = 16000 @@ -76,7 +77,7 @@ def _maybe_convert_sets(target_dir, extracted_data): samples.append((record_file, os.path.splitext(os.path.basename(record_file))[0])) # Keep track of how many samples are good vs. problematic - counter = {'all': 0, 'failed': 0, 'invalid_label': 0, 'too_short': 0, 'too_long': 0} + counter = {'all': 0, 'failed': 0, 'invalid_label': 0, 'too_short': 0, 'too_long': 0, 'total_time': 0} lock = RLock() num_samples = len(samples) rows = [] @@ -109,6 +110,7 @@ def _maybe_convert_sets(target_dir, extracted_data): # This one is good - keep it for the target CSV rows.append((wav_filename, file_size, label)) counter['all'] += 1 + counter['total_time'] += frames print("Importing ogg files...") pool = Pool(cpu_count()) @@ -156,6 +158,7 @@ def _maybe_convert_sets(target_dir, extracted_data): print('Skipped %d samples that were too short to match the transcript.' % counter['too_short']) if counter['too_long'] > 0: print('Skipped %d samples that were longer than %d seconds.' % (counter['too_long'], MAX_SECS)) + print('Final amount of imported audio: %s.' % secs_to_hours(counter['total_time'] / SAMPLE_RATE)) def _maybe_convert_wav(ogg_filename, wav_filename): if not path.exists(wav_filename): diff --git a/bin/import_ts.py b/bin/import_ts.py index 146d9380..235c052e 100755 --- a/bin/import_ts.py +++ b/bin/import_ts.py @@ -27,6 +27,7 @@ from os import path from util.downloader import maybe_download from util.text import validate_label +from util.feeding import secs_to_hours FIELDNAMES = ['wav_filename', 'wav_filesize', 'transcript'] SAMPLE_RATE = 16000 @@ -74,7 +75,7 @@ def _maybe_convert_sets(target_dir, extracted_data, english_compatible=False): ] # Keep track of how many samples are good vs. problematic - counter = {'all': 0, 'failed': 0, 'invalid_label': 0, 'too_short': 0, 'too_long': 0} + counter = {'all': 0, 'failed': 0, 'invalid_label': 0, 'too_short': 0, 'too_long': 0, 'total_time': 0} lock = RLock() num_samples = len(data) rows = [] @@ -109,6 +110,7 @@ def _maybe_convert_sets(target_dir, extracted_data, english_compatible=False): # This one is good - keep it for the target CSV rows.append((wav_filename, file_size, label)) counter['all'] += 1 + counter['total_time'] += frames print("Importing wav files...") pool = Pool(cpu_count()) @@ -157,6 +159,7 @@ def _maybe_convert_sets(target_dir, extracted_data, english_compatible=False): print('Skipped %d samples that were too short to match the transcript.' % counter['too_short']) if counter['too_long'] > 0: print('Skipped %d samples that were longer than %d seconds.' % (counter['too_long'], MAX_SECS)) + print('Final amount of imported audio: %s.' % secs_to_hours(counter['total_time'] / SAMPLE_RATE)) def _maybe_convert_wav(orig_filename, wav_filename): if not path.exists(wav_filename): diff --git a/stats.py b/stats.py new file mode 100644 index 00000000..466b78e5 --- /dev/null +++ b/stats.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 + +import argparse +import os + +from util.feeding import read_csvs, secs_to_hours + +def main(): + parser = argparse.ArgumentParser() + + parser.add_argument("-csv", "--csv-files", help="Str. Filenames as a comma separated list", required=True) + parser.add_argument("--sample-rate", type=int, default=16000, required=False, help="Audio sample rate") + parser.add_argument("--channels", type=int, default=1, required=False, help="Audio channels") + parser.add_argument("--bits-per-sample", type=int, default=16, required=False, help="Audio bits per sample") + args = parser.parse_args() + in_files = [os.path.abspath(i) for i in args.csv_files.split(",")] + + csv_dataframe = read_csvs(in_files) + total_bytes = csv_dataframe['wav_filesize'].sum() + total_files = len(csv_dataframe.index) + + bytes_without_headers = total_bytes - 44 * total_files + + total_time = bytes_without_headers / (args.sample_rate * args.channels * args.bits_per_sample / 8) + + print('total_bytes', total_bytes) + print('total_files', total_files) + print('bytes_without_headers', bytes_without_headers) + print('total_time', secs_to_hours(total_time)) + +if __name__ == '__main__': + main() diff --git a/util/feeding.py b/util/feeding.py index e15914ab..a88f3660 100644 --- a/util/feeding.py +++ b/util/feeding.py @@ -8,6 +8,7 @@ from functools import partial import numpy as np import pandas import tensorflow as tf +import datetime from tensorflow.contrib.framework.python.ops import audio_ops as contrib_audio @@ -98,3 +99,8 @@ def create_dataset(csvs, batch_size, cache_path=''): .prefetch(num_gpus)) return dataset + +def secs_to_hours(secs): + hours, remainder = divmod(secs, 3600) + minutes, seconds = divmod(remainder, 60) + return '%d:%02d:%02d' % (hours, minutes, seconds)