experiments
/
STT
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #2819 from tilmankamp/sdb 

Process pool for audio preparation
This commit is contained in:
Tilman Kamp 2020-03-12 15:13:25 +01:00 committed by GitHub
parent 60304da5a6 63bc695600
commit 87f70693c3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 9 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
util/audio.py
View File

@ -92,12 +92,15 @@ class Sample:
self.audio_type = new_audio_type self.audio_type = new_audio_type
def _change_audio_type(sample_and_audio_type):
sample, audio_type = sample_and_audio_type
sample.change_audio_type(audio_type)
return sample
def change_audio_types(samples, audio_type=AUDIO_TYPE_PCM, processes=None, process_ahead=None): def change_audio_types(samples, audio_type=AUDIO_TYPE_PCM, processes=None, process_ahead=None):
def change_audio_type(sample):
sample.change_audio_type(audio_type)
return sample
with LimitingPool(processes=processes, process_ahead=process_ahead) as pool: with LimitingPool(processes=processes, process_ahead=process_ahead) as pool:
yield from pool.imap(change_audio_type, samples) yield from pool.imap(_change_audio_type, map(lambda s: (s, audio_type), samples))
def read_audio_format_from_wav_file(wav_file): def read_audio_format_from_wav_file(wav_file):

4
util/helpers.py
View File

@ -4,7 +4,7 @@ import time
import heapq import heapq
import semver import semver
from multiprocessing.dummy import Pool as ThreadPool from multiprocessing import Pool
KILO = 1024 KILO = 1024
KILOBYTE = 1 * KILO KILOBYTE = 1 * KILO
@ -83,7 +83,7 @@ class LimitingPool:
self.process_ahead = os.cpu_count() if process_ahead is None else process_ahead self.process_ahead = os.cpu_count() if process_ahead is None else process_ahead
self.sleeping_for = sleeping_for self.sleeping_for = sleeping_for
self.processed = 0 self.processed = 0
self.pool = ThreadPool(processes=processes) self.pool = Pool(processes=processes)
def __enter__(self): def __enter__(self):
return self return self