From 396ac7fe4685c9eeeed5e1dd8a9c9d69e56019c7 Mon Sep 17 00:00:00 2001 From: CatalinVoss Date: Thu, 12 Nov 2020 10:48:49 -0800 Subject: [PATCH] Remote I/O for downloader --- training/deepspeech_training/util/downloader.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/training/deepspeech_training/util/downloader.py b/training/deepspeech_training/util/downloader.py index 9fcbf674..0a40c481 100644 --- a/training/deepspeech_training/util/downloader.py +++ b/training/deepspeech_training/util/downloader.py @@ -2,6 +2,7 @@ import requests import progressbar from os import path, makedirs +from .io import open_remote, path_exists_remote SIMPLE_BAR = ['Progress ', progressbar.Bar(), ' ', progressbar.Percentage(), ' completed'] @@ -9,16 +10,16 @@ def maybe_download(archive_name, target_dir, archive_url): # If archive file does not exist, download it... archive_path = path.join(target_dir, archive_name) - if not path.exists(target_dir): + if not path_exists_remote(target_dir): print('No path "%s" - creating ...' % target_dir) makedirs(target_dir) - if not path.exists(archive_path): + if not path_exists_remote(archive_path): print('No archive "%s" - downloading...' % archive_path) req = requests.get(archive_url, stream=True) total_size = int(req.headers.get('content-length', 0)) done = 0 - with open(archive_path, 'wb') as f: + with open_remote(archive_path, 'wb') as f: bar = progressbar.ProgressBar(max_value=total_size, widgets=SIMPLE_BAR) for data in req.iter_content(1024*1024): done += len(data)