experiments/STT
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix downloader and taskcluster directory mgmt with remote I/O

Browse Source
This commit is contained in:
CatalinVoss 2020-11-12 16:30:11 -08:00
parent 64d278560d
commit 783cdad8db
2 changed files with 10 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
training/deepspeech_training/util/downloader.py
View File

@ -2,7 +2,7 @@ import requests
import progressbar import progressbar
from os import path, makedirs from os import path, makedirs
from .io import open_remote, path_exists_remote from .io import open_remote, path_exists_remote, is_remote_path
SIMPLE_BAR = ['Progress ', progressbar.Bar(), ' ', progressbar.Percentage(), ' completed'] SIMPLE_BAR = ['Progress ', progressbar.Bar(), ' ', progressbar.Percentage(), ' completed']
@ -10,7 +10,7 @@ def maybe_download(archive_name, target_dir, archive_url):
# If archive file does not exist, download it... # If archive file does not exist, download it...
archive_path = path.join(target_dir, archive_name) archive_path = path.join(target_dir, archive_name)
if not path_exists_remote(target_dir): if not is_remote_path(target_dir) and not path.exists(target_dir):
print('No path "%s" - creating ...' % target_dir) print('No path "%s" - creating ...' % target_dir)
makedirs(target_dir) makedirs(target_dir)

6
training/deepspeech_training/util/taskcluster.py
View File

@ -14,7 +14,7 @@ import sys
from pkg_resources import parse_version from pkg_resources import parse_version
from .io import isdir_remote, open_remote from .io import isdir_remote, open_remote, is_remote_path
DEFAULT_SCHEMES = { DEFAULT_SCHEMES = {
'deepspeech': 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.deepspeech.native_client.%(branch_name)s.%(arch_string)s/artifacts/public/%(artifact_name)s', 'deepspeech': 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.deepspeech.native_client.%(branch_name)s.%(arch_string)s/artifacts/public/%(artifact_name)s',
@ -43,13 +43,13 @@ def maybe_download_tc(target_dir, tc_url, progress=True):
assert target_dir is not None assert target_dir is not None
target_dir = os.path.abspath(target_dir) if not is_remote_path(target_dir):
try: try:
os.makedirs(target_dir) os.makedirs(target_dir)
except OSError as e: except OSError as e:
if e.errno != errno.EEXIST: if e.errno != errno.EEXIST:
raise e raise e
assert isdir_remote(os.path.dirname(target_dir)) assert os.path.isdir(os.path.dirname(target_dir))
tc_filename = os.path.basename(tc_url) tc_filename = os.path.basename(tc_url)
target_file = os.path.join(target_dir, tc_filename) target_file = os.path.join(target_dir, tc_filename)