From 783cdad8db471cc33c0f9d9fa79b0b1c8d4c198b Mon Sep 17 00:00:00 2001 From: CatalinVoss Date: Thu, 12 Nov 2020 16:30:11 -0800 Subject: [PATCH] Fix downloader and taskcluster directory mgmt with remote I/O --- training/deepspeech_training/util/downloader.py | 4 ++-- training/deepspeech_training/util/taskcluster.py | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/training/deepspeech_training/util/downloader.py b/training/deepspeech_training/util/downloader.py index 0a40c481..b8fcdb8d 100644 --- a/training/deepspeech_training/util/downloader.py +++ b/training/deepspeech_training/util/downloader.py @@ -2,7 +2,7 @@ import requests import progressbar from os import path, makedirs -from .io import open_remote, path_exists_remote +from .io import open_remote, path_exists_remote, is_remote_path SIMPLE_BAR = ['Progress ', progressbar.Bar(), ' ', progressbar.Percentage(), ' completed'] @@ -10,7 +10,7 @@ def maybe_download(archive_name, target_dir, archive_url): # If archive file does not exist, download it... archive_path = path.join(target_dir, archive_name) - if not path_exists_remote(target_dir): + if not is_remote_path(target_dir) and not path.exists(target_dir): print('No path "%s" - creating ...' % target_dir) makedirs(target_dir) diff --git a/training/deepspeech_training/util/taskcluster.py b/training/deepspeech_training/util/taskcluster.py index 1a5200ab..ba4f2019 100644 --- a/training/deepspeech_training/util/taskcluster.py +++ b/training/deepspeech_training/util/taskcluster.py @@ -14,7 +14,7 @@ import sys from pkg_resources import parse_version -from .io import isdir_remote, open_remote +from .io import isdir_remote, open_remote, is_remote_path DEFAULT_SCHEMES = { 'deepspeech': 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.deepspeech.native_client.%(branch_name)s.%(arch_string)s/artifacts/public/%(artifact_name)s', @@ -43,13 +43,13 @@ def maybe_download_tc(target_dir, tc_url, progress=True): assert target_dir is not None - target_dir = os.path.abspath(target_dir) - try: - os.makedirs(target_dir) - except OSError as e: - if e.errno != errno.EEXIST: - raise e - assert isdir_remote(os.path.dirname(target_dir)) + if not is_remote_path(target_dir): + try: + os.makedirs(target_dir) + except OSError as e: + if e.errno != errno.EEXIST: + raise e + assert os.path.isdir(os.path.dirname(target_dir)) tc_filename = os.path.basename(tc_url) target_file = os.path.join(target_dir, tc_filename)