Linter seems unhappy with conditional imports. Make gfile a module-level import.
I usually do this as a conditional because tf takes a while to load and it's nice to skip it when you want to run a script that just preps data or something like that, but it doesn't seem like a big deal.
This commit is contained in:
parent
2332e7fb76
commit
3d2b09b951
@ -3,9 +3,8 @@ A set of I/O utils that allow us to open files on remote storage as if they were
|
||||
into HDFS storage using Tensorflow's C++ FileStream API.
|
||||
Currently only includes wrappers for Google's GCS, but this can easily be expanded for AWS S3 buckets.
|
||||
"""
|
||||
import inspect
|
||||
import os
|
||||
import sys
|
||||
from tensorflow.io import gfile
|
||||
|
||||
|
||||
def is_remote_path(path):
|
||||
@ -21,9 +20,7 @@ def path_exists_remote(path):
|
||||
Wrapper that allows existance check of local and remote paths like
|
||||
`gs://...`
|
||||
"""
|
||||
# Conditional import
|
||||
if is_remote_path(path):
|
||||
from tensorflow.io import gfile
|
||||
return gfile.exists(path)
|
||||
return os.path.exists(path)
|
||||
|
||||
@ -32,7 +29,6 @@ def copy_remote(src, dst, overwrite=False):
|
||||
"""
|
||||
Allows us to copy a file from local to remote or vice versa
|
||||
"""
|
||||
from tensorflow.io import gfile
|
||||
return gfile.copy(src, dst, overwrite)
|
||||
|
||||
|
||||
@ -47,9 +43,7 @@ def open_remote(path, mode='r', buffering=-1, encoding=None, newline=None, close
|
||||
with open_remote('gs://.....', mode='w+') as f:
|
||||
do something with the file f, whether or not we have local access to it
|
||||
"""
|
||||
# Conditional import
|
||||
if is_remote_path(path):
|
||||
from tensorflow.io import gfile
|
||||
return gfile.GFile(path, mode=mode)
|
||||
return open(path, mode, buffering=buffering, encoding=encoding, newline=newline, closefd=closefd, opener=opener)
|
||||
|
||||
@ -58,9 +52,7 @@ def isdir_remote(path):
|
||||
"""
|
||||
Wrapper to check if remote and local paths are directories
|
||||
"""
|
||||
# Conditional import
|
||||
if is_remote_path(path):
|
||||
from tensorflow.io import gfile
|
||||
return gfile.isdir(path)
|
||||
return os.path.isdir(path)
|
||||
|
||||
@ -69,9 +61,7 @@ def listdir_remote(path):
|
||||
"""
|
||||
Wrapper to list paths in local dirs (alternative to using a glob, I suppose)
|
||||
"""
|
||||
# Conditional import
|
||||
if is_remote_path(path):
|
||||
from tensorflow.io import gfile
|
||||
return gfile.listdir(path)
|
||||
return os.listdir(path)
|
||||
|
||||
@ -80,9 +70,6 @@ def glob_remote(filename):
|
||||
"""
|
||||
Wrapper that provides globs on local and remote paths like `gs://...`
|
||||
"""
|
||||
# Conditional import
|
||||
from tensorflow.io import gfile
|
||||
|
||||
return gfile.glob(filename)
|
||||
|
||||
|
||||
@ -91,6 +78,4 @@ def remove_remote(filename):
|
||||
Wrapper that can remove_remote local and remote files like `gs://...`
|
||||
"""
|
||||
# Conditional import
|
||||
from tensorflow.io import gfile
|
||||
|
||||
return gfile.remove_remote(filename)
|
Loading…
x
Reference in New Issue
Block a user