Linter seems unhappy with conditional imports. Make gfile a module-level import.

I usually do this as a conditional because tf takes a while to load and it's nice to skip it when you want to run a script that just preps data or something like that, but it doesn't seem like a big deal.
This commit is contained in:
CatalinVoss 2020-11-13 10:47:06 -08:00
parent 2332e7fb76
commit 3d2b09b951

View File

@ -3,9 +3,8 @@ A set of I/O utils that allow us to open files on remote storage as if they were
into HDFS storage using Tensorflow's C++ FileStream API.
Currently only includes wrappers for Google's GCS, but this can easily be expanded for AWS S3 buckets.
"""
import inspect
import os
import sys
from tensorflow.io import gfile
def is_remote_path(path):
@ -21,9 +20,7 @@ def path_exists_remote(path):
Wrapper that allows existance check of local and remote paths like
`gs://...`
"""
# Conditional import
if is_remote_path(path):
from tensorflow.io import gfile
return gfile.exists(path)
return os.path.exists(path)
@ -32,7 +29,6 @@ def copy_remote(src, dst, overwrite=False):
"""
Allows us to copy a file from local to remote or vice versa
"""
from tensorflow.io import gfile
return gfile.copy(src, dst, overwrite)
@ -47,9 +43,7 @@ def open_remote(path, mode='r', buffering=-1, encoding=None, newline=None, close
with open_remote('gs://.....', mode='w+') as f:
do something with the file f, whether or not we have local access to it
"""
# Conditional import
if is_remote_path(path):
from tensorflow.io import gfile
return gfile.GFile(path, mode=mode)
return open(path, mode, buffering=buffering, encoding=encoding, newline=newline, closefd=closefd, opener=opener)
@ -58,9 +52,7 @@ def isdir_remote(path):
"""
Wrapper to check if remote and local paths are directories
"""
# Conditional import
if is_remote_path(path):
from tensorflow.io import gfile
return gfile.isdir(path)
return os.path.isdir(path)
@ -69,9 +61,7 @@ def listdir_remote(path):
"""
Wrapper to list paths in local dirs (alternative to using a glob, I suppose)
"""
# Conditional import
if is_remote_path(path):
from tensorflow.io import gfile
return gfile.listdir(path)
return os.listdir(path)
@ -80,9 +70,6 @@ def glob_remote(filename):
"""
Wrapper that provides globs on local and remote paths like `gs://...`
"""
# Conditional import
from tensorflow.io import gfile
return gfile.glob(filename)
@ -91,6 +78,4 @@ def remove_remote(filename):
Wrapper that can remove_remote local and remote files like `gs://...`
"""
# Conditional import
from tensorflow.io import gfile
return gfile.remove_remote(filename)