Merge pull request #41173 from lgeiger:gfile-pathlib
PiperOrigin-RevId: 320407557 Change-Id: I48a9a00d474d12e97d9ab45d598c8db3dab6f0cd
This commit is contained in:
TensorFlower Gardener
commit
b91f0caa8e
@ -38,7 +38,8 @@ class FileIO(object):
|
||||
"""FileIO class that exposes methods to read / write to / from files.
|
||||
|
||||
The constructor takes the following arguments:
|
||||
name: name of the file
|
||||
name: [path-like object](https://docs.python.org/3/glossary.html#term-path-like-object)
|
||||
giving the pathname of the file to be opened.
|
||||
mode: one of `r`, `w`, `a`, `r+`, `w+`, `a+`. Append `b` for bytes mode.
|
||||
|
||||
Can be used as an iterator to iterate over lines in the file.
|
||||
@ -76,7 +77,7 @@ class FileIO(object):
|
||||
raise errors.PermissionDeniedError(None, None,
|
||||
"File isn't open for reading")
|
||||
self._read_buf = _pywrap_file_io.BufferedInputStream(
|
||||
self.__name, 1024 * 512)
|
||||
compat.path_to_str(self.__name), 1024 * 512)
|
||||
|
||||
def _prewrite_check(self):
|
||||
if not self._writable_file:
|
||||
@ -84,7 +85,7 @@ class FileIO(object):
|
||||
raise errors.PermissionDeniedError(None, None,
|
||||
"File isn't open for writing")
|
||||
self._writable_file = _pywrap_file_io.WritableFile(
|
||||
compat.as_bytes(self.__name), compat.as_bytes(self.__mode))
|
||||
compat.path_to_bytes(self.__name), compat.as_bytes(self.__mode))
|
||||
|
||||
def _prepare_value(self, val):
|
||||
if self._binary_mode:
|
||||
@ -264,7 +265,7 @@ def file_exists_v2(path):
|
||||
errors.OpError: Propagates any errors reported by the FileSystem API.
|
||||
"""
|
||||
try:
|
||||
_pywrap_file_io.FileExists(compat.as_bytes(path))
|
||||
_pywrap_file_io.FileExists(compat.path_to_bytes(path))
|
||||
except errors.NotFoundError:
|
||||
return False
|
||||
return True
|
||||
@ -295,7 +296,7 @@ def delete_file_v2(path):
|
||||
errors.OpError: Propagates any errors reported by the FileSystem API. E.g.,
|
||||
`NotFoundError` if the path does not exist.
|
||||
"""
|
||||
_pywrap_file_io.DeleteFile(compat.as_bytes(path))
|
||||
_pywrap_file_io.DeleteFile(compat.path_to_bytes(path))
|
||||
|
||||
|
||||
def read_file_to_string(filename, binary_mode=False):
|
||||
@ -447,7 +448,7 @@ def create_dir_v2(path):
|
||||
Raises:
|
||||
errors.OpError: If the operation fails.
|
||||
"""
|
||||
_pywrap_file_io.CreateDir(compat.as_bytes(path))
|
||||
_pywrap_file_io.CreateDir(compat.path_to_bytes(path))
|
||||
|
||||
|
||||
@tf_export(v1=["gfile.MakeDirs"])
|
||||
@ -477,7 +478,7 @@ def recursive_create_dir_v2(path):
|
||||
Raises:
|
||||
errors.OpError: If the operation fails.
|
||||
"""
|
||||
_pywrap_file_io.RecursivelyCreateDir(compat.as_bytes(path))
|
||||
_pywrap_file_io.RecursivelyCreateDir(compat.path_to_bytes(path))
|
||||
|
||||
|
||||
@tf_export(v1=["gfile.Copy"])
|
||||
@ -510,7 +511,7 @@ def copy_v2(src, dst, overwrite=False):
|
||||
errors.OpError: If the operation fails.
|
||||
"""
|
||||
_pywrap_file_io.CopyFile(
|
||||
compat.as_bytes(src), compat.as_bytes(dst), overwrite)
|
||||
compat.path_to_bytes(src), compat.path_to_bytes(dst), overwrite)
|
||||
|
||||
|
||||
@tf_export(v1=["gfile.Rename"])
|
||||
@ -543,7 +544,7 @@ def rename_v2(src, dst, overwrite=False):
|
||||
errors.OpError: If the operation fails.
|
||||
"""
|
||||
_pywrap_file_io.RenameFile(
|
||||
compat.as_bytes(src), compat.as_bytes(dst), overwrite)
|
||||
compat.path_to_bytes(src), compat.path_to_bytes(dst), overwrite)
|
||||
|
||||
|
||||
def atomic_write_string_to_file(filename, contents, overwrite=True):
|
||||
@ -596,7 +597,7 @@ def delete_recursively_v2(path):
|
||||
Raises:
|
||||
errors.OpError: If the operation fails.
|
||||
"""
|
||||
_pywrap_file_io.DeleteRecursively(compat.as_bytes(path))
|
||||
_pywrap_file_io.DeleteRecursively(compat.path_to_bytes(path))
|
||||
|
||||
|
||||
@tf_export(v1=["gfile.IsDirectory"])
|
||||
@ -623,7 +624,7 @@ def is_directory_v2(path):
|
||||
True, if the path is a directory; False otherwise
|
||||
"""
|
||||
try:
|
||||
return _pywrap_file_io.IsDirectory(compat.as_bytes(path))
|
||||
return _pywrap_file_io.IsDirectory(compat.path_to_bytes(path))
|
||||
except errors.OpError:
|
||||
return False
|
||||
|
||||
@ -646,7 +647,7 @@ def has_atomic_move(path):
|
||||
not to use temporary locations in this case.
|
||||
"""
|
||||
try:
|
||||
return _pywrap_file_io.HasAtomicMove(compat.as_bytes(path))
|
||||
return _pywrap_file_io.HasAtomicMove(compat.path_to_bytes(path))
|
||||
except errors.OpError:
|
||||
# defaults to True
|
||||
return True
|
||||
@ -697,7 +698,7 @@ def list_directory_v2(path):
|
||||
# vector of string should be interpreted as strings, not bytes.
|
||||
return [
|
||||
compat.as_str_any(filename)
|
||||
for filename in _pywrap_file_io.GetChildren(compat.as_bytes(path))
|
||||
for filename in _pywrap_file_io.GetChildren(compat.path_to_bytes(path))
|
||||
]
|
||||
|
||||
|
||||
@ -745,7 +746,7 @@ def walk_v2(top, topdown=True, onerror=None):
|
||||
return "".join([os.path.join(parent, ""), item])
|
||||
return os.path.join(parent, item)
|
||||
|
||||
top = compat.as_str_any(top)
|
||||
top = compat.as_str_any(compat.path_to_str(top))
|
||||
try:
|
||||
listing = list_directory(top)
|
||||
except errors.NotFoundError as err:
|
||||
@ -806,7 +807,7 @@ def stat_v2(path):
|
||||
Raises:
|
||||
errors.OpError: If the operation fails.
|
||||
"""
|
||||
return _pywrap_file_io.Stat(path)
|
||||
return _pywrap_file_io.Stat(compat.path_to_str(path))
|
||||
|
||||
|
||||
def filecmp(filename_a, filename_b):
|
||||
|
||||
@ -20,7 +20,9 @@ from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import os.path
|
||||
import pathlib
|
||||
|
||||
from absl.testing import parameterized
|
||||
import numpy as np
|
||||
|
||||
from tensorflow.python.framework import errors
|
||||
@ -29,7 +31,12 @@ from tensorflow.python.platform import gfile
|
||||
from tensorflow.python.platform import test
|
||||
|
||||
|
||||
class FileIoTest(test.TestCase):
|
||||
run_all_path_types = parameterized.named_parameters(
|
||||
("str", os.path.join),
|
||||
("pathlib", lambda *paths: pathlib.Path(os.path.join(*paths))))
|
||||
|
||||
|
||||
class FileIoTest(test.TestCase, parameterized.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self._base_dir = os.path.join(self.get_temp_dir(), "base_dir")
|
||||
@ -43,14 +50,16 @@ class FileIoTest(test.TestCase):
|
||||
with self.assertRaises(errors.NotFoundError):
|
||||
_ = f.read()
|
||||
|
||||
def testFileDoesntExist(self):
|
||||
file_path = os.path.join(self._base_dir, "temp_file")
|
||||
@run_all_path_types
|
||||
def testFileDoesntExist(self, join):
|
||||
file_path = join(self._base_dir, "temp_file")
|
||||
self.assertFalse(file_io.file_exists(file_path))
|
||||
with self.assertRaises(errors.NotFoundError):
|
||||
_ = file_io.read_file_to_string(file_path)
|
||||
|
||||
def testWriteToString(self):
|
||||
file_path = os.path.join(self._base_dir, "temp_file")
|
||||
@run_all_path_types
|
||||
def testWriteToString(self, join):
|
||||
file_path = join(self._base_dir, "temp_file")
|
||||
file_io.write_string_to_file(file_path, "testing")
|
||||
self.assertTrue(file_io.file_exists(file_path))
|
||||
file_contents = file_io.read_file_to_string(file_path)
|
||||
@ -75,14 +84,16 @@ class FileIoTest(test.TestCase):
|
||||
file_contents = file_io.read_file_to_string(file_path)
|
||||
self.assertEqual("new", file_contents)
|
||||
|
||||
def testReadBinaryMode(self):
|
||||
file_path = os.path.join(self._base_dir, "temp_file")
|
||||
@run_all_path_types
|
||||
def testReadBinaryMode(self, join):
|
||||
file_path = join(self._base_dir, "temp_file")
|
||||
file_io.write_string_to_file(file_path, "testing")
|
||||
with file_io.FileIO(file_path, mode="rb") as f:
|
||||
self.assertEqual(b"testing", f.read())
|
||||
|
||||
def testWriteBinaryMode(self):
|
||||
file_path = os.path.join(self._base_dir, "temp_file")
|
||||
@run_all_path_types
|
||||
def testWriteBinaryMode(self, join):
|
||||
file_path = join(self._base_dir, "temp_file")
|
||||
file_io.FileIO(file_path, "wb").write("testing")
|
||||
with file_io.FileIO(file_path, mode="r") as f:
|
||||
self.assertEqual("testing", f.read())
|
||||
@ -128,8 +139,9 @@ class FileIoTest(test.TestCase):
|
||||
with self.assertRaises(errors.PermissionDeniedError):
|
||||
file_io.FileIO(file_path, mode="w").read()
|
||||
|
||||
def testFileDelete(self):
|
||||
file_path = os.path.join(self._base_dir, "temp_file")
|
||||
@run_all_path_types
|
||||
def testFileDelete(self, join):
|
||||
file_path = join(self._base_dir, "temp_file")
|
||||
file_io.FileIO(file_path, mode="w").write("testing")
|
||||
file_io.delete_file(file_path)
|
||||
self.assertFalse(file_io.file_exists(file_path))
|
||||
@ -171,8 +183,9 @@ class FileIoTest(test.TestCase):
|
||||
self.assertItemsEqual(
|
||||
file_io.get_matching_files(glob_pattern), expected_match)
|
||||
|
||||
def testCreateRecursiveDir(self):
|
||||
dir_path = os.path.join(self._base_dir, "temp_dir/temp_dir1/temp_dir2")
|
||||
@run_all_path_types
|
||||
def testCreateRecursiveDir(self, join):
|
||||
dir_path = join(self._base_dir, "temp_dir/temp_dir1/temp_dir2")
|
||||
file_io.recursive_create_dir(dir_path)
|
||||
file_io.recursive_create_dir(dir_path) # repeat creation
|
||||
file_path = os.path.join(dir_path, "temp_file")
|
||||
@ -181,10 +194,11 @@ class FileIoTest(test.TestCase):
|
||||
file_io.delete_recursively(os.path.join(self._base_dir, "temp_dir"))
|
||||
self.assertFalse(file_io.file_exists(file_path))
|
||||
|
||||
def testCopy(self):
|
||||
file_path = os.path.join(self._base_dir, "temp_file")
|
||||
@run_all_path_types
|
||||
def testCopy(self, join):
|
||||
file_path = join(self._base_dir, "temp_file")
|
||||
file_io.FileIO(file_path, mode="w").write("testing")
|
||||
copy_path = os.path.join(self._base_dir, "copy_file")
|
||||
copy_path = join(self._base_dir, "copy_file")
|
||||
file_io.copy(file_path, copy_path)
|
||||
self.assertTrue(file_io.file_exists(copy_path))
|
||||
f = file_io.FileIO(file_path, mode="r")
|
||||
@ -208,10 +222,11 @@ class FileIoTest(test.TestCase):
|
||||
with self.assertRaises(errors.AlreadyExistsError):
|
||||
file_io.copy(file_path, copy_path, overwrite=False)
|
||||
|
||||
def testRename(self):
|
||||
file_path = os.path.join(self._base_dir, "temp_file")
|
||||
@run_all_path_types
|
||||
def testRename(self, join):
|
||||
file_path = join(self._base_dir, "temp_file")
|
||||
file_io.FileIO(file_path, mode="w").write("testing")
|
||||
rename_path = os.path.join(self._base_dir, "rename_file")
|
||||
rename_path = join(self._base_dir, "rename_file")
|
||||
file_io.rename(file_path, rename_path)
|
||||
self.assertTrue(file_io.file_exists(rename_path))
|
||||
self.assertFalse(file_io.file_exists(file_path))
|
||||
@ -240,13 +255,14 @@ class FileIoTest(test.TestCase):
|
||||
with self.assertRaises(errors.NotFoundError):
|
||||
file_io.delete_recursively(fake_dir_path)
|
||||
|
||||
def testIsDirectory(self):
|
||||
dir_path = os.path.join(self._base_dir, "test_dir")
|
||||
@run_all_path_types
|
||||
def testIsDirectory(self, join):
|
||||
dir_path = join(self._base_dir, "test_dir")
|
||||
# Failure for a non-existing dir.
|
||||
self.assertFalse(file_io.is_directory(dir_path))
|
||||
file_io.create_dir(dir_path)
|
||||
self.assertTrue(file_io.is_directory(dir_path))
|
||||
file_path = os.path.join(dir_path, "test_file")
|
||||
file_path = join(dir_path, "test_file")
|
||||
file_io.FileIO(file_path, mode="w").write("test")
|
||||
# False for a file.
|
||||
self.assertFalse(file_io.is_directory(file_path))
|
||||
@ -254,16 +270,17 @@ class FileIoTest(test.TestCase):
|
||||
file_statistics = file_io.stat(dir_path)
|
||||
self.assertTrue(file_statistics.is_directory)
|
||||
|
||||
def testListDirectory(self):
|
||||
dir_path = os.path.join(self._base_dir, "test_dir")
|
||||
@run_all_path_types
|
||||
def testListDirectory(self, join):
|
||||
dir_path = join(self._base_dir, "test_dir")
|
||||
file_io.create_dir(dir_path)
|
||||
files = ["file1.txt", "file2.txt", "file3.txt"]
|
||||
for name in files:
|
||||
file_path = os.path.join(dir_path, name)
|
||||
file_path = join(dir_path, name)
|
||||
file_io.FileIO(file_path, mode="w").write("testing")
|
||||
subdir_path = os.path.join(dir_path, "sub_dir")
|
||||
subdir_path = join(dir_path, "sub_dir")
|
||||
file_io.create_dir(subdir_path)
|
||||
subdir_file_path = os.path.join(subdir_path, "file4.txt")
|
||||
subdir_file_path = join(subdir_path, "file4.txt")
|
||||
file_io.FileIO(subdir_file_path, mode="w").write("testing")
|
||||
dir_list = file_io.list_directory(dir_path)
|
||||
self.assertItemsEqual(files + ["sub_dir"], dir_list)
|
||||
@ -289,8 +306,10 @@ class FileIoTest(test.TestCase):
|
||||
mode="w").write("testing")
|
||||
file_io.create_dir(os.path.join(dir_path, "subdir1_2/subdir2"))
|
||||
|
||||
def testWalkInOrder(self):
|
||||
dir_path = os.path.join(self._base_dir, "test_dir")
|
||||
@run_all_path_types
|
||||
def testWalkInOrder(self, join):
|
||||
dir_path_str = os.path.join(self._base_dir, "test_dir")
|
||||
dir_path = join(self._base_dir, "test_dir")
|
||||
self._setupWalkDirectories(dir_path)
|
||||
# Now test the walk (in_order = True)
|
||||
all_dirs = []
|
||||
@ -300,15 +319,14 @@ class FileIoTest(test.TestCase):
|
||||
all_dirs.append(w_dir)
|
||||
all_subdirs.append(w_subdirs)
|
||||
all_files.append(w_files)
|
||||
self.assertItemsEqual(all_dirs, [dir_path] + [
|
||||
os.path.join(dir_path, item)
|
||||
for item in
|
||||
self.assertItemsEqual(all_dirs, [dir_path_str] + [
|
||||
os.path.join(dir_path_str, item) for item in
|
||||
["subdir1_1", "subdir1_2", "subdir1_2/subdir2", "subdir1_3"]
|
||||
])
|
||||
self.assertEqual(dir_path, all_dirs[0])
|
||||
self.assertEqual(dir_path_str, all_dirs[0])
|
||||
self.assertLess(
|
||||
all_dirs.index(os.path.join(dir_path, "subdir1_2")),
|
||||
all_dirs.index(os.path.join(dir_path, "subdir1_2/subdir2")))
|
||||
all_dirs.index(os.path.join(dir_path_str, "subdir1_2")),
|
||||
all_dirs.index(os.path.join(dir_path_str, "subdir1_2/subdir2")))
|
||||
self.assertItemsEqual(all_subdirs[1:5], [[], ["subdir2"], [], []])
|
||||
self.assertItemsEqual(all_subdirs[0],
|
||||
["subdir1_1", "subdir1_2", "subdir1_3"])
|
||||
@ -357,8 +375,9 @@ class FileIoTest(test.TestCase):
|
||||
self.assertItemsEqual(all_subdirs, [])
|
||||
self.assertItemsEqual(all_files, [])
|
||||
|
||||
def testStat(self):
|
||||
file_path = os.path.join(self._base_dir, "temp_file")
|
||||
@run_all_path_types
|
||||
def testStat(self, join):
|
||||
file_path = join(self._base_dir, "temp_file")
|
||||
file_io.FileIO(file_path, mode="w").write("testing")
|
||||
file_statistics = file_io.stat(file_path)
|
||||
os_statistics = os.stat(file_path)
|
||||
@ -512,8 +531,9 @@ class FileIoTest(test.TestCase):
|
||||
f.flush()
|
||||
self.assertEqual(content, f.read(len(content) + 1))
|
||||
|
||||
def testUTF8StringPathExists(self):
|
||||
file_path = os.path.join(self._base_dir, "UTF8测试_file_exist")
|
||||
@run_all_path_types
|
||||
def testUTF8StringPathExists(self, join):
|
||||
file_path = join(self._base_dir, "UTF8测试_file_exist")
|
||||
file_io.write_string_to_file(file_path, "testing")
|
||||
v = file_io.file_exists(file_path)
|
||||
self.assertEqual(v, True)
|
||||
|
||||
@ -180,6 +180,27 @@ def path_to_str(path):
|
||||
return path
|
||||
|
||||
|
||||
def path_to_bytes(path):
|
||||
r"""Converts input which is a `PathLike` object to `bytes`.
|
||||
|
||||
Converts from any python constant representation of a `PathLike` object
|
||||
or `str` to bytes.
|
||||
|
||||
Args:
|
||||
path: An object that can be converted to path representation.
|
||||
|
||||
Returns:
|
||||
A `bytes` object.
|
||||
|
||||
Usage:
|
||||
In case a simplified `bytes` version of the path is needed from an
|
||||
`os.PathLike` object
|
||||
"""
|
||||
if hasattr(path, '__fspath__'):
|
||||
path = path.__fspath__()
|
||||
return as_bytes(path)
|
||||
|
||||
|
||||
# Numpy 1.8 scalars don't inherit from numbers.Integral in Python 3, so we
|
||||
# need to check them specifically. The same goes from Real and Complex.
|
||||
integral_types = (_numbers.Integral, _np.integer)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user