Merge pull request #41173 from lgeiger:gfile-pathlib

PiperOrigin-RevId: 320407557
Change-Id: I48a9a00d474d12e97d9ab45d598c8db3dab6f0cd

tensorflow/python/lib/io/file_io.py

@@ -38,7 +38,8 @@ class FileIO(object):
   """FileIO class that exposes methods to read / write to / from files.
 
   The constructor takes the following arguments:
-  name: name of the file
+  name: [path-like object](https://docs.python.org/3/glossary.html#term-path-like-object)
+    giving the pathname of the file to be opened.
   mode: one of `r`, `w`, `a`, `r+`, `w+`, `a+`. Append `b` for bytes mode.
 
   Can be used as an iterator to iterate over lines in the file.
@@ -76,7 +77,7 @@ class FileIO(object):
         raise errors.PermissionDeniedError(None, None,
                                            "File isn't open for reading")
       self._read_buf = _pywrap_file_io.BufferedInputStream(
-          self.__name, 1024 * 512)
+          compat.path_to_str(self.__name), 1024 * 512)
 
   def _prewrite_check(self):
     if not self._writable_file:
@@ -84,7 +85,7 @@ class FileIO(object):
         raise errors.PermissionDeniedError(None, None,
                                            "File isn't open for writing")
       self._writable_file = _pywrap_file_io.WritableFile(
-          compat.as_bytes(self.__name), compat.as_bytes(self.__mode))
+          compat.path_to_bytes(self.__name), compat.as_bytes(self.__mode))
 
   def _prepare_value(self, val):
     if self._binary_mode:
@@ -264,7 +265,7 @@ def file_exists_v2(path):
     errors.OpError: Propagates any errors reported by the FileSystem API.
   """
   try:
-    _pywrap_file_io.FileExists(compat.as_bytes(path))
+    _pywrap_file_io.FileExists(compat.path_to_bytes(path))
   except errors.NotFoundError:
     return False
   return True
@@ -295,7 +296,7 @@ def delete_file_v2(path):
     errors.OpError: Propagates any errors reported by the FileSystem API.  E.g.,
     `NotFoundError` if the path does not exist.
   """
-  _pywrap_file_io.DeleteFile(compat.as_bytes(path))
+  _pywrap_file_io.DeleteFile(compat.path_to_bytes(path))
 
 
 def read_file_to_string(filename, binary_mode=False):
@@ -447,7 +448,7 @@ def create_dir_v2(path):
   Raises:
     errors.OpError: If the operation fails.
   """
-  _pywrap_file_io.CreateDir(compat.as_bytes(path))
+  _pywrap_file_io.CreateDir(compat.path_to_bytes(path))
 
 
 @tf_export(v1=["gfile.MakeDirs"])
@@ -477,7 +478,7 @@ def recursive_create_dir_v2(path):
   Raises:
     errors.OpError: If the operation fails.
   """
-  _pywrap_file_io.RecursivelyCreateDir(compat.as_bytes(path))
+  _pywrap_file_io.RecursivelyCreateDir(compat.path_to_bytes(path))
 
 
 @tf_export(v1=["gfile.Copy"])
@@ -510,7 +511,7 @@ def copy_v2(src, dst, overwrite=False):
     errors.OpError: If the operation fails.
   """
   _pywrap_file_io.CopyFile(
-      compat.as_bytes(src), compat.as_bytes(dst), overwrite)
+      compat.path_to_bytes(src), compat.path_to_bytes(dst), overwrite)
 
 
 @tf_export(v1=["gfile.Rename"])
@@ -543,7 +544,7 @@ def rename_v2(src, dst, overwrite=False):
     errors.OpError: If the operation fails.
   """
   _pywrap_file_io.RenameFile(
-      compat.as_bytes(src), compat.as_bytes(dst), overwrite)
+      compat.path_to_bytes(src), compat.path_to_bytes(dst), overwrite)
 
 
 def atomic_write_string_to_file(filename, contents, overwrite=True):
@@ -596,7 +597,7 @@ def delete_recursively_v2(path):
   Raises:
     errors.OpError: If the operation fails.
   """
-  _pywrap_file_io.DeleteRecursively(compat.as_bytes(path))
+  _pywrap_file_io.DeleteRecursively(compat.path_to_bytes(path))
 
 
 @tf_export(v1=["gfile.IsDirectory"])
@@ -623,7 +624,7 @@ def is_directory_v2(path):
     True, if the path is a directory; False otherwise
   """
   try:
-    return _pywrap_file_io.IsDirectory(compat.as_bytes(path))
+    return _pywrap_file_io.IsDirectory(compat.path_to_bytes(path))
   except errors.OpError:
     return False
 
@@ -646,7 +647,7 @@ def has_atomic_move(path):
            not to use temporary locations in this case.
   """
   try:
-    return _pywrap_file_io.HasAtomicMove(compat.as_bytes(path))
+    return _pywrap_file_io.HasAtomicMove(compat.path_to_bytes(path))
   except errors.OpError:
     # defaults to True
     return True
@@ -697,7 +698,7 @@ def list_directory_v2(path):
   # vector of string should be interpreted as strings, not bytes.
   return [
       compat.as_str_any(filename)
-      for filename in _pywrap_file_io.GetChildren(compat.as_bytes(path))
+      for filename in _pywrap_file_io.GetChildren(compat.path_to_bytes(path))
   ]
 
 
@@ -745,7 +746,7 @@ def walk_v2(top, topdown=True, onerror=None):
       return "".join([os.path.join(parent, ""), item])
     return os.path.join(parent, item)
 
-  top = compat.as_str_any(top)
+  top = compat.as_str_any(compat.path_to_str(top))
   try:
     listing = list_directory(top)
   except errors.NotFoundError as err:
@@ -806,7 +807,7 @@ def stat_v2(path):
   Raises:
     errors.OpError: If the operation fails.
   """
-  return _pywrap_file_io.Stat(path)
+  return _pywrap_file_io.Stat(compat.path_to_str(path))
 
 
 def filecmp(filename_a, filename_b):

tensorflow/python/lib/io/file_io_test.py

@@ -20,7 +20,9 @@ from __future__ import division
 from __future__ import print_function
 
 import os.path
+import pathlib
 
+from absl.testing import parameterized
 import numpy as np
 
 from tensorflow.python.framework import errors
@@ -29,7 +31,12 @@ from tensorflow.python.platform import gfile
 from tensorflow.python.platform import test
 
 
-class FileIoTest(test.TestCase):
+run_all_path_types = parameterized.named_parameters(
+    ("str", os.path.join),
+    ("pathlib", lambda *paths: pathlib.Path(os.path.join(*paths))))
+
+
+class FileIoTest(test.TestCase, parameterized.TestCase):
 
   def setUp(self):
     self._base_dir = os.path.join(self.get_temp_dir(), "base_dir")
@@ -43,14 +50,16 @@ class FileIoTest(test.TestCase):
     with self.assertRaises(errors.NotFoundError):
       _ = f.read()
 
-  def testFileDoesntExist(self):
+  @run_all_path_types
-    file_path = os.path.join(self._base_dir, "temp_file")
+  def testFileDoesntExist(self, join):
+    file_path = join(self._base_dir, "temp_file")
     self.assertFalse(file_io.file_exists(file_path))
     with self.assertRaises(errors.NotFoundError):
       _ = file_io.read_file_to_string(file_path)
 
-  def testWriteToString(self):
+  @run_all_path_types
-    file_path = os.path.join(self._base_dir, "temp_file")
+  def testWriteToString(self, join):
+    file_path = join(self._base_dir, "temp_file")
     file_io.write_string_to_file(file_path, "testing")
     self.assertTrue(file_io.file_exists(file_path))
     file_contents = file_io.read_file_to_string(file_path)
@@ -75,14 +84,16 @@ class FileIoTest(test.TestCase):
     file_contents = file_io.read_file_to_string(file_path)
     self.assertEqual("new", file_contents)
 
-  def testReadBinaryMode(self):
+  @run_all_path_types
-    file_path = os.path.join(self._base_dir, "temp_file")
+  def testReadBinaryMode(self, join):
+    file_path = join(self._base_dir, "temp_file")
     file_io.write_string_to_file(file_path, "testing")
     with file_io.FileIO(file_path, mode="rb") as f:
       self.assertEqual(b"testing", f.read())
 
-  def testWriteBinaryMode(self):
+  @run_all_path_types
-    file_path = os.path.join(self._base_dir, "temp_file")
+  def testWriteBinaryMode(self, join):
+    file_path = join(self._base_dir, "temp_file")
     file_io.FileIO(file_path, "wb").write("testing")
     with file_io.FileIO(file_path, mode="r") as f:
       self.assertEqual("testing", f.read())
@@ -128,8 +139,9 @@ class FileIoTest(test.TestCase):
     with self.assertRaises(errors.PermissionDeniedError):
       file_io.FileIO(file_path, mode="w").read()
 
-  def testFileDelete(self):
+  @run_all_path_types
-    file_path = os.path.join(self._base_dir, "temp_file")
+  def testFileDelete(self, join):
+    file_path = join(self._base_dir, "temp_file")
     file_io.FileIO(file_path, mode="w").write("testing")
     file_io.delete_file(file_path)
     self.assertFalse(file_io.file_exists(file_path))
@@ -171,8 +183,9 @@ class FileIoTest(test.TestCase):
     self.assertItemsEqual(
         file_io.get_matching_files(glob_pattern), expected_match)
 
-  def testCreateRecursiveDir(self):
+  @run_all_path_types
-    dir_path = os.path.join(self._base_dir, "temp_dir/temp_dir1/temp_dir2")
+  def testCreateRecursiveDir(self, join):
+    dir_path = join(self._base_dir, "temp_dir/temp_dir1/temp_dir2")
     file_io.recursive_create_dir(dir_path)
     file_io.recursive_create_dir(dir_path)  # repeat creation
     file_path = os.path.join(dir_path, "temp_file")
@@ -181,10 +194,11 @@ class FileIoTest(test.TestCase):
     file_io.delete_recursively(os.path.join(self._base_dir, "temp_dir"))
     self.assertFalse(file_io.file_exists(file_path))
 
-  def testCopy(self):
+  @run_all_path_types
-    file_path = os.path.join(self._base_dir, "temp_file")
+  def testCopy(self, join):
+    file_path = join(self._base_dir, "temp_file")
     file_io.FileIO(file_path, mode="w").write("testing")
-    copy_path = os.path.join(self._base_dir, "copy_file")
+    copy_path = join(self._base_dir, "copy_file")
     file_io.copy(file_path, copy_path)
     self.assertTrue(file_io.file_exists(copy_path))
     f = file_io.FileIO(file_path, mode="r")
@@ -208,10 +222,11 @@ class FileIoTest(test.TestCase):
     with self.assertRaises(errors.AlreadyExistsError):
       file_io.copy(file_path, copy_path, overwrite=False)
 
-  def testRename(self):
+  @run_all_path_types
-    file_path = os.path.join(self._base_dir, "temp_file")
+  def testRename(self, join):
+    file_path = join(self._base_dir, "temp_file")
     file_io.FileIO(file_path, mode="w").write("testing")
-    rename_path = os.path.join(self._base_dir, "rename_file")
+    rename_path = join(self._base_dir, "rename_file")
     file_io.rename(file_path, rename_path)
     self.assertTrue(file_io.file_exists(rename_path))
     self.assertFalse(file_io.file_exists(file_path))
@@ -240,13 +255,14 @@ class FileIoTest(test.TestCase):
     with self.assertRaises(errors.NotFoundError):
       file_io.delete_recursively(fake_dir_path)
 
-  def testIsDirectory(self):
+  @run_all_path_types
-    dir_path = os.path.join(self._base_dir, "test_dir")
+  def testIsDirectory(self, join):
+    dir_path = join(self._base_dir, "test_dir")
     # Failure for a non-existing dir.
     self.assertFalse(file_io.is_directory(dir_path))
     file_io.create_dir(dir_path)
     self.assertTrue(file_io.is_directory(dir_path))
-    file_path = os.path.join(dir_path, "test_file")
+    file_path = join(dir_path, "test_file")
     file_io.FileIO(file_path, mode="w").write("test")
     # False for a file.
     self.assertFalse(file_io.is_directory(file_path))
@@ -254,16 +270,17 @@ class FileIoTest(test.TestCase):
     file_statistics = file_io.stat(dir_path)
     self.assertTrue(file_statistics.is_directory)
 
-  def testListDirectory(self):
+  @run_all_path_types
-    dir_path = os.path.join(self._base_dir, "test_dir")
+  def testListDirectory(self, join):
+    dir_path = join(self._base_dir, "test_dir")
     file_io.create_dir(dir_path)
     files = ["file1.txt", "file2.txt", "file3.txt"]
     for name in files:
-      file_path = os.path.join(dir_path, name)
+      file_path = join(dir_path, name)
       file_io.FileIO(file_path, mode="w").write("testing")
-    subdir_path = os.path.join(dir_path, "sub_dir")
+    subdir_path = join(dir_path, "sub_dir")
     file_io.create_dir(subdir_path)
-    subdir_file_path = os.path.join(subdir_path, "file4.txt")
+    subdir_file_path = join(subdir_path, "file4.txt")
     file_io.FileIO(subdir_file_path, mode="w").write("testing")
     dir_list = file_io.list_directory(dir_path)
     self.assertItemsEqual(files + ["sub_dir"], dir_list)
@@ -289,8 +306,10 @@ class FileIoTest(test.TestCase):
         mode="w").write("testing")
     file_io.create_dir(os.path.join(dir_path, "subdir1_2/subdir2"))
 
-  def testWalkInOrder(self):
+  @run_all_path_types
-    dir_path = os.path.join(self._base_dir, "test_dir")
+  def testWalkInOrder(self, join):
+    dir_path_str = os.path.join(self._base_dir, "test_dir")
+    dir_path = join(self._base_dir, "test_dir")
     self._setupWalkDirectories(dir_path)
     # Now test the walk (in_order = True)
     all_dirs = []
@@ -300,15 +319,14 @@ class FileIoTest(test.TestCase):
       all_dirs.append(w_dir)
       all_subdirs.append(w_subdirs)
       all_files.append(w_files)
-    self.assertItemsEqual(all_dirs, [dir_path] + [
+    self.assertItemsEqual(all_dirs, [dir_path_str] + [
-        os.path.join(dir_path, item)
+        os.path.join(dir_path_str, item) for item in
-        for item in
         ["subdir1_1", "subdir1_2", "subdir1_2/subdir2", "subdir1_3"]
     ])
-    self.assertEqual(dir_path, all_dirs[0])
+    self.assertEqual(dir_path_str, all_dirs[0])
     self.assertLess(
-        all_dirs.index(os.path.join(dir_path, "subdir1_2")),
+        all_dirs.index(os.path.join(dir_path_str, "subdir1_2")),
-        all_dirs.index(os.path.join(dir_path, "subdir1_2/subdir2")))
+        all_dirs.index(os.path.join(dir_path_str, "subdir1_2/subdir2")))
     self.assertItemsEqual(all_subdirs[1:5], [[], ["subdir2"], [], []])
     self.assertItemsEqual(all_subdirs[0],
                           ["subdir1_1", "subdir1_2", "subdir1_3"])
@@ -357,8 +375,9 @@ class FileIoTest(test.TestCase):
     self.assertItemsEqual(all_subdirs, [])
     self.assertItemsEqual(all_files, [])
 
-  def testStat(self):
+  @run_all_path_types
-    file_path = os.path.join(self._base_dir, "temp_file")
+  def testStat(self, join):
+    file_path = join(self._base_dir, "temp_file")
     file_io.FileIO(file_path, mode="w").write("testing")
     file_statistics = file_io.stat(file_path)
     os_statistics = os.stat(file_path)
@@ -512,8 +531,9 @@ class FileIoTest(test.TestCase):
     f.flush()
     self.assertEqual(content, f.read(len(content) + 1))
 
-  def testUTF8StringPathExists(self):
+  @run_all_path_types
-    file_path = os.path.join(self._base_dir, "UTF8测试_file_exist")
+  def testUTF8StringPathExists(self, join):
+    file_path = join(self._base_dir, "UTF8测试_file_exist")
     file_io.write_string_to_file(file_path, "testing")
     v = file_io.file_exists(file_path)
     self.assertEqual(v, True)

tensorflow/python/util/compat.py

@@ -180,6 +180,27 @@ def path_to_str(path):
   return path
 
 
+def path_to_bytes(path):
+  r"""Converts input which is a `PathLike` object to `bytes`.
+
+  Converts from any python constant representation of a `PathLike` object
+  or `str` to bytes.
+
+  Args:
+    path: An object that can be converted to path representation.
+
+  Returns:
+    A `bytes` object.
+
+  Usage:
+    In case a simplified `bytes` version of the path is needed from an
+    `os.PathLike` object
+  """
+  if hasattr(path, '__fspath__'):
+    path = path.__fspath__()
+  return as_bytes(path)
+
+
 # Numpy 1.8 scalars don't inherit from numbers.Integral in Python 3, so we
 # need to check them specifically.  The same goes from Real and Complex.
 integral_types = (_numbers.Integral, _np.integer)