From 9d829078824f4e1e8c0b622fc9f360899794b110 Mon Sep 17 00:00:00 2001 From: Sergei Lebedev Date: Thu, 19 Dec 2019 15:24:20 -0800 Subject: [PATCH] Converted file_io.i to pybind11 This is part of a larger effort to deprecate swig and eventually with modularization break pywrap_tensorflow into smaller components. Please refer to https://github.com/tensorflow/community/blob/master/rfcs/20190208-pybind11.md for more information. PiperOrigin-RevId: 286474536 Change-Id: Ic942a4480b1c1a19bdc3d6b65d3272221e47537b --- tensorflow/python/BUILD | 15 +- tensorflow/python/lib/io/file_io.i | 302 -------------------- tensorflow/python/lib/io/file_io.py | 93 +++--- tensorflow/python/lib/io/file_io_wrapper.cc | 205 +++++++++++++ tensorflow/python/platform/base.i | 1 + tensorflow/python/tensorflow.i | 9 +- 6 files changed, 267 insertions(+), 358 deletions(-) delete mode 100644 tensorflow/python/lib/io/file_io.i create mode 100644 tensorflow/python/lib/io/file_io_wrapper.cc diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index 47e989341e0..f42f8577d24 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -5496,7 +5496,6 @@ tf_py_wrap_cc( "grappler/item.i", "grappler/tf_optimizer.i", "lib/core/strings.i", - "lib/io/file_io.i", "lib/io/py_record_reader.i", "platform/base.i", "//tensorflow/compiler/mlir/python:mlir.i", @@ -5683,6 +5682,19 @@ cc_import( # ** Targets for Windows build (end) ** +tf_python_pybind_extension( + name = "_pywrap_file_io", + srcs = ["lib/io/file_io_wrapper.cc"], + module_name = "_pywrap_file_io", + deps = [ + ":pybind11_absl", + ":pybind11_status", + "//tensorflow/core:framework_headers_lib", + "//tensorflow/core:protos_all_cc", + "@pybind11", + ], +) + py_library( name = "lib", srcs = [ @@ -5692,6 +5704,7 @@ py_library( ], srcs_version = "PY2AND3", deps = [ + ":_pywrap_file_io", ":_pywrap_record_io", ":errors", ":pywrap_tensorflow", diff --git a/tensorflow/python/lib/io/file_io.i b/tensorflow/python/lib/io/file_io.i deleted file mode 100644 index cbd619bb764..00000000000 --- a/tensorflow/python/lib/io/file_io.i +++ /dev/null @@ -1,302 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -%include "tensorflow/python/lib/core/strings.i" -%include "tensorflow/python/platform/base.i" - -%{ -#include "tensorflow/c/tf_status_helper.h" -#include "tensorflow/core/framework/types.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/lib/core/stringpiece.h" -#include "tensorflow/core/lib/io/buffered_inputstream.h" -#include "tensorflow/core/lib/io/inputstream_interface.h" -#include "tensorflow/core/lib/io/random_inputstream.h" -#include "tensorflow/core/lib/io/path.h" -#include "tensorflow/core/platform/env.h" -#include "tensorflow/core/platform/file_statistics.h" -#include "tensorflow/core/platform/file_system.h" -#include "tensorflow/core/protobuf/meta_graph.pb.h" -%} - -%{ -inline void FileExists(const string& filename, TF_Status* status) { - tensorflow::Status s = tensorflow::Env::Default()->FileExists(filename); - if (!s.ok()) { - Set_TF_Status_from_Status(status, s); - } -} - -inline void FileExists(const tensorflow::StringPiece& filename, - TF_Status* status) { - tensorflow::Status s = - tensorflow::Env::Default()->FileExists(string(filename)); - if (!s.ok()) { - Set_TF_Status_from_Status(status, s); - } -} - -inline void DeleteFile(const string& filename, TF_Status* status) { - tensorflow::Status s = tensorflow::Env::Default()->DeleteFile(filename); - if (!s.ok()) { - Set_TF_Status_from_Status(status, s); - } -} - -string ReadFileToString(const string& filename, TF_Status* status) { - string file_content; - tensorflow::Status s = ReadFileToString(tensorflow::Env::Default(), - filename, &file_content); - if (!s.ok()) { - Set_TF_Status_from_Status(status, s); - } - return file_content; -} - -void WriteStringToFile(const string& filename, const string& file_content, - TF_Status* status) { - tensorflow::Status s = WriteStringToFile(tensorflow::Env::Default(), - filename, file_content); - if (!s.ok()) { - Set_TF_Status_from_Status(status, s); - } -} - -std::vector GetChildren(const string& dir, TF_Status* status) { - std::vector results; - tensorflow::Status s = tensorflow::Env::Default()->GetChildren( - dir, &results); - if (!s.ok()) { - Set_TF_Status_from_Status(status, s); - } - return results; -} - -std::vector GetMatchingFiles(const string& filename, TF_Status* status) { - std::vector results; - tensorflow::Status s = tensorflow::Env::Default()->GetMatchingPaths( - filename, &results); - if (!s.ok()) { - Set_TF_Status_from_Status(status, s); - } - return results; -} - -void CreateDir(const string& dirname, TF_Status* status) { - tensorflow::Status s = tensorflow::Env::Default()->CreateDir(dirname); - if (!s.ok() && s.code() != tensorflow::error::ALREADY_EXISTS) { - Set_TF_Status_from_Status(status, s); - } -} - -void RecursivelyCreateDir(const string& dirname, TF_Status* status) { - tensorflow::Status s = tensorflow::Env::Default()->RecursivelyCreateDir( - dirname); - if (!s.ok()) { - Set_TF_Status_from_Status(status, s); - } -} - -void CopyFile(const string& src, const string& target, bool overwrite, - TF_Status* status) { - // If overwrite is false and the target file exists then its an error. - if (!overwrite && tensorflow::Env::Default()->FileExists(target).ok()) { - TF_SetStatus(status, TF_ALREADY_EXISTS, "file already exists"); - return; - } - tensorflow::Status s = tensorflow::Env::Default()->CopyFile(src, target); - if (!s.ok()) { - Set_TF_Status_from_Status(status, s); - } -} - -void RenameFile(const string& src, const string& target, bool overwrite, - TF_Status* status) { - // If overwrite is false and the target file exists then its an error. - if (!overwrite && tensorflow::Env::Default()->FileExists(target).ok()) { - TF_SetStatus(status, TF_ALREADY_EXISTS, "file already exists"); - return; - } - tensorflow::Status s = tensorflow::Env::Default()->RenameFile(src, target); - if (!s.ok()) { - Set_TF_Status_from_Status(status, s); - } -} - -using tensorflow::int64; - -void DeleteRecursively(const string& dirname, TF_Status* status) { - int64 undeleted_files, undeleted_dirs; - tensorflow::Status s = tensorflow::Env::Default()->DeleteRecursively( - dirname, &undeleted_files, &undeleted_dirs); - if (!s.ok()) { - Set_TF_Status_from_Status(status, s); - return; - } - if (undeleted_files > 0 || undeleted_dirs > 0) { - TF_SetStatus(status, TF_PERMISSION_DENIED, "could not fully delete dir"); - return; - } -} - -bool IsDirectory(const string& dirname, TF_Status* out_status) { - tensorflow::Status status = tensorflow::Env::Default()->IsDirectory(dirname); - if (status.ok()) { - return true; - } - // FAILED_PRECONDITION Status response means path exists but isn't a dir. - if (status.code() != tensorflow::error::FAILED_PRECONDITION) { - Set_TF_Status_from_Status(out_status, status); - } - return false; -} - -using tensorflow::FileStatistics; - -void Stat(const string& filename, FileStatistics* stats, TF_Status* status) { - tensorflow::Status s = tensorflow::Env::Default()->Stat(filename, - stats); - if (!s.ok()) { - Set_TF_Status_from_Status(status, s); - } -} - -tensorflow::io::BufferedInputStream* CreateBufferedInputStream( - const string& filename, size_t buffer_size, TF_Status* status) { - std::unique_ptr file; - tensorflow::Status s = - tensorflow::Env::Default()->NewRandomAccessFile(filename, &file); - if (!s.ok()) { - Set_TF_Status_from_Status(status, s); - return nullptr; - } - std::unique_ptr input_stream( - new tensorflow::io::RandomAccessInputStream( - file.release(), true /* owns_file */)); - std::unique_ptr buffered_input_stream( - new tensorflow::io::BufferedInputStream( - input_stream.release(), buffer_size, true /* owns_input_stream */)); - return buffered_input_stream.release(); -} - -tensorflow::WritableFile* CreateWritableFile( - const string& filename, const string& mode, TF_Status* status) { - std::unique_ptr file; - tensorflow::Status s; - if (mode.find("a") != std::string::npos) { - s = tensorflow::Env::Default()->NewAppendableFile(filename, &file); - } else { - s = tensorflow::Env::Default()->NewWritableFile(filename, &file); - } - if (!s.ok()) { - Set_TF_Status_from_Status(status, s); - return nullptr; - } - return file.release(); -} - -void AppendToFile(const string& file_content, tensorflow::WritableFile* file, - TF_Status* status) { - tensorflow::Status s = file->Append(file_content); - if (!s.ok()) { - Set_TF_Status_from_Status(status, s); - } -} - -int64 TellFile(tensorflow::WritableFile* file, TF_Status* status) { - int64 position = -1; - tensorflow::Status s = file->Tell(&position); - if (!s.ok()) { - Set_TF_Status_from_Status(status, s); - } - return position; -} - - -string ReadFromStream(tensorflow::io::BufferedInputStream* stream, - size_t bytes, - TF_Status* status) { - tensorflow::tstring result; - tensorflow::Status s = stream->ReadNBytes(bytes, &result); - if (!s.ok() && s.code() != tensorflow::error::OUT_OF_RANGE) { - Set_TF_Status_from_Status(status, s); - result.clear(); - } - return result; -} - -%} - -// Ensure that the returned object is destroyed when its wrapper is -// garbage collected. -%newobject CreateBufferedInputStream; -%newobject CreateWritableFile; - -// Wrap the above functions. -inline void FileExists(const string& filename, TF_Status* status); -inline void DeleteFile(const string& filename, TF_Status* status); -string ReadFileToString(const string& filename, TF_Status* status); -void WriteStringToFile(const string& filename, const string& file_content, - TF_Status* status); -std::vector GetChildren(const string& dir, TF_Status* status); -std::vector GetMatchingFiles(const string& filename, - TF_Status* status); -void CreateDir(const string& dirname, TF_Status* status); -void RecursivelyCreateDir(const string& dirname, TF_Status* status); -void CopyFile(const string& oldpath, const string& newpath, bool overwrite, - TF_Status* status); -void RenameFile(const string& oldname, const string& newname, bool overwrite, - TF_Status* status); -void DeleteRecursively(const string& dirname, TF_Status* status); -bool IsDirectory(const string& dirname, TF_Status* out_status); -void Stat(const string& filename, tensorflow::FileStatistics* stats, - TF_Status* status); -tensorflow::io::BufferedInputStream* CreateBufferedInputStream( - const string& filename, size_t buffer_size, TF_Status* status); -tensorflow::WritableFile* CreateWritableFile(const string& filename, - const string& mode, - TF_Status* status); -void AppendToFile(const string& file_content, tensorflow::WritableFile* file, - TF_Status* status); -int64 TellFile(tensorflow::WritableFile* file, TF_Status* status); -string ReadFromStream(tensorflow::io::BufferedInputStream* stream, - size_t bytes, - TF_Status* status); - -%ignore tensorflow::Status::operator=; -%include "tensorflow/core/platform/status.h" - -%ignoreall -%unignore tensorflow::io; -%unignore tensorflow::io::BufferedInputStream; -%unignore tensorflow::io::BufferedInputStream::~BufferedInputStream; -%unignore tensorflow::io::BufferedInputStream::ReadLineAsString; -%unignore tensorflow::io::BufferedInputStream::Seek; -%unignore tensorflow::io::BufferedInputStream::Tell; -%unignore tensorflow::WritableFile; -%unignore tensorflow::WritableFile::Close; -%unignore tensorflow::WritableFile::Flush; -%unignore tensorflow::WritableFile::~WritableFile; -%include "tensorflow/core/platform/file_system.h" -%include "tensorflow/core/lib/io/inputstream_interface.h" -%include "tensorflow/core/lib/io/buffered_inputstream.h" -%unignoreall - -%include "tensorflow/c/tf_status_helper.h" - -%ignore tensorflow::io::internal::JoinPathImpl; -%include "tensorflow/core/lib/io/path.h" - -%include "tensorflow/core/platform/file_statistics.h" diff --git a/tensorflow/python/lib/io/file_io.py b/tensorflow/python/lib/io/file_io.py index 65c0f0810f1..55b4359d75b 100644 --- a/tensorflow/python/lib/io/file_io.py +++ b/tensorflow/python/lib/io/file_io.py @@ -12,11 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""File IO methods that wrap the C++ FileSystem API. - -The C++ FileSystem API is SWIG wrapped in file_io.i. These functions call those -to accomplish basic File IO operations. -""" +"""File IO methods that wrap the C++ FileSystem API.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function @@ -27,8 +23,7 @@ import uuid import six -from tensorflow.python import pywrap_tensorflow -from tensorflow.python.framework import c_api_util +from tensorflow.python import _pywrap_file_io from tensorflow.python.framework import errors from tensorflow.python.util import compat from tensorflow.python.util import deprecation @@ -80,15 +75,15 @@ class FileIO(object): if not self._read_check_passed: raise errors.PermissionDeniedError(None, None, "File isn't open for reading") - self._read_buf = pywrap_tensorflow.CreateBufferedInputStream( - compat.as_bytes(self.__name), 1024 * 512) + self._read_buf = _pywrap_file_io.BufferedInputStream( + self.__name, 1024 * 512) def _prewrite_check(self): if not self._writable_file: if not self._write_check_passed: raise errors.PermissionDeniedError(None, None, "File isn't open for writing") - self._writable_file = pywrap_tensorflow.CreateWritableFile( + self._writable_file = _pywrap_file_io.WritableFile( compat.as_bytes(self.__name), compat.as_bytes(self.__mode)) def _prepare_value(self, val): @@ -104,8 +99,7 @@ class FileIO(object): def write(self, file_content): """Writes file_content to the file. Appends to the end of the file.""" self._prewrite_check() - pywrap_tensorflow.AppendToFile( - compat.as_bytes(file_content), self._writable_file) + self._writable_file.append(compat.as_bytes(file_content)) def read(self, n=-1): """Returns the contents of a file as a string. @@ -124,8 +118,7 @@ class FileIO(object): length = self.size() - self.tell() else: length = n - return self._prepare_value( - pywrap_tensorflow.ReadFromStream(self._read_buf, length)) + return self._prepare_value(self._read_buf.read(length)) @deprecation.deprecated_args( None, "position is deprecated in favor of the offset argument.", @@ -158,25 +151,23 @@ class FileIO(object): if position is not None: offset = position - with errors.raise_exception_on_not_ok_status() as status: - if whence == 0: - pass - elif whence == 1: - offset += self.tell() - elif whence == 2: - offset += self.size() - else: - raise errors.InvalidArgumentError( - None, None, - "Invalid whence argument: {}. Valid values are 0, 1, or 2.".format( - whence)) - ret_status = self._read_buf.Seek(offset) - pywrap_tensorflow.Set_TF_Status_from_Status(status, ret_status) + if whence == 0: + pass + elif whence == 1: + offset += self.tell() + elif whence == 2: + offset += self.size() + else: + raise errors.InvalidArgumentError( + None, None, + "Invalid whence argument: {}. Valid values are 0, 1, or 2.".format( + whence)) + self._read_buf.seek(offset) def readline(self): r"""Reads the next line from the file. Leaves the '\n' at the end.""" self._preread_check() - return self._prepare_value(self._read_buf.ReadLineAsString()) + return self._prepare_value(self._read_buf.readline()) def readlines(self): """Returns all lines from the file in a list.""" @@ -193,11 +184,11 @@ class FileIO(object): """Returns the current position in the file.""" if self._read_check_passed: self._preread_check() - return self._read_buf.Tell() + return self._read_buf.tell() else: self._prewrite_check() - return pywrap_tensorflow.TellFile(self._writable_file) + return self._writable_file.tell() def __enter__(self): """Make usable with "with" statement.""" @@ -227,18 +218,14 @@ class FileIO(object): data would survive an application crash but not necessarily an OS crash. """ if self._writable_file: - with errors.raise_exception_on_not_ok_status() as status: - ret_status = self._writable_file.Flush() - pywrap_tensorflow.Set_TF_Status_from_Status(status, ret_status) + self._writable_file.flush() def close(self): """Closes FileIO. Should be called for the WritableFile to be flushed.""" self._read_buf = None if self._writable_file: - with errors.raise_exception_on_not_ok_status() as status: - ret_status = self._writable_file.Close() - pywrap_tensorflow.Set_TF_Status_from_Status(status, ret_status) - self._writable_file = None + self._writable_file.close() + self._writable_file = None def seekable(self): """Returns True as FileIO supports random access ops of seek()/tell()""" @@ -277,7 +264,7 @@ def file_exists_v2(path): errors.OpError: Propagates any errors reported by the FileSystem API. """ try: - pywrap_tensorflow.FileExists(compat.as_bytes(path)) + _pywrap_file_io.FileExists(compat.as_bytes(path)) except errors.NotFoundError: return False return True @@ -308,7 +295,7 @@ def delete_file_v2(path): errors.OpError: Propagates any errors reported by the FileSystem API. E.g., `NotFoundError` if the path does not exist. """ - pywrap_tensorflow.DeleteFile(compat.as_bytes(path)) + _pywrap_file_io.DeleteFile(compat.as_bytes(path)) def read_file_to_string(filename, binary_mode=False): @@ -380,7 +367,7 @@ def get_matching_files_v2(pattern): return [ # Convert the filenames to string from bytes. compat.as_str_any(matching_filename) - for matching_filename in pywrap_tensorflow.GetMatchingFiles( + for matching_filename in _pywrap_file_io.GetMatchingFiles( compat.as_bytes(pattern)) ] else: @@ -388,7 +375,7 @@ def get_matching_files_v2(pattern): # Convert the filenames to string from bytes. compat.as_str_any(matching_filename) # pylint: disable=g-complex-comprehension for single_filename in pattern - for matching_filename in pywrap_tensorflow.GetMatchingFiles( + for matching_filename in _pywrap_file_io.GetMatchingFiles( compat.as_bytes(single_filename)) ] @@ -422,7 +409,7 @@ def create_dir_v2(path): Raises: errors.OpError: If the operation fails. """ - pywrap_tensorflow.CreateDir(compat.as_bytes(path)) + _pywrap_file_io.CreateDir(compat.as_bytes(path)) @tf_export(v1=["gfile.MakeDirs"]) @@ -452,7 +439,7 @@ def recursive_create_dir_v2(path): Raises: errors.OpError: If the operation fails. """ - pywrap_tensorflow.RecursivelyCreateDir(compat.as_bytes(path)) + _pywrap_file_io.RecursivelyCreateDir(compat.as_bytes(path)) @tf_export(v1=["gfile.Copy"]) @@ -484,7 +471,7 @@ def copy_v2(src, dst, overwrite=False): Raises: errors.OpError: If the operation fails. """ - pywrap_tensorflow.CopyFile( + _pywrap_file_io.CopyFile( compat.as_bytes(src), compat.as_bytes(dst), overwrite) @@ -517,7 +504,7 @@ def rename_v2(src, dst, overwrite=False): Raises: errors.OpError: If the operation fails. """ - pywrap_tensorflow.RenameFile( + _pywrap_file_io.RenameFile( compat.as_bytes(src), compat.as_bytes(dst), overwrite) @@ -568,7 +555,7 @@ def delete_recursively_v2(path): Raises: errors.OpError: If the operation fails. """ - pywrap_tensorflow.DeleteRecursively(compat.as_bytes(path)) + _pywrap_file_io.DeleteRecursively(compat.as_bytes(path)) @tf_export(v1=["gfile.IsDirectory"]) @@ -594,8 +581,10 @@ def is_directory_v2(path): Returns: True, if the path is a directory; False otherwise """ - status = c_api_util.ScopedTFStatus() - return pywrap_tensorflow.IsDirectory(compat.as_bytes(path), status) + try: + return _pywrap_file_io.IsDirectory(compat.as_bytes(path)) + except errors.OpError: + return False @tf_export(v1=["gfile.ListDirectory"]) @@ -643,7 +632,7 @@ def list_directory_v2(path): # vector of string should be interpreted as strings, not bytes. return [ compat.as_str_any(filename) - for filename in pywrap_tensorflow.GetChildren(compat.as_bytes(path)) + for filename in _pywrap_file_io.GetChildren(compat.as_bytes(path)) ] @@ -742,9 +731,7 @@ def stat_v2(path): Raises: errors.OpError: If the operation fails. """ - file_statistics = pywrap_tensorflow.FileStatistics() - pywrap_tensorflow.Stat(compat.as_bytes(path), file_statistics) - return file_statistics + return _pywrap_file_io.Stat(path) def filecmp(filename_a, filename_b): diff --git a/tensorflow/python/lib/io/file_io_wrapper.cc b/tensorflow/python/lib/io/file_io_wrapper.cc new file mode 100644 index 00000000000..28e55f1d8a3 --- /dev/null +++ b/tensorflow/python/lib/io/file_io_wrapper.cc @@ -0,0 +1,205 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include + +#include "include/pybind11/pybind11.h" +#include "include/pybind11/stl.h" +#include "tensorflow/core/lib/core/error_codes.pb.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/io/buffered_inputstream.h" +#include "tensorflow/core/lib/io/random_inputstream.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/file_statistics.h" +#include "tensorflow/core/platform/file_system.h" +#include "tensorflow/core/platform/stringpiece.h" +#include "tensorflow/core/platform/tstring.h" +#include "tensorflow/python/lib/core/pybind11_absl.h" +#include "tensorflow/python/lib/core/pybind11_status.h" + +namespace { +namespace py = pybind11; + +PYBIND11_MODULE(_pywrap_file_io, m) { + m.def("FileExists", [](const std::string& filename) { + tensorflow::MaybeRaiseRegisteredFromStatus( + tensorflow::Env::Default()->FileExists(filename)); + }); + m.def("DeleteFile", [](const std::string& filename) { + tensorflow::MaybeRaiseRegisteredFromStatus( + tensorflow::Env::Default()->DeleteFile(filename)); + }); + m.def("ReadFileToString", [](const std::string& filename) { + std::string data; + const auto status = + ReadFileToString(tensorflow::Env::Default(), filename, &data); + tensorflow::MaybeRaiseRegisteredFromStatus(status); + return py::bytes(data); + }); + m.def("WriteStringToFile", + [](const std::string& filename, tensorflow::StringPiece data) { + return WriteStringToFile(tensorflow::Env::Default(), filename, data); + }); + m.def("GetChildren", [](const std::string& dirname) { + std::vector results; + const auto status = + tensorflow::Env::Default()->GetChildren(dirname, &results); + tensorflow::MaybeRaiseRegisteredFromStatus(status); + return results; + }); + m.def("GetMatchingFiles", [](const std::string& pattern) { + std::vector results; + const auto status = + tensorflow::Env::Default()->GetMatchingPaths(pattern, &results); + tensorflow::MaybeRaiseRegisteredFromStatus(status); + return results; + }); + m.def("CreateDir", [](const std::string& dirname) { + const auto status = tensorflow::Env::Default()->CreateDir(dirname); + if (tensorflow::errors::IsAlreadyExists(status)) { + return; + } + tensorflow::MaybeRaiseRegisteredFromStatus(status); + }); + m.def("RecursivelyCreateDir", [](const std::string& dirname) { + tensorflow::MaybeRaiseRegisteredFromStatus( + tensorflow::Env::Default()->RecursivelyCreateDir(dirname)); + }); + m.def("CopyFile", + [](const std::string& src, const std::string& target, bool overwrite) { + auto* env = tensorflow::Env::Default(); + tensorflow::Status status; + if (!overwrite && env->FileExists(target).ok()) { + status = tensorflow::errors::AlreadyExists("file already exists"); + } else { + status = env->CopyFile(src, target); + } + tensorflow::MaybeRaiseRegisteredFromStatus(status); + }); + m.def("RenameFile", + [](const std::string& src, const std::string& target, bool overwrite) { + auto* env = tensorflow::Env::Default(); + tensorflow::Status status; + if (!overwrite && env->FileExists(target).ok()) { + status = tensorflow::errors::AlreadyExists("file already exists"); + } else { + status = env->RenameFile(src, target); + } + tensorflow::MaybeRaiseRegisteredFromStatus(status); + }); + m.def("DeleteRecursively", [](const std::string& dirname) { + tensorflow::int64 undeleted_files; + tensorflow::int64 undeleted_dirs; + auto status = tensorflow::Env::Default()->DeleteRecursively( + dirname, &undeleted_files, &undeleted_dirs); + if (status.ok() && (undeleted_files > 0 || undeleted_dirs > 0)) { + status = + tensorflow::errors::PermissionDenied("could not fully delete dir"); + } + tensorflow::MaybeRaiseRegisteredFromStatus(status); + }); + m.def("IsDirectory", [](const std::string& dirname) { + const auto status = tensorflow::Env::Default()->IsDirectory(dirname); + // FAILED_PRECONDITION response means path exists but isn't a dir. + if (tensorflow::errors::IsFailedPrecondition(status)) { + return false; + } + + tensorflow::MaybeRaiseRegisteredFromStatus(status); + return true; + }); + + py::class_(m, "FileStatistics") + .def_readonly("length", &tensorflow::FileStatistics::length) + .def_readonly("mtime_nsec", &tensorflow::FileStatistics::mtime_nsec) + .def_readonly("is_directory", &tensorflow::FileStatistics::is_directory); + + m.def("Stat", [](const std::string& filename) { + std::unique_ptr self( + new tensorflow::FileStatistics); + const auto status = tensorflow::Env::Default()->Stat(filename, self.get()); + tensorflow::MaybeRaiseRegisteredFromStatus(status); + return self.release(); + }); + + using tensorflow::WritableFile; + py::class_(m, "WritableFile") + .def(py::init([](const std::string& filename, const std::string& mode) { + auto* env = tensorflow::Env::Default(); + std::unique_ptr self; + const auto status = mode.find("a") == std::string::npos + ? env->NewWritableFile(filename, &self) + : env->NewAppendableFile(filename, &self); + tensorflow::MaybeRaiseRegisteredFromStatus(status); + return self.release(); + })) + .def("append", + [](WritableFile* self, tensorflow::StringPiece data) { + tensorflow::MaybeRaiseRegisteredFromStatus(self->Append(data)); + }) + // TODO(slebedev): Make WritableFile::Tell const and change self + // to be a reference. + .def("tell", + [](WritableFile* self) { + tensorflow::int64 pos = -1; + const auto status = self->Tell(&pos); + tensorflow::MaybeRaiseRegisteredFromStatus(status); + return pos; + }) + .def("flush", + [](WritableFile* self) { + tensorflow::MaybeRaiseRegisteredFromStatus(self->Flush()); + }) + .def("close", [](WritableFile* self) { + tensorflow::MaybeRaiseRegisteredFromStatus(self->Close()); + }); + + using tensorflow::io::BufferedInputStream; + py::class_(m, "BufferedInputStream") + .def(py::init([](const std::string& filename, size_t buffer_size) { + std::unique_ptr file; + const auto status = + tensorflow::Env::Default()->NewRandomAccessFile(filename, &file); + tensorflow::MaybeRaiseRegisteredFromStatus(status); + std::unique_ptr input_stream( + new tensorflow::io::RandomAccessInputStream(file.release(), + /*owns_file=*/true)); + return new BufferedInputStream(input_stream.release(), buffer_size, + /*owns_input_stream=*/true); + })) + .def("read", + [](BufferedInputStream* self, tensorflow::int64 bytes_to_read) { + tensorflow::tstring result; + const auto status = self->ReadNBytes(bytes_to_read, &result); + if (!status.ok() && !tensorflow::errors::IsOutOfRange(status)) { + result.clear(); + tensorflow::MaybeRaiseRegisteredFromStatus(status); + } + return py::bytes(result); + }) + .def("readline", + [](BufferedInputStream* self) { + return py::bytes(self->ReadLineAsString()); + }) + .def("seek", + [](BufferedInputStream* self, tensorflow::int64 pos) { + tensorflow::MaybeRaiseRegisteredFromStatus(self->Seek(pos)); + }) + .def("tell", [](BufferedInputStream* self) { return self->Tell(); }); +} +} // namespace diff --git a/tensorflow/python/platform/base.i b/tensorflow/python/platform/base.i index 65a56f91b93..92f7d8bf987 100644 --- a/tensorflow/python/platform/base.i +++ b/tensorflow/python/platform/base.i @@ -23,6 +23,7 @@ limitations under the License. #include "tensorflow/c/tf_datatype.h" #include "tensorflow/python/lib/core/py_exception_registry.h" + using tensorflow::int64; using tensorflow::uint64; using tensorflow::string; diff --git a/tensorflow/python/tensorflow.i b/tensorflow/python/tensorflow.i index 4e65120353e..2faff274498 100644 --- a/tensorflow/python/tensorflow.i +++ b/tensorflow/python/tensorflow.i @@ -19,8 +19,6 @@ limitations under the License. %include "tensorflow/python/client/tf_session.i" -%include "tensorflow/python/lib/io/file_io.i" - %include "tensorflow/python/lib/io/py_record_reader.i" %include "tensorflow/python/grappler/cluster.i" @@ -29,3 +27,10 @@ limitations under the License. %include "tensorflow/python/grappler/cost_analyzer.i" %include "tensorflow/compiler/mlir/python/mlir.i" + +// TODO(slebedev): This is a temporary workaround for projects implicitly +// relying on TensorFlow exposing tensorflow::Status. +%unignoreall + +%ignore tensorflow::Status::operator=; +%include "tensorflow/core/platform/status.h"