diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index 1937e9dc049..a424ba25ceb 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -5479,6 +5479,7 @@ tf_py_wrap_cc( "grappler/item.i", "grappler/tf_optimizer.i", "lib/core/strings.i", + "lib/io/file_io.i", "lib/io/py_record_reader.i", "platform/base.i", "//tensorflow/compiler/mlir/python:mlir.i", @@ -5665,19 +5666,6 @@ cc_import( # ** Targets for Windows build (end) ** -tf_python_pybind_extension( - name = "_pywrap_file_io", - srcs = ["lib/io/file_io_wrapper.cc"], - module_name = "_pywrap_file_io", - deps = [ - ":pybind11_absl", - ":pybind11_status", - "//tensorflow/core:framework_headers_lib", - "//tensorflow/core:protos_all_cc", - "@pybind11", - ], -) - py_library( name = "lib", srcs = [ @@ -5687,7 +5675,6 @@ py_library( ], srcs_version = "PY2AND3", deps = [ - ":_pywrap_file_io", ":_pywrap_record_io", ":errors", ":pywrap_tensorflow", diff --git a/tensorflow/python/lib/io/file_io.i b/tensorflow/python/lib/io/file_io.i new file mode 100644 index 00000000000..cbd619bb764 --- /dev/null +++ b/tensorflow/python/lib/io/file_io.i @@ -0,0 +1,302 @@ +/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +%include "tensorflow/python/lib/core/strings.i" +%include "tensorflow/python/platform/base.i" + +%{ +#include "tensorflow/c/tf_status_helper.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/core/stringpiece.h" +#include "tensorflow/core/lib/io/buffered_inputstream.h" +#include "tensorflow/core/lib/io/inputstream_interface.h" +#include "tensorflow/core/lib/io/random_inputstream.h" +#include "tensorflow/core/lib/io/path.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/file_statistics.h" +#include "tensorflow/core/platform/file_system.h" +#include "tensorflow/core/protobuf/meta_graph.pb.h" +%} + +%{ +inline void FileExists(const string& filename, TF_Status* status) { + tensorflow::Status s = tensorflow::Env::Default()->FileExists(filename); + if (!s.ok()) { + Set_TF_Status_from_Status(status, s); + } +} + +inline void FileExists(const tensorflow::StringPiece& filename, + TF_Status* status) { + tensorflow::Status s = + tensorflow::Env::Default()->FileExists(string(filename)); + if (!s.ok()) { + Set_TF_Status_from_Status(status, s); + } +} + +inline void DeleteFile(const string& filename, TF_Status* status) { + tensorflow::Status s = tensorflow::Env::Default()->DeleteFile(filename); + if (!s.ok()) { + Set_TF_Status_from_Status(status, s); + } +} + +string ReadFileToString(const string& filename, TF_Status* status) { + string file_content; + tensorflow::Status s = ReadFileToString(tensorflow::Env::Default(), + filename, &file_content); + if (!s.ok()) { + Set_TF_Status_from_Status(status, s); + } + return file_content; +} + +void WriteStringToFile(const string& filename, const string& file_content, + TF_Status* status) { + tensorflow::Status s = WriteStringToFile(tensorflow::Env::Default(), + filename, file_content); + if (!s.ok()) { + Set_TF_Status_from_Status(status, s); + } +} + +std::vector GetChildren(const string& dir, TF_Status* status) { + std::vector results; + tensorflow::Status s = tensorflow::Env::Default()->GetChildren( + dir, &results); + if (!s.ok()) { + Set_TF_Status_from_Status(status, s); + } + return results; +} + +std::vector GetMatchingFiles(const string& filename, TF_Status* status) { + std::vector results; + tensorflow::Status s = tensorflow::Env::Default()->GetMatchingPaths( + filename, &results); + if (!s.ok()) { + Set_TF_Status_from_Status(status, s); + } + return results; +} + +void CreateDir(const string& dirname, TF_Status* status) { + tensorflow::Status s = tensorflow::Env::Default()->CreateDir(dirname); + if (!s.ok() && s.code() != tensorflow::error::ALREADY_EXISTS) { + Set_TF_Status_from_Status(status, s); + } +} + +void RecursivelyCreateDir(const string& dirname, TF_Status* status) { + tensorflow::Status s = tensorflow::Env::Default()->RecursivelyCreateDir( + dirname); + if (!s.ok()) { + Set_TF_Status_from_Status(status, s); + } +} + +void CopyFile(const string& src, const string& target, bool overwrite, + TF_Status* status) { + // If overwrite is false and the target file exists then its an error. + if (!overwrite && tensorflow::Env::Default()->FileExists(target).ok()) { + TF_SetStatus(status, TF_ALREADY_EXISTS, "file already exists"); + return; + } + tensorflow::Status s = tensorflow::Env::Default()->CopyFile(src, target); + if (!s.ok()) { + Set_TF_Status_from_Status(status, s); + } +} + +void RenameFile(const string& src, const string& target, bool overwrite, + TF_Status* status) { + // If overwrite is false and the target file exists then its an error. + if (!overwrite && tensorflow::Env::Default()->FileExists(target).ok()) { + TF_SetStatus(status, TF_ALREADY_EXISTS, "file already exists"); + return; + } + tensorflow::Status s = tensorflow::Env::Default()->RenameFile(src, target); + if (!s.ok()) { + Set_TF_Status_from_Status(status, s); + } +} + +using tensorflow::int64; + +void DeleteRecursively(const string& dirname, TF_Status* status) { + int64 undeleted_files, undeleted_dirs; + tensorflow::Status s = tensorflow::Env::Default()->DeleteRecursively( + dirname, &undeleted_files, &undeleted_dirs); + if (!s.ok()) { + Set_TF_Status_from_Status(status, s); + return; + } + if (undeleted_files > 0 || undeleted_dirs > 0) { + TF_SetStatus(status, TF_PERMISSION_DENIED, "could not fully delete dir"); + return; + } +} + +bool IsDirectory(const string& dirname, TF_Status* out_status) { + tensorflow::Status status = tensorflow::Env::Default()->IsDirectory(dirname); + if (status.ok()) { + return true; + } + // FAILED_PRECONDITION Status response means path exists but isn't a dir. + if (status.code() != tensorflow::error::FAILED_PRECONDITION) { + Set_TF_Status_from_Status(out_status, status); + } + return false; +} + +using tensorflow::FileStatistics; + +void Stat(const string& filename, FileStatistics* stats, TF_Status* status) { + tensorflow::Status s = tensorflow::Env::Default()->Stat(filename, + stats); + if (!s.ok()) { + Set_TF_Status_from_Status(status, s); + } +} + +tensorflow::io::BufferedInputStream* CreateBufferedInputStream( + const string& filename, size_t buffer_size, TF_Status* status) { + std::unique_ptr file; + tensorflow::Status s = + tensorflow::Env::Default()->NewRandomAccessFile(filename, &file); + if (!s.ok()) { + Set_TF_Status_from_Status(status, s); + return nullptr; + } + std::unique_ptr input_stream( + new tensorflow::io::RandomAccessInputStream( + file.release(), true /* owns_file */)); + std::unique_ptr buffered_input_stream( + new tensorflow::io::BufferedInputStream( + input_stream.release(), buffer_size, true /* owns_input_stream */)); + return buffered_input_stream.release(); +} + +tensorflow::WritableFile* CreateWritableFile( + const string& filename, const string& mode, TF_Status* status) { + std::unique_ptr file; + tensorflow::Status s; + if (mode.find("a") != std::string::npos) { + s = tensorflow::Env::Default()->NewAppendableFile(filename, &file); + } else { + s = tensorflow::Env::Default()->NewWritableFile(filename, &file); + } + if (!s.ok()) { + Set_TF_Status_from_Status(status, s); + return nullptr; + } + return file.release(); +} + +void AppendToFile(const string& file_content, tensorflow::WritableFile* file, + TF_Status* status) { + tensorflow::Status s = file->Append(file_content); + if (!s.ok()) { + Set_TF_Status_from_Status(status, s); + } +} + +int64 TellFile(tensorflow::WritableFile* file, TF_Status* status) { + int64 position = -1; + tensorflow::Status s = file->Tell(&position); + if (!s.ok()) { + Set_TF_Status_from_Status(status, s); + } + return position; +} + + +string ReadFromStream(tensorflow::io::BufferedInputStream* stream, + size_t bytes, + TF_Status* status) { + tensorflow::tstring result; + tensorflow::Status s = stream->ReadNBytes(bytes, &result); + if (!s.ok() && s.code() != tensorflow::error::OUT_OF_RANGE) { + Set_TF_Status_from_Status(status, s); + result.clear(); + } + return result; +} + +%} + +// Ensure that the returned object is destroyed when its wrapper is +// garbage collected. +%newobject CreateBufferedInputStream; +%newobject CreateWritableFile; + +// Wrap the above functions. +inline void FileExists(const string& filename, TF_Status* status); +inline void DeleteFile(const string& filename, TF_Status* status); +string ReadFileToString(const string& filename, TF_Status* status); +void WriteStringToFile(const string& filename, const string& file_content, + TF_Status* status); +std::vector GetChildren(const string& dir, TF_Status* status); +std::vector GetMatchingFiles(const string& filename, + TF_Status* status); +void CreateDir(const string& dirname, TF_Status* status); +void RecursivelyCreateDir(const string& dirname, TF_Status* status); +void CopyFile(const string& oldpath, const string& newpath, bool overwrite, + TF_Status* status); +void RenameFile(const string& oldname, const string& newname, bool overwrite, + TF_Status* status); +void DeleteRecursively(const string& dirname, TF_Status* status); +bool IsDirectory(const string& dirname, TF_Status* out_status); +void Stat(const string& filename, tensorflow::FileStatistics* stats, + TF_Status* status); +tensorflow::io::BufferedInputStream* CreateBufferedInputStream( + const string& filename, size_t buffer_size, TF_Status* status); +tensorflow::WritableFile* CreateWritableFile(const string& filename, + const string& mode, + TF_Status* status); +void AppendToFile(const string& file_content, tensorflow::WritableFile* file, + TF_Status* status); +int64 TellFile(tensorflow::WritableFile* file, TF_Status* status); +string ReadFromStream(tensorflow::io::BufferedInputStream* stream, + size_t bytes, + TF_Status* status); + +%ignore tensorflow::Status::operator=; +%include "tensorflow/core/platform/status.h" + +%ignoreall +%unignore tensorflow::io; +%unignore tensorflow::io::BufferedInputStream; +%unignore tensorflow::io::BufferedInputStream::~BufferedInputStream; +%unignore tensorflow::io::BufferedInputStream::ReadLineAsString; +%unignore tensorflow::io::BufferedInputStream::Seek; +%unignore tensorflow::io::BufferedInputStream::Tell; +%unignore tensorflow::WritableFile; +%unignore tensorflow::WritableFile::Close; +%unignore tensorflow::WritableFile::Flush; +%unignore tensorflow::WritableFile::~WritableFile; +%include "tensorflow/core/platform/file_system.h" +%include "tensorflow/core/lib/io/inputstream_interface.h" +%include "tensorflow/core/lib/io/buffered_inputstream.h" +%unignoreall + +%include "tensorflow/c/tf_status_helper.h" + +%ignore tensorflow::io::internal::JoinPathImpl; +%include "tensorflow/core/lib/io/path.h" + +%include "tensorflow/core/platform/file_statistics.h" diff --git a/tensorflow/python/lib/io/file_io.py b/tensorflow/python/lib/io/file_io.py index 55b4359d75b..65c0f0810f1 100644 --- a/tensorflow/python/lib/io/file_io.py +++ b/tensorflow/python/lib/io/file_io.py @@ -12,7 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""File IO methods that wrap the C++ FileSystem API.""" +"""File IO methods that wrap the C++ FileSystem API. + +The C++ FileSystem API is SWIG wrapped in file_io.i. These functions call those +to accomplish basic File IO operations. +""" from __future__ import absolute_import from __future__ import division from __future__ import print_function @@ -23,7 +27,8 @@ import uuid import six -from tensorflow.python import _pywrap_file_io +from tensorflow.python import pywrap_tensorflow +from tensorflow.python.framework import c_api_util from tensorflow.python.framework import errors from tensorflow.python.util import compat from tensorflow.python.util import deprecation @@ -75,15 +80,15 @@ class FileIO(object): if not self._read_check_passed: raise errors.PermissionDeniedError(None, None, "File isn't open for reading") - self._read_buf = _pywrap_file_io.BufferedInputStream( - self.__name, 1024 * 512) + self._read_buf = pywrap_tensorflow.CreateBufferedInputStream( + compat.as_bytes(self.__name), 1024 * 512) def _prewrite_check(self): if not self._writable_file: if not self._write_check_passed: raise errors.PermissionDeniedError(None, None, "File isn't open for writing") - self._writable_file = _pywrap_file_io.WritableFile( + self._writable_file = pywrap_tensorflow.CreateWritableFile( compat.as_bytes(self.__name), compat.as_bytes(self.__mode)) def _prepare_value(self, val): @@ -99,7 +104,8 @@ class FileIO(object): def write(self, file_content): """Writes file_content to the file. Appends to the end of the file.""" self._prewrite_check() - self._writable_file.append(compat.as_bytes(file_content)) + pywrap_tensorflow.AppendToFile( + compat.as_bytes(file_content), self._writable_file) def read(self, n=-1): """Returns the contents of a file as a string. @@ -118,7 +124,8 @@ class FileIO(object): length = self.size() - self.tell() else: length = n - return self._prepare_value(self._read_buf.read(length)) + return self._prepare_value( + pywrap_tensorflow.ReadFromStream(self._read_buf, length)) @deprecation.deprecated_args( None, "position is deprecated in favor of the offset argument.", @@ -151,23 +158,25 @@ class FileIO(object): if position is not None: offset = position - if whence == 0: - pass - elif whence == 1: - offset += self.tell() - elif whence == 2: - offset += self.size() - else: - raise errors.InvalidArgumentError( - None, None, - "Invalid whence argument: {}. Valid values are 0, 1, or 2.".format( - whence)) - self._read_buf.seek(offset) + with errors.raise_exception_on_not_ok_status() as status: + if whence == 0: + pass + elif whence == 1: + offset += self.tell() + elif whence == 2: + offset += self.size() + else: + raise errors.InvalidArgumentError( + None, None, + "Invalid whence argument: {}. Valid values are 0, 1, or 2.".format( + whence)) + ret_status = self._read_buf.Seek(offset) + pywrap_tensorflow.Set_TF_Status_from_Status(status, ret_status) def readline(self): r"""Reads the next line from the file. Leaves the '\n' at the end.""" self._preread_check() - return self._prepare_value(self._read_buf.readline()) + return self._prepare_value(self._read_buf.ReadLineAsString()) def readlines(self): """Returns all lines from the file in a list.""" @@ -184,11 +193,11 @@ class FileIO(object): """Returns the current position in the file.""" if self._read_check_passed: self._preread_check() - return self._read_buf.tell() + return self._read_buf.Tell() else: self._prewrite_check() - return self._writable_file.tell() + return pywrap_tensorflow.TellFile(self._writable_file) def __enter__(self): """Make usable with "with" statement.""" @@ -218,14 +227,18 @@ class FileIO(object): data would survive an application crash but not necessarily an OS crash. """ if self._writable_file: - self._writable_file.flush() + with errors.raise_exception_on_not_ok_status() as status: + ret_status = self._writable_file.Flush() + pywrap_tensorflow.Set_TF_Status_from_Status(status, ret_status) def close(self): """Closes FileIO. Should be called for the WritableFile to be flushed.""" self._read_buf = None if self._writable_file: - self._writable_file.close() - self._writable_file = None + with errors.raise_exception_on_not_ok_status() as status: + ret_status = self._writable_file.Close() + pywrap_tensorflow.Set_TF_Status_from_Status(status, ret_status) + self._writable_file = None def seekable(self): """Returns True as FileIO supports random access ops of seek()/tell()""" @@ -264,7 +277,7 @@ def file_exists_v2(path): errors.OpError: Propagates any errors reported by the FileSystem API. """ try: - _pywrap_file_io.FileExists(compat.as_bytes(path)) + pywrap_tensorflow.FileExists(compat.as_bytes(path)) except errors.NotFoundError: return False return True @@ -295,7 +308,7 @@ def delete_file_v2(path): errors.OpError: Propagates any errors reported by the FileSystem API. E.g., `NotFoundError` if the path does not exist. """ - _pywrap_file_io.DeleteFile(compat.as_bytes(path)) + pywrap_tensorflow.DeleteFile(compat.as_bytes(path)) def read_file_to_string(filename, binary_mode=False): @@ -367,7 +380,7 @@ def get_matching_files_v2(pattern): return [ # Convert the filenames to string from bytes. compat.as_str_any(matching_filename) - for matching_filename in _pywrap_file_io.GetMatchingFiles( + for matching_filename in pywrap_tensorflow.GetMatchingFiles( compat.as_bytes(pattern)) ] else: @@ -375,7 +388,7 @@ def get_matching_files_v2(pattern): # Convert the filenames to string from bytes. compat.as_str_any(matching_filename) # pylint: disable=g-complex-comprehension for single_filename in pattern - for matching_filename in _pywrap_file_io.GetMatchingFiles( + for matching_filename in pywrap_tensorflow.GetMatchingFiles( compat.as_bytes(single_filename)) ] @@ -409,7 +422,7 @@ def create_dir_v2(path): Raises: errors.OpError: If the operation fails. """ - _pywrap_file_io.CreateDir(compat.as_bytes(path)) + pywrap_tensorflow.CreateDir(compat.as_bytes(path)) @tf_export(v1=["gfile.MakeDirs"]) @@ -439,7 +452,7 @@ def recursive_create_dir_v2(path): Raises: errors.OpError: If the operation fails. """ - _pywrap_file_io.RecursivelyCreateDir(compat.as_bytes(path)) + pywrap_tensorflow.RecursivelyCreateDir(compat.as_bytes(path)) @tf_export(v1=["gfile.Copy"]) @@ -471,7 +484,7 @@ def copy_v2(src, dst, overwrite=False): Raises: errors.OpError: If the operation fails. """ - _pywrap_file_io.CopyFile( + pywrap_tensorflow.CopyFile( compat.as_bytes(src), compat.as_bytes(dst), overwrite) @@ -504,7 +517,7 @@ def rename_v2(src, dst, overwrite=False): Raises: errors.OpError: If the operation fails. """ - _pywrap_file_io.RenameFile( + pywrap_tensorflow.RenameFile( compat.as_bytes(src), compat.as_bytes(dst), overwrite) @@ -555,7 +568,7 @@ def delete_recursively_v2(path): Raises: errors.OpError: If the operation fails. """ - _pywrap_file_io.DeleteRecursively(compat.as_bytes(path)) + pywrap_tensorflow.DeleteRecursively(compat.as_bytes(path)) @tf_export(v1=["gfile.IsDirectory"]) @@ -581,10 +594,8 @@ def is_directory_v2(path): Returns: True, if the path is a directory; False otherwise """ - try: - return _pywrap_file_io.IsDirectory(compat.as_bytes(path)) - except errors.OpError: - return False + status = c_api_util.ScopedTFStatus() + return pywrap_tensorflow.IsDirectory(compat.as_bytes(path), status) @tf_export(v1=["gfile.ListDirectory"]) @@ -632,7 +643,7 @@ def list_directory_v2(path): # vector of string should be interpreted as strings, not bytes. return [ compat.as_str_any(filename) - for filename in _pywrap_file_io.GetChildren(compat.as_bytes(path)) + for filename in pywrap_tensorflow.GetChildren(compat.as_bytes(path)) ] @@ -731,7 +742,9 @@ def stat_v2(path): Raises: errors.OpError: If the operation fails. """ - return _pywrap_file_io.Stat(path) + file_statistics = pywrap_tensorflow.FileStatistics() + pywrap_tensorflow.Stat(compat.as_bytes(path), file_statistics) + return file_statistics def filecmp(filename_a, filename_b): diff --git a/tensorflow/python/lib/io/file_io_wrapper.cc b/tensorflow/python/lib/io/file_io_wrapper.cc deleted file mode 100644 index 28e55f1d8a3..00000000000 --- a/tensorflow/python/lib/io/file_io_wrapper.cc +++ /dev/null @@ -1,205 +0,0 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include -#include -#include - -#include "include/pybind11/pybind11.h" -#include "include/pybind11/stl.h" -#include "tensorflow/core/lib/core/error_codes.pb.h" -#include "tensorflow/core/lib/core/errors.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/lib/io/buffered_inputstream.h" -#include "tensorflow/core/lib/io/random_inputstream.h" -#include "tensorflow/core/platform/env.h" -#include "tensorflow/core/platform/file_statistics.h" -#include "tensorflow/core/platform/file_system.h" -#include "tensorflow/core/platform/stringpiece.h" -#include "tensorflow/core/platform/tstring.h" -#include "tensorflow/python/lib/core/pybind11_absl.h" -#include "tensorflow/python/lib/core/pybind11_status.h" - -namespace { -namespace py = pybind11; - -PYBIND11_MODULE(_pywrap_file_io, m) { - m.def("FileExists", [](const std::string& filename) { - tensorflow::MaybeRaiseRegisteredFromStatus( - tensorflow::Env::Default()->FileExists(filename)); - }); - m.def("DeleteFile", [](const std::string& filename) { - tensorflow::MaybeRaiseRegisteredFromStatus( - tensorflow::Env::Default()->DeleteFile(filename)); - }); - m.def("ReadFileToString", [](const std::string& filename) { - std::string data; - const auto status = - ReadFileToString(tensorflow::Env::Default(), filename, &data); - tensorflow::MaybeRaiseRegisteredFromStatus(status); - return py::bytes(data); - }); - m.def("WriteStringToFile", - [](const std::string& filename, tensorflow::StringPiece data) { - return WriteStringToFile(tensorflow::Env::Default(), filename, data); - }); - m.def("GetChildren", [](const std::string& dirname) { - std::vector results; - const auto status = - tensorflow::Env::Default()->GetChildren(dirname, &results); - tensorflow::MaybeRaiseRegisteredFromStatus(status); - return results; - }); - m.def("GetMatchingFiles", [](const std::string& pattern) { - std::vector results; - const auto status = - tensorflow::Env::Default()->GetMatchingPaths(pattern, &results); - tensorflow::MaybeRaiseRegisteredFromStatus(status); - return results; - }); - m.def("CreateDir", [](const std::string& dirname) { - const auto status = tensorflow::Env::Default()->CreateDir(dirname); - if (tensorflow::errors::IsAlreadyExists(status)) { - return; - } - tensorflow::MaybeRaiseRegisteredFromStatus(status); - }); - m.def("RecursivelyCreateDir", [](const std::string& dirname) { - tensorflow::MaybeRaiseRegisteredFromStatus( - tensorflow::Env::Default()->RecursivelyCreateDir(dirname)); - }); - m.def("CopyFile", - [](const std::string& src, const std::string& target, bool overwrite) { - auto* env = tensorflow::Env::Default(); - tensorflow::Status status; - if (!overwrite && env->FileExists(target).ok()) { - status = tensorflow::errors::AlreadyExists("file already exists"); - } else { - status = env->CopyFile(src, target); - } - tensorflow::MaybeRaiseRegisteredFromStatus(status); - }); - m.def("RenameFile", - [](const std::string& src, const std::string& target, bool overwrite) { - auto* env = tensorflow::Env::Default(); - tensorflow::Status status; - if (!overwrite && env->FileExists(target).ok()) { - status = tensorflow::errors::AlreadyExists("file already exists"); - } else { - status = env->RenameFile(src, target); - } - tensorflow::MaybeRaiseRegisteredFromStatus(status); - }); - m.def("DeleteRecursively", [](const std::string& dirname) { - tensorflow::int64 undeleted_files; - tensorflow::int64 undeleted_dirs; - auto status = tensorflow::Env::Default()->DeleteRecursively( - dirname, &undeleted_files, &undeleted_dirs); - if (status.ok() && (undeleted_files > 0 || undeleted_dirs > 0)) { - status = - tensorflow::errors::PermissionDenied("could not fully delete dir"); - } - tensorflow::MaybeRaiseRegisteredFromStatus(status); - }); - m.def("IsDirectory", [](const std::string& dirname) { - const auto status = tensorflow::Env::Default()->IsDirectory(dirname); - // FAILED_PRECONDITION response means path exists but isn't a dir. - if (tensorflow::errors::IsFailedPrecondition(status)) { - return false; - } - - tensorflow::MaybeRaiseRegisteredFromStatus(status); - return true; - }); - - py::class_(m, "FileStatistics") - .def_readonly("length", &tensorflow::FileStatistics::length) - .def_readonly("mtime_nsec", &tensorflow::FileStatistics::mtime_nsec) - .def_readonly("is_directory", &tensorflow::FileStatistics::is_directory); - - m.def("Stat", [](const std::string& filename) { - std::unique_ptr self( - new tensorflow::FileStatistics); - const auto status = tensorflow::Env::Default()->Stat(filename, self.get()); - tensorflow::MaybeRaiseRegisteredFromStatus(status); - return self.release(); - }); - - using tensorflow::WritableFile; - py::class_(m, "WritableFile") - .def(py::init([](const std::string& filename, const std::string& mode) { - auto* env = tensorflow::Env::Default(); - std::unique_ptr self; - const auto status = mode.find("a") == std::string::npos - ? env->NewWritableFile(filename, &self) - : env->NewAppendableFile(filename, &self); - tensorflow::MaybeRaiseRegisteredFromStatus(status); - return self.release(); - })) - .def("append", - [](WritableFile* self, tensorflow::StringPiece data) { - tensorflow::MaybeRaiseRegisteredFromStatus(self->Append(data)); - }) - // TODO(slebedev): Make WritableFile::Tell const and change self - // to be a reference. - .def("tell", - [](WritableFile* self) { - tensorflow::int64 pos = -1; - const auto status = self->Tell(&pos); - tensorflow::MaybeRaiseRegisteredFromStatus(status); - return pos; - }) - .def("flush", - [](WritableFile* self) { - tensorflow::MaybeRaiseRegisteredFromStatus(self->Flush()); - }) - .def("close", [](WritableFile* self) { - tensorflow::MaybeRaiseRegisteredFromStatus(self->Close()); - }); - - using tensorflow::io::BufferedInputStream; - py::class_(m, "BufferedInputStream") - .def(py::init([](const std::string& filename, size_t buffer_size) { - std::unique_ptr file; - const auto status = - tensorflow::Env::Default()->NewRandomAccessFile(filename, &file); - tensorflow::MaybeRaiseRegisteredFromStatus(status); - std::unique_ptr input_stream( - new tensorflow::io::RandomAccessInputStream(file.release(), - /*owns_file=*/true)); - return new BufferedInputStream(input_stream.release(), buffer_size, - /*owns_input_stream=*/true); - })) - .def("read", - [](BufferedInputStream* self, tensorflow::int64 bytes_to_read) { - tensorflow::tstring result; - const auto status = self->ReadNBytes(bytes_to_read, &result); - if (!status.ok() && !tensorflow::errors::IsOutOfRange(status)) { - result.clear(); - tensorflow::MaybeRaiseRegisteredFromStatus(status); - } - return py::bytes(result); - }) - .def("readline", - [](BufferedInputStream* self) { - return py::bytes(self->ReadLineAsString()); - }) - .def("seek", - [](BufferedInputStream* self, tensorflow::int64 pos) { - tensorflow::MaybeRaiseRegisteredFromStatus(self->Seek(pos)); - }) - .def("tell", [](BufferedInputStream* self) { return self->Tell(); }); -} -} // namespace diff --git a/tensorflow/python/platform/base.i b/tensorflow/python/platform/base.i index 92f7d8bf987..65a56f91b93 100644 --- a/tensorflow/python/platform/base.i +++ b/tensorflow/python/platform/base.i @@ -23,7 +23,6 @@ limitations under the License. #include "tensorflow/c/tf_datatype.h" #include "tensorflow/python/lib/core/py_exception_registry.h" - using tensorflow::int64; using tensorflow::uint64; using tensorflow::string; diff --git a/tensorflow/python/tensorflow.i b/tensorflow/python/tensorflow.i index 52f2eadca25..4e65120353e 100644 --- a/tensorflow/python/tensorflow.i +++ b/tensorflow/python/tensorflow.i @@ -19,6 +19,8 @@ limitations under the License. %include "tensorflow/python/client/tf_session.i" +%include "tensorflow/python/lib/io/file_io.i" + %include "tensorflow/python/lib/io/py_record_reader.i" %include "tensorflow/python/grappler/cluster.i"