Converted file_io.i to pybind11
This is part of a larger effort to deprecate swig and eventually with modularization break pywrap_tensorflow into smaller components. Please refer to https://github.com/tensorflow/community/blob/master/rfcs/20190208-pybind11.md for more information. PiperOrigin-RevId: 286474536 Change-Id: Ic942a4480b1c1a19bdc3d6b65d3272221e47537b
This commit is contained in:
Sergei Lebedev
TensorFlower Gardener
parent
00526b3758
commit
9d82907882
@ -5496,7 +5496,6 @@ tf_py_wrap_cc(
|
||||
"grappler/item.i",
|
||||
"grappler/tf_optimizer.i",
|
||||
"lib/core/strings.i",
|
||||
"lib/io/file_io.i",
|
||||
"lib/io/py_record_reader.i",
|
||||
"platform/base.i",
|
||||
"//tensorflow/compiler/mlir/python:mlir.i",
|
||||
@ -5683,6 +5682,19 @@ cc_import(
|
||||
|
||||
# ** Targets for Windows build (end) **
|
||||
|
||||
tf_python_pybind_extension(
|
||||
name = "_pywrap_file_io",
|
||||
srcs = ["lib/io/file_io_wrapper.cc"],
|
||||
module_name = "_pywrap_file_io",
|
||||
deps = [
|
||||
":pybind11_absl",
|
||||
":pybind11_status",
|
||||
"//tensorflow/core:framework_headers_lib",
|
||||
"//tensorflow/core:protos_all_cc",
|
||||
"@pybind11",
|
||||
],
|
||||
)
|
||||
|
||||
py_library(
|
||||
name = "lib",
|
||||
srcs = [
|
||||
@ -5692,6 +5704,7 @@ py_library(
|
||||
],
|
||||
srcs_version = "PY2AND3",
|
||||
deps = [
|
||||
":_pywrap_file_io",
|
||||
":_pywrap_record_io",
|
||||
":errors",
|
||||
":pywrap_tensorflow",
|
||||
|
||||
@ -1,302 +0,0 @@
|
||||
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
%include "tensorflow/python/lib/core/strings.i"
|
||||
%include "tensorflow/python/platform/base.i"
|
||||
|
||||
%{
|
||||
#include "tensorflow/c/tf_status_helper.h"
|
||||
#include "tensorflow/core/framework/types.h"
|
||||
#include "tensorflow/core/lib/core/status.h"
|
||||
#include "tensorflow/core/lib/core/stringpiece.h"
|
||||
#include "tensorflow/core/lib/io/buffered_inputstream.h"
|
||||
#include "tensorflow/core/lib/io/inputstream_interface.h"
|
||||
#include "tensorflow/core/lib/io/random_inputstream.h"
|
||||
#include "tensorflow/core/lib/io/path.h"
|
||||
#include "tensorflow/core/platform/env.h"
|
||||
#include "tensorflow/core/platform/file_statistics.h"
|
||||
#include "tensorflow/core/platform/file_system.h"
|
||||
#include "tensorflow/core/protobuf/meta_graph.pb.h"
|
||||
%}
|
||||
|
||||
%{
|
||||
inline void FileExists(const string& filename, TF_Status* status) {
|
||||
tensorflow::Status s = tensorflow::Env::Default()->FileExists(filename);
|
||||
if (!s.ok()) {
|
||||
Set_TF_Status_from_Status(status, s);
|
||||
}
|
||||
}
|
||||
|
||||
inline void FileExists(const tensorflow::StringPiece& filename,
|
||||
TF_Status* status) {
|
||||
tensorflow::Status s =
|
||||
tensorflow::Env::Default()->FileExists(string(filename));
|
||||
if (!s.ok()) {
|
||||
Set_TF_Status_from_Status(status, s);
|
||||
}
|
||||
}
|
||||
|
||||
inline void DeleteFile(const string& filename, TF_Status* status) {
|
||||
tensorflow::Status s = tensorflow::Env::Default()->DeleteFile(filename);
|
||||
if (!s.ok()) {
|
||||
Set_TF_Status_from_Status(status, s);
|
||||
}
|
||||
}
|
||||
|
||||
string ReadFileToString(const string& filename, TF_Status* status) {
|
||||
string file_content;
|
||||
tensorflow::Status s = ReadFileToString(tensorflow::Env::Default(),
|
||||
filename, &file_content);
|
||||
if (!s.ok()) {
|
||||
Set_TF_Status_from_Status(status, s);
|
||||
}
|
||||
return file_content;
|
||||
}
|
||||
|
||||
void WriteStringToFile(const string& filename, const string& file_content,
|
||||
TF_Status* status) {
|
||||
tensorflow::Status s = WriteStringToFile(tensorflow::Env::Default(),
|
||||
filename, file_content);
|
||||
if (!s.ok()) {
|
||||
Set_TF_Status_from_Status(status, s);
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<string> GetChildren(const string& dir, TF_Status* status) {
|
||||
std::vector<string> results;
|
||||
tensorflow::Status s = tensorflow::Env::Default()->GetChildren(
|
||||
dir, &results);
|
||||
if (!s.ok()) {
|
||||
Set_TF_Status_from_Status(status, s);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
std::vector<string> GetMatchingFiles(const string& filename, TF_Status* status) {
|
||||
std::vector<string> results;
|
||||
tensorflow::Status s = tensorflow::Env::Default()->GetMatchingPaths(
|
||||
filename, &results);
|
||||
if (!s.ok()) {
|
||||
Set_TF_Status_from_Status(status, s);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
void CreateDir(const string& dirname, TF_Status* status) {
|
||||
tensorflow::Status s = tensorflow::Env::Default()->CreateDir(dirname);
|
||||
if (!s.ok() && s.code() != tensorflow::error::ALREADY_EXISTS) {
|
||||
Set_TF_Status_from_Status(status, s);
|
||||
}
|
||||
}
|
||||
|
||||
void RecursivelyCreateDir(const string& dirname, TF_Status* status) {
|
||||
tensorflow::Status s = tensorflow::Env::Default()->RecursivelyCreateDir(
|
||||
dirname);
|
||||
if (!s.ok()) {
|
||||
Set_TF_Status_from_Status(status, s);
|
||||
}
|
||||
}
|
||||
|
||||
void CopyFile(const string& src, const string& target, bool overwrite,
|
||||
TF_Status* status) {
|
||||
// If overwrite is false and the target file exists then its an error.
|
||||
if (!overwrite && tensorflow::Env::Default()->FileExists(target).ok()) {
|
||||
TF_SetStatus(status, TF_ALREADY_EXISTS, "file already exists");
|
||||
return;
|
||||
}
|
||||
tensorflow::Status s = tensorflow::Env::Default()->CopyFile(src, target);
|
||||
if (!s.ok()) {
|
||||
Set_TF_Status_from_Status(status, s);
|
||||
}
|
||||
}
|
||||
|
||||
void RenameFile(const string& src, const string& target, bool overwrite,
|
||||
TF_Status* status) {
|
||||
// If overwrite is false and the target file exists then its an error.
|
||||
if (!overwrite && tensorflow::Env::Default()->FileExists(target).ok()) {
|
||||
TF_SetStatus(status, TF_ALREADY_EXISTS, "file already exists");
|
||||
return;
|
||||
}
|
||||
tensorflow::Status s = tensorflow::Env::Default()->RenameFile(src, target);
|
||||
if (!s.ok()) {
|
||||
Set_TF_Status_from_Status(status, s);
|
||||
}
|
||||
}
|
||||
|
||||
using tensorflow::int64;
|
||||
|
||||
void DeleteRecursively(const string& dirname, TF_Status* status) {
|
||||
int64 undeleted_files, undeleted_dirs;
|
||||
tensorflow::Status s = tensorflow::Env::Default()->DeleteRecursively(
|
||||
dirname, &undeleted_files, &undeleted_dirs);
|
||||
if (!s.ok()) {
|
||||
Set_TF_Status_from_Status(status, s);
|
||||
return;
|
||||
}
|
||||
if (undeleted_files > 0 || undeleted_dirs > 0) {
|
||||
TF_SetStatus(status, TF_PERMISSION_DENIED, "could not fully delete dir");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
bool IsDirectory(const string& dirname, TF_Status* out_status) {
|
||||
tensorflow::Status status = tensorflow::Env::Default()->IsDirectory(dirname);
|
||||
if (status.ok()) {
|
||||
return true;
|
||||
}
|
||||
// FAILED_PRECONDITION Status response means path exists but isn't a dir.
|
||||
if (status.code() != tensorflow::error::FAILED_PRECONDITION) {
|
||||
Set_TF_Status_from_Status(out_status, status);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
using tensorflow::FileStatistics;
|
||||
|
||||
void Stat(const string& filename, FileStatistics* stats, TF_Status* status) {
|
||||
tensorflow::Status s = tensorflow::Env::Default()->Stat(filename,
|
||||
stats);
|
||||
if (!s.ok()) {
|
||||
Set_TF_Status_from_Status(status, s);
|
||||
}
|
||||
}
|
||||
|
||||
tensorflow::io::BufferedInputStream* CreateBufferedInputStream(
|
||||
const string& filename, size_t buffer_size, TF_Status* status) {
|
||||
std::unique_ptr<tensorflow::RandomAccessFile> file;
|
||||
tensorflow::Status s =
|
||||
tensorflow::Env::Default()->NewRandomAccessFile(filename, &file);
|
||||
if (!s.ok()) {
|
||||
Set_TF_Status_from_Status(status, s);
|
||||
return nullptr;
|
||||
}
|
||||
std::unique_ptr<tensorflow::io::RandomAccessInputStream> input_stream(
|
||||
new tensorflow::io::RandomAccessInputStream(
|
||||
file.release(), true /* owns_file */));
|
||||
std::unique_ptr<tensorflow::io::BufferedInputStream> buffered_input_stream(
|
||||
new tensorflow::io::BufferedInputStream(
|
||||
input_stream.release(), buffer_size, true /* owns_input_stream */));
|
||||
return buffered_input_stream.release();
|
||||
}
|
||||
|
||||
tensorflow::WritableFile* CreateWritableFile(
|
||||
const string& filename, const string& mode, TF_Status* status) {
|
||||
std::unique_ptr<tensorflow::WritableFile> file;
|
||||
tensorflow::Status s;
|
||||
if (mode.find("a") != std::string::npos) {
|
||||
s = tensorflow::Env::Default()->NewAppendableFile(filename, &file);
|
||||
} else {
|
||||
s = tensorflow::Env::Default()->NewWritableFile(filename, &file);
|
||||
}
|
||||
if (!s.ok()) {
|
||||
Set_TF_Status_from_Status(status, s);
|
||||
return nullptr;
|
||||
}
|
||||
return file.release();
|
||||
}
|
||||
|
||||
void AppendToFile(const string& file_content, tensorflow::WritableFile* file,
|
||||
TF_Status* status) {
|
||||
tensorflow::Status s = file->Append(file_content);
|
||||
if (!s.ok()) {
|
||||
Set_TF_Status_from_Status(status, s);
|
||||
}
|
||||
}
|
||||
|
||||
int64 TellFile(tensorflow::WritableFile* file, TF_Status* status) {
|
||||
int64 position = -1;
|
||||
tensorflow::Status s = file->Tell(&position);
|
||||
if (!s.ok()) {
|
||||
Set_TF_Status_from_Status(status, s);
|
||||
}
|
||||
return position;
|
||||
}
|
||||
|
||||
|
||||
string ReadFromStream(tensorflow::io::BufferedInputStream* stream,
|
||||
size_t bytes,
|
||||
TF_Status* status) {
|
||||
tensorflow::tstring result;
|
||||
tensorflow::Status s = stream->ReadNBytes(bytes, &result);
|
||||
if (!s.ok() && s.code() != tensorflow::error::OUT_OF_RANGE) {
|
||||
Set_TF_Status_from_Status(status, s);
|
||||
result.clear();
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
%}
|
||||
|
||||
// Ensure that the returned object is destroyed when its wrapper is
|
||||
// garbage collected.
|
||||
%newobject CreateBufferedInputStream;
|
||||
%newobject CreateWritableFile;
|
||||
|
||||
// Wrap the above functions.
|
||||
inline void FileExists(const string& filename, TF_Status* status);
|
||||
inline void DeleteFile(const string& filename, TF_Status* status);
|
||||
string ReadFileToString(const string& filename, TF_Status* status);
|
||||
void WriteStringToFile(const string& filename, const string& file_content,
|
||||
TF_Status* status);
|
||||
std::vector<string> GetChildren(const string& dir, TF_Status* status);
|
||||
std::vector<string> GetMatchingFiles(const string& filename,
|
||||
TF_Status* status);
|
||||
void CreateDir(const string& dirname, TF_Status* status);
|
||||
void RecursivelyCreateDir(const string& dirname, TF_Status* status);
|
||||
void CopyFile(const string& oldpath, const string& newpath, bool overwrite,
|
||||
TF_Status* status);
|
||||
void RenameFile(const string& oldname, const string& newname, bool overwrite,
|
||||
TF_Status* status);
|
||||
void DeleteRecursively(const string& dirname, TF_Status* status);
|
||||
bool IsDirectory(const string& dirname, TF_Status* out_status);
|
||||
void Stat(const string& filename, tensorflow::FileStatistics* stats,
|
||||
TF_Status* status);
|
||||
tensorflow::io::BufferedInputStream* CreateBufferedInputStream(
|
||||
const string& filename, size_t buffer_size, TF_Status* status);
|
||||
tensorflow::WritableFile* CreateWritableFile(const string& filename,
|
||||
const string& mode,
|
||||
TF_Status* status);
|
||||
void AppendToFile(const string& file_content, tensorflow::WritableFile* file,
|
||||
TF_Status* status);
|
||||
int64 TellFile(tensorflow::WritableFile* file, TF_Status* status);
|
||||
string ReadFromStream(tensorflow::io::BufferedInputStream* stream,
|
||||
size_t bytes,
|
||||
TF_Status* status);
|
||||
|
||||
%ignore tensorflow::Status::operator=;
|
||||
%include "tensorflow/core/platform/status.h"
|
||||
|
||||
%ignoreall
|
||||
%unignore tensorflow::io;
|
||||
%unignore tensorflow::io::BufferedInputStream;
|
||||
%unignore tensorflow::io::BufferedInputStream::~BufferedInputStream;
|
||||
%unignore tensorflow::io::BufferedInputStream::ReadLineAsString;
|
||||
%unignore tensorflow::io::BufferedInputStream::Seek;
|
||||
%unignore tensorflow::io::BufferedInputStream::Tell;
|
||||
%unignore tensorflow::WritableFile;
|
||||
%unignore tensorflow::WritableFile::Close;
|
||||
%unignore tensorflow::WritableFile::Flush;
|
||||
%unignore tensorflow::WritableFile::~WritableFile;
|
||||
%include "tensorflow/core/platform/file_system.h"
|
||||
%include "tensorflow/core/lib/io/inputstream_interface.h"
|
||||
%include "tensorflow/core/lib/io/buffered_inputstream.h"
|
||||
%unignoreall
|
||||
|
||||
%include "tensorflow/c/tf_status_helper.h"
|
||||
|
||||
%ignore tensorflow::io::internal::JoinPathImpl;
|
||||
%include "tensorflow/core/lib/io/path.h"
|
||||
|
||||
%include "tensorflow/core/platform/file_statistics.h"
|
||||
@ -12,11 +12,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""File IO methods that wrap the C++ FileSystem API.
|
||||
|
||||
The C++ FileSystem API is SWIG wrapped in file_io.i. These functions call those
|
||||
to accomplish basic File IO operations.
|
||||
"""
|
||||
"""File IO methods that wrap the C++ FileSystem API."""
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
@ -27,8 +23,7 @@ import uuid
|
||||
|
||||
import six
|
||||
|
||||
from tensorflow.python import pywrap_tensorflow
|
||||
from tensorflow.python.framework import c_api_util
|
||||
from tensorflow.python import _pywrap_file_io
|
||||
from tensorflow.python.framework import errors
|
||||
from tensorflow.python.util import compat
|
||||
from tensorflow.python.util import deprecation
|
||||
@ -80,15 +75,15 @@ class FileIO(object):
|
||||
if not self._read_check_passed:
|
||||
raise errors.PermissionDeniedError(None, None,
|
||||
"File isn't open for reading")
|
||||
self._read_buf = pywrap_tensorflow.CreateBufferedInputStream(
|
||||
compat.as_bytes(self.__name), 1024 * 512)
|
||||
self._read_buf = _pywrap_file_io.BufferedInputStream(
|
||||
self.__name, 1024 * 512)
|
||||
|
||||
def _prewrite_check(self):
|
||||
if not self._writable_file:
|
||||
if not self._write_check_passed:
|
||||
raise errors.PermissionDeniedError(None, None,
|
||||
"File isn't open for writing")
|
||||
self._writable_file = pywrap_tensorflow.CreateWritableFile(
|
||||
self._writable_file = _pywrap_file_io.WritableFile(
|
||||
compat.as_bytes(self.__name), compat.as_bytes(self.__mode))
|
||||
|
||||
def _prepare_value(self, val):
|
||||
@ -104,8 +99,7 @@ class FileIO(object):
|
||||
def write(self, file_content):
|
||||
"""Writes file_content to the file. Appends to the end of the file."""
|
||||
self._prewrite_check()
|
||||
pywrap_tensorflow.AppendToFile(
|
||||
compat.as_bytes(file_content), self._writable_file)
|
||||
self._writable_file.append(compat.as_bytes(file_content))
|
||||
|
||||
def read(self, n=-1):
|
||||
"""Returns the contents of a file as a string.
|
||||
@ -124,8 +118,7 @@ class FileIO(object):
|
||||
length = self.size() - self.tell()
|
||||
else:
|
||||
length = n
|
||||
return self._prepare_value(
|
||||
pywrap_tensorflow.ReadFromStream(self._read_buf, length))
|
||||
return self._prepare_value(self._read_buf.read(length))
|
||||
|
||||
@deprecation.deprecated_args(
|
||||
None, "position is deprecated in favor of the offset argument.",
|
||||
@ -158,25 +151,23 @@ class FileIO(object):
|
||||
if position is not None:
|
||||
offset = position
|
||||
|
||||
with errors.raise_exception_on_not_ok_status() as status:
|
||||
if whence == 0:
|
||||
pass
|
||||
elif whence == 1:
|
||||
offset += self.tell()
|
||||
elif whence == 2:
|
||||
offset += self.size()
|
||||
else:
|
||||
raise errors.InvalidArgumentError(
|
||||
None, None,
|
||||
"Invalid whence argument: {}. Valid values are 0, 1, or 2.".format(
|
||||
whence))
|
||||
ret_status = self._read_buf.Seek(offset)
|
||||
pywrap_tensorflow.Set_TF_Status_from_Status(status, ret_status)
|
||||
if whence == 0:
|
||||
pass
|
||||
elif whence == 1:
|
||||
offset += self.tell()
|
||||
elif whence == 2:
|
||||
offset += self.size()
|
||||
else:
|
||||
raise errors.InvalidArgumentError(
|
||||
None, None,
|
||||
"Invalid whence argument: {}. Valid values are 0, 1, or 2.".format(
|
||||
whence))
|
||||
self._read_buf.seek(offset)
|
||||
|
||||
def readline(self):
|
||||
r"""Reads the next line from the file. Leaves the '\n' at the end."""
|
||||
self._preread_check()
|
||||
return self._prepare_value(self._read_buf.ReadLineAsString())
|
||||
return self._prepare_value(self._read_buf.readline())
|
||||
|
||||
def readlines(self):
|
||||
"""Returns all lines from the file in a list."""
|
||||
@ -193,11 +184,11 @@ class FileIO(object):
|
||||
"""Returns the current position in the file."""
|
||||
if self._read_check_passed:
|
||||
self._preread_check()
|
||||
return self._read_buf.Tell()
|
||||
return self._read_buf.tell()
|
||||
else:
|
||||
self._prewrite_check()
|
||||
|
||||
return pywrap_tensorflow.TellFile(self._writable_file)
|
||||
return self._writable_file.tell()
|
||||
|
||||
def __enter__(self):
|
||||
"""Make usable with "with" statement."""
|
||||
@ -227,18 +218,14 @@ class FileIO(object):
|
||||
data would survive an application crash but not necessarily an OS crash.
|
||||
"""
|
||||
if self._writable_file:
|
||||
with errors.raise_exception_on_not_ok_status() as status:
|
||||
ret_status = self._writable_file.Flush()
|
||||
pywrap_tensorflow.Set_TF_Status_from_Status(status, ret_status)
|
||||
self._writable_file.flush()
|
||||
|
||||
def close(self):
|
||||
"""Closes FileIO. Should be called for the WritableFile to be flushed."""
|
||||
self._read_buf = None
|
||||
if self._writable_file:
|
||||
with errors.raise_exception_on_not_ok_status() as status:
|
||||
ret_status = self._writable_file.Close()
|
||||
pywrap_tensorflow.Set_TF_Status_from_Status(status, ret_status)
|
||||
self._writable_file = None
|
||||
self._writable_file.close()
|
||||
self._writable_file = None
|
||||
|
||||
def seekable(self):
|
||||
"""Returns True as FileIO supports random access ops of seek()/tell()"""
|
||||
@ -277,7 +264,7 @@ def file_exists_v2(path):
|
||||
errors.OpError: Propagates any errors reported by the FileSystem API.
|
||||
"""
|
||||
try:
|
||||
pywrap_tensorflow.FileExists(compat.as_bytes(path))
|
||||
_pywrap_file_io.FileExists(compat.as_bytes(path))
|
||||
except errors.NotFoundError:
|
||||
return False
|
||||
return True
|
||||
@ -308,7 +295,7 @@ def delete_file_v2(path):
|
||||
errors.OpError: Propagates any errors reported by the FileSystem API. E.g.,
|
||||
`NotFoundError` if the path does not exist.
|
||||
"""
|
||||
pywrap_tensorflow.DeleteFile(compat.as_bytes(path))
|
||||
_pywrap_file_io.DeleteFile(compat.as_bytes(path))
|
||||
|
||||
|
||||
def read_file_to_string(filename, binary_mode=False):
|
||||
@ -380,7 +367,7 @@ def get_matching_files_v2(pattern):
|
||||
return [
|
||||
# Convert the filenames to string from bytes.
|
||||
compat.as_str_any(matching_filename)
|
||||
for matching_filename in pywrap_tensorflow.GetMatchingFiles(
|
||||
for matching_filename in _pywrap_file_io.GetMatchingFiles(
|
||||
compat.as_bytes(pattern))
|
||||
]
|
||||
else:
|
||||
@ -388,7 +375,7 @@ def get_matching_files_v2(pattern):
|
||||
# Convert the filenames to string from bytes.
|
||||
compat.as_str_any(matching_filename) # pylint: disable=g-complex-comprehension
|
||||
for single_filename in pattern
|
||||
for matching_filename in pywrap_tensorflow.GetMatchingFiles(
|
||||
for matching_filename in _pywrap_file_io.GetMatchingFiles(
|
||||
compat.as_bytes(single_filename))
|
||||
]
|
||||
|
||||
@ -422,7 +409,7 @@ def create_dir_v2(path):
|
||||
Raises:
|
||||
errors.OpError: If the operation fails.
|
||||
"""
|
||||
pywrap_tensorflow.CreateDir(compat.as_bytes(path))
|
||||
_pywrap_file_io.CreateDir(compat.as_bytes(path))
|
||||
|
||||
|
||||
@tf_export(v1=["gfile.MakeDirs"])
|
||||
@ -452,7 +439,7 @@ def recursive_create_dir_v2(path):
|
||||
Raises:
|
||||
errors.OpError: If the operation fails.
|
||||
"""
|
||||
pywrap_tensorflow.RecursivelyCreateDir(compat.as_bytes(path))
|
||||
_pywrap_file_io.RecursivelyCreateDir(compat.as_bytes(path))
|
||||
|
||||
|
||||
@tf_export(v1=["gfile.Copy"])
|
||||
@ -484,7 +471,7 @@ def copy_v2(src, dst, overwrite=False):
|
||||
Raises:
|
||||
errors.OpError: If the operation fails.
|
||||
"""
|
||||
pywrap_tensorflow.CopyFile(
|
||||
_pywrap_file_io.CopyFile(
|
||||
compat.as_bytes(src), compat.as_bytes(dst), overwrite)
|
||||
|
||||
|
||||
@ -517,7 +504,7 @@ def rename_v2(src, dst, overwrite=False):
|
||||
Raises:
|
||||
errors.OpError: If the operation fails.
|
||||
"""
|
||||
pywrap_tensorflow.RenameFile(
|
||||
_pywrap_file_io.RenameFile(
|
||||
compat.as_bytes(src), compat.as_bytes(dst), overwrite)
|
||||
|
||||
|
||||
@ -568,7 +555,7 @@ def delete_recursively_v2(path):
|
||||
Raises:
|
||||
errors.OpError: If the operation fails.
|
||||
"""
|
||||
pywrap_tensorflow.DeleteRecursively(compat.as_bytes(path))
|
||||
_pywrap_file_io.DeleteRecursively(compat.as_bytes(path))
|
||||
|
||||
|
||||
@tf_export(v1=["gfile.IsDirectory"])
|
||||
@ -594,8 +581,10 @@ def is_directory_v2(path):
|
||||
Returns:
|
||||
True, if the path is a directory; False otherwise
|
||||
"""
|
||||
status = c_api_util.ScopedTFStatus()
|
||||
return pywrap_tensorflow.IsDirectory(compat.as_bytes(path), status)
|
||||
try:
|
||||
return _pywrap_file_io.IsDirectory(compat.as_bytes(path))
|
||||
except errors.OpError:
|
||||
return False
|
||||
|
||||
|
||||
@tf_export(v1=["gfile.ListDirectory"])
|
||||
@ -643,7 +632,7 @@ def list_directory_v2(path):
|
||||
# vector of string should be interpreted as strings, not bytes.
|
||||
return [
|
||||
compat.as_str_any(filename)
|
||||
for filename in pywrap_tensorflow.GetChildren(compat.as_bytes(path))
|
||||
for filename in _pywrap_file_io.GetChildren(compat.as_bytes(path))
|
||||
]
|
||||
|
||||
|
||||
@ -742,9 +731,7 @@ def stat_v2(path):
|
||||
Raises:
|
||||
errors.OpError: If the operation fails.
|
||||
"""
|
||||
file_statistics = pywrap_tensorflow.FileStatistics()
|
||||
pywrap_tensorflow.Stat(compat.as_bytes(path), file_statistics)
|
||||
return file_statistics
|
||||
return _pywrap_file_io.Stat(path)
|
||||
|
||||
|
||||
def filecmp(filename_a, filename_b):
|
||||
|
||||
205
tensorflow/python/lib/io/file_io_wrapper.cc
Normal file
205
tensorflow/python/lib/io/file_io_wrapper.cc
Normal file
@ -0,0 +1,205 @@
|
||||
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "include/pybind11/pybind11.h"
|
||||
#include "include/pybind11/stl.h"
|
||||
#include "tensorflow/core/lib/core/error_codes.pb.h"
|
||||
#include "tensorflow/core/lib/core/errors.h"
|
||||
#include "tensorflow/core/lib/core/status.h"
|
||||
#include "tensorflow/core/lib/io/buffered_inputstream.h"
|
||||
#include "tensorflow/core/lib/io/random_inputstream.h"
|
||||
#include "tensorflow/core/platform/env.h"
|
||||
#include "tensorflow/core/platform/file_statistics.h"
|
||||
#include "tensorflow/core/platform/file_system.h"
|
||||
#include "tensorflow/core/platform/stringpiece.h"
|
||||
#include "tensorflow/core/platform/tstring.h"
|
||||
#include "tensorflow/python/lib/core/pybind11_absl.h"
|
||||
#include "tensorflow/python/lib/core/pybind11_status.h"
|
||||
|
||||
namespace {
|
||||
namespace py = pybind11;
|
||||
|
||||
PYBIND11_MODULE(_pywrap_file_io, m) {
|
||||
m.def("FileExists", [](const std::string& filename) {
|
||||
tensorflow::MaybeRaiseRegisteredFromStatus(
|
||||
tensorflow::Env::Default()->FileExists(filename));
|
||||
});
|
||||
m.def("DeleteFile", [](const std::string& filename) {
|
||||
tensorflow::MaybeRaiseRegisteredFromStatus(
|
||||
tensorflow::Env::Default()->DeleteFile(filename));
|
||||
});
|
||||
m.def("ReadFileToString", [](const std::string& filename) {
|
||||
std::string data;
|
||||
const auto status =
|
||||
ReadFileToString(tensorflow::Env::Default(), filename, &data);
|
||||
tensorflow::MaybeRaiseRegisteredFromStatus(status);
|
||||
return py::bytes(data);
|
||||
});
|
||||
m.def("WriteStringToFile",
|
||||
[](const std::string& filename, tensorflow::StringPiece data) {
|
||||
return WriteStringToFile(tensorflow::Env::Default(), filename, data);
|
||||
});
|
||||
m.def("GetChildren", [](const std::string& dirname) {
|
||||
std::vector<std::string> results;
|
||||
const auto status =
|
||||
tensorflow::Env::Default()->GetChildren(dirname, &results);
|
||||
tensorflow::MaybeRaiseRegisteredFromStatus(status);
|
||||
return results;
|
||||
});
|
||||
m.def("GetMatchingFiles", [](const std::string& pattern) {
|
||||
std::vector<std::string> results;
|
||||
const auto status =
|
||||
tensorflow::Env::Default()->GetMatchingPaths(pattern, &results);
|
||||
tensorflow::MaybeRaiseRegisteredFromStatus(status);
|
||||
return results;
|
||||
});
|
||||
m.def("CreateDir", [](const std::string& dirname) {
|
||||
const auto status = tensorflow::Env::Default()->CreateDir(dirname);
|
||||
if (tensorflow::errors::IsAlreadyExists(status)) {
|
||||
return;
|
||||
}
|
||||
tensorflow::MaybeRaiseRegisteredFromStatus(status);
|
||||
});
|
||||
m.def("RecursivelyCreateDir", [](const std::string& dirname) {
|
||||
tensorflow::MaybeRaiseRegisteredFromStatus(
|
||||
tensorflow::Env::Default()->RecursivelyCreateDir(dirname));
|
||||
});
|
||||
m.def("CopyFile",
|
||||
[](const std::string& src, const std::string& target, bool overwrite) {
|
||||
auto* env = tensorflow::Env::Default();
|
||||
tensorflow::Status status;
|
||||
if (!overwrite && env->FileExists(target).ok()) {
|
||||
status = tensorflow::errors::AlreadyExists("file already exists");
|
||||
} else {
|
||||
status = env->CopyFile(src, target);
|
||||
}
|
||||
tensorflow::MaybeRaiseRegisteredFromStatus(status);
|
||||
});
|
||||
m.def("RenameFile",
|
||||
[](const std::string& src, const std::string& target, bool overwrite) {
|
||||
auto* env = tensorflow::Env::Default();
|
||||
tensorflow::Status status;
|
||||
if (!overwrite && env->FileExists(target).ok()) {
|
||||
status = tensorflow::errors::AlreadyExists("file already exists");
|
||||
} else {
|
||||
status = env->RenameFile(src, target);
|
||||
}
|
||||
tensorflow::MaybeRaiseRegisteredFromStatus(status);
|
||||
});
|
||||
m.def("DeleteRecursively", [](const std::string& dirname) {
|
||||
tensorflow::int64 undeleted_files;
|
||||
tensorflow::int64 undeleted_dirs;
|
||||
auto status = tensorflow::Env::Default()->DeleteRecursively(
|
||||
dirname, &undeleted_files, &undeleted_dirs);
|
||||
if (status.ok() && (undeleted_files > 0 || undeleted_dirs > 0)) {
|
||||
status =
|
||||
tensorflow::errors::PermissionDenied("could not fully delete dir");
|
||||
}
|
||||
tensorflow::MaybeRaiseRegisteredFromStatus(status);
|
||||
});
|
||||
m.def("IsDirectory", [](const std::string& dirname) {
|
||||
const auto status = tensorflow::Env::Default()->IsDirectory(dirname);
|
||||
// FAILED_PRECONDITION response means path exists but isn't a dir.
|
||||
if (tensorflow::errors::IsFailedPrecondition(status)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
tensorflow::MaybeRaiseRegisteredFromStatus(status);
|
||||
return true;
|
||||
});
|
||||
|
||||
py::class_<tensorflow::FileStatistics>(m, "FileStatistics")
|
||||
.def_readonly("length", &tensorflow::FileStatistics::length)
|
||||
.def_readonly("mtime_nsec", &tensorflow::FileStatistics::mtime_nsec)
|
||||
.def_readonly("is_directory", &tensorflow::FileStatistics::is_directory);
|
||||
|
||||
m.def("Stat", [](const std::string& filename) {
|
||||
std::unique_ptr<tensorflow::FileStatistics> self(
|
||||
new tensorflow::FileStatistics);
|
||||
const auto status = tensorflow::Env::Default()->Stat(filename, self.get());
|
||||
tensorflow::MaybeRaiseRegisteredFromStatus(status);
|
||||
return self.release();
|
||||
});
|
||||
|
||||
using tensorflow::WritableFile;
|
||||
py::class_<WritableFile>(m, "WritableFile")
|
||||
.def(py::init([](const std::string& filename, const std::string& mode) {
|
||||
auto* env = tensorflow::Env::Default();
|
||||
std::unique_ptr<WritableFile> self;
|
||||
const auto status = mode.find("a") == std::string::npos
|
||||
? env->NewWritableFile(filename, &self)
|
||||
: env->NewAppendableFile(filename, &self);
|
||||
tensorflow::MaybeRaiseRegisteredFromStatus(status);
|
||||
return self.release();
|
||||
}))
|
||||
.def("append",
|
||||
[](WritableFile* self, tensorflow::StringPiece data) {
|
||||
tensorflow::MaybeRaiseRegisteredFromStatus(self->Append(data));
|
||||
})
|
||||
// TODO(slebedev): Make WritableFile::Tell const and change self
|
||||
// to be a reference.
|
||||
.def("tell",
|
||||
[](WritableFile* self) {
|
||||
tensorflow::int64 pos = -1;
|
||||
const auto status = self->Tell(&pos);
|
||||
tensorflow::MaybeRaiseRegisteredFromStatus(status);
|
||||
return pos;
|
||||
})
|
||||
.def("flush",
|
||||
[](WritableFile* self) {
|
||||
tensorflow::MaybeRaiseRegisteredFromStatus(self->Flush());
|
||||
})
|
||||
.def("close", [](WritableFile* self) {
|
||||
tensorflow::MaybeRaiseRegisteredFromStatus(self->Close());
|
||||
});
|
||||
|
||||
using tensorflow::io::BufferedInputStream;
|
||||
py::class_<BufferedInputStream>(m, "BufferedInputStream")
|
||||
.def(py::init([](const std::string& filename, size_t buffer_size) {
|
||||
std::unique_ptr<tensorflow::RandomAccessFile> file;
|
||||
const auto status =
|
||||
tensorflow::Env::Default()->NewRandomAccessFile(filename, &file);
|
||||
tensorflow::MaybeRaiseRegisteredFromStatus(status);
|
||||
std::unique_ptr<tensorflow::io::RandomAccessInputStream> input_stream(
|
||||
new tensorflow::io::RandomAccessInputStream(file.release(),
|
||||
/*owns_file=*/true));
|
||||
return new BufferedInputStream(input_stream.release(), buffer_size,
|
||||
/*owns_input_stream=*/true);
|
||||
}))
|
||||
.def("read",
|
||||
[](BufferedInputStream* self, tensorflow::int64 bytes_to_read) {
|
||||
tensorflow::tstring result;
|
||||
const auto status = self->ReadNBytes(bytes_to_read, &result);
|
||||
if (!status.ok() && !tensorflow::errors::IsOutOfRange(status)) {
|
||||
result.clear();
|
||||
tensorflow::MaybeRaiseRegisteredFromStatus(status);
|
||||
}
|
||||
return py::bytes(result);
|
||||
})
|
||||
.def("readline",
|
||||
[](BufferedInputStream* self) {
|
||||
return py::bytes(self->ReadLineAsString());
|
||||
})
|
||||
.def("seek",
|
||||
[](BufferedInputStream* self, tensorflow::int64 pos) {
|
||||
tensorflow::MaybeRaiseRegisteredFromStatus(self->Seek(pos));
|
||||
})
|
||||
.def("tell", [](BufferedInputStream* self) { return self->Tell(); });
|
||||
}
|
||||
} // namespace
|
||||
@ -23,6 +23,7 @@ limitations under the License.
|
||||
#include "tensorflow/c/tf_datatype.h"
|
||||
#include "tensorflow/python/lib/core/py_exception_registry.h"
|
||||
|
||||
using tensorflow::int64;
|
||||
using tensorflow::uint64;
|
||||
using tensorflow::string;
|
||||
|
||||
|
||||
@ -19,8 +19,6 @@ limitations under the License.
|
||||
|
||||
%include "tensorflow/python/client/tf_session.i"
|
||||
|
||||
%include "tensorflow/python/lib/io/file_io.i"
|
||||
|
||||
%include "tensorflow/python/lib/io/py_record_reader.i"
|
||||
|
||||
%include "tensorflow/python/grappler/cluster.i"
|
||||
@ -29,3 +27,10 @@ limitations under the License.
|
||||
%include "tensorflow/python/grappler/cost_analyzer.i"
|
||||
|
||||
%include "tensorflow/compiler/mlir/python/mlir.i"
|
||||
|
||||
// TODO(slebedev): This is a temporary workaround for projects implicitly
|
||||
// relying on TensorFlow exposing tensorflow::Status.
|
||||
%unignoreall
|
||||
|
||||
%ignore tensorflow::Status::operator=;
|
||||
%include "tensorflow/core/platform/status.h"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user