From 46a8319ee74337182c7aadf80acbeb7f01eb7ffd Mon Sep 17 00:00:00 2001 From: Sami Kama Date: Tue, 4 Aug 2020 18:04:52 -0700 Subject: [PATCH] Moving rest of the filesystems to Transactional API --- .../platform/default/posix_file_system.cc | 59 +++++++------- .../core/platform/default/posix_file_system.h | 68 +++++++---------- .../platform/hadoop/hadoop_file_system.cc | 57 +++++++------- .../core/platform/hadoop/hadoop_file_system.h | 67 ++++++---------- .../platform/windows/windows_file_system.cc | 60 +++++++-------- .../platform/windows/windows_file_system.h | 76 +++++++------------ tensorflow/core/util/memmapped_file_system.cc | 62 +++++++-------- tensorflow/core/util/memmapped_file_system.h | 63 ++++++--------- .../asset_manager_filesystem.cc | 61 +++++++-------- .../asset_manager_filesystem.h | 66 +++++++--------- 10 files changed, 276 insertions(+), 363 deletions(-) diff --git a/tensorflow/core/platform/default/posix_file_system.cc b/tensorflow/core/platform/default/posix_file_system.cc index 8533e34fc3f..18fea3fe15d 100644 --- a/tensorflow/core/platform/default/posix_file_system.cc +++ b/tensorflow/core/platform/default/posix_file_system.cc @@ -178,8 +178,8 @@ class PosixReadOnlyMemoryRegion : public ReadOnlyMemoryRegion { }; Status PosixFileSystem::NewRandomAccessFile( - const string& fname, - std::unique_ptr* result /*, TransactionToken* token */) { + const string& fname, TransactionToken* token, + std::unique_ptr* result) { string translated_fname = TranslateName(fname); Status s; int fd = open(translated_fname.c_str(), O_RDONLY); @@ -191,9 +191,9 @@ Status PosixFileSystem::NewRandomAccessFile( return s; } -Status PosixFileSystem::NewWritableFile( - const string& fname, - std::unique_ptr* result /*, TransactionToken* token */) { +Status PosixFileSystem::NewWritableFile(const string& fname, + TransactionToken* token, + std::unique_ptr* result) { string translated_fname = TranslateName(fname); Status s; FILE* f = fopen(translated_fname.c_str(), "w"); @@ -206,8 +206,8 @@ Status PosixFileSystem::NewWritableFile( } Status PosixFileSystem::NewAppendableFile( - const string& fname, - std::unique_ptr* result /*, TransactionToken* token */) { + const string& fname, TransactionToken* token, + std::unique_ptr* result) { string translated_fname = TranslateName(fname); Status s; FILE* f = fopen(translated_fname.c_str(), "a"); @@ -220,8 +220,8 @@ Status PosixFileSystem::NewAppendableFile( } Status PosixFileSystem::NewReadOnlyMemoryRegionFromFile( - const string& fname, std::unique_ptr* - result /*, TransactionToken* token */) { + const string& fname, TransactionToken* token, + std::unique_ptr* result) { string translated_fname = TranslateName(fname); Status s = Status::OK(); int fd = open(translated_fname.c_str(), O_RDONLY); @@ -244,17 +244,16 @@ Status PosixFileSystem::NewReadOnlyMemoryRegionFromFile( return s; } -Status PosixFileSystem::FileExists( - const string& fname /*, TransactionToken* token */) { +Status PosixFileSystem::FileExists(const string& fname, + TransactionToken* token) { if (access(TranslateName(fname).c_str(), F_OK) == 0) { return Status::OK(); } return errors::NotFound(fname, " not found"); } -Status PosixFileSystem::GetChildren( - const string& dir, - std::vector* result /*, TransactionToken* token */) { +Status PosixFileSystem::GetChildren(const string& dir, TransactionToken* token, + std::vector* result) { string translated_dir = TranslateName(dir); result->clear(); DIR* d = opendir(translated_dir.c_str()); @@ -274,14 +273,14 @@ Status PosixFileSystem::GetChildren( return Status::OK(); } -Status PosixFileSystem::GetMatchingPaths( - const string& pattern, - std::vector* results /*, TransactionToken* token */) { +Status PosixFileSystem::GetMatchingPaths(const string& pattern, + TransactionToken* token, + std::vector* results) { return internal::GetMatchingPaths(this, Env::Default(), pattern, results); } -Status PosixFileSystem::DeleteFile( - const string& fname /*, TransactionToken* token */) { +Status PosixFileSystem::DeleteFile(const string& fname, + TransactionToken* token) { Status result; if (unlink(TranslateName(fname).c_str()) != 0) { result = IOError(fname, errno); @@ -289,8 +288,7 @@ Status PosixFileSystem::DeleteFile( return result; } -Status PosixFileSystem::CreateDir( - const string& name /*, TransactionToken* token */) { +Status PosixFileSystem::CreateDir(const string& name, TransactionToken* token) { string translated = TranslateName(name); if (translated.empty()) { return errors::AlreadyExists(name); @@ -301,8 +299,7 @@ Status PosixFileSystem::CreateDir( return Status::OK(); } -Status PosixFileSystem::DeleteDir( - const string& name /*, TransactionToken* token */) { +Status PosixFileSystem::DeleteDir(const string& name, TransactionToken* token) { Status result; if (rmdir(TranslateName(name).c_str()) != 0) { result = IOError(name, errno); @@ -310,8 +307,8 @@ Status PosixFileSystem::DeleteDir( return result; } -Status PosixFileSystem::GetFileSize( - const string& fname, uint64* size /*, TransactionToken* token */) { +Status PosixFileSystem::GetFileSize(const string& fname, + TransactionToken* token, uint64* size) { Status s; struct stat sbuf; if (stat(TranslateName(fname).c_str(), &sbuf) != 0) { @@ -323,8 +320,8 @@ Status PosixFileSystem::GetFileSize( return s; } -Status PosixFileSystem::Stat( - const string& fname, FileStatistics* stats /*, TransactionToken* token */) { +Status PosixFileSystem::Stat(const string& fname, TransactionToken* token, + FileStatistics* stats) { Status s; struct stat sbuf; if (stat(TranslateName(fname).c_str(), &sbuf) != 0) { @@ -337,8 +334,8 @@ Status PosixFileSystem::Stat( return s; } -Status PosixFileSystem::RenameFile( - const string& src, const string& target /*, TransactionToken* token */) { +Status PosixFileSystem::RenameFile(const string& src, const string& target, + TransactionToken* token) { Status result; if (rename(TranslateName(src).c_str(), TranslateName(target).c_str()) != 0) { result = IOError(src, errno); @@ -346,8 +343,8 @@ Status PosixFileSystem::RenameFile( return result; } -Status PosixFileSystem::CopyFile( - const string& src, const string& target /*, TransactionToken* token */) { +Status PosixFileSystem::CopyFile(const string& src, const string& target, + TransactionToken* token) { string translated_src = TranslateName(src); struct stat sbuf; if (stat(translated_src.c_str(), &sbuf) != 0) { diff --git a/tensorflow/core/platform/default/posix_file_system.h b/tensorflow/core/platform/default/posix_file_system.h index a1c6f34ad65..8e301c8b2e4 100644 --- a/tensorflow/core/platform/default/posix_file_system.h +++ b/tensorflow/core/platform/default/posix_file_system.h @@ -27,63 +27,47 @@ class PosixFileSystem : public FileSystem { ~PosixFileSystem() {} + TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT; + Status NewRandomAccessFile( - const string& filename, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override; + const string& filename, TransactionToken* token, + std::unique_ptr* result) override; - Status NewWritableFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override; + Status NewWritableFile(const string& fname, TransactionToken* token, + std::unique_ptr* result) override; - Status NewAppendableFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override; + Status NewAppendableFile(const string& fname, TransactionToken* token, + std::unique_ptr* result) override; Status NewReadOnlyMemoryRegionFromFile( - const string& filename, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override; + const string& filename, TransactionToken* token, + std::unique_ptr* result) override; - Status FileExists( - const string& fname /*, TransactionToken* token = nullptr */) override; + Status FileExists(const string& fname, TransactionToken* token) override; - Status GetChildren( - const string& dir, - std::vector* result /*, TransactionToken* token = nullptr */) - override; + Status GetChildren(const string& dir, TransactionToken* token, + std::vector* result) override; - Status Stat( - const string& fname, - FileStatistics* stats /*, TransactionToken* token = nullptr */) override; + Status Stat(const string& fname, TransactionToken* token, + FileStatistics* stats) override; - Status GetMatchingPaths( - const string& pattern, - std::vector* results /*, TransactionToken* token = nullptr */) - override; + Status GetMatchingPaths(const string& pattern, TransactionToken* token, + std::vector* results) override; - Status DeleteFile( - const string& fname /*, TransactionToken* token = nullptr */) override; + Status DeleteFile(const string& fname, TransactionToken* token) override; - Status CreateDir( - const string& name /*, TransactionToken* token = nullptr */) override; + Status CreateDir(const string& name, TransactionToken* token) override; - Status DeleteDir( - const string& name /*, TransactionToken* token = nullptr */) override; + Status DeleteDir(const string& name, TransactionToken* token) override; - Status GetFileSize( - const string& fname, - uint64* size /*, TransactionToken* token = nullptr */) override; + Status GetFileSize(const string& fname, TransactionToken* token, + uint64* size) override; - Status RenameFile( - const string& src, - const string& target /*, TransactionToken* token = nullptr */) override; + Status RenameFile(const string& src, const string& target, + TransactionToken* token) override; - Status CopyFile( - const string& src, - const string& target /*, TransactionToken* token = nullptr */) override; + Status CopyFile(const string& src, const string& target, + TransactionToken* token) override; }; Status IOError(const string& context, int err_number); diff --git a/tensorflow/core/platform/hadoop/hadoop_file_system.cc b/tensorflow/core/platform/hadoop/hadoop_file_system.cc index 5b2c5a76aae..f8ed61c3ac9 100644 --- a/tensorflow/core/platform/hadoop/hadoop_file_system.cc +++ b/tensorflow/core/platform/hadoop/hadoop_file_system.cc @@ -280,8 +280,8 @@ class HDFSRandomAccessFile : public RandomAccessFile { }; Status HadoopFileSystem::NewRandomAccessFile( - const string& fname, - std::unique_ptr* result /*, TransactionToken* token */) { + const string& fname, TransactionToken* token, + std::unique_ptr* result) { hdfsFS fs = nullptr; TF_RETURN_IF_ERROR(Connect(fname, &fs)); @@ -373,8 +373,8 @@ class HDFSWritableFile : public WritableFile { }; Status HadoopFileSystem::NewWritableFile( - const string& fname, - std::unique_ptr* result /*, TransactionToken* token */) { + const string& fname, TransactionToken* token, + std::unique_ptr* result) { hdfsFS fs = nullptr; TF_RETURN_IF_ERROR(Connect(fname, &fs)); @@ -388,8 +388,8 @@ Status HadoopFileSystem::NewWritableFile( } Status HadoopFileSystem::NewAppendableFile( - const string& fname, - std::unique_ptr* result /*, TransactionToken* token */) { + const string& fname, TransactionToken* token, + std::unique_ptr* result) { hdfsFS fs = nullptr; TF_RETURN_IF_ERROR(Connect(fname, &fs)); @@ -403,8 +403,8 @@ Status HadoopFileSystem::NewAppendableFile( } Status HadoopFileSystem::NewReadOnlyMemoryRegionFromFile( - const string& fname, std::unique_ptr* - result /*, TransactionToken* token */) { + const string& fname, TransactionToken* token, + std::unique_ptr* result) { // hadoopReadZero() technically supports this call with the following // caveats: // - It only works up to 2 GB. We'd have to Stat() the file to ensure that @@ -414,8 +414,8 @@ Status HadoopFileSystem::NewReadOnlyMemoryRegionFromFile( return errors::Unimplemented("HDFS does not support ReadOnlyMemoryRegion"); } -Status HadoopFileSystem::FileExists( - const string& fname /*, TransactionToken* token */) { +Status HadoopFileSystem::FileExists(const string& fname, + TransactionToken* token) { hdfsFS fs = nullptr; TF_RETURN_IF_ERROR(Connect(fname, &fs)); if (libhdfs()->hdfsExists(fs, TranslateName(fname).c_str()) == 0) { @@ -424,9 +424,8 @@ Status HadoopFileSystem::FileExists( return errors::NotFound(fname, " not found."); } -Status HadoopFileSystem::GetChildren( - const string& dir, - std::vector* result /*, TransactionToken* token */) { +Status HadoopFileSystem::GetChildren(const string& dir, TransactionToken* token, + std::vector* result) { result->clear(); hdfsFS fs = nullptr; TF_RETURN_IF_ERROR(Connect(dir, &fs)); @@ -434,7 +433,7 @@ Status HadoopFileSystem::GetChildren( // hdfsListDirectory returns nullptr if the directory is empty. Do a separate // check to verify the directory exists first. FileStatistics stat; - TF_RETURN_IF_ERROR(Stat(dir, &stat)); + TF_RETURN_IF_ERROR(Stat(dir, token, &stat)); int entries = 0; hdfsFileInfo* info = @@ -453,14 +452,14 @@ Status HadoopFileSystem::GetChildren( return Status::OK(); } -Status HadoopFileSystem::GetMatchingPaths( - const string& pattern, - std::vector* results /*, TransactionToken* token */) { +Status HadoopFileSystem::GetMatchingPaths(const string& pattern, + TransactionToken* token, + std::vector* results) { return internal::GetMatchingPaths(this, Env::Default(), pattern, results); } -Status HadoopFileSystem::DeleteFile( - const string& fname /*, TransactionToken* token */) { +Status HadoopFileSystem::DeleteFile(const string& fname, + TransactionToken* token) { hdfsFS fs = nullptr; TF_RETURN_IF_ERROR(Connect(fname, &fs)); @@ -471,8 +470,7 @@ Status HadoopFileSystem::DeleteFile( return Status::OK(); } -Status HadoopFileSystem::CreateDir( - const string& dir /*, TransactionToken* token */) { +Status HadoopFileSystem::CreateDir(const string& dir, TransactionToken* token) { hdfsFS fs = nullptr; TF_RETURN_IF_ERROR(Connect(dir, &fs)); @@ -482,8 +480,7 @@ Status HadoopFileSystem::CreateDir( return Status::OK(); } -Status HadoopFileSystem::DeleteDir( - const string& dir /*, TransactionToken* token */) { +Status HadoopFileSystem::DeleteDir(const string& dir, TransactionToken* token) { hdfsFS fs = nullptr; TF_RETURN_IF_ERROR(Connect(dir, &fs)); @@ -502,7 +499,7 @@ Status HadoopFileSystem::DeleteDir( // the call is actually successful. Check again by Stat. if (info == nullptr && errno != 0) { FileStatistics stat; - TF_RETURN_IF_ERROR(Stat(dir, &stat)); + TF_RETURN_IF_ERROR(Stat(dir, token, &stat)); } if (entries > 0) { @@ -515,8 +512,8 @@ Status HadoopFileSystem::DeleteDir( return Status::OK(); } -Status HadoopFileSystem::GetFileSize( - const string& fname, uint64* size /*, TransactionToken* token */) { +Status HadoopFileSystem::GetFileSize(const string& fname, + TransactionToken* token, uint64* size) { hdfsFS fs = nullptr; TF_RETURN_IF_ERROR(Connect(fname, &fs)); @@ -530,8 +527,8 @@ Status HadoopFileSystem::GetFileSize( return Status::OK(); } -Status HadoopFileSystem::RenameFile( - const string& src, const string& target /*, TransactionToken* token */) { +Status HadoopFileSystem::RenameFile(const string& src, const string& target, + TransactionToken* token) { hdfsFS fs = nullptr; TF_RETURN_IF_ERROR(Connect(src, &fs)); @@ -548,8 +545,8 @@ Status HadoopFileSystem::RenameFile( return Status::OK(); } -Status HadoopFileSystem::Stat( - const string& fname, FileStatistics* stats /*, TransactionToken* token */) { +Status HadoopFileSystem::Stat(const string& fname, TransactionToken* token, + FileStatistics* stats) { hdfsFS fs = nullptr; TF_RETURN_IF_ERROR(Connect(fname, &fs)); diff --git a/tensorflow/core/platform/hadoop/hadoop_file_system.h b/tensorflow/core/platform/hadoop/hadoop_file_system.h index 13abc067cd8..5e7233633a6 100644 --- a/tensorflow/core/platform/hadoop/hadoop_file_system.h +++ b/tensorflow/core/platform/hadoop/hadoop_file_system.h @@ -32,63 +32,46 @@ class HadoopFileSystem : public FileSystem { HadoopFileSystem(); ~HadoopFileSystem(); + TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT; + Status NewRandomAccessFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr*/) override; + const string& fname, TransactionToken* token, + std::unique_ptr* result) override; - Status NewWritableFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr*/) override; + Status NewWritableFile(const string& fname, TransactionToken* token, + std::unique_ptr* result) override; - Status NewAppendableFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr*/) override; + Status NewAppendableFile(const string& fname, TransactionToken* token, + std::unique_ptr* result) override; Status NewReadOnlyMemoryRegionFromFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr*/) override; + const string& fname, TransactionToken* token, + std::unique_ptr* result) override; - Status FileExists( - const string& fname /*, TransactionToken* token = nullptr*/) override; + Status FileExists(const string& fname, TransactionToken* token) override; - Status GetChildren( - const string& dir, - std::vector* result /*, TransactionToken* token = nullptr*/) - override; + Status GetChildren(const string& dir, TransactionToken* token, + std::vector* result) override; - Status GetMatchingPaths( - const string& pattern, - std::vector* results /*, TransactionToken* token = nullptr*/) - override; + Status GetMatchingPaths(const string& pattern, TransactionToken* token, + std::vector* results) override; - Status DeleteFile( - const string& fname /*, TransactionToken* token = nullptr*/) override; + Status DeleteFile(const string& fname, TransactionToken* token) override; - Status CreateDir( - const string& name /*, TransactionToken* token = nullptr*/) override; + Status CreateDir(const string& name, TransactionToken* token) override; - Status DeleteDir( - const string& name /*, TransactionToken* token = nullptr*/) override; + Status DeleteDir(const string& name, TransactionToken* token) override; - Status GetFileSize( - const string& fname, - uint64* size /*, TransactionToken* token = nullptr*/) override; + Status GetFileSize(const string& fname, TransactionToken* token, + uint64* size) override; - Status RenameFile( - const string& src, - const string& target /*, TransactionToken* token = nullptr*/) override; + Status RenameFile(const string& src, const string& target, + TransactionToken* token) override; - Status Stat( - const string& fname, - FileStatistics* stat /*, TransactionToken* token = nullptr*/) override; + Status Stat(const string& fname, TransactionToken* token, + FileStatistics* stat) override; - string TranslateName( - const string& name /*, TransactionToken* token = nullptr*/) - const override; + string TranslateName(const string& name) const override; private: Status Connect(StringPiece fname, hdfsFS* fs); diff --git a/tensorflow/core/platform/windows/windows_file_system.cc b/tensorflow/core/platform/windows/windows_file_system.cc index e1e8656bce5..475f8791144 100644 --- a/tensorflow/core/platform/windows/windows_file_system.cc +++ b/tensorflow/core/platform/windows/windows_file_system.cc @@ -261,8 +261,8 @@ class WinReadOnlyMemoryRegion : public ReadOnlyMemoryRegion { } // namespace Status WindowsFileSystem::NewRandomAccessFile( - const string& fname, - std::unique_ptr* result /*, TransactionToken* token */) { + const string& fname, TransactionToken* token, + std::unique_ptr* result) { string translated_fname = TranslateName(fname); std::wstring ws_translated_fname = Utf8ToWideChar(translated_fname); result->reset(); @@ -289,8 +289,8 @@ Status WindowsFileSystem::NewRandomAccessFile( } Status WindowsFileSystem::NewWritableFile( - const string& fname, - std::unique_ptr* result /*, TransactionToken* token */) { + const string& fname, TransactionToken* token, + std::unique_ptr* result) { string translated_fname = TranslateName(fname); std::wstring ws_translated_fname = Utf8ToWideChar(translated_fname); result->reset(); @@ -310,8 +310,8 @@ Status WindowsFileSystem::NewWritableFile( } Status WindowsFileSystem::NewAppendableFile( - const string& fname, - std::unique_ptr* result /*, TransactionToken* token */) { + const string& fname, TransactionToken* token, + std::unique_ptr* result) { string translated_fname = TranslateName(fname); std::wstring ws_translated_fname = Utf8ToWideChar(translated_fname); result->reset(); @@ -341,8 +341,8 @@ Status WindowsFileSystem::NewAppendableFile( } Status WindowsFileSystem::NewReadOnlyMemoryRegionFromFile( - const string& fname, std::unique_ptr* - result /*, TransactionToken* token */) { + const string& fname, TransactionToken* token, + std::unique_ptr* result) { string translated_fname = TranslateName(fname); std::wstring ws_translated_fname = Utf8ToWideChar(translated_fname); result->reset(); @@ -418,8 +418,8 @@ Status WindowsFileSystem::NewReadOnlyMemoryRegionFromFile( return s; } -Status WindowsFileSystem::FileExists( - const string& fname /*, TransactionToken* token */) { +Status WindowsFileSystem::FileExists(const string& fname, + TransactionToken* token) { constexpr int kOk = 0; std::wstring ws_translated_fname = Utf8ToWideChar(TranslateName(fname)); if (_waccess(ws_translated_fname.c_str(), kOk) == 0) { @@ -428,9 +428,9 @@ Status WindowsFileSystem::FileExists( return errors::NotFound(fname, " not found"); } -Status WindowsFileSystem::GetChildren( - const string& dir, - std::vector* result /*, TransactionToken* token */) { +Status WindowsFileSystem::GetChildren(const string& dir, + TransactionToken* token, + std::vector* result) { string translated_dir = TranslateName(dir); std::wstring ws_translated_dir = Utf8ToWideChar(translated_dir); result->clear(); @@ -465,8 +465,8 @@ Status WindowsFileSystem::GetChildren( return Status::OK(); } -Status WindowsFileSystem::DeleteFile( - const string& fname /*, TransactionToken* token */) { +Status WindowsFileSystem::DeleteFile(const string& fname, + TransactionToken* token) { Status result; std::wstring file_name = Utf8ToWideChar(fname); if (_wunlink(file_name.c_str()) != 0) { @@ -475,8 +475,8 @@ Status WindowsFileSystem::DeleteFile( return result; } -Status WindowsFileSystem::CreateDir( - const string& name /*, TransactionToken* token */) { +Status WindowsFileSystem::CreateDir(const string& name, + TransactionToken* token) { Status result; std::wstring ws_name = Utf8ToWideChar(name); if (ws_name.empty()) { @@ -488,8 +488,8 @@ Status WindowsFileSystem::CreateDir( return result; } -Status WindowsFileSystem::DeleteDir( - const string& name /*, TransactionToken* token */) { +Status WindowsFileSystem::DeleteDir(const string& name, + TransactionToken* token) { Status result; std::wstring ws_name = Utf8ToWideChar(name); if (_wrmdir(ws_name.c_str()) != 0) { @@ -498,8 +498,8 @@ Status WindowsFileSystem::DeleteDir( return result; } -Status WindowsFileSystem::GetFileSize( - const string& fname, uint64* size /*, TransactionToken* token */) { +Status WindowsFileSystem::GetFileSize(const string& fname, + TransactionToken* token, uint64* size) { string translated_fname = TranslateName(fname); std::wstring ws_translated_dir = Utf8ToWideChar(translated_fname); Status result; @@ -517,8 +517,8 @@ Status WindowsFileSystem::GetFileSize( return result; } -Status WindowsFileSystem::IsDirectory( - const string& fname /*, TransactionToken* token */) { +Status WindowsFileSystem::IsDirectory(const string& fname, + TransactionToken* token) { TF_RETURN_IF_ERROR(FileExists(fname)); std::wstring ws_translated_fname = Utf8ToWideChar(TranslateName(fname)); if (PathIsDirectoryW(ws_translated_fname.c_str())) { @@ -527,8 +527,8 @@ Status WindowsFileSystem::IsDirectory( return Status(tensorflow::error::FAILED_PRECONDITION, "Not a directory"); } -Status WindowsFileSystem::RenameFile( - const string& src, const string& target /*, TransactionToken* token */) { +Status WindowsFileSystem::RenameFile(const string& src, const string& target, + TransactionToken* token) { Status result; // rename() is not capable of replacing the existing file as on Linux // so use OS API directly @@ -542,9 +542,9 @@ Status WindowsFileSystem::RenameFile( return result; } -Status WindowsFileSystem::GetMatchingPaths( - const string& pattern, - std::vector* results /*, TransactionToken* token */) { +Status WindowsFileSystem::GetMatchingPaths(const string& pattern, + TransactionToken* token, + std::vector* results) { // NOTE(mrry): The existing implementation of FileSystem::GetMatchingPaths() // does not handle Windows paths containing backslashes correctly. Since // Windows APIs will accept forward and backslashes equivalently, we @@ -567,8 +567,8 @@ bool WindowsFileSystem::Match(const string& filename, const string& pattern) { return PathMatchSpecW(ws_path.c_str(), ws_pattern.c_str()) == TRUE; } -Status WindowsFileSystem::Stat( - const string& fname, FileStatistics* stat /*, TransactionToken* token */) { +Status WindowsFileSystem::Stat(const string& fname, TransactionToken* token, + FileStatistics* stat) { Status result; struct _stat sbuf; std::wstring ws_translated_fname = Utf8ToWideChar(TranslateName(fname)); diff --git a/tensorflow/core/platform/windows/windows_file_system.h b/tensorflow/core/platform/windows/windows_file_system.h index 604cd141e40..8c550f53b84 100644 --- a/tensorflow/core/platform/windows/windows_file_system.h +++ b/tensorflow/core/platform/windows/windows_file_system.h @@ -32,72 +32,50 @@ class WindowsFileSystem : public FileSystem { ~WindowsFileSystem() {} + TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT; + Status NewRandomAccessFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override; + const string& fname, TransactionToken* token, + std::unique_ptr* result) override; - Status NewWritableFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override; + Status NewWritableFile(const string& fname, TransactionToken* token, + std::unique_ptr* result) override; - Status NewAppendableFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override; + Status NewAppendableFile(const string& fname, TransactionToken* token, + std::unique_ptr* result) override; Status NewReadOnlyMemoryRegionFromFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override; + const string& fname, TransactionToken* token, + std::unique_ptr* result) override; - Status FileExists( - const string& fname /*, TransactionToken* token = nullptr */) override; + Status FileExists(const string& fname, TransactionToken* token) override; - Status GetChildren( - const string& dir, - std::vector* result /*, TransactionToken* token = nullptr */) - override; + Status GetChildren(const string& dir, TransactionToken* token, + std::vector* result) override; - Status GetMatchingPaths( - const string& pattern, - std::vector* result /*, TransactionToken* token = nullptr */) - override; + Status GetMatchingPaths(const string& pattern, TransactionToken* token, + std::vector* result) override; - bool Match( - const string& filename, - const string& pattern /*, TransactionToken* token = nullptr */) override; + bool Match(const string& filename, const string& pattern) override; - Status Stat( - const string& fname, - FileStatistics* stat /*, TransactionToken* token = nullptr */) override; + Status Stat(const string& fname, TransactionToken* token, + FileStatistics* stat) override; - Status DeleteFile( - const string& fname /*, TransactionToken* token = nullptr */) override; + Status DeleteFile(const string& fname, TransactionToken* token) override; - Status CreateDir( - const string& name /*, TransactionToken* token = nullptr */) override; + Status CreateDir(const string& name, TransactionToken* token) override; - Status DeleteDir( - const string& name /*, TransactionToken* token = nullptr */) override; + Status DeleteDir(const string& name, TransactionToken* token) override; - Status GetFileSize( - const string& fname, - uint64* size /*, TransactionToken* token = nullptr */) override; + Status GetFileSize(const string& fname, TransactionToken* token, + uint64* size) override; - Status IsDirectory( - const string& fname /*, TransactionToken* token = nullptr */) override; + Status IsDirectory(const string& fname, TransactionToken* token) override; - Status RenameFile( - const string& src, - const string& target /*, TransactionToken* token = nullptr */) override; + Status RenameFile(const string& src, const string& target, + TransactionToken* token) override; - string TranslateName( - const string& name /*, TransactionToken* token = nullptr */) - const override { - return name; - } + string TranslateName(const string& name) const override { return name; } char Separator() const override { return '\\'; }; }; diff --git a/tensorflow/core/util/memmapped_file_system.cc b/tensorflow/core/util/memmapped_file_system.cc index 1451d6350ce..c6bda8b07e9 100644 --- a/tensorflow/core/util/memmapped_file_system.cc +++ b/tensorflow/core/util/memmapped_file_system.cc @@ -86,8 +86,8 @@ class RandomAccessFileFromMemmapped : public RandomAccessFile { MemmappedFileSystem::MemmappedFileSystem() {} -Status MemmappedFileSystem::FileExists( - const string& fname /*, TransactionToken* token */) { +Status MemmappedFileSystem::FileExists(const string& fname, + TransactionToken* token) { if (!mapped_memory_) { return errors::FailedPrecondition("MemmappedEnv is not initialized"); } @@ -99,8 +99,8 @@ Status MemmappedFileSystem::FileExists( } Status MemmappedFileSystem::NewRandomAccessFile( - const string& filename, - std::unique_ptr* result /*, TransactionToken* token */) { + const string& filename, TransactionToken* token, + std::unique_ptr* result) { if (!mapped_memory_) { return errors::FailedPrecondition("MemmappedEnv is not initialized"); } @@ -115,8 +115,8 @@ Status MemmappedFileSystem::NewRandomAccessFile( } Status MemmappedFileSystem::NewReadOnlyMemoryRegionFromFile( - const string& filename, std::unique_ptr* - result /*, TransactionToken* token */) { + const string& filename, TransactionToken* token, + std::unique_ptr* result) { if (!mapped_memory_) { return errors::FailedPrecondition("MemmappedEnv is not initialized"); } @@ -130,8 +130,8 @@ Status MemmappedFileSystem::NewReadOnlyMemoryRegionFromFile( return Status::OK(); } -Status MemmappedFileSystem::GetFileSize( - const string& filename, uint64* size /*, TransactionToken* token */) { +Status MemmappedFileSystem::GetFileSize(const string& filename, + TransactionToken* token, uint64* size) { if (!mapped_memory_) { return errors::FailedPrecondition("MemmappedEnv is not initialized"); } @@ -143,59 +143,59 @@ Status MemmappedFileSystem::GetFileSize( return Status::OK(); } -Status MemmappedFileSystem::Stat( - const string& fname, FileStatistics* stat /*, TransactionToken* token */) { +Status MemmappedFileSystem::Stat(const string& fname, TransactionToken* token, + FileStatistics* stat) { uint64 size; - auto status = GetFileSize(fname, &size); + auto status = GetFileSize(fname, token, &size); if (status.ok()) { stat->length = size; } return status; } -Status MemmappedFileSystem::NewWritableFile( - const string& filename, - std::unique_ptr* wf /*, TransactionToken* token */) { +Status MemmappedFileSystem::NewWritableFile(const string& filename, + TransactionToken* token, + std::unique_ptr* wf) { return errors::Unimplemented("memmapped format doesn't support writing"); } Status MemmappedFileSystem::NewAppendableFile( - const string& filename, - std::unique_ptr* result /*, TransactionToken* token */) { + const string& filename, TransactionToken* token, + std::unique_ptr* result) { return errors::Unimplemented("memmapped format doesn't support writing"); } -Status MemmappedFileSystem::GetChildren( - const string& filename, - std::vector* strings /*, TransactionToken* token */) { +Status MemmappedFileSystem::GetChildren(const string& filename, + TransactionToken* token, + std::vector* strings) { return errors::Unimplemented("memmapped format doesn't support GetChildren"); } -Status MemmappedFileSystem::GetMatchingPaths( - const string& pattern, - std::vector* results /*, TransactionToken* token */) { +Status MemmappedFileSystem::GetMatchingPaths(const string& pattern, + TransactionToken* token, + std::vector* results) { return errors::Unimplemented( "memmapped format doesn't support GetMatchingPaths"); } -Status MemmappedFileSystem::DeleteFile( - const string& filename /*, TransactionToken* token */) { +Status MemmappedFileSystem::DeleteFile(const string& filename, + TransactionToken* token) { return errors::Unimplemented("memmapped format doesn't support DeleteFile"); } -Status MemmappedFileSystem::CreateDir( - const string& dirname /*, TransactionToken* token */) { +Status MemmappedFileSystem::CreateDir(const string& dirname, + TransactionToken* token) { return errors::Unimplemented("memmapped format doesn't support CreateDir"); } -Status MemmappedFileSystem::DeleteDir( - const string& dirname /*, TransactionToken* token */) { +Status MemmappedFileSystem::DeleteDir(const string& dirname, + TransactionToken* token) { return errors::Unimplemented("memmapped format doesn't support DeleteDir"); } -Status MemmappedFileSystem::RenameFile( - const string& filename_from, - const string& filename_to /*, TransactionToken* token */) { +Status MemmappedFileSystem::RenameFile(const string& filename_from, + const string& filename_to, + TransactionToken* token) { return errors::Unimplemented("memmapped format doesn't support RenameFile"); } diff --git a/tensorflow/core/util/memmapped_file_system.h b/tensorflow/core/util/memmapped_file_system.h index d8f19444454..27305a500f5 100644 --- a/tensorflow/core/util/memmapped_file_system.h +++ b/tensorflow/core/util/memmapped_file_system.h @@ -60,52 +60,39 @@ class MemmappedFileSystem : public FileSystem { MemmappedFileSystem(); ~MemmappedFileSystem() override = default; - Status FileExists( - const string& fname /*, TransactionToken* token = nullptr */) override; + + TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT; + + Status FileExists(const string& fname, TransactionToken* token) override; Status NewRandomAccessFile( - const string& filename, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override; + const string& filename, TransactionToken* token, + std::unique_ptr* result) override; Status NewReadOnlyMemoryRegionFromFile( - const string& filename, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override; + const string& filename, TransactionToken* token, + std::unique_ptr* result) override; // All these functions return Unimplemented error, the memmapped storage is // read only. - Status NewWritableFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override; - Status NewAppendableFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override; - Status GetChildren(const string& dir, - std::vector* - r /*, TransactionToken* token = nullptr */) override; - Status GetMatchingPaths( - const string& pattern, - std::vector* results /*, TransactionToken* token = nullptr */) - override; - Status DeleteFile( - const string& f /*, TransactionToken* token = nullptr */) override; - Status CreateDir( - const string& d /*, TransactionToken* token = nullptr */) override; - Status DeleteDir( - const string& d /*, TransactionToken* token = nullptr */) override; - Status RenameFile( - const string& s, - const string& t /*, TransactionToken* token = nullptr */) override; + Status NewWritableFile(const string& fname, TransactionToken* token, + std::unique_ptr* result) override; + Status NewAppendableFile(const string& fname, TransactionToken* token, + std::unique_ptr* result) override; + Status GetChildren(const string& dir, TransactionToken* token, + std::vector* r) override; + Status GetMatchingPaths(const string& pattern, TransactionToken* token, + std::vector* results) override; + Status DeleteFile(const string& f, TransactionToken* token) override; + Status CreateDir(const string& d, TransactionToken* token) override; + Status DeleteDir(const string& d, TransactionToken* token) override; + Status RenameFile(const string& s, const string& t, + TransactionToken* token) override; // These functions are implemented. - Status GetFileSize( - const string& f, - uint64* s /*, TransactionToken* token = nullptr */) override; + Status GetFileSize(const string& f, TransactionToken* token, + uint64* s) override; // Currently just returns size. - Status Stat( - const string& fname, - FileStatistics* stat /*, TransactionToken* token = nullptr */) override; + Status Stat(const string& fname, TransactionToken* token, + FileStatistics* stat) override; // Initializes filesystem from a file in memmapped format. Status InitializeFromFile(Env* env, const string& filename); diff --git a/tensorflow/tools/android/inference_interface/asset_manager_filesystem.cc b/tensorflow/tools/android/inference_interface/asset_manager_filesystem.cc index 04d5774adb8..648affc9926 100644 --- a/tensorflow/tools/android/inference_interface/asset_manager_filesystem.cc +++ b/tensorflow/tools/android/inference_interface/asset_manager_filesystem.cc @@ -124,8 +124,8 @@ AssetManagerFileSystem::AssetManagerFileSystem(AAssetManager* asset_manager, const string& prefix) : asset_manager_(asset_manager), prefix_(prefix) {} -Status AssetManagerFileSystem::FileExists( - const string& fname /*, TransactionToken* token */) { +Status AssetManagerFileSystem::FileExists(const string& fname, + TransactionToken* token) { string path = RemoveAssetPrefix(fname); auto asset = ScopedAsset( AAssetManager_open(asset_manager_, path.c_str(), AASSET_MODE_RANDOM)); @@ -136,8 +136,8 @@ Status AssetManagerFileSystem::FileExists( } Status AssetManagerFileSystem::NewRandomAccessFile( - const string& fname, - std::unique_ptr* result /*, TransactionToken* token */) { + const string& fname, TransactionToken* token, + std::unique_ptr* result) { string path = RemoveAssetPrefix(fname); auto asset = ScopedAsset( AAssetManager_open(asset_manager_, path.c_str(), AASSET_MODE_RANDOM)); @@ -149,8 +149,8 @@ Status AssetManagerFileSystem::NewRandomAccessFile( } Status AssetManagerFileSystem::NewReadOnlyMemoryRegionFromFile( - const string& fname, std::unique_ptr* - result /*, TransactionToken* token */) { + const string& fname, TransactionToken* token, + std::unique_ptr* result) { string path = RemoveAssetPrefix(fname); auto asset = ScopedAsset( AAssetManager_open(asset_manager_, path.c_str(), AASSET_MODE_STREAMING)); @@ -186,9 +186,9 @@ Status AssetManagerFileSystem::NewReadOnlyMemoryRegionFromFile( return Status::OK(); } -Status AssetManagerFileSystem::GetChildren( - const string& prefixed_dir, - std::vector* r /*, TransactionToken* token */) { +Status AssetManagerFileSystem::GetChildren(const string& prefixed_dir, + TransactionToken* token, + std::vector* r) { std::string path = NormalizeDirectoryPath(prefixed_dir); auto dir = ScopedAssetDir(AAssetManager_openDir(asset_manager_, path.c_str())); @@ -203,8 +203,8 @@ Status AssetManagerFileSystem::GetChildren( return Status::OK(); } -Status AssetManagerFileSystem::GetFileSize( - const string& fname, uint64* s /*, TransactionToken* token */) { +Status AssetManagerFileSystem::GetFileSize(const string& fname, + TransactionToken* token, uint64* s) { // If fname corresponds to a directory, return early. It doesn't map to an // AAsset, and would otherwise return NotFound. if (DirectoryExists(fname)) { @@ -221,8 +221,9 @@ Status AssetManagerFileSystem::GetFileSize( return Status::OK(); } -Status AssetManagerFileSystem::Stat( - const string& fname, FileStatistics* stat /*, TransactionToken* token */) { +Status AssetManagerFileSystem::Stat(const string& fname, + TransactionToken* token, + FileStatistics* stat) { uint64 size; stat->is_directory = DirectoryExists(fname); TF_RETURN_IF_ERROR(GetFileSize(fname, &size)); @@ -240,8 +241,8 @@ string AssetManagerFileSystem::RemoveAssetPrefix(const string& name) { return string(piece); } -bool AssetManagerFileSystem::DirectoryExists( - const std::string& fname /*, TransactionToken* token */) { +bool AssetManagerFileSystem::DirectoryExists(const std::string& fname, + TransactionToken* token) { std::string path = NormalizeDirectoryPath(fname); auto dir = ScopedAssetDir(AAssetManager_openDir(asset_manager_, path.c_str())); @@ -250,36 +251,36 @@ bool AssetManagerFileSystem::DirectoryExists( return AAssetDir_getNextFileName(dir.get()) != NULL; } -Status AssetManagerFileSystem::GetMatchingPaths( - const string& pattern, - std::vector* results /*, TransactionToken* token */) { +Status AssetManagerFileSystem::GetMatchingPaths(const string& pattern, + TransactionToken* token, + std::vector* results) { return internal::GetMatchingPaths(this, Env::Default(), pattern, results); } Status AssetManagerFileSystem::NewWritableFile( - const string& fname, - std::unique_ptr* result /*, TransactionToken* token */) { + const string& fname, TransactionToken* token, + std::unique_ptr* result) { return errors::Unimplemented("Asset storage is read only."); } Status AssetManagerFileSystem::NewAppendableFile( - const string& fname, - std::unique_ptr* result /*, TransactionToken* token */) { + const string& fname, TransactionToken* token, + std::unique_ptr* result) { return errors::Unimplemented("Asset storage is read only."); } -Status AssetManagerFileSystem::DeleteFile( - const string& f /*, TransactionToken* token */) { +Status AssetManagerFileSystem::DeleteFile(const string& f, + TransactionToken* token) { return errors::Unimplemented("Asset storage is read only."); } -Status AssetManagerFileSystem::CreateDir( - const string& d /*, TransactionToken* token */) { +Status AssetManagerFileSystem::CreateDir(const string& d, + TransactionToken* token) { return errors::Unimplemented("Asset storage is read only."); } -Status AssetManagerFileSystem::DeleteDir( - const string& d /*, TransactionToken* token */) { +Status AssetManagerFileSystem::DeleteDir(const string& d, + TransactionToken* token) { return errors::Unimplemented("Asset storage is read only."); } -Status AssetManagerFileSystem::RenameFile( - const string& s, const string& t /*, TransactionToken* token */) { +Status AssetManagerFileSystem::RenameFile(const string& s, const string& t, + TransactionToken* token) { return errors::Unimplemented("Asset storage is read only."); } diff --git a/tensorflow/tools/android/inference_interface/asset_manager_filesystem.h b/tensorflow/tools/android/inference_interface/asset_manager_filesystem.h index 329e55d6cc7..893d5ccb90a 100644 --- a/tensorflow/tools/android/inference_interface/asset_manager_filesystem.h +++ b/tensorflow/tools/android/inference_interface/asset_manager_filesystem.h @@ -42,52 +42,38 @@ class AssetManagerFileSystem : public FileSystem { AssetManagerFileSystem(AAssetManager* asset_manager, const string& prefix); ~AssetManagerFileSystem() override = default; - Status FileExists( - const string& fname /*, TransactionToken* token = nullptr*/) override; - Status NewRandomAccessFile( - const string& filename, - std::unique_ptr* - result /*, TransactionToken* token = nullptr*/) override; - Status NewReadOnlyMemoryRegionFromFile( - const string& filename, - std::unique_ptr* - result /*, TransactionToken* token = nullptr*/) override; + TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT; - Status GetFileSize( - const string& f, - uint64* s /*, TransactionToken* token = nullptr*/) override; + Status FileExists(const string& fname, TransactionToken* token) override; + Status NewRandomAccessFile( + const string& filename, TransactionToken* token, + std::unique_ptr* result) override; + Status NewReadOnlyMemoryRegionFromFile( + const string& filename, TransactionToken* token, + std::unique_ptr* result) override; + + Status GetFileSize(const string& f, TransactionToken* token, + uint64* s) override; // Currently just returns size. - Status Stat( - const string& fname, - FileStatistics* stat /*, TransactionToken* token = nullptr*/) override; - Status GetChildren( - const string& dir, - std::vector* r /*, TransactionToken* token = nullptr*/) override; + Status Stat(const string& fname, TransactionToken* token, + FileStatistics* stat) override; + Status GetChildren(const string& dir, TransactionToken* token, + std::vector* r) override; // All these functions return Unimplemented error. Asset storage is // read only. - Status NewWritableFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr*/) override; - Status NewAppendableFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr*/) override; - Status DeleteFile( - const string& f /*, TransactionToken* token = nullptr*/) override; - Status CreateDir( - const string& d /*, TransactionToken* token = nullptr*/) override; - Status DeleteDir( - const string& d /*, TransactionToken* token = nullptr*/) override; - Status RenameFile( - const string& s, - const string& t /*, TransactionToken* token = nullptr*/) override; + Status NewWritableFile(const string& fname, TransactionToken* token, + std::unique_ptr* result) override; + Status NewAppendableFile(const string& fname, TransactionToken* token, + std::unique_ptr* result) override; + Status DeleteFile(const string& f, TransactionToken* token) override; + Status CreateDir(const string& d, TransactionToken* token) override; + Status DeleteDir(const string& d, TransactionToken* token) override; + Status RenameFile(const string& s, const string& t, + TransactionToken* token) override; - Status GetMatchingPaths( - const string& pattern, - std::vector* results /*, TransactionToken* token = nullptr*/) - override; + Status GetMatchingPaths(const string& pattern, TransactionToken* token, + std::vector* results) override; private: string RemoveAssetPrefix(const string& name);