Moving rest of the filesystems to Transactional API

This commit is contained in:
Sami Kama 2020-08-04 18:04:52 -07:00
parent 79594069bb
commit 46a8319ee7
10 changed files with 276 additions and 363 deletions

View File

@ -178,8 +178,8 @@ class PosixReadOnlyMemoryRegion : public ReadOnlyMemoryRegion {
};
Status PosixFileSystem::NewRandomAccessFile(
const string& fname,
std::unique_ptr<RandomAccessFile>* result /*, TransactionToken* token */) {
const string& fname, TransactionToken* token,
std::unique_ptr<RandomAccessFile>* result) {
string translated_fname = TranslateName(fname);
Status s;
int fd = open(translated_fname.c_str(), O_RDONLY);
@ -191,9 +191,9 @@ Status PosixFileSystem::NewRandomAccessFile(
return s;
}
Status PosixFileSystem::NewWritableFile(
const string& fname,
std::unique_ptr<WritableFile>* result /*, TransactionToken* token */) {
Status PosixFileSystem::NewWritableFile(const string& fname,
TransactionToken* token,
std::unique_ptr<WritableFile>* result) {
string translated_fname = TranslateName(fname);
Status s;
FILE* f = fopen(translated_fname.c_str(), "w");
@ -206,8 +206,8 @@ Status PosixFileSystem::NewWritableFile(
}
Status PosixFileSystem::NewAppendableFile(
const string& fname,
std::unique_ptr<WritableFile>* result /*, TransactionToken* token */) {
const string& fname, TransactionToken* token,
std::unique_ptr<WritableFile>* result) {
string translated_fname = TranslateName(fname);
Status s;
FILE* f = fopen(translated_fname.c_str(), "a");
@ -220,8 +220,8 @@ Status PosixFileSystem::NewAppendableFile(
}
Status PosixFileSystem::NewReadOnlyMemoryRegionFromFile(
const string& fname, std::unique_ptr<ReadOnlyMemoryRegion>*
result /*, TransactionToken* token */) {
const string& fname, TransactionToken* token,
std::unique_ptr<ReadOnlyMemoryRegion>* result) {
string translated_fname = TranslateName(fname);
Status s = Status::OK();
int fd = open(translated_fname.c_str(), O_RDONLY);
@ -244,17 +244,16 @@ Status PosixFileSystem::NewReadOnlyMemoryRegionFromFile(
return s;
}
Status PosixFileSystem::FileExists(
const string& fname /*, TransactionToken* token */) {
Status PosixFileSystem::FileExists(const string& fname,
TransactionToken* token) {
if (access(TranslateName(fname).c_str(), F_OK) == 0) {
return Status::OK();
}
return errors::NotFound(fname, " not found");
}
Status PosixFileSystem::GetChildren(
const string& dir,
std::vector<string>* result /*, TransactionToken* token */) {
Status PosixFileSystem::GetChildren(const string& dir, TransactionToken* token,
std::vector<string>* result) {
string translated_dir = TranslateName(dir);
result->clear();
DIR* d = opendir(translated_dir.c_str());
@ -274,14 +273,14 @@ Status PosixFileSystem::GetChildren(
return Status::OK();
}
Status PosixFileSystem::GetMatchingPaths(
const string& pattern,
std::vector<string>* results /*, TransactionToken* token */) {
Status PosixFileSystem::GetMatchingPaths(const string& pattern,
TransactionToken* token,
std::vector<string>* results) {
return internal::GetMatchingPaths(this, Env::Default(), pattern, results);
}
Status PosixFileSystem::DeleteFile(
const string& fname /*, TransactionToken* token */) {
Status PosixFileSystem::DeleteFile(const string& fname,
TransactionToken* token) {
Status result;
if (unlink(TranslateName(fname).c_str()) != 0) {
result = IOError(fname, errno);
@ -289,8 +288,7 @@ Status PosixFileSystem::DeleteFile(
return result;
}
Status PosixFileSystem::CreateDir(
const string& name /*, TransactionToken* token */) {
Status PosixFileSystem::CreateDir(const string& name, TransactionToken* token) {
string translated = TranslateName(name);
if (translated.empty()) {
return errors::AlreadyExists(name);
@ -301,8 +299,7 @@ Status PosixFileSystem::CreateDir(
return Status::OK();
}
Status PosixFileSystem::DeleteDir(
const string& name /*, TransactionToken* token */) {
Status PosixFileSystem::DeleteDir(const string& name, TransactionToken* token) {
Status result;
if (rmdir(TranslateName(name).c_str()) != 0) {
result = IOError(name, errno);
@ -310,8 +307,8 @@ Status PosixFileSystem::DeleteDir(
return result;
}
Status PosixFileSystem::GetFileSize(
const string& fname, uint64* size /*, TransactionToken* token */) {
Status PosixFileSystem::GetFileSize(const string& fname,
TransactionToken* token, uint64* size) {
Status s;
struct stat sbuf;
if (stat(TranslateName(fname).c_str(), &sbuf) != 0) {
@ -323,8 +320,8 @@ Status PosixFileSystem::GetFileSize(
return s;
}
Status PosixFileSystem::Stat(
const string& fname, FileStatistics* stats /*, TransactionToken* token */) {
Status PosixFileSystem::Stat(const string& fname, TransactionToken* token,
FileStatistics* stats) {
Status s;
struct stat sbuf;
if (stat(TranslateName(fname).c_str(), &sbuf) != 0) {
@ -337,8 +334,8 @@ Status PosixFileSystem::Stat(
return s;
}
Status PosixFileSystem::RenameFile(
const string& src, const string& target /*, TransactionToken* token */) {
Status PosixFileSystem::RenameFile(const string& src, const string& target,
TransactionToken* token) {
Status result;
if (rename(TranslateName(src).c_str(), TranslateName(target).c_str()) != 0) {
result = IOError(src, errno);
@ -346,8 +343,8 @@ Status PosixFileSystem::RenameFile(
return result;
}
Status PosixFileSystem::CopyFile(
const string& src, const string& target /*, TransactionToken* token */) {
Status PosixFileSystem::CopyFile(const string& src, const string& target,
TransactionToken* token) {
string translated_src = TranslateName(src);
struct stat sbuf;
if (stat(translated_src.c_str(), &sbuf) != 0) {

View File

@ -27,63 +27,47 @@ class PosixFileSystem : public FileSystem {
~PosixFileSystem() {}
TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT;
Status NewRandomAccessFile(
const string& filename,
std::unique_ptr<RandomAccessFile>*
result /*, TransactionToken* token = nullptr */) override;
const string& filename, TransactionToken* token,
std::unique_ptr<RandomAccessFile>* result) override;
Status NewWritableFile(
const string& fname,
std::unique_ptr<WritableFile>*
result /*, TransactionToken* token = nullptr */) override;
Status NewWritableFile(const string& fname, TransactionToken* token,
std::unique_ptr<WritableFile>* result) override;
Status NewAppendableFile(
const string& fname,
std::unique_ptr<WritableFile>*
result /*, TransactionToken* token = nullptr */) override;
Status NewAppendableFile(const string& fname, TransactionToken* token,
std::unique_ptr<WritableFile>* result) override;
Status NewReadOnlyMemoryRegionFromFile(
const string& filename,
std::unique_ptr<ReadOnlyMemoryRegion>*
result /*, TransactionToken* token = nullptr */) override;
const string& filename, TransactionToken* token,
std::unique_ptr<ReadOnlyMemoryRegion>* result) override;
Status FileExists(
const string& fname /*, TransactionToken* token = nullptr */) override;
Status FileExists(const string& fname, TransactionToken* token) override;
Status GetChildren(
const string& dir,
std::vector<string>* result /*, TransactionToken* token = nullptr */)
override;
Status GetChildren(const string& dir, TransactionToken* token,
std::vector<string>* result) override;
Status Stat(
const string& fname,
FileStatistics* stats /*, TransactionToken* token = nullptr */) override;
Status Stat(const string& fname, TransactionToken* token,
FileStatistics* stats) override;
Status GetMatchingPaths(
const string& pattern,
std::vector<string>* results /*, TransactionToken* token = nullptr */)
override;
Status GetMatchingPaths(const string& pattern, TransactionToken* token,
std::vector<string>* results) override;
Status DeleteFile(
const string& fname /*, TransactionToken* token = nullptr */) override;
Status DeleteFile(const string& fname, TransactionToken* token) override;
Status CreateDir(
const string& name /*, TransactionToken* token = nullptr */) override;
Status CreateDir(const string& name, TransactionToken* token) override;
Status DeleteDir(
const string& name /*, TransactionToken* token = nullptr */) override;
Status DeleteDir(const string& name, TransactionToken* token) override;
Status GetFileSize(
const string& fname,
uint64* size /*, TransactionToken* token = nullptr */) override;
Status GetFileSize(const string& fname, TransactionToken* token,
uint64* size) override;
Status RenameFile(
const string& src,
const string& target /*, TransactionToken* token = nullptr */) override;
Status RenameFile(const string& src, const string& target,
TransactionToken* token) override;
Status CopyFile(
const string& src,
const string& target /*, TransactionToken* token = nullptr */) override;
Status CopyFile(const string& src, const string& target,
TransactionToken* token) override;
};
Status IOError(const string& context, int err_number);

View File

@ -280,8 +280,8 @@ class HDFSRandomAccessFile : public RandomAccessFile {
};
Status HadoopFileSystem::NewRandomAccessFile(
const string& fname,
std::unique_ptr<RandomAccessFile>* result /*, TransactionToken* token */) {
const string& fname, TransactionToken* token,
std::unique_ptr<RandomAccessFile>* result) {
hdfsFS fs = nullptr;
TF_RETURN_IF_ERROR(Connect(fname, &fs));
@ -373,8 +373,8 @@ class HDFSWritableFile : public WritableFile {
};
Status HadoopFileSystem::NewWritableFile(
const string& fname,
std::unique_ptr<WritableFile>* result /*, TransactionToken* token */) {
const string& fname, TransactionToken* token,
std::unique_ptr<WritableFile>* result) {
hdfsFS fs = nullptr;
TF_RETURN_IF_ERROR(Connect(fname, &fs));
@ -388,8 +388,8 @@ Status HadoopFileSystem::NewWritableFile(
}
Status HadoopFileSystem::NewAppendableFile(
const string& fname,
std::unique_ptr<WritableFile>* result /*, TransactionToken* token */) {
const string& fname, TransactionToken* token,
std::unique_ptr<WritableFile>* result) {
hdfsFS fs = nullptr;
TF_RETURN_IF_ERROR(Connect(fname, &fs));
@ -403,8 +403,8 @@ Status HadoopFileSystem::NewAppendableFile(
}
Status HadoopFileSystem::NewReadOnlyMemoryRegionFromFile(
const string& fname, std::unique_ptr<ReadOnlyMemoryRegion>*
result /*, TransactionToken* token */) {
const string& fname, TransactionToken* token,
std::unique_ptr<ReadOnlyMemoryRegion>* result) {
// hadoopReadZero() technically supports this call with the following
// caveats:
// - It only works up to 2 GB. We'd have to Stat() the file to ensure that
@ -414,8 +414,8 @@ Status HadoopFileSystem::NewReadOnlyMemoryRegionFromFile(
return errors::Unimplemented("HDFS does not support ReadOnlyMemoryRegion");
}
Status HadoopFileSystem::FileExists(
const string& fname /*, TransactionToken* token */) {
Status HadoopFileSystem::FileExists(const string& fname,
TransactionToken* token) {
hdfsFS fs = nullptr;
TF_RETURN_IF_ERROR(Connect(fname, &fs));
if (libhdfs()->hdfsExists(fs, TranslateName(fname).c_str()) == 0) {
@ -424,9 +424,8 @@ Status HadoopFileSystem::FileExists(
return errors::NotFound(fname, " not found.");
}
Status HadoopFileSystem::GetChildren(
const string& dir,
std::vector<string>* result /*, TransactionToken* token */) {
Status HadoopFileSystem::GetChildren(const string& dir, TransactionToken* token,
std::vector<string>* result) {
result->clear();
hdfsFS fs = nullptr;
TF_RETURN_IF_ERROR(Connect(dir, &fs));
@ -434,7 +433,7 @@ Status HadoopFileSystem::GetChildren(
// hdfsListDirectory returns nullptr if the directory is empty. Do a separate
// check to verify the directory exists first.
FileStatistics stat;
TF_RETURN_IF_ERROR(Stat(dir, &stat));
TF_RETURN_IF_ERROR(Stat(dir, token, &stat));
int entries = 0;
hdfsFileInfo* info =
@ -453,14 +452,14 @@ Status HadoopFileSystem::GetChildren(
return Status::OK();
}
Status HadoopFileSystem::GetMatchingPaths(
const string& pattern,
std::vector<string>* results /*, TransactionToken* token */) {
Status HadoopFileSystem::GetMatchingPaths(const string& pattern,
TransactionToken* token,
std::vector<string>* results) {
return internal::GetMatchingPaths(this, Env::Default(), pattern, results);
}
Status HadoopFileSystem::DeleteFile(
const string& fname /*, TransactionToken* token */) {
Status HadoopFileSystem::DeleteFile(const string& fname,
TransactionToken* token) {
hdfsFS fs = nullptr;
TF_RETURN_IF_ERROR(Connect(fname, &fs));
@ -471,8 +470,7 @@ Status HadoopFileSystem::DeleteFile(
return Status::OK();
}
Status HadoopFileSystem::CreateDir(
const string& dir /*, TransactionToken* token */) {
Status HadoopFileSystem::CreateDir(const string& dir, TransactionToken* token) {
hdfsFS fs = nullptr;
TF_RETURN_IF_ERROR(Connect(dir, &fs));
@ -482,8 +480,7 @@ Status HadoopFileSystem::CreateDir(
return Status::OK();
}
Status HadoopFileSystem::DeleteDir(
const string& dir /*, TransactionToken* token */) {
Status HadoopFileSystem::DeleteDir(const string& dir, TransactionToken* token) {
hdfsFS fs = nullptr;
TF_RETURN_IF_ERROR(Connect(dir, &fs));
@ -502,7 +499,7 @@ Status HadoopFileSystem::DeleteDir(
// the call is actually successful. Check again by Stat.
if (info == nullptr && errno != 0) {
FileStatistics stat;
TF_RETURN_IF_ERROR(Stat(dir, &stat));
TF_RETURN_IF_ERROR(Stat(dir, token, &stat));
}
if (entries > 0) {
@ -515,8 +512,8 @@ Status HadoopFileSystem::DeleteDir(
return Status::OK();
}
Status HadoopFileSystem::GetFileSize(
const string& fname, uint64* size /*, TransactionToken* token */) {
Status HadoopFileSystem::GetFileSize(const string& fname,
TransactionToken* token, uint64* size) {
hdfsFS fs = nullptr;
TF_RETURN_IF_ERROR(Connect(fname, &fs));
@ -530,8 +527,8 @@ Status HadoopFileSystem::GetFileSize(
return Status::OK();
}
Status HadoopFileSystem::RenameFile(
const string& src, const string& target /*, TransactionToken* token */) {
Status HadoopFileSystem::RenameFile(const string& src, const string& target,
TransactionToken* token) {
hdfsFS fs = nullptr;
TF_RETURN_IF_ERROR(Connect(src, &fs));
@ -548,8 +545,8 @@ Status HadoopFileSystem::RenameFile(
return Status::OK();
}
Status HadoopFileSystem::Stat(
const string& fname, FileStatistics* stats /*, TransactionToken* token */) {
Status HadoopFileSystem::Stat(const string& fname, TransactionToken* token,
FileStatistics* stats) {
hdfsFS fs = nullptr;
TF_RETURN_IF_ERROR(Connect(fname, &fs));

View File

@ -32,63 +32,46 @@ class HadoopFileSystem : public FileSystem {
HadoopFileSystem();
~HadoopFileSystem();
TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT;
Status NewRandomAccessFile(
const string& fname,
std::unique_ptr<RandomAccessFile>*
result /*, TransactionToken* token = nullptr*/) override;
const string& fname, TransactionToken* token,
std::unique_ptr<RandomAccessFile>* result) override;
Status NewWritableFile(
const string& fname,
std::unique_ptr<WritableFile>*
result /*, TransactionToken* token = nullptr*/) override;
Status NewWritableFile(const string& fname, TransactionToken* token,
std::unique_ptr<WritableFile>* result) override;
Status NewAppendableFile(
const string& fname,
std::unique_ptr<WritableFile>*
result /*, TransactionToken* token = nullptr*/) override;
Status NewAppendableFile(const string& fname, TransactionToken* token,
std::unique_ptr<WritableFile>* result) override;
Status NewReadOnlyMemoryRegionFromFile(
const string& fname,
std::unique_ptr<ReadOnlyMemoryRegion>*
result /*, TransactionToken* token = nullptr*/) override;
const string& fname, TransactionToken* token,
std::unique_ptr<ReadOnlyMemoryRegion>* result) override;
Status FileExists(
const string& fname /*, TransactionToken* token = nullptr*/) override;
Status FileExists(const string& fname, TransactionToken* token) override;
Status GetChildren(
const string& dir,
std::vector<string>* result /*, TransactionToken* token = nullptr*/)
override;
Status GetChildren(const string& dir, TransactionToken* token,
std::vector<string>* result) override;
Status GetMatchingPaths(
const string& pattern,
std::vector<string>* results /*, TransactionToken* token = nullptr*/)
override;
Status GetMatchingPaths(const string& pattern, TransactionToken* token,
std::vector<string>* results) override;
Status DeleteFile(
const string& fname /*, TransactionToken* token = nullptr*/) override;
Status DeleteFile(const string& fname, TransactionToken* token) override;
Status CreateDir(
const string& name /*, TransactionToken* token = nullptr*/) override;
Status CreateDir(const string& name, TransactionToken* token) override;
Status DeleteDir(
const string& name /*, TransactionToken* token = nullptr*/) override;
Status DeleteDir(const string& name, TransactionToken* token) override;
Status GetFileSize(
const string& fname,
uint64* size /*, TransactionToken* token = nullptr*/) override;
Status GetFileSize(const string& fname, TransactionToken* token,
uint64* size) override;
Status RenameFile(
const string& src,
const string& target /*, TransactionToken* token = nullptr*/) override;
Status RenameFile(const string& src, const string& target,
TransactionToken* token) override;
Status Stat(
const string& fname,
FileStatistics* stat /*, TransactionToken* token = nullptr*/) override;
Status Stat(const string& fname, TransactionToken* token,
FileStatistics* stat) override;
string TranslateName(
const string& name /*, TransactionToken* token = nullptr*/)
const override;
string TranslateName(const string& name) const override;
private:
Status Connect(StringPiece fname, hdfsFS* fs);

View File

@ -261,8 +261,8 @@ class WinReadOnlyMemoryRegion : public ReadOnlyMemoryRegion {
} // namespace
Status WindowsFileSystem::NewRandomAccessFile(
const string& fname,
std::unique_ptr<RandomAccessFile>* result /*, TransactionToken* token */) {
const string& fname, TransactionToken* token,
std::unique_ptr<RandomAccessFile>* result) {
string translated_fname = TranslateName(fname);
std::wstring ws_translated_fname = Utf8ToWideChar(translated_fname);
result->reset();
@ -289,8 +289,8 @@ Status WindowsFileSystem::NewRandomAccessFile(
}
Status WindowsFileSystem::NewWritableFile(
const string& fname,
std::unique_ptr<WritableFile>* result /*, TransactionToken* token */) {
const string& fname, TransactionToken* token,
std::unique_ptr<WritableFile>* result) {
string translated_fname = TranslateName(fname);
std::wstring ws_translated_fname = Utf8ToWideChar(translated_fname);
result->reset();
@ -310,8 +310,8 @@ Status WindowsFileSystem::NewWritableFile(
}
Status WindowsFileSystem::NewAppendableFile(
const string& fname,
std::unique_ptr<WritableFile>* result /*, TransactionToken* token */) {
const string& fname, TransactionToken* token,
std::unique_ptr<WritableFile>* result) {
string translated_fname = TranslateName(fname);
std::wstring ws_translated_fname = Utf8ToWideChar(translated_fname);
result->reset();
@ -341,8 +341,8 @@ Status WindowsFileSystem::NewAppendableFile(
}
Status WindowsFileSystem::NewReadOnlyMemoryRegionFromFile(
const string& fname, std::unique_ptr<ReadOnlyMemoryRegion>*
result /*, TransactionToken* token */) {
const string& fname, TransactionToken* token,
std::unique_ptr<ReadOnlyMemoryRegion>* result) {
string translated_fname = TranslateName(fname);
std::wstring ws_translated_fname = Utf8ToWideChar(translated_fname);
result->reset();
@ -418,8 +418,8 @@ Status WindowsFileSystem::NewReadOnlyMemoryRegionFromFile(
return s;
}
Status WindowsFileSystem::FileExists(
const string& fname /*, TransactionToken* token */) {
Status WindowsFileSystem::FileExists(const string& fname,
TransactionToken* token) {
constexpr int kOk = 0;
std::wstring ws_translated_fname = Utf8ToWideChar(TranslateName(fname));
if (_waccess(ws_translated_fname.c_str(), kOk) == 0) {
@ -428,9 +428,9 @@ Status WindowsFileSystem::FileExists(
return errors::NotFound(fname, " not found");
}
Status WindowsFileSystem::GetChildren(
const string& dir,
std::vector<string>* result /*, TransactionToken* token */) {
Status WindowsFileSystem::GetChildren(const string& dir,
TransactionToken* token,
std::vector<string>* result) {
string translated_dir = TranslateName(dir);
std::wstring ws_translated_dir = Utf8ToWideChar(translated_dir);
result->clear();
@ -465,8 +465,8 @@ Status WindowsFileSystem::GetChildren(
return Status::OK();
}
Status WindowsFileSystem::DeleteFile(
const string& fname /*, TransactionToken* token */) {
Status WindowsFileSystem::DeleteFile(const string& fname,
TransactionToken* token) {
Status result;
std::wstring file_name = Utf8ToWideChar(fname);
if (_wunlink(file_name.c_str()) != 0) {
@ -475,8 +475,8 @@ Status WindowsFileSystem::DeleteFile(
return result;
}
Status WindowsFileSystem::CreateDir(
const string& name /*, TransactionToken* token */) {
Status WindowsFileSystem::CreateDir(const string& name,
TransactionToken* token) {
Status result;
std::wstring ws_name = Utf8ToWideChar(name);
if (ws_name.empty()) {
@ -488,8 +488,8 @@ Status WindowsFileSystem::CreateDir(
return result;
}
Status WindowsFileSystem::DeleteDir(
const string& name /*, TransactionToken* token */) {
Status WindowsFileSystem::DeleteDir(const string& name,
TransactionToken* token) {
Status result;
std::wstring ws_name = Utf8ToWideChar(name);
if (_wrmdir(ws_name.c_str()) != 0) {
@ -498,8 +498,8 @@ Status WindowsFileSystem::DeleteDir(
return result;
}
Status WindowsFileSystem::GetFileSize(
const string& fname, uint64* size /*, TransactionToken* token */) {
Status WindowsFileSystem::GetFileSize(const string& fname,
TransactionToken* token, uint64* size) {
string translated_fname = TranslateName(fname);
std::wstring ws_translated_dir = Utf8ToWideChar(translated_fname);
Status result;
@ -517,8 +517,8 @@ Status WindowsFileSystem::GetFileSize(
return result;
}
Status WindowsFileSystem::IsDirectory(
const string& fname /*, TransactionToken* token */) {
Status WindowsFileSystem::IsDirectory(const string& fname,
TransactionToken* token) {
TF_RETURN_IF_ERROR(FileExists(fname));
std::wstring ws_translated_fname = Utf8ToWideChar(TranslateName(fname));
if (PathIsDirectoryW(ws_translated_fname.c_str())) {
@ -527,8 +527,8 @@ Status WindowsFileSystem::IsDirectory(
return Status(tensorflow::error::FAILED_PRECONDITION, "Not a directory");
}
Status WindowsFileSystem::RenameFile(
const string& src, const string& target /*, TransactionToken* token */) {
Status WindowsFileSystem::RenameFile(const string& src, const string& target,
TransactionToken* token) {
Status result;
// rename() is not capable of replacing the existing file as on Linux
// so use OS API directly
@ -542,9 +542,9 @@ Status WindowsFileSystem::RenameFile(
return result;
}
Status WindowsFileSystem::GetMatchingPaths(
const string& pattern,
std::vector<string>* results /*, TransactionToken* token */) {
Status WindowsFileSystem::GetMatchingPaths(const string& pattern,
TransactionToken* token,
std::vector<string>* results) {
// NOTE(mrry): The existing implementation of FileSystem::GetMatchingPaths()
// does not handle Windows paths containing backslashes correctly. Since
// Windows APIs will accept forward and backslashes equivalently, we
@ -567,8 +567,8 @@ bool WindowsFileSystem::Match(const string& filename, const string& pattern) {
return PathMatchSpecW(ws_path.c_str(), ws_pattern.c_str()) == TRUE;
}
Status WindowsFileSystem::Stat(
const string& fname, FileStatistics* stat /*, TransactionToken* token */) {
Status WindowsFileSystem::Stat(const string& fname, TransactionToken* token,
FileStatistics* stat) {
Status result;
struct _stat sbuf;
std::wstring ws_translated_fname = Utf8ToWideChar(TranslateName(fname));

View File

@ -32,72 +32,50 @@ class WindowsFileSystem : public FileSystem {
~WindowsFileSystem() {}
TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT;
Status NewRandomAccessFile(
const string& fname,
std::unique_ptr<RandomAccessFile>*
result /*, TransactionToken* token = nullptr */) override;
const string& fname, TransactionToken* token,
std::unique_ptr<RandomAccessFile>* result) override;
Status NewWritableFile(
const string& fname,
std::unique_ptr<WritableFile>*
result /*, TransactionToken* token = nullptr */) override;
Status NewWritableFile(const string& fname, TransactionToken* token,
std::unique_ptr<WritableFile>* result) override;
Status NewAppendableFile(
const string& fname,
std::unique_ptr<WritableFile>*
result /*, TransactionToken* token = nullptr */) override;
Status NewAppendableFile(const string& fname, TransactionToken* token,
std::unique_ptr<WritableFile>* result) override;
Status NewReadOnlyMemoryRegionFromFile(
const string& fname,
std::unique_ptr<ReadOnlyMemoryRegion>*
result /*, TransactionToken* token = nullptr */) override;
const string& fname, TransactionToken* token,
std::unique_ptr<ReadOnlyMemoryRegion>* result) override;
Status FileExists(
const string& fname /*, TransactionToken* token = nullptr */) override;
Status FileExists(const string& fname, TransactionToken* token) override;
Status GetChildren(
const string& dir,
std::vector<string>* result /*, TransactionToken* token = nullptr */)
override;
Status GetChildren(const string& dir, TransactionToken* token,
std::vector<string>* result) override;
Status GetMatchingPaths(
const string& pattern,
std::vector<string>* result /*, TransactionToken* token = nullptr */)
override;
Status GetMatchingPaths(const string& pattern, TransactionToken* token,
std::vector<string>* result) override;
bool Match(
const string& filename,
const string& pattern /*, TransactionToken* token = nullptr */) override;
bool Match(const string& filename, const string& pattern) override;
Status Stat(
const string& fname,
FileStatistics* stat /*, TransactionToken* token = nullptr */) override;
Status Stat(const string& fname, TransactionToken* token,
FileStatistics* stat) override;
Status DeleteFile(
const string& fname /*, TransactionToken* token = nullptr */) override;
Status DeleteFile(const string& fname, TransactionToken* token) override;
Status CreateDir(
const string& name /*, TransactionToken* token = nullptr */) override;
Status CreateDir(const string& name, TransactionToken* token) override;
Status DeleteDir(
const string& name /*, TransactionToken* token = nullptr */) override;
Status DeleteDir(const string& name, TransactionToken* token) override;
Status GetFileSize(
const string& fname,
uint64* size /*, TransactionToken* token = nullptr */) override;
Status GetFileSize(const string& fname, TransactionToken* token,
uint64* size) override;
Status IsDirectory(
const string& fname /*, TransactionToken* token = nullptr */) override;
Status IsDirectory(const string& fname, TransactionToken* token) override;
Status RenameFile(
const string& src,
const string& target /*, TransactionToken* token = nullptr */) override;
Status RenameFile(const string& src, const string& target,
TransactionToken* token) override;
string TranslateName(
const string& name /*, TransactionToken* token = nullptr */)
const override {
return name;
}
string TranslateName(const string& name) const override { return name; }
char Separator() const override { return '\\'; };
};

View File

@ -86,8 +86,8 @@ class RandomAccessFileFromMemmapped : public RandomAccessFile {
MemmappedFileSystem::MemmappedFileSystem() {}
Status MemmappedFileSystem::FileExists(
const string& fname /*, TransactionToken* token */) {
Status MemmappedFileSystem::FileExists(const string& fname,
TransactionToken* token) {
if (!mapped_memory_) {
return errors::FailedPrecondition("MemmappedEnv is not initialized");
}
@ -99,8 +99,8 @@ Status MemmappedFileSystem::FileExists(
}
Status MemmappedFileSystem::NewRandomAccessFile(
const string& filename,
std::unique_ptr<RandomAccessFile>* result /*, TransactionToken* token */) {
const string& filename, TransactionToken* token,
std::unique_ptr<RandomAccessFile>* result) {
if (!mapped_memory_) {
return errors::FailedPrecondition("MemmappedEnv is not initialized");
}
@ -115,8 +115,8 @@ Status MemmappedFileSystem::NewRandomAccessFile(
}
Status MemmappedFileSystem::NewReadOnlyMemoryRegionFromFile(
const string& filename, std::unique_ptr<ReadOnlyMemoryRegion>*
result /*, TransactionToken* token */) {
const string& filename, TransactionToken* token,
std::unique_ptr<ReadOnlyMemoryRegion>* result) {
if (!mapped_memory_) {
return errors::FailedPrecondition("MemmappedEnv is not initialized");
}
@ -130,8 +130,8 @@ Status MemmappedFileSystem::NewReadOnlyMemoryRegionFromFile(
return Status::OK();
}
Status MemmappedFileSystem::GetFileSize(
const string& filename, uint64* size /*, TransactionToken* token */) {
Status MemmappedFileSystem::GetFileSize(const string& filename,
TransactionToken* token, uint64* size) {
if (!mapped_memory_) {
return errors::FailedPrecondition("MemmappedEnv is not initialized");
}
@ -143,59 +143,59 @@ Status MemmappedFileSystem::GetFileSize(
return Status::OK();
}
Status MemmappedFileSystem::Stat(
const string& fname, FileStatistics* stat /*, TransactionToken* token */) {
Status MemmappedFileSystem::Stat(const string& fname, TransactionToken* token,
FileStatistics* stat) {
uint64 size;
auto status = GetFileSize(fname, &size);
auto status = GetFileSize(fname, token, &size);
if (status.ok()) {
stat->length = size;
}
return status;
}
Status MemmappedFileSystem::NewWritableFile(
const string& filename,
std::unique_ptr<WritableFile>* wf /*, TransactionToken* token */) {
Status MemmappedFileSystem::NewWritableFile(const string& filename,
TransactionToken* token,
std::unique_ptr<WritableFile>* wf) {
return errors::Unimplemented("memmapped format doesn't support writing");
}
Status MemmappedFileSystem::NewAppendableFile(
const string& filename,
std::unique_ptr<WritableFile>* result /*, TransactionToken* token */) {
const string& filename, TransactionToken* token,
std::unique_ptr<WritableFile>* result) {
return errors::Unimplemented("memmapped format doesn't support writing");
}
Status MemmappedFileSystem::GetChildren(
const string& filename,
std::vector<string>* strings /*, TransactionToken* token */) {
Status MemmappedFileSystem::GetChildren(const string& filename,
TransactionToken* token,
std::vector<string>* strings) {
return errors::Unimplemented("memmapped format doesn't support GetChildren");
}
Status MemmappedFileSystem::GetMatchingPaths(
const string& pattern,
std::vector<string>* results /*, TransactionToken* token */) {
Status MemmappedFileSystem::GetMatchingPaths(const string& pattern,
TransactionToken* token,
std::vector<string>* results) {
return errors::Unimplemented(
"memmapped format doesn't support GetMatchingPaths");
}
Status MemmappedFileSystem::DeleteFile(
const string& filename /*, TransactionToken* token */) {
Status MemmappedFileSystem::DeleteFile(const string& filename,
TransactionToken* token) {
return errors::Unimplemented("memmapped format doesn't support DeleteFile");
}
Status MemmappedFileSystem::CreateDir(
const string& dirname /*, TransactionToken* token */) {
Status MemmappedFileSystem::CreateDir(const string& dirname,
TransactionToken* token) {
return errors::Unimplemented("memmapped format doesn't support CreateDir");
}
Status MemmappedFileSystem::DeleteDir(
const string& dirname /*, TransactionToken* token */) {
Status MemmappedFileSystem::DeleteDir(const string& dirname,
TransactionToken* token) {
return errors::Unimplemented("memmapped format doesn't support DeleteDir");
}
Status MemmappedFileSystem::RenameFile(
const string& filename_from,
const string& filename_to /*, TransactionToken* token */) {
Status MemmappedFileSystem::RenameFile(const string& filename_from,
const string& filename_to,
TransactionToken* token) {
return errors::Unimplemented("memmapped format doesn't support RenameFile");
}

View File

@ -60,52 +60,39 @@ class MemmappedFileSystem : public FileSystem {
MemmappedFileSystem();
~MemmappedFileSystem() override = default;
Status FileExists(
const string& fname /*, TransactionToken* token = nullptr */) override;
TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT;
Status FileExists(const string& fname, TransactionToken* token) override;
Status NewRandomAccessFile(
const string& filename,
std::unique_ptr<RandomAccessFile>*
result /*, TransactionToken* token = nullptr */) override;
const string& filename, TransactionToken* token,
std::unique_ptr<RandomAccessFile>* result) override;
Status NewReadOnlyMemoryRegionFromFile(
const string& filename,
std::unique_ptr<ReadOnlyMemoryRegion>*
result /*, TransactionToken* token = nullptr */) override;
const string& filename, TransactionToken* token,
std::unique_ptr<ReadOnlyMemoryRegion>* result) override;
// All these functions return Unimplemented error, the memmapped storage is
// read only.
Status NewWritableFile(
const string& fname,
std::unique_ptr<WritableFile>*
result /*, TransactionToken* token = nullptr */) override;
Status NewAppendableFile(
const string& fname,
std::unique_ptr<WritableFile>*
result /*, TransactionToken* token = nullptr */) override;
Status GetChildren(const string& dir,
std::vector<string>*
r /*, TransactionToken* token = nullptr */) override;
Status GetMatchingPaths(
const string& pattern,
std::vector<string>* results /*, TransactionToken* token = nullptr */)
override;
Status DeleteFile(
const string& f /*, TransactionToken* token = nullptr */) override;
Status CreateDir(
const string& d /*, TransactionToken* token = nullptr */) override;
Status DeleteDir(
const string& d /*, TransactionToken* token = nullptr */) override;
Status RenameFile(
const string& s,
const string& t /*, TransactionToken* token = nullptr */) override;
Status NewWritableFile(const string& fname, TransactionToken* token,
std::unique_ptr<WritableFile>* result) override;
Status NewAppendableFile(const string& fname, TransactionToken* token,
std::unique_ptr<WritableFile>* result) override;
Status GetChildren(const string& dir, TransactionToken* token,
std::vector<string>* r) override;
Status GetMatchingPaths(const string& pattern, TransactionToken* token,
std::vector<string>* results) override;
Status DeleteFile(const string& f, TransactionToken* token) override;
Status CreateDir(const string& d, TransactionToken* token) override;
Status DeleteDir(const string& d, TransactionToken* token) override;
Status RenameFile(const string& s, const string& t,
TransactionToken* token) override;
// These functions are implemented.
Status GetFileSize(
const string& f,
uint64* s /*, TransactionToken* token = nullptr */) override;
Status GetFileSize(const string& f, TransactionToken* token,
uint64* s) override;
// Currently just returns size.
Status Stat(
const string& fname,
FileStatistics* stat /*, TransactionToken* token = nullptr */) override;
Status Stat(const string& fname, TransactionToken* token,
FileStatistics* stat) override;
// Initializes filesystem from a file in memmapped format.
Status InitializeFromFile(Env* env, const string& filename);

View File

@ -124,8 +124,8 @@ AssetManagerFileSystem::AssetManagerFileSystem(AAssetManager* asset_manager,
const string& prefix)
: asset_manager_(asset_manager), prefix_(prefix) {}
Status AssetManagerFileSystem::FileExists(
const string& fname /*, TransactionToken* token */) {
Status AssetManagerFileSystem::FileExists(const string& fname,
TransactionToken* token) {
string path = RemoveAssetPrefix(fname);
auto asset = ScopedAsset(
AAssetManager_open(asset_manager_, path.c_str(), AASSET_MODE_RANDOM));
@ -136,8 +136,8 @@ Status AssetManagerFileSystem::FileExists(
}
Status AssetManagerFileSystem::NewRandomAccessFile(
const string& fname,
std::unique_ptr<RandomAccessFile>* result /*, TransactionToken* token */) {
const string& fname, TransactionToken* token,
std::unique_ptr<RandomAccessFile>* result) {
string path = RemoveAssetPrefix(fname);
auto asset = ScopedAsset(
AAssetManager_open(asset_manager_, path.c_str(), AASSET_MODE_RANDOM));
@ -149,8 +149,8 @@ Status AssetManagerFileSystem::NewRandomAccessFile(
}
Status AssetManagerFileSystem::NewReadOnlyMemoryRegionFromFile(
const string& fname, std::unique_ptr<ReadOnlyMemoryRegion>*
result /*, TransactionToken* token */) {
const string& fname, TransactionToken* token,
std::unique_ptr<ReadOnlyMemoryRegion>* result) {
string path = RemoveAssetPrefix(fname);
auto asset = ScopedAsset(
AAssetManager_open(asset_manager_, path.c_str(), AASSET_MODE_STREAMING));
@ -186,9 +186,9 @@ Status AssetManagerFileSystem::NewReadOnlyMemoryRegionFromFile(
return Status::OK();
}
Status AssetManagerFileSystem::GetChildren(
const string& prefixed_dir,
std::vector<string>* r /*, TransactionToken* token */) {
Status AssetManagerFileSystem::GetChildren(const string& prefixed_dir,
TransactionToken* token,
std::vector<string>* r) {
std::string path = NormalizeDirectoryPath(prefixed_dir);
auto dir =
ScopedAssetDir(AAssetManager_openDir(asset_manager_, path.c_str()));
@ -203,8 +203,8 @@ Status AssetManagerFileSystem::GetChildren(
return Status::OK();
}
Status AssetManagerFileSystem::GetFileSize(
const string& fname, uint64* s /*, TransactionToken* token */) {
Status AssetManagerFileSystem::GetFileSize(const string& fname,
TransactionToken* token, uint64* s) {
// If fname corresponds to a directory, return early. It doesn't map to an
// AAsset, and would otherwise return NotFound.
if (DirectoryExists(fname)) {
@ -221,8 +221,9 @@ Status AssetManagerFileSystem::GetFileSize(
return Status::OK();
}
Status AssetManagerFileSystem::Stat(
const string& fname, FileStatistics* stat /*, TransactionToken* token */) {
Status AssetManagerFileSystem::Stat(const string& fname,
TransactionToken* token,
FileStatistics* stat) {
uint64 size;
stat->is_directory = DirectoryExists(fname);
TF_RETURN_IF_ERROR(GetFileSize(fname, &size));
@ -240,8 +241,8 @@ string AssetManagerFileSystem::RemoveAssetPrefix(const string& name) {
return string(piece);
}
bool AssetManagerFileSystem::DirectoryExists(
const std::string& fname /*, TransactionToken* token */) {
bool AssetManagerFileSystem::DirectoryExists(const std::string& fname,
TransactionToken* token) {
std::string path = NormalizeDirectoryPath(fname);
auto dir =
ScopedAssetDir(AAssetManager_openDir(asset_manager_, path.c_str()));
@ -250,36 +251,36 @@ bool AssetManagerFileSystem::DirectoryExists(
return AAssetDir_getNextFileName(dir.get()) != NULL;
}
Status AssetManagerFileSystem::GetMatchingPaths(
const string& pattern,
std::vector<string>* results /*, TransactionToken* token */) {
Status AssetManagerFileSystem::GetMatchingPaths(const string& pattern,
TransactionToken* token,
std::vector<string>* results) {
return internal::GetMatchingPaths(this, Env::Default(), pattern, results);
}
Status AssetManagerFileSystem::NewWritableFile(
const string& fname,
std::unique_ptr<WritableFile>* result /*, TransactionToken* token */) {
const string& fname, TransactionToken* token,
std::unique_ptr<WritableFile>* result) {
return errors::Unimplemented("Asset storage is read only.");
}
Status AssetManagerFileSystem::NewAppendableFile(
const string& fname,
std::unique_ptr<WritableFile>* result /*, TransactionToken* token */) {
const string& fname, TransactionToken* token,
std::unique_ptr<WritableFile>* result) {
return errors::Unimplemented("Asset storage is read only.");
}
Status AssetManagerFileSystem::DeleteFile(
const string& f /*, TransactionToken* token */) {
Status AssetManagerFileSystem::DeleteFile(const string& f,
TransactionToken* token) {
return errors::Unimplemented("Asset storage is read only.");
}
Status AssetManagerFileSystem::CreateDir(
const string& d /*, TransactionToken* token */) {
Status AssetManagerFileSystem::CreateDir(const string& d,
TransactionToken* token) {
return errors::Unimplemented("Asset storage is read only.");
}
Status AssetManagerFileSystem::DeleteDir(
const string& d /*, TransactionToken* token */) {
Status AssetManagerFileSystem::DeleteDir(const string& d,
TransactionToken* token) {
return errors::Unimplemented("Asset storage is read only.");
}
Status AssetManagerFileSystem::RenameFile(
const string& s, const string& t /*, TransactionToken* token */) {
Status AssetManagerFileSystem::RenameFile(const string& s, const string& t,
TransactionToken* token) {
return errors::Unimplemented("Asset storage is read only.");
}

View File

@ -42,52 +42,38 @@ class AssetManagerFileSystem : public FileSystem {
AssetManagerFileSystem(AAssetManager* asset_manager, const string& prefix);
~AssetManagerFileSystem() override = default;
Status FileExists(
const string& fname /*, TransactionToken* token = nullptr*/) override;
Status NewRandomAccessFile(
const string& filename,
std::unique_ptr<RandomAccessFile>*
result /*, TransactionToken* token = nullptr*/) override;
Status NewReadOnlyMemoryRegionFromFile(
const string& filename,
std::unique_ptr<ReadOnlyMemoryRegion>*
result /*, TransactionToken* token = nullptr*/) override;
TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT;
Status GetFileSize(
const string& f,
uint64* s /*, TransactionToken* token = nullptr*/) override;
Status FileExists(const string& fname, TransactionToken* token) override;
Status NewRandomAccessFile(
const string& filename, TransactionToken* token,
std::unique_ptr<RandomAccessFile>* result) override;
Status NewReadOnlyMemoryRegionFromFile(
const string& filename, TransactionToken* token,
std::unique_ptr<ReadOnlyMemoryRegion>* result) override;
Status GetFileSize(const string& f, TransactionToken* token,
uint64* s) override;
// Currently just returns size.
Status Stat(
const string& fname,
FileStatistics* stat /*, TransactionToken* token = nullptr*/) override;
Status GetChildren(
const string& dir,
std::vector<string>* r /*, TransactionToken* token = nullptr*/) override;
Status Stat(const string& fname, TransactionToken* token,
FileStatistics* stat) override;
Status GetChildren(const string& dir, TransactionToken* token,
std::vector<string>* r) override;
// All these functions return Unimplemented error. Asset storage is
// read only.
Status NewWritableFile(
const string& fname,
std::unique_ptr<WritableFile>*
result /*, TransactionToken* token = nullptr*/) override;
Status NewAppendableFile(
const string& fname,
std::unique_ptr<WritableFile>*
result /*, TransactionToken* token = nullptr*/) override;
Status DeleteFile(
const string& f /*, TransactionToken* token = nullptr*/) override;
Status CreateDir(
const string& d /*, TransactionToken* token = nullptr*/) override;
Status DeleteDir(
const string& d /*, TransactionToken* token = nullptr*/) override;
Status RenameFile(
const string& s,
const string& t /*, TransactionToken* token = nullptr*/) override;
Status NewWritableFile(const string& fname, TransactionToken* token,
std::unique_ptr<WritableFile>* result) override;
Status NewAppendableFile(const string& fname, TransactionToken* token,
std::unique_ptr<WritableFile>* result) override;
Status DeleteFile(const string& f, TransactionToken* token) override;
Status CreateDir(const string& d, TransactionToken* token) override;
Status DeleteDir(const string& d, TransactionToken* token) override;
Status RenameFile(const string& s, const string& t,
TransactionToken* token) override;
Status GetMatchingPaths(
const string& pattern,
std::vector<string>* results /*, TransactionToken* token = nullptr*/)
override;
Status GetMatchingPaths(const string& pattern, TransactionToken* token,
std::vector<string>* results) override;
private:
string RemoveAssetPrefix(const string& name);