experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #42050 from samikama:Transactions_part12

Browse Source 
PiperOrigin-RevId: 325251453
Change-Id: I99328f0cad8021226d8a049b4ccf7caf422eab42
This commit is contained in:
TensorFlower Gardener 2020-08-06 10:01:09 -07:00
commit 91c0645cbb
10 changed files with 275 additions and 363 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
tensorflow/core/platform/default/posix_file_system.cc
View File

@ -178,8 +178,8 @@ class PosixReadOnlyMemoryRegion : public ReadOnlyMemoryRegion {
}; };
Status PosixFileSystem::NewRandomAccessFile( Status PosixFileSystem::NewRandomAccessFile(
const string& fname, const string& fname, TransactionToken* token,
std::unique_ptr<RandomAccessFile>* result /*, TransactionToken* token */) { std::unique_ptr<RandomAccessFile>* result) {
string translated_fname = TranslateName(fname); string translated_fname = TranslateName(fname);
Status s; Status s;
int fd = open(translated_fname.c_str(), O_RDONLY); int fd = open(translated_fname.c_str(), O_RDONLY);
@ -191,9 +191,9 @@ Status PosixFileSystem::NewRandomAccessFile(
return s; return s;
} }
Status PosixFileSystem::NewWritableFile( Status PosixFileSystem::NewWritableFile(const string& fname,
const string& fname, TransactionToken* token,
std::unique_ptr<WritableFile>* result /*, TransactionToken* token */) { std::unique_ptr<WritableFile>* result) {
string translated_fname = TranslateName(fname); string translated_fname = TranslateName(fname);
Status s; Status s;
FILE* f = fopen(translated_fname.c_str(), "w"); FILE* f = fopen(translated_fname.c_str(), "w");
@ -206,8 +206,8 @@ Status PosixFileSystem::NewWritableFile(
} }
Status PosixFileSystem::NewAppendableFile( Status PosixFileSystem::NewAppendableFile(
const string& fname, const string& fname, TransactionToken* token,
std::unique_ptr<WritableFile>* result /*, TransactionToken* token */) { std::unique_ptr<WritableFile>* result) {
string translated_fname = TranslateName(fname); string translated_fname = TranslateName(fname);
Status s; Status s;
FILE* f = fopen(translated_fname.c_str(), "a"); FILE* f = fopen(translated_fname.c_str(), "a");
@ -220,8 +220,8 @@ Status PosixFileSystem::NewAppendableFile(
} }
Status PosixFileSystem::NewReadOnlyMemoryRegionFromFile( Status PosixFileSystem::NewReadOnlyMemoryRegionFromFile(
const string& fname, std::unique_ptr<ReadOnlyMemoryRegion>* const string& fname, TransactionToken* token,
result /*, TransactionToken* token */) { std::unique_ptr<ReadOnlyMemoryRegion>* result) {
string translated_fname = TranslateName(fname); string translated_fname = TranslateName(fname);
Status s = Status::OK(); Status s = Status::OK();
int fd = open(translated_fname.c_str(), O_RDONLY); int fd = open(translated_fname.c_str(), O_RDONLY);
@ -244,17 +244,16 @@ Status PosixFileSystem::NewReadOnlyMemoryRegionFromFile(
return s; return s;
} }
Status PosixFileSystem::FileExists( Status PosixFileSystem::FileExists(const string& fname,
const string& fname /*, TransactionToken* token */) { TransactionToken* token) {
if (access(TranslateName(fname).c_str(), F_OK) == 0) { if (access(TranslateName(fname).c_str(), F_OK) == 0) {
return Status::OK(); return Status::OK();
} }
return errors::NotFound(fname, " not found"); return errors::NotFound(fname, " not found");
} }
Status PosixFileSystem::GetChildren( Status PosixFileSystem::GetChildren(const string& dir, TransactionToken* token,
const string& dir, std::vector<string>* result) {
std::vector<string>* result /*, TransactionToken* token */) {
string translated_dir = TranslateName(dir); string translated_dir = TranslateName(dir);
result->clear(); result->clear();
DIR* d = opendir(translated_dir.c_str()); DIR* d = opendir(translated_dir.c_str());
@ -274,14 +273,14 @@ Status PosixFileSystem::GetChildren(
return Status::OK(); return Status::OK();
} }
Status PosixFileSystem::GetMatchingPaths( Status PosixFileSystem::GetMatchingPaths(const string& pattern,
const string& pattern, TransactionToken* token,
std::vector<string>* results /*, TransactionToken* token */) { std::vector<string>* results) {
return internal::GetMatchingPaths(this, Env::Default(), pattern, results); return internal::GetMatchingPaths(this, Env::Default(), pattern, results);
} }
Status PosixFileSystem::DeleteFile( Status PosixFileSystem::DeleteFile(const string& fname,
const string& fname /*, TransactionToken* token */) { TransactionToken* token) {
Status result; Status result;
if (unlink(TranslateName(fname).c_str()) != 0) { if (unlink(TranslateName(fname).c_str()) != 0) {
result = IOError(fname, errno); result = IOError(fname, errno);
@ -289,8 +288,7 @@ Status PosixFileSystem::DeleteFile(
return result; return result;
} }
Status PosixFileSystem::CreateDir( Status PosixFileSystem::CreateDir(const string& name, TransactionToken* token) {
const string& name /*, TransactionToken* token */) {
string translated = TranslateName(name); string translated = TranslateName(name);
if (translated.empty()) { if (translated.empty()) {
return errors::AlreadyExists(name); return errors::AlreadyExists(name);
@ -301,8 +299,7 @@ Status PosixFileSystem::CreateDir(
return Status::OK(); return Status::OK();
} }
Status PosixFileSystem::DeleteDir( Status PosixFileSystem::DeleteDir(const string& name, TransactionToken* token) {
const string& name /*, TransactionToken* token */) {
Status result; Status result;
if (rmdir(TranslateName(name).c_str()) != 0) { if (rmdir(TranslateName(name).c_str()) != 0) {
result = IOError(name, errno); result = IOError(name, errno);
@ -310,8 +307,8 @@ Status PosixFileSystem::DeleteDir(
return result; return result;
} }
Status PosixFileSystem::GetFileSize( Status PosixFileSystem::GetFileSize(const string& fname,
const string& fname, uint64* size /*, TransactionToken* token */) { TransactionToken* token, uint64* size) {
Status s; Status s;
struct stat sbuf; struct stat sbuf;
if (stat(TranslateName(fname).c_str(), &sbuf) != 0) { if (stat(TranslateName(fname).c_str(), &sbuf) != 0) {
@ -323,8 +320,8 @@ Status PosixFileSystem::GetFileSize(
return s; return s;
} }
Status PosixFileSystem::Stat( Status PosixFileSystem::Stat(const string& fname, TransactionToken* token,
const string& fname, FileStatistics* stats /*, TransactionToken* token */) { FileStatistics* stats) {
Status s; Status s;
struct stat sbuf; struct stat sbuf;
if (stat(TranslateName(fname).c_str(), &sbuf) != 0) { if (stat(TranslateName(fname).c_str(), &sbuf) != 0) {
@ -337,8 +334,8 @@ Status PosixFileSystem::Stat(
return s; return s;
} }
Status PosixFileSystem::RenameFile( Status PosixFileSystem::RenameFile(const string& src, const string& target,
const string& src, const string& target /*, TransactionToken* token */) { TransactionToken* token) {
Status result; Status result;
if (rename(TranslateName(src).c_str(), TranslateName(target).c_str()) != 0) { if (rename(TranslateName(src).c_str(), TranslateName(target).c_str()) != 0) {
result = IOError(src, errno); result = IOError(src, errno);
@ -346,8 +343,8 @@ Status PosixFileSystem::RenameFile(
return result; return result;
} }
Status PosixFileSystem::CopyFile( Status PosixFileSystem::CopyFile(const string& src, const string& target,
const string& src, const string& target /*, TransactionToken* token */) { TransactionToken* token) {
string translated_src = TranslateName(src); string translated_src = TranslateName(src);
struct stat sbuf; struct stat sbuf;
if (stat(translated_src.c_str(), &sbuf) != 0) { if (stat(translated_src.c_str(), &sbuf) != 0) {

68
tensorflow/core/platform/default/posix_file_system.h
View File

@ -27,63 +27,47 @@ class PosixFileSystem : public FileSystem {
~PosixFileSystem() {} ~PosixFileSystem() {}
TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT;
Status NewRandomAccessFile( Status NewRandomAccessFile(
const string& filename, const string& filename, TransactionToken* token,
std::unique_ptr<RandomAccessFile>* std::unique_ptr<RandomAccessFile>* result) override;
result /*, TransactionToken* token = nullptr */) override;
Status NewWritableFile( Status NewWritableFile(const string& fname, TransactionToken* token,
const string& fname, std::unique_ptr<WritableFile>* result) override;
std::unique_ptr<WritableFile>*
result /*, TransactionToken* token = nullptr */) override;
Status NewAppendableFile( Status NewAppendableFile(const string& fname, TransactionToken* token,
const string& fname, std::unique_ptr<WritableFile>* result) override;
std::unique_ptr<WritableFile>*
result /*, TransactionToken* token = nullptr */) override;
Status NewReadOnlyMemoryRegionFromFile( Status NewReadOnlyMemoryRegionFromFile(
const string& filename, const string& filename, TransactionToken* token,
std::unique_ptr<ReadOnlyMemoryRegion>* std::unique_ptr<ReadOnlyMemoryRegion>* result) override;
result /*, TransactionToken* token = nullptr */) override;
Status FileExists( Status FileExists(const string& fname, TransactionToken* token) override;
const string& fname /*, TransactionToken* token = nullptr */) override;
Status GetChildren( Status GetChildren(const string& dir, TransactionToken* token,
const string& dir, std::vector<string>* result) override;
std::vector<string>* result /*, TransactionToken* token = nullptr */)
override;
Status Stat( Status Stat(const string& fname, TransactionToken* token,
const string& fname, FileStatistics* stats) override;
FileStatistics* stats /*, TransactionToken* token = nullptr */) override;
Status GetMatchingPaths( Status GetMatchingPaths(const string& pattern, TransactionToken* token,
const string& pattern, std::vector<string>* results) override;
std::vector<string>* results /*, TransactionToken* token = nullptr */)
override;
Status DeleteFile( Status DeleteFile(const string& fname, TransactionToken* token) override;
const string& fname /*, TransactionToken* token = nullptr */) override;
Status CreateDir( Status CreateDir(const string& name, TransactionToken* token) override;
const string& name /*, TransactionToken* token = nullptr */) override;
Status DeleteDir( Status DeleteDir(const string& name, TransactionToken* token) override;
const string& name /*, TransactionToken* token = nullptr */) override;
Status GetFileSize( Status GetFileSize(const string& fname, TransactionToken* token,
const string& fname, uint64* size) override;
uint64* size /*, TransactionToken* token = nullptr */) override;
Status RenameFile( Status RenameFile(const string& src, const string& target,
const string& src, TransactionToken* token) override;
const string& target /*, TransactionToken* token = nullptr */) override;
Status CopyFile( Status CopyFile(const string& src, const string& target,
const string& src, TransactionToken* token) override;
const string& target /*, TransactionToken* token = nullptr */) override;
}; };
Status IOError(const string& context, int err_number); Status IOError(const string& context, int err_number);

57
tensorflow/core/platform/hadoop/hadoop_file_system.cc
View File

@ -280,8 +280,8 @@ class HDFSRandomAccessFile : public RandomAccessFile {
}; };
Status HadoopFileSystem::NewRandomAccessFile( Status HadoopFileSystem::NewRandomAccessFile(
const string& fname, const string& fname, TransactionToken* token,
std::unique_ptr<RandomAccessFile>* result /*, TransactionToken* token */) { std::unique_ptr<RandomAccessFile>* result) {
hdfsFS fs = nullptr; hdfsFS fs = nullptr;
TF_RETURN_IF_ERROR(Connect(fname, &fs)); TF_RETURN_IF_ERROR(Connect(fname, &fs));
@ -373,8 +373,8 @@ class HDFSWritableFile : public WritableFile {
}; };
Status HadoopFileSystem::NewWritableFile( Status HadoopFileSystem::NewWritableFile(
const string& fname, const string& fname, TransactionToken* token,
std::unique_ptr<WritableFile>* result /*, TransactionToken* token */) { std::unique_ptr<WritableFile>* result) {
hdfsFS fs = nullptr; hdfsFS fs = nullptr;
TF_RETURN_IF_ERROR(Connect(fname, &fs)); TF_RETURN_IF_ERROR(Connect(fname, &fs));
@ -388,8 +388,8 @@ Status HadoopFileSystem::NewWritableFile(
} }
Status HadoopFileSystem::NewAppendableFile( Status HadoopFileSystem::NewAppendableFile(
const string& fname, const string& fname, TransactionToken* token,
std::unique_ptr<WritableFile>* result /*, TransactionToken* token */) { std::unique_ptr<WritableFile>* result) {
hdfsFS fs = nullptr; hdfsFS fs = nullptr;
TF_RETURN_IF_ERROR(Connect(fname, &fs)); TF_RETURN_IF_ERROR(Connect(fname, &fs));
@ -403,8 +403,8 @@ Status HadoopFileSystem::NewAppendableFile(
} }
Status HadoopFileSystem::NewReadOnlyMemoryRegionFromFile( Status HadoopFileSystem::NewReadOnlyMemoryRegionFromFile(
const string& fname, std::unique_ptr<ReadOnlyMemoryRegion>* const string& fname, TransactionToken* token,
result /*, TransactionToken* token */) { std::unique_ptr<ReadOnlyMemoryRegion>* result) {
// hadoopReadZero() technically supports this call with the following // hadoopReadZero() technically supports this call with the following
// caveats: // caveats:
// - It only works up to 2 GB. We'd have to Stat() the file to ensure that // - It only works up to 2 GB. We'd have to Stat() the file to ensure that
@ -414,8 +414,8 @@ Status HadoopFileSystem::NewReadOnlyMemoryRegionFromFile(
return errors::Unimplemented("HDFS does not support ReadOnlyMemoryRegion"); return errors::Unimplemented("HDFS does not support ReadOnlyMemoryRegion");
} }
Status HadoopFileSystem::FileExists( Status HadoopFileSystem::FileExists(const string& fname,
const string& fname /*, TransactionToken* token */) { TransactionToken* token) {
hdfsFS fs = nullptr; hdfsFS fs = nullptr;
TF_RETURN_IF_ERROR(Connect(fname, &fs)); TF_RETURN_IF_ERROR(Connect(fname, &fs));
if (libhdfs()->hdfsExists(fs, TranslateName(fname).c_str()) == 0) { if (libhdfs()->hdfsExists(fs, TranslateName(fname).c_str()) == 0) {
@ -424,9 +424,8 @@ Status HadoopFileSystem::FileExists(
return errors::NotFound(fname, " not found."); return errors::NotFound(fname, " not found.");
} }
Status HadoopFileSystem::GetChildren( Status HadoopFileSystem::GetChildren(const string& dir, TransactionToken* token,
const string& dir, std::vector<string>* result) {
std::vector<string>* result /*, TransactionToken* token */) {
result->clear(); result->clear();
hdfsFS fs = nullptr; hdfsFS fs = nullptr;
TF_RETURN_IF_ERROR(Connect(dir, &fs)); TF_RETURN_IF_ERROR(Connect(dir, &fs));
@ -434,7 +433,7 @@ Status HadoopFileSystem::GetChildren(
// hdfsListDirectory returns nullptr if the directory is empty. Do a separate // hdfsListDirectory returns nullptr if the directory is empty. Do a separate
// check to verify the directory exists first. // check to verify the directory exists first.
FileStatistics stat; FileStatistics stat;
TF_RETURN_IF_ERROR(Stat(dir, &stat)); TF_RETURN_IF_ERROR(Stat(dir, token, &stat));
int entries = 0; int entries = 0;
hdfsFileInfo* info = hdfsFileInfo* info =
@ -453,14 +452,14 @@ Status HadoopFileSystem::GetChildren(
return Status::OK(); return Status::OK();
} }
Status HadoopFileSystem::GetMatchingPaths( Status HadoopFileSystem::GetMatchingPaths(const string& pattern,
const string& pattern, TransactionToken* token,
std::vector<string>* results /*, TransactionToken* token */) { std::vector<string>* results) {
return internal::GetMatchingPaths(this, Env::Default(), pattern, results); return internal::GetMatchingPaths(this, Env::Default(), pattern, results);
} }
Status HadoopFileSystem::DeleteFile( Status HadoopFileSystem::DeleteFile(const string& fname,
const string& fname /*, TransactionToken* token */) { TransactionToken* token) {
hdfsFS fs = nullptr; hdfsFS fs = nullptr;
TF_RETURN_IF_ERROR(Connect(fname, &fs)); TF_RETURN_IF_ERROR(Connect(fname, &fs));
@ -471,8 +470,7 @@ Status HadoopFileSystem::DeleteFile(
return Status::OK(); return Status::OK();
} }
Status HadoopFileSystem::CreateDir( Status HadoopFileSystem::CreateDir(const string& dir, TransactionToken* token) {
const string& dir /*, TransactionToken* token */) {
hdfsFS fs = nullptr; hdfsFS fs = nullptr;
TF_RETURN_IF_ERROR(Connect(dir, &fs)); TF_RETURN_IF_ERROR(Connect(dir, &fs));
@ -482,8 +480,7 @@ Status HadoopFileSystem::CreateDir(
return Status::OK(); return Status::OK();
} }
Status HadoopFileSystem::DeleteDir( Status HadoopFileSystem::DeleteDir(const string& dir, TransactionToken* token) {
const string& dir /*, TransactionToken* token */) {
hdfsFS fs = nullptr; hdfsFS fs = nullptr;
TF_RETURN_IF_ERROR(Connect(dir, &fs)); TF_RETURN_IF_ERROR(Connect(dir, &fs));
@ -502,7 +499,7 @@ Status HadoopFileSystem::DeleteDir(
// the call is actually successful. Check again by Stat. // the call is actually successful. Check again by Stat.
if (info == nullptr && errno != 0) { if (info == nullptr && errno != 0) {
FileStatistics stat; FileStatistics stat;
TF_RETURN_IF_ERROR(Stat(dir, &stat)); TF_RETURN_IF_ERROR(Stat(dir, token, &stat));
} }
if (entries > 0) { if (entries > 0) {
@ -515,8 +512,8 @@ Status HadoopFileSystem::DeleteDir(
return Status::OK(); return Status::OK();
} }
Status HadoopFileSystem::GetFileSize( Status HadoopFileSystem::GetFileSize(const string& fname,
const string& fname, uint64* size /*, TransactionToken* token */) { TransactionToken* token, uint64* size) {
hdfsFS fs = nullptr; hdfsFS fs = nullptr;
TF_RETURN_IF_ERROR(Connect(fname, &fs)); TF_RETURN_IF_ERROR(Connect(fname, &fs));
@ -530,8 +527,8 @@ Status HadoopFileSystem::GetFileSize(
return Status::OK(); return Status::OK();
} }
Status HadoopFileSystem::RenameFile( Status HadoopFileSystem::RenameFile(const string& src, const string& target,
const string& src, const string& target /*, TransactionToken* token */) { TransactionToken* token) {
hdfsFS fs = nullptr; hdfsFS fs = nullptr;
TF_RETURN_IF_ERROR(Connect(src, &fs)); TF_RETURN_IF_ERROR(Connect(src, &fs));
@ -548,8 +545,8 @@ Status HadoopFileSystem::RenameFile(
return Status::OK(); return Status::OK();
} }
Status HadoopFileSystem::Stat( Status HadoopFileSystem::Stat(const string& fname, TransactionToken* token,
const string& fname, FileStatistics* stats /*, TransactionToken* token */) { FileStatistics* stats) {
hdfsFS fs = nullptr; hdfsFS fs = nullptr;
TF_RETURN_IF_ERROR(Connect(fname, &fs)); TF_RETURN_IF_ERROR(Connect(fname, &fs));

67
tensorflow/core/platform/hadoop/hadoop_file_system.h
View File

@ -32,63 +32,46 @@ class HadoopFileSystem : public FileSystem {
HadoopFileSystem(); HadoopFileSystem();
~HadoopFileSystem(); ~HadoopFileSystem();
TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT;
Status NewRandomAccessFile( Status NewRandomAccessFile(
const string& fname, const string& fname, TransactionToken* token,
std::unique_ptr<RandomAccessFile>* std::unique_ptr<RandomAccessFile>* result) override;
result /*, TransactionToken* token = nullptr*/) override;
Status NewWritableFile( Status NewWritableFile(const string& fname, TransactionToken* token,
const string& fname, std::unique_ptr<WritableFile>* result) override;
std::unique_ptr<WritableFile>*
result /*, TransactionToken* token = nullptr*/) override;
Status NewAppendableFile( Status NewAppendableFile(const string& fname, TransactionToken* token,
const string& fname, std::unique_ptr<WritableFile>* result) override;
std::unique_ptr<WritableFile>*
result /*, TransactionToken* token = nullptr*/) override;
Status NewReadOnlyMemoryRegionFromFile( Status NewReadOnlyMemoryRegionFromFile(
const string& fname, const string& fname, TransactionToken* token,
std::unique_ptr<ReadOnlyMemoryRegion>* std::unique_ptr<ReadOnlyMemoryRegion>* result) override;
result /*, TransactionToken* token = nullptr*/) override;
Status FileExists( Status FileExists(const string& fname, TransactionToken* token) override;
const string& fname /*, TransactionToken* token = nullptr*/) override;
Status GetChildren( Status GetChildren(const string& dir, TransactionToken* token,
const string& dir, std::vector<string>* result) override;
std::vector<string>* result /*, TransactionToken* token = nullptr*/)
override;
Status GetMatchingPaths( Status GetMatchingPaths(const string& pattern, TransactionToken* token,
const string& pattern, std::vector<string>* results) override;
std::vector<string>* results /*, TransactionToken* token = nullptr*/)
override;
Status DeleteFile( Status DeleteFile(const string& fname, TransactionToken* token) override;
const string& fname /*, TransactionToken* token = nullptr*/) override;
Status CreateDir( Status CreateDir(const string& dir, TransactionToken* token) override;
const string& name /*, TransactionToken* token = nullptr*/) override;
Status DeleteDir( Status DeleteDir(const string& dir, TransactionToken* token) override;
const string& name /*, TransactionToken* token = nullptr*/) override;
Status GetFileSize( Status GetFileSize(const string& fname, TransactionToken* token,
const string& fname, uint64* size) override;
uint64* size /*, TransactionToken* token = nullptr*/) override;
Status RenameFile( Status RenameFile(const string& src, const string& target,
const string& src, TransactionToken* token) override;
const string& target /*, TransactionToken* token = nullptr*/) override;
Status Stat( Status Stat(const string& fname, TransactionToken* token,
const string& fname, FileStatistics* stat) override;
FileStatistics* stat /*, TransactionToken* token = nullptr*/) override;
string TranslateName( string TranslateName(const string& name) const override;
const string& name /*, TransactionToken* token = nullptr*/)
const override;
private: private:
Status Connect(StringPiece fname, hdfsFS* fs); Status Connect(StringPiece fname, hdfsFS* fs);

60
tensorflow/core/platform/windows/windows_file_system.cc
View File

@ -261,8 +261,8 @@ class WinReadOnlyMemoryRegion : public ReadOnlyMemoryRegion {
} // namespace } // namespace
Status WindowsFileSystem::NewRandomAccessFile( Status WindowsFileSystem::NewRandomAccessFile(
const string& fname, const string& fname, TransactionToken* token,
std::unique_ptr<RandomAccessFile>* result /*, TransactionToken* token */) { std::unique_ptr<RandomAccessFile>* result) {
string translated_fname = TranslateName(fname); string translated_fname = TranslateName(fname);
std::wstring ws_translated_fname = Utf8ToWideChar(translated_fname); std::wstring ws_translated_fname = Utf8ToWideChar(translated_fname);
result->reset(); result->reset();
@ -289,8 +289,8 @@ Status WindowsFileSystem::NewRandomAccessFile(
} }
Status WindowsFileSystem::NewWritableFile( Status WindowsFileSystem::NewWritableFile(
const string& fname, const string& fname, TransactionToken* token,
std::unique_ptr<WritableFile>* result /*, TransactionToken* token */) { std::unique_ptr<WritableFile>* result) {
string translated_fname = TranslateName(fname); string translated_fname = TranslateName(fname);
std::wstring ws_translated_fname = Utf8ToWideChar(translated_fname); std::wstring ws_translated_fname = Utf8ToWideChar(translated_fname);
result->reset(); result->reset();
@ -310,8 +310,8 @@ Status WindowsFileSystem::NewWritableFile(
} }
Status WindowsFileSystem::NewAppendableFile( Status WindowsFileSystem::NewAppendableFile(
const string& fname, const string& fname, TransactionToken* token,
std::unique_ptr<WritableFile>* result /*, TransactionToken* token */) { std::unique_ptr<WritableFile>* result) {
string translated_fname = TranslateName(fname); string translated_fname = TranslateName(fname);
std::wstring ws_translated_fname = Utf8ToWideChar(translated_fname); std::wstring ws_translated_fname = Utf8ToWideChar(translated_fname);
result->reset(); result->reset();
@ -341,8 +341,8 @@ Status WindowsFileSystem::NewAppendableFile(
} }
Status WindowsFileSystem::NewReadOnlyMemoryRegionFromFile( Status WindowsFileSystem::NewReadOnlyMemoryRegionFromFile(
const string& fname, std::unique_ptr<ReadOnlyMemoryRegion>* const string& fname, TransactionToken* token,
result /*, TransactionToken* token */) { std::unique_ptr<ReadOnlyMemoryRegion>* result) {
string translated_fname = TranslateName(fname); string translated_fname = TranslateName(fname);
std::wstring ws_translated_fname = Utf8ToWideChar(translated_fname); std::wstring ws_translated_fname = Utf8ToWideChar(translated_fname);
result->reset(); result->reset();
@ -418,8 +418,8 @@ Status WindowsFileSystem::NewReadOnlyMemoryRegionFromFile(
return s; return s;
} }
Status WindowsFileSystem::FileExists( Status WindowsFileSystem::FileExists(const string& fname,
const string& fname /*, TransactionToken* token */) { TransactionToken* token) {
constexpr int kOk = 0; constexpr int kOk = 0;
std::wstring ws_translated_fname = Utf8ToWideChar(TranslateName(fname)); std::wstring ws_translated_fname = Utf8ToWideChar(TranslateName(fname));
if (_waccess(ws_translated_fname.c_str(), kOk) == 0) { if (_waccess(ws_translated_fname.c_str(), kOk) == 0) {
@ -428,9 +428,9 @@ Status WindowsFileSystem::FileExists(
return errors::NotFound(fname, " not found"); return errors::NotFound(fname, " not found");
} }
Status WindowsFileSystem::GetChildren( Status WindowsFileSystem::GetChildren(const string& dir,
const string& dir, TransactionToken* token,
std::vector<string>* result /*, TransactionToken* token */) { std::vector<string>* result) {
string translated_dir = TranslateName(dir); string translated_dir = TranslateName(dir);
std::wstring ws_translated_dir = Utf8ToWideChar(translated_dir); std::wstring ws_translated_dir = Utf8ToWideChar(translated_dir);
result->clear(); result->clear();
@ -465,8 +465,8 @@ Status WindowsFileSystem::GetChildren(
return Status::OK(); return Status::OK();
} }
Status WindowsFileSystem::DeleteFile( Status WindowsFileSystem::DeleteFile(const string& fname,
const string& fname /*, TransactionToken* token */) { TransactionToken* token) {
Status result; Status result;
std::wstring file_name = Utf8ToWideChar(fname); std::wstring file_name = Utf8ToWideChar(fname);
if (_wunlink(file_name.c_str()) != 0) { if (_wunlink(file_name.c_str()) != 0) {
@ -475,8 +475,8 @@ Status WindowsFileSystem::DeleteFile(
return result; return result;
} }
Status WindowsFileSystem::CreateDir( Status WindowsFileSystem::CreateDir(const string& name,
const string& name /*, TransactionToken* token */) { TransactionToken* token) {
Status result; Status result;
std::wstring ws_name = Utf8ToWideChar(name); std::wstring ws_name = Utf8ToWideChar(name);
if (ws_name.empty()) { if (ws_name.empty()) {
@ -488,8 +488,8 @@ Status WindowsFileSystem::CreateDir(
return result; return result;
} }
Status WindowsFileSystem::DeleteDir( Status WindowsFileSystem::DeleteDir(const string& name,
const string& name /*, TransactionToken* token */) { TransactionToken* token) {
Status result; Status result;
std::wstring ws_name = Utf8ToWideChar(name); std::wstring ws_name = Utf8ToWideChar(name);
if (_wrmdir(ws_name.c_str()) != 0) { if (_wrmdir(ws_name.c_str()) != 0) {
@ -498,8 +498,8 @@ Status WindowsFileSystem::DeleteDir(
return result; return result;
} }
Status WindowsFileSystem::GetFileSize( Status WindowsFileSystem::GetFileSize(const string& fname,
const string& fname, uint64* size /*, TransactionToken* token */) { TransactionToken* token, uint64* size) {
string translated_fname = TranslateName(fname); string translated_fname = TranslateName(fname);
std::wstring ws_translated_dir = Utf8ToWideChar(translated_fname); std::wstring ws_translated_dir = Utf8ToWideChar(translated_fname);
Status result; Status result;
@ -517,8 +517,8 @@ Status WindowsFileSystem::GetFileSize(
return result; return result;
} }
Status WindowsFileSystem::IsDirectory( Status WindowsFileSystem::IsDirectory(const string& fname,
const string& fname /*, TransactionToken* token */) { TransactionToken* token) {
TF_RETURN_IF_ERROR(FileExists(fname)); TF_RETURN_IF_ERROR(FileExists(fname));
std::wstring ws_translated_fname = Utf8ToWideChar(TranslateName(fname)); std::wstring ws_translated_fname = Utf8ToWideChar(TranslateName(fname));
if (PathIsDirectoryW(ws_translated_fname.c_str())) { if (PathIsDirectoryW(ws_translated_fname.c_str())) {
@ -527,8 +527,8 @@ Status WindowsFileSystem::IsDirectory(
return Status(tensorflow::error::FAILED_PRECONDITION, "Not a directory"); return Status(tensorflow::error::FAILED_PRECONDITION, "Not a directory");
} }
Status WindowsFileSystem::RenameFile( Status WindowsFileSystem::RenameFile(const string& src, const string& target,
const string& src, const string& target /*, TransactionToken* token */) { TransactionToken* token) {
Status result; Status result;
// rename() is not capable of replacing the existing file as on Linux // rename() is not capable of replacing the existing file as on Linux
// so use OS API directly // so use OS API directly
@ -542,9 +542,9 @@ Status WindowsFileSystem::RenameFile(
return result; return result;
} }
Status WindowsFileSystem::GetMatchingPaths( Status WindowsFileSystem::GetMatchingPaths(const string& pattern,
const string& pattern, TransactionToken* token,
std::vector<string>* results /*, TransactionToken* token */) { std::vector<string>* results) {
// NOTE(mrry): The existing implementation of FileSystem::GetMatchingPaths() // NOTE(mrry): The existing implementation of FileSystem::GetMatchingPaths()
// does not handle Windows paths containing backslashes correctly. Since // does not handle Windows paths containing backslashes correctly. Since
// Windows APIs will accept forward and backslashes equivalently, we // Windows APIs will accept forward and backslashes equivalently, we
@ -567,8 +567,8 @@ bool WindowsFileSystem::Match(const string& filename, const string& pattern) {
return PathMatchSpecW(ws_path.c_str(), ws_pattern.c_str()) == TRUE; return PathMatchSpecW(ws_path.c_str(), ws_pattern.c_str()) == TRUE;
} }
Status WindowsFileSystem::Stat( Status WindowsFileSystem::Stat(const string& fname, TransactionToken* token,
const string& fname, FileStatistics* stat /*, TransactionToken* token */) { FileStatistics* stat) {
Status result; Status result;
struct _stat sbuf; struct _stat sbuf;
std::wstring ws_translated_fname = Utf8ToWideChar(TranslateName(fname)); std::wstring ws_translated_fname = Utf8ToWideChar(TranslateName(fname));

76
tensorflow/core/platform/windows/windows_file_system.h
View File

@ -32,72 +32,50 @@ class WindowsFileSystem : public FileSystem {
~WindowsFileSystem() {} ~WindowsFileSystem() {}
TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT;
Status NewRandomAccessFile( Status NewRandomAccessFile(
const string& fname, const string& fname, TransactionToken* token,
std::unique_ptr<RandomAccessFile>* std::unique_ptr<RandomAccessFile>* result) override;
result /*, TransactionToken* token = nullptr */) override;
Status NewWritableFile( Status NewWritableFile(const string& fname, TransactionToken* token,
const string& fname, std::unique_ptr<WritableFile>* result) override;
std::unique_ptr<WritableFile>*
result /*, TransactionToken* token = nullptr */) override;
Status NewAppendableFile( Status NewAppendableFile(const string& fname, TransactionToken* token,
const string& fname, std::unique_ptr<WritableFile>* result) override;
std::unique_ptr<WritableFile>*
result /*, TransactionToken* token = nullptr */) override;
Status NewReadOnlyMemoryRegionFromFile( Status NewReadOnlyMemoryRegionFromFile(
const string& fname, const string& fname, TransactionToken* token,
std::unique_ptr<ReadOnlyMemoryRegion>* std::unique_ptr<ReadOnlyMemoryRegion>* result) override;
result /*, TransactionToken* token = nullptr */) override;
Status FileExists( Status FileExists(const string& fname, TransactionToken* token) override;
const string& fname /*, TransactionToken* token = nullptr */) override;
Status GetChildren( Status GetChildren(const string& dir, TransactionToken* token,
const string& dir, std::vector<string>* result) override;
std::vector<string>* result /*, TransactionToken* token = nullptr */)
override;
Status GetMatchingPaths( Status GetMatchingPaths(const string& pattern, TransactionToken* token,
const string& pattern, std::vector<string>* result) override;
std::vector<string>* result /*, TransactionToken* token = nullptr */)
override;
bool Match( bool Match(const string& filename, const string& pattern) override;
const string& filename,
const string& pattern /*, TransactionToken* token = nullptr */) override;
Status Stat( Status Stat(const string& fname, TransactionToken* token,
const string& fname, FileStatistics* stat) override;
FileStatistics* stat /*, TransactionToken* token = nullptr */) override;
Status DeleteFile( Status DeleteFile(const string& fname, TransactionToken* token) override;
const string& fname /*, TransactionToken* token = nullptr */) override;
Status CreateDir( Status CreateDir(const string& name, TransactionToken* token) override;
const string& name /*, TransactionToken* token = nullptr */) override;
Status DeleteDir( Status DeleteDir(const string& name, TransactionToken* token) override;
const string& name /*, TransactionToken* token = nullptr */) override;
Status GetFileSize( Status GetFileSize(const string& fname, TransactionToken* token,
const string& fname, uint64* size) override;
uint64* size /*, TransactionToken* token = nullptr */) override;
Status IsDirectory( Status IsDirectory(const string& fname, TransactionToken* token) override;
const string& fname /*, TransactionToken* token = nullptr */) override;
Status RenameFile( Status RenameFile(const string& src, const string& target,
const string& src, TransactionToken* token) override;
const string& target /*, TransactionToken* token = nullptr */) override;
string TranslateName( string TranslateName(const string& name) const override { return name; }
const string& name /*, TransactionToken* token = nullptr */)
const override {
return name;
}
char Separator() const override { return '\\'; }; char Separator() const override { return '\\'; };
}; };

62
tensorflow/core/util/memmapped_file_system.cc
View File

@ -86,8 +86,8 @@ class RandomAccessFileFromMemmapped : public RandomAccessFile {
MemmappedFileSystem::MemmappedFileSystem() {} MemmappedFileSystem::MemmappedFileSystem() {}
Status MemmappedFileSystem::FileExists( Status MemmappedFileSystem::FileExists(const string& fname,
const string& fname /*, TransactionToken* token */) { TransactionToken* token) {
if (!mapped_memory_) { if (!mapped_memory_) {
return errors::FailedPrecondition("MemmappedEnv is not initialized"); return errors::FailedPrecondition("MemmappedEnv is not initialized");
} }
@ -99,8 +99,8 @@ Status MemmappedFileSystem::FileExists(
} }
Status MemmappedFileSystem::NewRandomAccessFile( Status MemmappedFileSystem::NewRandomAccessFile(
const string& filename, const string& filename, TransactionToken* token,
std::unique_ptr<RandomAccessFile>* result /*, TransactionToken* token */) { std::unique_ptr<RandomAccessFile>* result) {
if (!mapped_memory_) { if (!mapped_memory_) {
return errors::FailedPrecondition("MemmappedEnv is not initialized"); return errors::FailedPrecondition("MemmappedEnv is not initialized");
} }
@ -115,8 +115,8 @@ Status MemmappedFileSystem::NewRandomAccessFile(
} }
Status MemmappedFileSystem::NewReadOnlyMemoryRegionFromFile( Status MemmappedFileSystem::NewReadOnlyMemoryRegionFromFile(
const string& filename, std::unique_ptr<ReadOnlyMemoryRegion>* const string& filename, TransactionToken* token,
result /*, TransactionToken* token */) { std::unique_ptr<ReadOnlyMemoryRegion>* result) {
if (!mapped_memory_) { if (!mapped_memory_) {
return errors::FailedPrecondition("MemmappedEnv is not initialized"); return errors::FailedPrecondition("MemmappedEnv is not initialized");
} }
@ -130,8 +130,8 @@ Status MemmappedFileSystem::NewReadOnlyMemoryRegionFromFile(
return Status::OK(); return Status::OK();
} }
Status MemmappedFileSystem::GetFileSize( Status MemmappedFileSystem::GetFileSize(const string& filename,
const string& filename, uint64* size /*, TransactionToken* token */) { TransactionToken* token, uint64* size) {
if (!mapped_memory_) { if (!mapped_memory_) {
return errors::FailedPrecondition("MemmappedEnv is not initialized"); return errors::FailedPrecondition("MemmappedEnv is not initialized");
} }
@ -143,59 +143,59 @@ Status MemmappedFileSystem::GetFileSize(
return Status::OK(); return Status::OK();
} }
Status MemmappedFileSystem::Stat( Status MemmappedFileSystem::Stat(const string& fname, TransactionToken* token,
const string& fname, FileStatistics* stat /*, TransactionToken* token */) { FileStatistics* stat) {
uint64 size; uint64 size;
auto status = GetFileSize(fname, &size); auto status = GetFileSize(fname, token, &size);
if (status.ok()) { if (status.ok()) {
stat->length = size; stat->length = size;
} }
return status; return status;
} }
Status MemmappedFileSystem::NewWritableFile( Status MemmappedFileSystem::NewWritableFile(const string& filename,
const string& filename, TransactionToken* token,
std::unique_ptr<WritableFile>* wf /*, TransactionToken* token */) { std::unique_ptr<WritableFile>* wf) {
return errors::Unimplemented("memmapped format doesn't support writing"); return errors::Unimplemented("memmapped format doesn't support writing");
} }
Status MemmappedFileSystem::NewAppendableFile( Status MemmappedFileSystem::NewAppendableFile(
const string& filename, const string& filename, TransactionToken* token,
std::unique_ptr<WritableFile>* result /*, TransactionToken* token */) { std::unique_ptr<WritableFile>* result) {
return errors::Unimplemented("memmapped format doesn't support writing"); return errors::Unimplemented("memmapped format doesn't support writing");
} }
Status MemmappedFileSystem::GetChildren( Status MemmappedFileSystem::GetChildren(const string& filename,
const string& filename, TransactionToken* token,
std::vector<string>* strings /*, TransactionToken* token */) { std::vector<string>* strings) {
return errors::Unimplemented("memmapped format doesn't support GetChildren"); return errors::Unimplemented("memmapped format doesn't support GetChildren");
} }
Status MemmappedFileSystem::GetMatchingPaths( Status MemmappedFileSystem::GetMatchingPaths(const string& pattern,
const string& pattern, TransactionToken* token,
std::vector<string>* results /*, TransactionToken* token */) { std::vector<string>* results) {
return errors::Unimplemented( return errors::Unimplemented(
"memmapped format doesn't support GetMatchingPaths"); "memmapped format doesn't support GetMatchingPaths");
} }
Status MemmappedFileSystem::DeleteFile( Status MemmappedFileSystem::DeleteFile(const string& filename,
const string& filename /*, TransactionToken* token */) { TransactionToken* token) {
return errors::Unimplemented("memmapped format doesn't support DeleteFile"); return errors::Unimplemented("memmapped format doesn't support DeleteFile");
} }
Status MemmappedFileSystem::CreateDir( Status MemmappedFileSystem::CreateDir(const string& dirname,
const string& dirname /*, TransactionToken* token */) { TransactionToken* token) {
return errors::Unimplemented("memmapped format doesn't support CreateDir"); return errors::Unimplemented("memmapped format doesn't support CreateDir");
} }
Status MemmappedFileSystem::DeleteDir( Status MemmappedFileSystem::DeleteDir(const string& dirname,
const string& dirname /*, TransactionToken* token */) { TransactionToken* token) {
return errors::Unimplemented("memmapped format doesn't support DeleteDir"); return errors::Unimplemented("memmapped format doesn't support DeleteDir");
} }
Status MemmappedFileSystem::RenameFile( Status MemmappedFileSystem::RenameFile(const string& filename_from,
const string& filename_from, const string& filename_to,
const string& filename_to /*, TransactionToken* token */) { TransactionToken* token) {
return errors::Unimplemented("memmapped format doesn't support RenameFile"); return errors::Unimplemented("memmapped format doesn't support RenameFile");
} }

63
tensorflow/core/util/memmapped_file_system.h
View File

@ -60,52 +60,39 @@ class MemmappedFileSystem : public FileSystem {
MemmappedFileSystem(); MemmappedFileSystem();
~MemmappedFileSystem() override = default; ~MemmappedFileSystem() override = default;
Status FileExists(
const string& fname /*, TransactionToken* token = nullptr */) override; TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT;
Status FileExists(const string& fname, TransactionToken* token) override;
Status NewRandomAccessFile( Status NewRandomAccessFile(
const string& filename, const string& filename, TransactionToken* token,
std::unique_ptr<RandomAccessFile>* std::unique_ptr<RandomAccessFile>* result) override;
result /*, TransactionToken* token = nullptr */) override;
Status NewReadOnlyMemoryRegionFromFile( Status NewReadOnlyMemoryRegionFromFile(
const string& filename, const string& filename, TransactionToken* token,
std::unique_ptr<ReadOnlyMemoryRegion>* std::unique_ptr<ReadOnlyMemoryRegion>* result) override;
result /*, TransactionToken* token = nullptr */) override;
// All these functions return Unimplemented error, the memmapped storage is // All these functions return Unimplemented error, the memmapped storage is
// read only. // read only.
Status NewWritableFile( Status NewWritableFile(const string& fname, TransactionToken* token,
const string& fname, std::unique_ptr<WritableFile>* result) override;
std::unique_ptr<WritableFile>* Status NewAppendableFile(const string& fname, TransactionToken* token,
result /*, TransactionToken* token = nullptr */) override; std::unique_ptr<WritableFile>* result) override;
Status NewAppendableFile( Status GetChildren(const string& dir, TransactionToken* token,
const string& fname, std::vector<string>* r) override;
std::unique_ptr<WritableFile>* Status GetMatchingPaths(const string& pattern, TransactionToken* token,
result /*, TransactionToken* token = nullptr */) override; std::vector<string>* results) override;
Status GetChildren(const string& dir, Status DeleteFile(const string& f, TransactionToken* token) override;
std::vector<string>* Status CreateDir(const string& d, TransactionToken* token) override;
r /*, TransactionToken* token = nullptr */) override; Status DeleteDir(const string& d, TransactionToken* token) override;
Status GetMatchingPaths( Status RenameFile(const string& s, const string& t,
const string& pattern, TransactionToken* token) override;
std::vector<string>* results /*, TransactionToken* token = nullptr */)
override;
Status DeleteFile(
const string& f /*, TransactionToken* token = nullptr */) override;
Status CreateDir(
const string& d /*, TransactionToken* token = nullptr */) override;
Status DeleteDir(
const string& d /*, TransactionToken* token = nullptr */) override;
Status RenameFile(
const string& s,
const string& t /*, TransactionToken* token = nullptr */) override;
// These functions are implemented. // These functions are implemented.
Status GetFileSize( Status GetFileSize(const string& f, TransactionToken* token,
const string& f, uint64* s) override;
uint64* s /*, TransactionToken* token = nullptr */) override;
// Currently just returns size. // Currently just returns size.
Status Stat( Status Stat(const string& fname, TransactionToken* token,
const string& fname, FileStatistics* stat) override;
FileStatistics* stat /*, TransactionToken* token = nullptr */) override;
// Initializes filesystem from a file in memmapped format. // Initializes filesystem from a file in memmapped format.
Status InitializeFromFile(Env* env, const string& filename); Status InitializeFromFile(Env* env, const string& filename);

60
tensorflow/tools/android/inference_interface/asset_manager_filesystem.cc
View File

@ -124,8 +124,8 @@ AssetManagerFileSystem::AssetManagerFileSystem(AAssetManager* asset_manager,
const string& prefix) const string& prefix)
: asset_manager_(asset_manager), prefix_(prefix) {} : asset_manager_(asset_manager), prefix_(prefix) {}
Status AssetManagerFileSystem::FileExists( Status AssetManagerFileSystem::FileExists(const string& fname,
const string& fname /*, TransactionToken* token */) { TransactionToken* token) {
string path = RemoveAssetPrefix(fname); string path = RemoveAssetPrefix(fname);
auto asset = ScopedAsset( auto asset = ScopedAsset(
AAssetManager_open(asset_manager_, path.c_str(), AASSET_MODE_RANDOM)); AAssetManager_open(asset_manager_, path.c_str(), AASSET_MODE_RANDOM));
@ -136,8 +136,8 @@ Status AssetManagerFileSystem::FileExists(
} }
Status AssetManagerFileSystem::NewRandomAccessFile( Status AssetManagerFileSystem::NewRandomAccessFile(
const string& fname, const string& fname, TransactionToken* token,
std::unique_ptr<RandomAccessFile>* result /*, TransactionToken* token */) { std::unique_ptr<RandomAccessFile>* result) {
string path = RemoveAssetPrefix(fname); string path = RemoveAssetPrefix(fname);
auto asset = ScopedAsset( auto asset = ScopedAsset(
AAssetManager_open(asset_manager_, path.c_str(), AASSET_MODE_RANDOM)); AAssetManager_open(asset_manager_, path.c_str(), AASSET_MODE_RANDOM));
@ -149,8 +149,8 @@ Status AssetManagerFileSystem::NewRandomAccessFile(
} }
Status AssetManagerFileSystem::NewReadOnlyMemoryRegionFromFile( Status AssetManagerFileSystem::NewReadOnlyMemoryRegionFromFile(
const string& fname, std::unique_ptr<ReadOnlyMemoryRegion>* const string& fname, TransactionToken* token,
result /*, TransactionToken* token */) { std::unique_ptr<ReadOnlyMemoryRegion>* result) {
string path = RemoveAssetPrefix(fname); string path = RemoveAssetPrefix(fname);
auto asset = ScopedAsset( auto asset = ScopedAsset(
AAssetManager_open(asset_manager_, path.c_str(), AASSET_MODE_STREAMING)); AAssetManager_open(asset_manager_, path.c_str(), AASSET_MODE_STREAMING));
@ -186,9 +186,9 @@ Status AssetManagerFileSystem::NewReadOnlyMemoryRegionFromFile(
return Status::OK(); return Status::OK();
} }
Status AssetManagerFileSystem::GetChildren( Status AssetManagerFileSystem::GetChildren(const string& prefixed_dir,
const string& prefixed_dir, TransactionToken* token,
std::vector<string>* r /*, TransactionToken* token */) { std::vector<string>* r) {
std::string path = NormalizeDirectoryPath(prefixed_dir); std::string path = NormalizeDirectoryPath(prefixed_dir);
auto dir = auto dir =
ScopedAssetDir(AAssetManager_openDir(asset_manager_, path.c_str())); ScopedAssetDir(AAssetManager_openDir(asset_manager_, path.c_str()));
@ -203,8 +203,8 @@ Status AssetManagerFileSystem::GetChildren(
return Status::OK(); return Status::OK();
} }
Status AssetManagerFileSystem::GetFileSize( Status AssetManagerFileSystem::GetFileSize(const string& fname,
const string& fname, uint64* s /*, TransactionToken* token */) { TransactionToken* token, uint64* s) {
// If fname corresponds to a directory, return early. It doesn't map to an // If fname corresponds to a directory, return early. It doesn't map to an
// AAsset, and would otherwise return NotFound. // AAsset, and would otherwise return NotFound.
if (DirectoryExists(fname)) { if (DirectoryExists(fname)) {
@ -221,8 +221,9 @@ Status AssetManagerFileSystem::GetFileSize(
return Status::OK(); return Status::OK();
} }
Status AssetManagerFileSystem::Stat( Status AssetManagerFileSystem::Stat(const string& fname,
const string& fname, FileStatistics* stat /*, TransactionToken* token */) { TransactionToken* token,
FileStatistics* stat) {
uint64 size; uint64 size;
stat->is_directory = DirectoryExists(fname); stat->is_directory = DirectoryExists(fname);
TF_RETURN_IF_ERROR(GetFileSize(fname, &size)); TF_RETURN_IF_ERROR(GetFileSize(fname, &size));
@ -240,8 +241,7 @@ string AssetManagerFileSystem::RemoveAssetPrefix(const string& name) {
return string(piece); return string(piece);
} }
bool AssetManagerFileSystem::DirectoryExists( bool AssetManagerFileSystem::DirectoryExists(const std::string& fname) {
const std::string& fname /*, TransactionToken* token */) {
std::string path = NormalizeDirectoryPath(fname); std::string path = NormalizeDirectoryPath(fname);
auto dir = auto dir =
ScopedAssetDir(AAssetManager_openDir(asset_manager_, path.c_str())); ScopedAssetDir(AAssetManager_openDir(asset_manager_, path.c_str()));
@ -250,36 +250,36 @@ bool AssetManagerFileSystem::DirectoryExists(
return AAssetDir_getNextFileName(dir.get()) != NULL; return AAssetDir_getNextFileName(dir.get()) != NULL;
} }
Status AssetManagerFileSystem::GetMatchingPaths( Status AssetManagerFileSystem::GetMatchingPaths(const string& pattern,
const string& pattern, TransactionToken* token,
std::vector<string>* results /*, TransactionToken* token */) { std::vector<string>* results) {
return internal::GetMatchingPaths(this, Env::Default(), pattern, results); return internal::GetMatchingPaths(this, Env::Default(), pattern, results);
} }
Status AssetManagerFileSystem::NewWritableFile( Status AssetManagerFileSystem::NewWritableFile(
const string& fname, const string& fname, TransactionToken* token,
std::unique_ptr<WritableFile>* result /*, TransactionToken* token */) { std::unique_ptr<WritableFile>* result) {
return errors::Unimplemented("Asset storage is read only."); return errors::Unimplemented("Asset storage is read only.");
} }
Status AssetManagerFileSystem::NewAppendableFile( Status AssetManagerFileSystem::NewAppendableFile(
const string& fname, const string& fname, TransactionToken* token,
std::unique_ptr<WritableFile>* result /*, TransactionToken* token */) { std::unique_ptr<WritableFile>* result) {
return errors::Unimplemented("Asset storage is read only."); return errors::Unimplemented("Asset storage is read only.");
} }
Status AssetManagerFileSystem::DeleteFile( Status AssetManagerFileSystem::DeleteFile(const string& f,
const string& f /*, TransactionToken* token */) { TransactionToken* token) {
return errors::Unimplemented("Asset storage is read only."); return errors::Unimplemented("Asset storage is read only.");
} }
Status AssetManagerFileSystem::CreateDir( Status AssetManagerFileSystem::CreateDir(const string& d,
const string& d /*, TransactionToken* token */) { TransactionToken* token) {
return errors::Unimplemented("Asset storage is read only."); return errors::Unimplemented("Asset storage is read only.");
} }
Status AssetManagerFileSystem::DeleteDir( Status AssetManagerFileSystem::DeleteDir(const string& d,
const string& d /*, TransactionToken* token */) { TransactionToken* token) {
return errors::Unimplemented("Asset storage is read only."); return errors::Unimplemented("Asset storage is read only.");
} }
Status AssetManagerFileSystem::RenameFile( Status AssetManagerFileSystem::RenameFile(const string& s, const string& t,
const string& s, const string& t /*, TransactionToken* token */) { TransactionToken* token) {
return errors::Unimplemented("Asset storage is read only."); return errors::Unimplemented("Asset storage is read only.");
} }

66
tensorflow/tools/android/inference_interface/asset_manager_filesystem.h
View File

@ -42,52 +42,38 @@ class AssetManagerFileSystem : public FileSystem {
AssetManagerFileSystem(AAssetManager* asset_manager, const string& prefix); AssetManagerFileSystem(AAssetManager* asset_manager, const string& prefix);
~AssetManagerFileSystem() override = default; ~AssetManagerFileSystem() override = default;
Status FileExists( TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT;
const string& fname /*, TransactionToken* token = nullptr*/) override;
Status NewRandomAccessFile(
const string& filename,
std::unique_ptr<RandomAccessFile>*
result /*, TransactionToken* token = nullptr*/) override;
Status NewReadOnlyMemoryRegionFromFile(
const string& filename,
std::unique_ptr<ReadOnlyMemoryRegion>*
result /*, TransactionToken* token = nullptr*/) override;
Status GetFileSize( Status FileExists(const string& fname, TransactionToken* token) override;
const string& f, Status NewRandomAccessFile(
uint64* s /*, TransactionToken* token = nullptr*/) override; const string& filename, TransactionToken* token,
std::unique_ptr<RandomAccessFile>* result) override;
Status NewReadOnlyMemoryRegionFromFile(
const string& filename, TransactionToken* token,
std::unique_ptr<ReadOnlyMemoryRegion>* result) override;
Status GetFileSize(const string& f, TransactionToken* token,
uint64* s) override;
// Currently just returns size. // Currently just returns size.
Status Stat( Status Stat(const string& fname, TransactionToken* token,
const string& fname, FileStatistics* stat) override;
FileStatistics* stat /*, TransactionToken* token = nullptr*/) override; Status GetChildren(const string& dir, TransactionToken* token,
Status GetChildren( std::vector<string>* r) override;
const string& dir,
std::vector<string>* r /*, TransactionToken* token = nullptr*/) override;
// All these functions return Unimplemented error. Asset storage is // All these functions return Unimplemented error. Asset storage is
// read only. // read only.
Status NewWritableFile( Status NewWritableFile(const string& fname, TransactionToken* token,
const string& fname, std::unique_ptr<WritableFile>* result) override;
std::unique_ptr<WritableFile>* Status NewAppendableFile(const string& fname, TransactionToken* token,
result /*, TransactionToken* token = nullptr*/) override; std::unique_ptr<WritableFile>* result) override;
Status NewAppendableFile( Status DeleteFile(const string& f, TransactionToken* token) override;
const string& fname, Status CreateDir(const string& d, TransactionToken* token) override;
std::unique_ptr<WritableFile>* Status DeleteDir(const string& d, TransactionToken* token) override;
result /*, TransactionToken* token = nullptr*/) override; Status RenameFile(const string& s, const string& t,
Status DeleteFile( TransactionToken* token) override;
const string& f /*, TransactionToken* token = nullptr*/) override;
Status CreateDir(
const string& d /*, TransactionToken* token = nullptr*/) override;
Status DeleteDir(
const string& d /*, TransactionToken* token = nullptr*/) override;
Status RenameFile(
const string& s,
const string& t /*, TransactionToken* token = nullptr*/) override;
Status GetMatchingPaths( Status GetMatchingPaths(const string& pattern, TransactionToken* token,
const string& pattern, std::vector<string>* results) override;
std::vector<string>* results /*, TransactionToken* token = nullptr*/)
override;
private: private:
string RemoveAssetPrefix(const string& name); string RemoveAssetPrefix(const string& name);