experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Allow modular POSIX filesystem to get file/dir statistics.

Browse Source 
We also provide tests to make sure all API requirements are satisfied.

Just a small sized part of work for modular filesystem plugins. For more details, consult the RFC at https://github.com/tensorflow/community/blob/master/rfcs/20190506-filesystem-plugin-modular-tensorflow.md

PiperOrigin-RevId: 281110604
Change-Id: I644b8d6da400e1639f7e8ec91ceaa440d4cb193d
This commit is contained in:
Mihai Maruseac 2019-11-18 11:13:04 -08:00 committed by TensorFlower Gardener
parent de21278334
commit 87451d7147
3 changed files with 225 additions and 9 deletions
tensorflow/c/experimental/filesystem
modular_filesystem.ccmodular_filesystem_test.cc
plugins/posix
posix_filesystem.cc

52
tensorflow/c/experimental/filesystem/modular_filesystem.cc
View File

@ -220,22 +220,56 @@ Status ModularFileSystem::CreateDir(const std::string& dirname) {
}
Status ModularFileSystem::Stat(const std::string& fname, FileStatistics* stat) {
// TODO(mihaimaruseac): Implementation to come in a new change
return Status(error::UNIMPLEMENTED,
"Modular filesystem stub not implemented yet");
if (ops_->stat == nullptr)
return errors::Unimplemented(tensorflow::strings::StrCat(
"Filesystem for ", fname, " does not support Stat()"));
if (stat == nullptr)
return errors::InvalidArgument("FileStatistics pointer must not be NULL");
UniquePtrTo_TF_Status plugin_status(TF_NewStatus(), TF_DeleteStatus);
std::string translated_name = TranslateName(fname);
TF_FileStatistics stats;
ops_->stat(filesystem_.get(), translated_name.c_str(), &stats,
plugin_status.get());
if (TF_GetCode(plugin_status.get()) == TF_OK) {
stat->length = stats.length;
stat->mtime_nsec = stats.mtime_nsec;
stat->is_directory = stats.is_directory;
}
return StatusFromTF_Status(plugin_status.get());
}
Status ModularFileSystem::IsDirectory(const std::string& name) {
// TODO(mihaimaruseac): Implementation to come in a new change
return Status(error::UNIMPLEMENTED,
"Modular filesystem stub not implemented yet");
if (ops_->is_directory == nullptr) return FileSystem::IsDirectory(name);
UniquePtrTo_TF_Status plugin_status(TF_NewStatus(), TF_DeleteStatus);
std::string translated_name = TranslateName(name);
ops_->is_directory(filesystem_.get(), translated_name.c_str(),
plugin_status.get());
return StatusFromTF_Status(plugin_status.get());
}
Status ModularFileSystem::GetFileSize(const std::string& fname,
uint64* file_size) {
// TODO(mihaimaruseac): Implementation to come in a new change
return Status(error::UNIMPLEMENTED,
"Modular filesystem stub not implemented yet");
if (ops_->get_file_size == nullptr) {
FileStatistics stat;
Status status = Stat(fname, &stat);
if (!status.ok()) return status;
if (stat.is_directory)
return errors::FailedPrecondition("Called GetFileSize on a directory");
*file_size = stat.length;
return status;
}
UniquePtrTo_TF_Status plugin_status(TF_NewStatus(), TF_DeleteStatus);
std::string translated_name = TranslateName(fname);
*file_size = ops_->get_file_size(filesystem_.get(), translated_name.c_str(),
plugin_status.get());
return StatusFromTF_Status(plugin_status.get());
}
Status ModularFileSystem::RenameFile(const std::string& src,

166
tensorflow/c/experimental/filesystem/modular_filesystem_test.cc
View File

@ -628,6 +628,172 @@ TEST_P(ModularFileSystemTest, TestFilesExistsNoFiles) {
EXPECT_TRUE(statuses.empty());
}
TEST_P(ModularFileSystemTest, TestStatEmptyFile) {
const std::string filepath = GetURIForPath("a_file");
std::unique_ptr<WritableFile> file;
Status status = env_->NewWritableFile(filepath, &file);
if (!status.ok()) GTEST_SKIP() << "NewWritableFile() not supported";
FileStatistics stat;
status = env_->Stat(filepath, &stat);
EXPECT_PRED2(UninmplementedOrReturnsCode, status, Code::OK);
if (!status.ok()) GTEST_SKIP() << "Stat() not supported";
EXPECT_FALSE(stat.is_directory);
EXPECT_EQ(stat.length, 0);
}
TEST_P(ModularFileSystemTest, TestStatNonEmptyFile) {
const std::string filepath = GetURIForPath("a_file");
std::unique_ptr<WritableFile> file;
Status status = env_->NewWritableFile(filepath, &file);
if (!status.ok()) GTEST_SKIP() << "NewWritableFile() not supported";
const std::string test_data("asdf");
status = file->Append(test_data);
if (!status.ok()) GTEST_SKIP() << "Append() not supported";
status = file->Flush();
if (!status.ok()) GTEST_SKIP() << "Flush() not supported";
status = file->Close();
if (!status.ok()) GTEST_SKIP() << "Close() not supported";
FileStatistics stat;
status = env_->Stat(filepath, &stat);
EXPECT_PRED2(UninmplementedOrReturnsCode, status, Code::OK);
if (!status.ok()) GTEST_SKIP() << "Stat() not supported";
EXPECT_FALSE(stat.is_directory);
EXPECT_EQ(stat.length, test_data.size());
}
TEST_P(ModularFileSystemTest, TestStatDirectory) {
const std::string dirpath = GetURIForPath("a_dir");
Status status = env_->CreateDir(dirpath);
if (!status.ok()) GTEST_SKIP() << "CreateDir() not supported";
FileStatistics stat;
status = env_->Stat(dirpath, &stat);
EXPECT_PRED2(UninmplementedOrReturnsCode, status, Code::OK);
if (!status.ok()) GTEST_SKIP() << "Stat() not supported";
EXPECT_TRUE(stat.is_directory);
}
TEST_P(ModularFileSystemTest, TestStatNotFound) {
const std::string dirpath = GetURIForPath("a_dir");
FileStatistics stat;
Status status = env_->Stat(dirpath, &stat);
EXPECT_PRED2(UninmplementedOrReturnsCode, status, Code::NOT_FOUND);
}
TEST_P(ModularFileSystemTest, TestStatPathIsInvalid) {
const std::string filepath = GetURIForPath("a_file");
std::unique_ptr<WritableFile> file;
Status status = env_->NewWritableFile(filepath, &file);
if (!status.ok()) GTEST_SKIP() << "NewWritableFile() not supported";
const std::string target_path = GetURIForPath("a_file/a_new_file");
FileStatistics stat;
status = env_->Stat(target_path, &stat);
EXPECT_PRED2(UninmplementedOrReturnsCode, status, Code::FAILED_PRECONDITION);
}
TEST_P(ModularFileSystemTest, TestIsDirectory) {
const std::string dirpath = GetURIForPath("a_dir");
Status status = env_->CreateDir(dirpath);
if (!status.ok()) GTEST_SKIP() << "CreateDir() not supported";
status = env_->IsDirectory(dirpath);
EXPECT_PRED2(UninmplementedOrReturnsCode, status, Code::OK);
}
TEST_P(ModularFileSystemTest, TestIsDirectoryFile) {
const std::string filepath = GetURIForPath("a_file");
std::unique_ptr<WritableFile> file;
Status status = env_->NewWritableFile(filepath, &file);
if (!status.ok()) GTEST_SKIP() << "NewWritableFile() not supported";
status = env_->IsDirectory(filepath);
EXPECT_PRED2(UninmplementedOrReturnsCode, status, Code::FAILED_PRECONDITION);
}
TEST_P(ModularFileSystemTest, TestIsDirectoryNotFound) {
const std::string dirpath = GetURIForPath("a_dir");
Status status = env_->IsDirectory(dirpath);
EXPECT_PRED2(UninmplementedOrReturnsCode, status, Code::NOT_FOUND);
}
TEST_P(ModularFileSystemTest, TestIsDirectoryPathIsInvalid) {
const std::string filepath = GetURIForPath("a_file");
std::unique_ptr<WritableFile> file;
Status status = env_->NewWritableFile(filepath, &file);
if (!status.ok()) GTEST_SKIP() << "NewWritableFile() not supported";
const std::string target_path = GetURIForPath("a_file/a_new_file");
status = env_->IsDirectory(target_path);
EXPECT_PRED2(UninmplementedOrReturnsCode, status, Code::FAILED_PRECONDITION);
}
TEST_P(ModularFileSystemTest, TestGetFileSizeEmptyFile) {
const std::string filepath = GetURIForPath("a_file");
std::unique_ptr<WritableFile> file;
Status status = env_->NewWritableFile(filepath, &file);
if (!status.ok()) GTEST_SKIP() << "NewWritableFile() not supported";
uint64 size;
status = env_->GetFileSize(filepath, &size);
EXPECT_PRED2(UninmplementedOrReturnsCode, status, Code::OK);
if (!status.ok()) GTEST_SKIP() << "GetFileSize() not supported";
EXPECT_EQ(size, 0);
}
TEST_P(ModularFileSystemTest, TestGetFileSizeNonEmptyFile) {
const std::string filepath = GetURIForPath("a_file");
std::unique_ptr<WritableFile> file;
Status status = env_->NewWritableFile(filepath, &file);
if (!status.ok()) GTEST_SKIP() << "NewWritableFile() not supported";
const std::string test_data("asdf");
status = file->Append(test_data);
if (!status.ok()) GTEST_SKIP() << "Append() not supported";
status = file->Flush();
if (!status.ok()) GTEST_SKIP() << "Flush() not supported";
status = file->Close();
if (!status.ok()) GTEST_SKIP() << "Close() not supported";
uint64 size;
status = env_->GetFileSize(filepath, &size);
EXPECT_PRED2(UninmplementedOrReturnsCode, status, Code::OK);
if (!status.ok()) GTEST_SKIP() << "GetFileSize() not supported";
EXPECT_EQ(size, test_data.size());
}
TEST_P(ModularFileSystemTest, TestGetFileSizeDirectory) {
const std::string dirpath = GetURIForPath("a_dir");
Status status = env_->CreateDir(dirpath);
if (!status.ok()) GTEST_SKIP() << "CreateDir() not supported";
uint64 size;
status = env_->GetFileSize(dirpath, &size);
EXPECT_PRED2(UninmplementedOrReturnsCode, status, Code::FAILED_PRECONDITION);
}
TEST_P(ModularFileSystemTest, TestGetFileSizeNotFound) {
const std::string filepath = GetURIForPath("a_dir");
uint64 size;
Status status = env_->GetFileSize(filepath, &size);
EXPECT_PRED2(UninmplementedOrReturnsCode, status, Code::NOT_FOUND);
}
TEST_P(ModularFileSystemTest, TestGetFileSizePathIsInvalid) {
const std::string filepath = GetURIForPath("a_file");
std::unique_ptr<WritableFile> file;
Status status = env_->NewWritableFile(filepath, &file);
if (!status.ok()) GTEST_SKIP() << "NewWritableFile() not supported";
const std::string target_path = GetURIForPath("a_file/a_new_file");
uint64 size;
status = env_->GetFileSize(target_path, &size);
EXPECT_PRED2(UninmplementedOrReturnsCode, status, Code::FAILED_PRECONDITION);
}
TEST_P(ModularFileSystemTest, TestAppendAndTell) {
const std::string filename = GetURIForPath("a_file");
std::unique_ptr<WritableFile> file;

16
tensorflow/c/experimental/filesystem/plugins/posix/posix_filesystem.cc
View File

@ -295,6 +295,19 @@ static void PathExists(const TF_Filesystem* filesystem, const char* path,
TF_SetStatus(status, TF_OK, "");
}
static void Stat(const TF_Filesystem* filesystem, const char* path,
TF_FileStatistics* stats, TF_Status* status) {
struct stat sbuf;
if (stat(path, &sbuf) != 0) {
TF_SetStatusFromIOError(status, errno, path);
} else {
stats->length = sbuf.st_size;
stats->mtime_nsec = sbuf.st_mtime * (1000 * 1000 * 1000);
stats->is_directory = S_ISDIR(sbuf.st_mode);
TF_SetStatus(status, TF_OK, "");
}
}
} // namespace tf_posix_filesystem
void TF_InitPlugin(TF_Status* status) {
@ -328,6 +341,9 @@ void TF_InitPlugin(TF_Status* status) {
/*copy_file=*/nullptr,
tf_posix_filesystem::PathExists,
/*paths_exist=*/nullptr,
tf_posix_filesystem::Stat,
/*is_directory=*/nullptr,
/*get_file_size=*/nullptr,
nullptr,
};