experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Implement modular POSIX filesystem support for deleting files and directories.

Browse Source 
We also provide tests to make sure all API requirements are satisfied.

Just a small sized part of work for modular filesystem plugins. For more details, consult the RFC at https://github.com/tensorflow/community/blob/master/rfcs/20190506-filesystem-plugin-modular-tensorflow.md

PiperOrigin-RevId: 281107956
Change-Id: I0be89ff20d2c865fe552a9eeee12573eff63eb01
This commit is contained in:
Mihai Maruseac 2019-11-18 11:02:46 -08:00 committed by TensorFlower Gardener
parent 837b26bd57
commit c455ab4555
3 changed files with 149 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
tensorflow/c/experimental/filesystem/modular_filesystem.cc
View File

@ -137,9 +137,15 @@ Status ModularFileSystem::GetMatchingPaths(const std::string& pattern,
}
Status ModularFileSystem::DeleteFile(const std::string& fname) {
// TODO(mihaimaruseac): Implementation to come in a new change
return Status(error::UNIMPLEMENTED,
"Modular filesystem stub not implemented yet");
if (ops_->delete_file == nullptr)
return errors::Unimplemented(tensorflow::strings::StrCat(
"Filesystem for ", fname, " does not support DeleteFile()"));
UniquePtrTo_TF_Status plugin_status(TF_NewStatus(), TF_DeleteStatus);
std::string translated_name = TranslateName(fname);
ops_->delete_file(filesystem_.get(), translated_name.c_str(),
plugin_status.get());
return StatusFromTF_Status(plugin_status.get());
}
Status ModularFileSystem::DeleteRecursively(const std::string& dirname,
@ -151,9 +157,15 @@ Status ModularFileSystem::DeleteRecursively(const std::string& dirname,
}
Status ModularFileSystem::DeleteDir(const std::string& dirname) {
// TODO(mihaimaruseac): Implementation to come in a new change
return Status(error::UNIMPLEMENTED,
"Modular filesystem stub not implemented yet");
if (ops_->delete_dir == nullptr)
return errors::Unimplemented(tensorflow::strings::StrCat(
"Filesystem for ", dirname, " does not support DeleteDir()"));
UniquePtrTo_TF_Status plugin_status(TF_NewStatus(), TF_DeleteStatus);
std::string translated_name = TranslateName(dirname);
ops_->delete_dir(filesystem_.get(), translated_name.c_str(),
plugin_status.get());
return StatusFromTF_Status(plugin_status.get());
}
Status ModularFileSystem::RecursivelyCreateDir(const std::string& dirname) {

112
tensorflow/c/experimental/filesystem/modular_filesystem_test.cc
View File

@ -427,6 +427,118 @@ TEST_P(ModularFileSystemTest, TestCreateDirPathIsInvalid) {
EXPECT_PRED2(UninmplementedOrReturnsCode, status, Code::FAILED_PRECONDITION);
}
TEST_P(ModularFileSystemTest, TestDeleteFile) {
const std::string filepath = GetURIForPath("a_file");
std::unique_ptr<WritableFile> new_file;
Status status = env_->NewWritableFile(filepath, &new_file);
if (!status.ok()) GTEST_SKIP() << "NewWritableFile() not supported";
status = env_->DeleteFile(filepath);
EXPECT_PRED2(UninmplementedOrReturnsCode, status, Code::OK);
}
TEST_P(ModularFileSystemTest, TestDeleteFileFromDirectory) {
const std::string dirpath = GetURIForPath("a_dir");
Status status = env_->CreateDir(dirpath);
if (!status.ok()) GTEST_SKIP() << "CreateDir() not supported";
const std::string filepath = GetURIForPath("a_dir/a_file");
std::unique_ptr<WritableFile> new_file;
status = env_->NewWritableFile(filepath, &new_file);
if (!status.ok()) GTEST_SKIP() << "NewWritableFile() not supported";
status = env_->DeleteFile(filepath);
EXPECT_PRED2(UninmplementedOrReturnsCode, status, Code::OK);
}
TEST_P(ModularFileSystemTest, TestDeleteFileDoesNotExist) {
const std::string filepath = GetURIForPath("a_file");
Status status = env_->DeleteFile(filepath);
EXPECT_PRED2(UninmplementedOrReturnsCode, status, Code::NOT_FOUND);
}
TEST_P(ModularFileSystemTest, TestDeleteFileWhichIsDirectory) {
const std::string dirpath = GetURIForPath("a_dir");
Status status = env_->CreateDir(dirpath);
if (!status.ok()) GTEST_SKIP() << "CreateDir() not supported";
status = env_->DeleteFile(dirpath);
EXPECT_PRED2(UninmplementedOrReturnsCode, status, Code::FAILED_PRECONDITION);
}
TEST_P(ModularFileSystemTest, TestDeleteFilePathIsInvalid) {
const std::string filepath = GetURIForPath("a_file");
std::unique_ptr<WritableFile> file;
Status status = env_->NewWritableFile(filepath, &file);
if (!status.ok()) GTEST_SKIP() << "NewWritableFile() not supported";
const std::string new_path = GetURIForPath("a_file/a_new_file");
status = env_->DeleteFile(new_path);
EXPECT_PRED2(UninmplementedOrReturnsCode, status, Code::FAILED_PRECONDITION);
}
TEST_P(ModularFileSystemTest, TestDeleteDirectory) {
const std::string dirpath = GetURIForPath("a_dir");
Status status = env_->CreateDir(dirpath);
if (!status.ok()) GTEST_SKIP() << "CreateDir() not supported";
status = env_->DeleteDir(dirpath);
EXPECT_PRED2(UninmplementedOrReturnsCode, status, Code::OK);
}
TEST_P(ModularFileSystemTest, TestDeleteDirectoryFromDirectory) {
const std::string dirpath = GetURIForPath("a_dir");
Status status = env_->CreateDir(dirpath);
if (!status.ok()) GTEST_SKIP() << "CreateDir() not supported";
const std::string target_path = GetURIForPath("a_dir/another_dir");
EXPECT_EQ(env_->CreateDir(target_path).code(), Code::OK);
status = env_->DeleteDir(target_path);
EXPECT_PRED2(UninmplementedOrReturnsCode, status, Code::OK);
}
TEST_P(ModularFileSystemTest, TestDeleteDirectoryDoesNotExist) {
const std::string dirpath = GetURIForPath("a_dir");
Status status = env_->DeleteDir(dirpath);
EXPECT_PRED2(UninmplementedOrReturnsCode, status, Code::NOT_FOUND);
}
TEST_P(ModularFileSystemTest, TestDeleteDirectoryNotEmpty) {
const std::string dirpath = GetURIForPath("a_dir");
Status status = env_->CreateDir(dirpath);
if (!status.ok()) GTEST_SKIP() << "CreateDir() not supported";
const std::string filepath = GetURIForPath("a_dir/a_file");
std::unique_ptr<WritableFile> new_file;
status = env_->NewWritableFile(filepath, &new_file);
if (!status.ok()) GTEST_SKIP() << "NewWritableFile() not supported";
status = env_->DeleteDir(dirpath);
EXPECT_PRED2(UninmplementedOrReturnsCode, status, Code::FAILED_PRECONDITION);
}
TEST_P(ModularFileSystemTest, TestDeleteDirectoryWhichIsFile) {
const std::string filepath = GetURIForPath("a_file");
std::unique_ptr<WritableFile> new_file;
Status status = env_->NewWritableFile(filepath, &new_file);
if (!status.ok()) GTEST_SKIP() << "NewWritableFile() not supported";
status = env_->DeleteDir(filepath);
EXPECT_PRED2(UninmplementedOrReturnsCode, status, Code::FAILED_PRECONDITION);
}
TEST_P(ModularFileSystemTest, TestDeleteDirectoryPathIsInvalid) {
const std::string filepath = GetURIForPath("a_file");
std::unique_ptr<WritableFile> file;
Status status = env_->NewWritableFile(filepath, &file);
if (!status.ok()) GTEST_SKIP() << "NewWritableFile() not supported";
const std::string new_path = GetURIForPath("a_file/a_dir");
status = env_->DeleteDir(new_path);
EXPECT_PRED2(UninmplementedOrReturnsCode, status, Code::FAILED_PRECONDITION);
}
TEST_P(ModularFileSystemTest, TestAppendAndTell) {
const std::string filename = GetURIForPath("a_file");
std::unique_ptr<WritableFile> file;

19
tensorflow/c/experimental/filesystem/plugins/posix/posix_filesystem.cc
View File

@ -271,6 +271,22 @@ static void CreateDir(const TF_Filesystem* filesystem, const char* path,
TF_SetStatus(status, TF_OK, "");
}
static void DeleteFile(const TF_Filesystem* filesystem, const char* path,
TF_Status* status) {
if (unlink(path) != 0)
TF_SetStatusFromIOError(status, errno, path);
else
TF_SetStatus(status, TF_OK, "");
}
static void DeleteDir(const TF_Filesystem* filesystem, const char* path,
TF_Status* status) {
if (rmdir(path) != 0)
TF_SetStatusFromIOError(status, errno, path);
else
TF_SetStatus(status, TF_OK, "");
}
// TODO(mihaimaruseac): More implementations to follow in subsequent changes.
} // namespace tf_posix_filesystem
@ -298,6 +314,9 @@ void TF_InitPlugin(TF_Status* status) {
tf_posix_filesystem::NewAppendableFile,
tf_posix_filesystem::NewReadOnlyMemoryRegionFromFile,
tf_posix_filesystem::CreateDir,
/*recursively_create_dir=*/nullptr,
tf_posix_filesystem::DeleteFile,
tf_posix_filesystem::DeleteDir,
nullptr,
};