diff --git a/tensorflow/c/experimental/filesystem/plugins/s3/s3_filesystem.cc b/tensorflow/c/experimental/filesystem/plugins/s3/s3_filesystem.cc index 8420b6ec013..c7f2ef8b03b 100644 --- a/tensorflow/c/experimental/filesystem/plugins/s3/s3_filesystem.cc +++ b/tensorflow/c/experimental/filesystem/plugins/s3/s3_filesystem.cc @@ -646,7 +646,8 @@ void Stat(const TF_Filesystem* filesystem, const char* path, head_object_outcome.GetResult().GetLastModified().Millis() * 1e6; found = true; } else { - return TF_SetStatusFromAWSError(head_object_outcome.GetError(), status); + TF_SetStatusFromAWSError(head_object_outcome.GetError(), status); + if (TF_GetCode(status) == TF_FAILED_PRECONDITION) return; } auto prefix = object; @@ -1110,6 +1111,7 @@ int GetChildren(const TF_Filesystem* filesystem, const char* path, for (int i = 0; i < num_entries; i++) (*entries)[i] = strdup(result[i].c_str()); TF_SetStatus(status, TF_OK, ""); + return num_entries; } static char* TranslateName(const TF_Filesystem* filesystem, const char* uri) { diff --git a/tensorflow/c/experimental/filesystem/plugins/s3/s3_filesystem.h b/tensorflow/c/experimental/filesystem/plugins/s3/s3_filesystem.h index 05c79a333c1..daaeacedceb 100644 --- a/tensorflow/c/experimental/filesystem/plugins/s3/s3_filesystem.h +++ b/tensorflow/c/experimental/filesystem/plugins/s3/s3_filesystem.h @@ -45,6 +45,12 @@ void Flush(const TF_WritableFile* file, TF_Status* status); void Close(const TF_WritableFile* file, TF_Status* status); } // namespace tf_writable_file +namespace tf_read_only_memory_region { +void Cleanup(TF_ReadOnlyMemoryRegion* region); +const void* Data(const TF_ReadOnlyMemoryRegion* region); +uint64_t Length(const TF_ReadOnlyMemoryRegion* region); +} // namespace tf_read_only_memory_region + namespace tf_s3_filesystem { typedef struct S3File { std::shared_ptr s3_client; @@ -70,6 +76,22 @@ void NewAppendableFile(const TF_Filesystem* filesystem, const char* path, TF_WritableFile* file, TF_Status* status); int64_t GetFileSize(const TF_Filesystem* filesystem, const char* path, TF_Status* status); +void NewReadOnlyMemoryRegionFromFile(const TF_Filesystem* filesystem, + const char* path, + TF_ReadOnlyMemoryRegion* region, + TF_Status* status); +void PathExists(const TF_Filesystem* filesystem, const char* path, + TF_Status* status); +void CreateDir(const TF_Filesystem* filesystem, const char* path, + TF_Status* status); +int GetChildren(const TF_Filesystem* filesystem, const char* path, + char*** entries, TF_Status* status); +void DeleteFile(const TF_Filesystem* filesystem, const char* path, + TF_Status* status); +void Stat(const TF_Filesystem* filesystem, const char* path, + TF_FileStatistics* stats, TF_Status* status); +void DeleteDir(const TF_Filesystem* filesystem, const char* path, + TF_Status* status); } // namespace tf_s3_filesystem #endif // TENSORFLOW_C_EXPERIMENTAL_FILESYSTEM_PLUGINS_S3_S3_FILESYSTEM_H_ diff --git a/tensorflow/c/experimental/filesystem/plugins/s3/s3_filesystem_test.cc b/tensorflow/c/experimental/filesystem/plugins/s3/s3_filesystem_test.cc index f367d0b6a98..4610e0b3b53 100644 --- a/tensorflow/c/experimental/filesystem/plugins/s3/s3_filesystem_test.cc +++ b/tensorflow/c/experimental/filesystem/plugins/s3/s3_filesystem_test.cc @@ -209,6 +209,143 @@ TEST_F(S3FilesystemTest, NewWritableFile) { EXPECT_EQ("content1,content2", content); } +TEST_F(S3FilesystemTest, NewAppendableFile) { + const std::string path = GetURIForPath("AppendableFile"); + WriteString(path, "test"); + ASSERT_TF_OK(status_); + + auto writer = GetWriter(); + tf_s3_filesystem::NewAppendableFile(filesystem_, path.c_str(), writer.get(), + status_); + EXPECT_TF_OK(status_); + tf_writable_file::Append(writer.get(), "content", strlen("content"), status_); + EXPECT_TF_OK(status_); + tf_writable_file::Close(writer.get(), status_); + EXPECT_TF_OK(status_); +} + +TEST_F(S3FilesystemTest, NewReadOnlyMemoryRegionFromFile) { + const std::string path = GetURIForPath("MemoryFile"); + const std::string content = "content"; + WriteString(path, content); + ASSERT_TF_OK(status_); + + std::unique_ptr + region(new TF_ReadOnlyMemoryRegion, [](TF_ReadOnlyMemoryRegion* file) { + if (file != nullptr) { + if (file->plugin_memory_region != nullptr) + tf_read_only_memory_region::Cleanup(file); + delete file; + } + }); + region->plugin_memory_region = nullptr; + tf_s3_filesystem::NewReadOnlyMemoryRegionFromFile(filesystem_, path.c_str(), + region.get(), status_); + EXPECT_TF_OK(status_); + std::string result(reinterpret_cast( + tf_read_only_memory_region::Data(region.get())), + tf_read_only_memory_region::Length(region.get())); + EXPECT_EQ(content, result); +} + +TEST_F(S3FilesystemTest, PathExists) { + const std::string path = GetURIForPath("PathExists"); + tf_s3_filesystem::PathExists(filesystem_, path.c_str(), status_); + EXPECT_EQ(TF_NOT_FOUND, TF_GetCode(status_)) << TF_Message(status_); + TF_SetStatus(status_, TF_OK, ""); + WriteString(path, "test"); + ASSERT_TF_OK(status_); + tf_s3_filesystem::PathExists(filesystem_, path.c_str(), status_); + EXPECT_TF_OK(status_); +} + +TEST_F(S3FilesystemTest, GetChildren) { + const std::string base = GetURIForPath("GetChildren"); + tf_s3_filesystem::CreateDir(filesystem_, base.c_str(), status_); + EXPECT_TF_OK(status_); + + const std::string file = io::JoinPath(base, "TestFile.csv"); + WriteString(file, "test"); + EXPECT_TF_OK(status_); + + const std::string subdir = io::JoinPath(base, "SubDir"); + tf_s3_filesystem::CreateDir(filesystem_, subdir.c_str(), status_); + EXPECT_TF_OK(status_); + const std::string subfile = io::JoinPath(subdir, "TestSubFile.csv"); + WriteString(subfile, "test"); + EXPECT_TF_OK(status_); + + char** entries; + auto num_entries = tf_s3_filesystem::GetChildren(filesystem_, base.c_str(), + &entries, status_); + EXPECT_TF_OK(status_); + + std::vector childrens; + for (int i = 0; i < num_entries; ++i) { + childrens.push_back(entries[i]); + } + std::sort(childrens.begin(), childrens.end()); + EXPECT_EQ(std::vector({"SubDir", "TestFile.csv"}), childrens); +} + +TEST_F(S3FilesystemTest, DeleteFile) { + const std::string path = GetURIForPath("DeleteFile"); + WriteString(path, "test"); + ASSERT_TF_OK(status_); + tf_s3_filesystem::DeleteFile(filesystem_, path.c_str(), status_); + EXPECT_TF_OK(status_); +} + +TEST_F(S3FilesystemTest, CreateDir) { + // s3 object storage doesn't support empty directory, we create file in the + // directory + const std::string dir = GetURIForPath("CreateDir"); + tf_s3_filesystem::CreateDir(filesystem_, dir.c_str(), status_); + EXPECT_TF_OK(status_); + + const std::string file = io::JoinPath(dir, "CreateDirFile.csv"); + WriteString(file, "test"); + ASSERT_TF_OK(status_); + + TF_FileStatistics stat; + tf_s3_filesystem::Stat(filesystem_, dir.c_str(), &stat, status_); + EXPECT_TF_OK(status_); + EXPECT_TRUE(stat.is_directory); +} + +TEST_F(S3FilesystemTest, DeleteDir) { + // s3 object storage doesn't support empty directory, we create file in the + // directory + const std::string dir = GetURIForPath("DeleteDir"); + const std::string file = io::JoinPath(dir, "DeleteDirFile.csv"); + WriteString(file, "test"); + ASSERT_TF_OK(status_); + tf_s3_filesystem::DeleteDir(filesystem_, dir.c_str(), status_); + EXPECT_NE(TF_GetCode(status_), TF_OK); + + TF_SetStatus(status_, TF_OK, ""); + tf_s3_filesystem::DeleteFile(filesystem_, file.c_str(), status_); + EXPECT_TF_OK(status_); + tf_s3_filesystem::DeleteDir(filesystem_, dir.c_str(), status_); + EXPECT_TF_OK(status_); + TF_FileStatistics stat; + tf_s3_filesystem::Stat(filesystem_, dir.c_str(), &stat, status_); + EXPECT_EQ(TF_GetCode(status_), TF_NOT_FOUND) << TF_Message(status_); +} + +TEST_F(S3FilesystemTest, StatFile) { + const std::string path = GetURIForPath("StatFile"); + WriteString(path, "test"); + ASSERT_TF_OK(status_); + + TF_FileStatistics stat; + tf_s3_filesystem::Stat(filesystem_, path.c_str(), &stat, status_); + EXPECT_TF_OK(status_); + EXPECT_EQ(4, stat.length); + EXPECT_FALSE(stat.is_directory); +} + } // namespace } // namespace tensorflow