experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #41868 from vnvo2409:s3-filesystem-test

Browse Source 
PiperOrigin-RevId: 323943244
Change-Id: I9bde5f2d53a43da2801aae01e8c96e13beceb7f7
This commit is contained in:
TensorFlower Gardener 2020-07-29 23:11:33 -07:00
commit 2c1b1b87f0
3 changed files with 162 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
tensorflow/c/experimental/filesystem/plugins/s3/s3_filesystem.cc
View File

@ -646,7 +646,8 @@ void Stat(const TF_Filesystem* filesystem, const char* path,
head_object_outcome.GetResult().GetLastModified().Millis() * 1e6;
found = true;
} else {
return TF_SetStatusFromAWSError(head_object_outcome.GetError(), status);
TF_SetStatusFromAWSError(head_object_outcome.GetError(), status);
if (TF_GetCode(status) == TF_FAILED_PRECONDITION) return;
}
auto prefix = object;
@ -1110,6 +1111,7 @@ int GetChildren(const TF_Filesystem* filesystem, const char* path,
for (int i = 0; i < num_entries; i++)
(*entries)[i] = strdup(result[i].c_str());
TF_SetStatus(status, TF_OK, "");
return num_entries;
}
static char* TranslateName(const TF_Filesystem* filesystem, const char* uri) {

22
tensorflow/c/experimental/filesystem/plugins/s3/s3_filesystem.h
View File

@ -45,6 +45,12 @@ void Flush(const TF_WritableFile* file, TF_Status* status);
void Close(const TF_WritableFile* file, TF_Status* status);
} // namespace tf_writable_file
namespace tf_read_only_memory_region {
void Cleanup(TF_ReadOnlyMemoryRegion* region);
const void* Data(const TF_ReadOnlyMemoryRegion* region);
uint64_t Length(const TF_ReadOnlyMemoryRegion* region);
} // namespace tf_read_only_memory_region
namespace tf_s3_filesystem {
typedef struct S3File {
std::shared_ptr<Aws::S3::S3Client> s3_client;
@ -70,6 +76,22 @@ void NewAppendableFile(const TF_Filesystem* filesystem, const char* path,
TF_WritableFile* file, TF_Status* status);
int64_t GetFileSize(const TF_Filesystem* filesystem, const char* path,
TF_Status* status);
void NewReadOnlyMemoryRegionFromFile(const TF_Filesystem* filesystem,
const char* path,
TF_ReadOnlyMemoryRegion* region,
TF_Status* status);
void PathExists(const TF_Filesystem* filesystem, const char* path,
TF_Status* status);
void CreateDir(const TF_Filesystem* filesystem, const char* path,
TF_Status* status);
int GetChildren(const TF_Filesystem* filesystem, const char* path,
char*** entries, TF_Status* status);
void DeleteFile(const TF_Filesystem* filesystem, const char* path,
TF_Status* status);
void Stat(const TF_Filesystem* filesystem, const char* path,
TF_FileStatistics* stats, TF_Status* status);
void DeleteDir(const TF_Filesystem* filesystem, const char* path,
TF_Status* status);
} // namespace tf_s3_filesystem
#endif // TENSORFLOW_C_EXPERIMENTAL_FILESYSTEM_PLUGINS_S3_S3_FILESYSTEM_H_

137
tensorflow/c/experimental/filesystem/plugins/s3/s3_filesystem_test.cc
View File

@ -209,6 +209,143 @@ TEST_F(S3FilesystemTest, NewWritableFile) {
EXPECT_EQ("content1,content2", content);
}
TEST_F(S3FilesystemTest, NewAppendableFile) {
const std::string path = GetURIForPath("AppendableFile");
WriteString(path, "test");
ASSERT_TF_OK(status_);
auto writer = GetWriter();
tf_s3_filesystem::NewAppendableFile(filesystem_, path.c_str(), writer.get(),
status_);
EXPECT_TF_OK(status_);
tf_writable_file::Append(writer.get(), "content", strlen("content"), status_);
EXPECT_TF_OK(status_);
tf_writable_file::Close(writer.get(), status_);
EXPECT_TF_OK(status_);
}
TEST_F(S3FilesystemTest, NewReadOnlyMemoryRegionFromFile) {
const std::string path = GetURIForPath("MemoryFile");
const std::string content = "content";
WriteString(path, content);
ASSERT_TF_OK(status_);
std::unique_ptr<TF_ReadOnlyMemoryRegion,
void (*)(TF_ReadOnlyMemoryRegion * file)>
region(new TF_ReadOnlyMemoryRegion, [](TF_ReadOnlyMemoryRegion* file) {
if (file != nullptr) {
if (file->plugin_memory_region != nullptr)
tf_read_only_memory_region::Cleanup(file);
delete file;
}
});
region->plugin_memory_region = nullptr;
tf_s3_filesystem::NewReadOnlyMemoryRegionFromFile(filesystem_, path.c_str(),
region.get(), status_);
EXPECT_TF_OK(status_);
std::string result(reinterpret_cast<const char*>(
tf_read_only_memory_region::Data(region.get())),
tf_read_only_memory_region::Length(region.get()));
EXPECT_EQ(content, result);
}
TEST_F(S3FilesystemTest, PathExists) {
const std::string path = GetURIForPath("PathExists");
tf_s3_filesystem::PathExists(filesystem_, path.c_str(), status_);
EXPECT_EQ(TF_NOT_FOUND, TF_GetCode(status_)) << TF_Message(status_);
TF_SetStatus(status_, TF_OK, "");
WriteString(path, "test");
ASSERT_TF_OK(status_);
tf_s3_filesystem::PathExists(filesystem_, path.c_str(), status_);
EXPECT_TF_OK(status_);
}
TEST_F(S3FilesystemTest, GetChildren) {
const std::string base = GetURIForPath("GetChildren");
tf_s3_filesystem::CreateDir(filesystem_, base.c_str(), status_);
EXPECT_TF_OK(status_);
const std::string file = io::JoinPath(base, "TestFile.csv");
WriteString(file, "test");
EXPECT_TF_OK(status_);
const std::string subdir = io::JoinPath(base, "SubDir");
tf_s3_filesystem::CreateDir(filesystem_, subdir.c_str(), status_);
EXPECT_TF_OK(status_);
const std::string subfile = io::JoinPath(subdir, "TestSubFile.csv");
WriteString(subfile, "test");
EXPECT_TF_OK(status_);
char** entries;
auto num_entries = tf_s3_filesystem::GetChildren(filesystem_, base.c_str(),
&entries, status_);
EXPECT_TF_OK(status_);
std::vector<std::string> childrens;
for (int i = 0; i < num_entries; ++i) {
childrens.push_back(entries[i]);
}
std::sort(childrens.begin(), childrens.end());
EXPECT_EQ(std::vector<string>({"SubDir", "TestFile.csv"}), childrens);
}
TEST_F(S3FilesystemTest, DeleteFile) {
const std::string path = GetURIForPath("DeleteFile");
WriteString(path, "test");
ASSERT_TF_OK(status_);
tf_s3_filesystem::DeleteFile(filesystem_, path.c_str(), status_);
EXPECT_TF_OK(status_);
}
TEST_F(S3FilesystemTest, CreateDir) {
// s3 object storage doesn't support empty directory, we create file in the
// directory
const std::string dir = GetURIForPath("CreateDir");
tf_s3_filesystem::CreateDir(filesystem_, dir.c_str(), status_);
EXPECT_TF_OK(status_);
const std::string file = io::JoinPath(dir, "CreateDirFile.csv");
WriteString(file, "test");
ASSERT_TF_OK(status_);
TF_FileStatistics stat;
tf_s3_filesystem::Stat(filesystem_, dir.c_str(), &stat, status_);
EXPECT_TF_OK(status_);
EXPECT_TRUE(stat.is_directory);
}
TEST_F(S3FilesystemTest, DeleteDir) {
// s3 object storage doesn't support empty directory, we create file in the
// directory
const std::string dir = GetURIForPath("DeleteDir");
const std::string file = io::JoinPath(dir, "DeleteDirFile.csv");
WriteString(file, "test");
ASSERT_TF_OK(status_);
tf_s3_filesystem::DeleteDir(filesystem_, dir.c_str(), status_);
EXPECT_NE(TF_GetCode(status_), TF_OK);
TF_SetStatus(status_, TF_OK, "");
tf_s3_filesystem::DeleteFile(filesystem_, file.c_str(), status_);
EXPECT_TF_OK(status_);
tf_s3_filesystem::DeleteDir(filesystem_, dir.c_str(), status_);
EXPECT_TF_OK(status_);
TF_FileStatistics stat;
tf_s3_filesystem::Stat(filesystem_, dir.c_str(), &stat, status_);
EXPECT_EQ(TF_GetCode(status_), TF_NOT_FOUND) << TF_Message(status_);
}
TEST_F(S3FilesystemTest, StatFile) {
const std::string path = GetURIForPath("StatFile");
WriteString(path, "test");
ASSERT_TF_OK(status_);
TF_FileStatistics stat;
tf_s3_filesystem::Stat(filesystem_, path.c_str(), &stat, status_);
EXPECT_TF_OK(status_);
EXPECT_EQ(4, stat.length);
EXPECT_FALSE(stat.is_directory);
}
} // namespace
} // namespace tensorflow