Allow modular POSIX filesystem to get file/dir statistics.

We also provide tests to make sure all API requirements are satisfied.

Just a small sized part of work for modular filesystem plugins. For more details, consult the RFC at https://github.com/tensorflow/community/blob/master/rfcs/20190506-filesystem-plugin-modular-tensorflow.md

PiperOrigin-RevId: 281110604
Change-Id: I644b8d6da400e1639f7e8ec91ceaa440d4cb193d

tensorflow/c/experimental/filesystem/modular_filesystem.cc

@@ -220,22 +220,56 @@ Status ModularFileSystem::CreateDir(const std::string& dirname) {
 }
 
 Status ModularFileSystem::Stat(const std::string& fname, FileStatistics* stat) {
-  // TODO(mihaimaruseac): Implementation to come in a new change
+  if (ops_->stat == nullptr)
-  return Status(error::UNIMPLEMENTED,
+    return errors::Unimplemented(tensorflow::strings::StrCat(
-                "Modular filesystem stub not implemented yet");
+        "Filesystem for ", fname, " does not support Stat()"));
+
+  if (stat == nullptr)
+    return errors::InvalidArgument("FileStatistics pointer must not be NULL");
+
+  UniquePtrTo_TF_Status plugin_status(TF_NewStatus(), TF_DeleteStatus);
+  std::string translated_name = TranslateName(fname);
+  TF_FileStatistics stats;
+  ops_->stat(filesystem_.get(), translated_name.c_str(), &stats,
+             plugin_status.get());
+
+  if (TF_GetCode(plugin_status.get()) == TF_OK) {
+    stat->length = stats.length;
+    stat->mtime_nsec = stats.mtime_nsec;
+    stat->is_directory = stats.is_directory;
+  }
+
+  return StatusFromTF_Status(plugin_status.get());
 }
 
 Status ModularFileSystem::IsDirectory(const std::string& name) {
-  // TODO(mihaimaruseac): Implementation to come in a new change
+  if (ops_->is_directory == nullptr) return FileSystem::IsDirectory(name);
-  return Status(error::UNIMPLEMENTED,
+
-                "Modular filesystem stub not implemented yet");
+  UniquePtrTo_TF_Status plugin_status(TF_NewStatus(), TF_DeleteStatus);
+  std::string translated_name = TranslateName(name);
+  ops_->is_directory(filesystem_.get(), translated_name.c_str(),
+                     plugin_status.get());
+  return StatusFromTF_Status(plugin_status.get());
 }
 
 Status ModularFileSystem::GetFileSize(const std::string& fname,
                                       uint64* file_size) {
-  // TODO(mihaimaruseac): Implementation to come in a new change
+  if (ops_->get_file_size == nullptr) {
-  return Status(error::UNIMPLEMENTED,
+    FileStatistics stat;
-                "Modular filesystem stub not implemented yet");
+    Status status = Stat(fname, &stat);
+    if (!status.ok()) return status;
+    if (stat.is_directory)
+      return errors::FailedPrecondition("Called GetFileSize on a directory");
+
+    *file_size = stat.length;
+    return status;
+  }
+
+  UniquePtrTo_TF_Status plugin_status(TF_NewStatus(), TF_DeleteStatus);
+  std::string translated_name = TranslateName(fname);
+  *file_size = ops_->get_file_size(filesystem_.get(), translated_name.c_str(),
+                                   plugin_status.get());
+  return StatusFromTF_Status(plugin_status.get());
 }
 
 Status ModularFileSystem::RenameFile(const std::string& src,

tensorflow/c/experimental/filesystem/modular_filesystem_test.cc

@@ -628,6 +628,172 @@ TEST_P(ModularFileSystemTest, TestFilesExistsNoFiles) {
   EXPECT_TRUE(statuses.empty());
 }
 
+TEST_P(ModularFileSystemTest, TestStatEmptyFile) {
+  const std::string filepath = GetURIForPath("a_file");
+  std::unique_ptr<WritableFile> file;
+  Status status = env_->NewWritableFile(filepath, &file);
+  if (!status.ok()) GTEST_SKIP() << "NewWritableFile() not supported";
+
+  FileStatistics stat;
+  status = env_->Stat(filepath, &stat);
+  EXPECT_PRED2(UninmplementedOrReturnsCode, status, Code::OK);
+  if (!status.ok()) GTEST_SKIP() << "Stat() not supported";
+  EXPECT_FALSE(stat.is_directory);
+  EXPECT_EQ(stat.length, 0);
+}
+
+TEST_P(ModularFileSystemTest, TestStatNonEmptyFile) {
+  const std::string filepath = GetURIForPath("a_file");
+  std::unique_ptr<WritableFile> file;
+  Status status = env_->NewWritableFile(filepath, &file);
+  if (!status.ok()) GTEST_SKIP() << "NewWritableFile() not supported";
+
+  const std::string test_data("asdf");
+  status = file->Append(test_data);
+  if (!status.ok()) GTEST_SKIP() << "Append() not supported";
+  status = file->Flush();
+  if (!status.ok()) GTEST_SKIP() << "Flush() not supported";
+  status = file->Close();
+  if (!status.ok()) GTEST_SKIP() << "Close() not supported";
+
+  FileStatistics stat;
+  status = env_->Stat(filepath, &stat);
+  EXPECT_PRED2(UninmplementedOrReturnsCode, status, Code::OK);
+  if (!status.ok()) GTEST_SKIP() << "Stat() not supported";
+  EXPECT_FALSE(stat.is_directory);
+  EXPECT_EQ(stat.length, test_data.size());
+}
+
+TEST_P(ModularFileSystemTest, TestStatDirectory) {
+  const std::string dirpath = GetURIForPath("a_dir");
+  Status status = env_->CreateDir(dirpath);
+  if (!status.ok()) GTEST_SKIP() << "CreateDir() not supported";
+
+  FileStatistics stat;
+  status = env_->Stat(dirpath, &stat);
+  EXPECT_PRED2(UninmplementedOrReturnsCode, status, Code::OK);
+  if (!status.ok()) GTEST_SKIP() << "Stat() not supported";
+  EXPECT_TRUE(stat.is_directory);
+}
+
+TEST_P(ModularFileSystemTest, TestStatNotFound) {
+  const std::string dirpath = GetURIForPath("a_dir");
+  FileStatistics stat;
+  Status status = env_->Stat(dirpath, &stat);
+  EXPECT_PRED2(UninmplementedOrReturnsCode, status, Code::NOT_FOUND);
+}
+
+TEST_P(ModularFileSystemTest, TestStatPathIsInvalid) {
+  const std::string filepath = GetURIForPath("a_file");
+  std::unique_ptr<WritableFile> file;
+  Status status = env_->NewWritableFile(filepath, &file);
+  if (!status.ok()) GTEST_SKIP() << "NewWritableFile() not supported";
+
+  const std::string target_path = GetURIForPath("a_file/a_new_file");
+  FileStatistics stat;
+  status = env_->Stat(target_path, &stat);
+  EXPECT_PRED2(UninmplementedOrReturnsCode, status, Code::FAILED_PRECONDITION);
+}
+
+TEST_P(ModularFileSystemTest, TestIsDirectory) {
+  const std::string dirpath = GetURIForPath("a_dir");
+  Status status = env_->CreateDir(dirpath);
+  if (!status.ok()) GTEST_SKIP() << "CreateDir() not supported";
+
+  status = env_->IsDirectory(dirpath);
+  EXPECT_PRED2(UninmplementedOrReturnsCode, status, Code::OK);
+}
+
+TEST_P(ModularFileSystemTest, TestIsDirectoryFile) {
+  const std::string filepath = GetURIForPath("a_file");
+  std::unique_ptr<WritableFile> file;
+  Status status = env_->NewWritableFile(filepath, &file);
+  if (!status.ok()) GTEST_SKIP() << "NewWritableFile() not supported";
+
+  status = env_->IsDirectory(filepath);
+  EXPECT_PRED2(UninmplementedOrReturnsCode, status, Code::FAILED_PRECONDITION);
+}
+
+TEST_P(ModularFileSystemTest, TestIsDirectoryNotFound) {
+  const std::string dirpath = GetURIForPath("a_dir");
+  Status status = env_->IsDirectory(dirpath);
+  EXPECT_PRED2(UninmplementedOrReturnsCode, status, Code::NOT_FOUND);
+}
+
+TEST_P(ModularFileSystemTest, TestIsDirectoryPathIsInvalid) {
+  const std::string filepath = GetURIForPath("a_file");
+  std::unique_ptr<WritableFile> file;
+  Status status = env_->NewWritableFile(filepath, &file);
+  if (!status.ok()) GTEST_SKIP() << "NewWritableFile() not supported";
+
+  const std::string target_path = GetURIForPath("a_file/a_new_file");
+  status = env_->IsDirectory(target_path);
+  EXPECT_PRED2(UninmplementedOrReturnsCode, status, Code::FAILED_PRECONDITION);
+}
+
+TEST_P(ModularFileSystemTest, TestGetFileSizeEmptyFile) {
+  const std::string filepath = GetURIForPath("a_file");
+  std::unique_ptr<WritableFile> file;
+  Status status = env_->NewWritableFile(filepath, &file);
+  if (!status.ok()) GTEST_SKIP() << "NewWritableFile() not supported";
+
+  uint64 size;
+  status = env_->GetFileSize(filepath, &size);
+  EXPECT_PRED2(UninmplementedOrReturnsCode, status, Code::OK);
+  if (!status.ok()) GTEST_SKIP() << "GetFileSize() not supported";
+  EXPECT_EQ(size, 0);
+}
+
+TEST_P(ModularFileSystemTest, TestGetFileSizeNonEmptyFile) {
+  const std::string filepath = GetURIForPath("a_file");
+  std::unique_ptr<WritableFile> file;
+  Status status = env_->NewWritableFile(filepath, &file);
+  if (!status.ok()) GTEST_SKIP() << "NewWritableFile() not supported";
+
+  const std::string test_data("asdf");
+  status = file->Append(test_data);
+  if (!status.ok()) GTEST_SKIP() << "Append() not supported";
+  status = file->Flush();
+  if (!status.ok()) GTEST_SKIP() << "Flush() not supported";
+  status = file->Close();
+  if (!status.ok()) GTEST_SKIP() << "Close() not supported";
+
+  uint64 size;
+  status = env_->GetFileSize(filepath, &size);
+  EXPECT_PRED2(UninmplementedOrReturnsCode, status, Code::OK);
+  if (!status.ok()) GTEST_SKIP() << "GetFileSize() not supported";
+  EXPECT_EQ(size, test_data.size());
+}
+
+TEST_P(ModularFileSystemTest, TestGetFileSizeDirectory) {
+  const std::string dirpath = GetURIForPath("a_dir");
+  Status status = env_->CreateDir(dirpath);
+  if (!status.ok()) GTEST_SKIP() << "CreateDir() not supported";
+
+  uint64 size;
+  status = env_->GetFileSize(dirpath, &size);
+  EXPECT_PRED2(UninmplementedOrReturnsCode, status, Code::FAILED_PRECONDITION);
+}
+
+TEST_P(ModularFileSystemTest, TestGetFileSizeNotFound) {
+  const std::string filepath = GetURIForPath("a_dir");
+  uint64 size;
+  Status status = env_->GetFileSize(filepath, &size);
+  EXPECT_PRED2(UninmplementedOrReturnsCode, status, Code::NOT_FOUND);
+}
+
+TEST_P(ModularFileSystemTest, TestGetFileSizePathIsInvalid) {
+  const std::string filepath = GetURIForPath("a_file");
+  std::unique_ptr<WritableFile> file;
+  Status status = env_->NewWritableFile(filepath, &file);
+  if (!status.ok()) GTEST_SKIP() << "NewWritableFile() not supported";
+
+  const std::string target_path = GetURIForPath("a_file/a_new_file");
+  uint64 size;
+  status = env_->GetFileSize(target_path, &size);
+  EXPECT_PRED2(UninmplementedOrReturnsCode, status, Code::FAILED_PRECONDITION);
+}
+
 TEST_P(ModularFileSystemTest, TestAppendAndTell) {
   const std::string filename = GetURIForPath("a_file");
   std::unique_ptr<WritableFile> file;

tensorflow/c/experimental/filesystem/plugins/posix/posix_filesystem.cc

@@ -295,6 +295,19 @@ static void PathExists(const TF_Filesystem* filesystem, const char* path,
     TF_SetStatus(status, TF_OK, "");
 }
 
+static void Stat(const TF_Filesystem* filesystem, const char* path,
+                 TF_FileStatistics* stats, TF_Status* status) {
+  struct stat sbuf;
+  if (stat(path, &sbuf) != 0) {
+    TF_SetStatusFromIOError(status, errno, path);
+  } else {
+    stats->length = sbuf.st_size;
+    stats->mtime_nsec = sbuf.st_mtime * (1000 * 1000 * 1000);
+    stats->is_directory = S_ISDIR(sbuf.st_mode);
+    TF_SetStatus(status, TF_OK, "");
+  }
+}
+
 }  // namespace tf_posix_filesystem
 
 void TF_InitPlugin(TF_Status* status) {
@@ -328,6 +341,9 @@ void TF_InitPlugin(TF_Status* status) {
       /*copy_file=*/nullptr,
       tf_posix_filesystem::PathExists,
       /*paths_exist=*/nullptr,
+      tf_posix_filesystem::Stat,
+      /*is_directory=*/nullptr,
+      /*get_file_size=*/nullptr,
       nullptr,
   };