From c16e21ae3641a4c11e1766b4355c4eefcdcc132d Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Fri, 5 Feb 2021 02:59:22 +0000 Subject: [PATCH] Deprecate legacy hdfs file system This PR is part of the effort to switch to modular file system support by deprecates hdfs file system and direct user to use modular file system from tensorflow-io instead. A `TF_ENABLE_LEGACY_FILESYSTEM=1` will allow the usage of legacy hdfs file system the same way as before (a warning will be displayed). Signed-off-by: Yong Tang --- tensorflow/core/platform/env.h | 34 ++++++++++++++----- .../platform/hadoop/hadoop_file_system.cc | 6 ++-- 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/tensorflow/core/platform/env.h b/tensorflow/core/platform/env.h index 308d8a09fa7..f695cccbfb0 100644 --- a/tensorflow/core/platform/env.h +++ b/tensorflow/core/platform/env.h @@ -632,7 +632,22 @@ namespace register_file_system { template struct Register { - Register(Env* env, const std::string& scheme) { + Register(Env* env, const std::string& scheme, bool legacy) { + // TODO(yongtang): Remove legacy file system registration for hdfs/s3/gcs + // after TF 2.6+. + if (legacy) { + const char* enable_legacy_env = getenv("TF_ENABLE_LEGACY_FILESYSTEM"); + string enable_legacy = + enable_legacy_env ? absl::AsciiStrToLower(enable_legacy_env) : ""; + if (!(enable_legacy == "true" || enable_legacy == "1")) { + return; + } + LOG(WARNING) << "Legacy file system for '" << scheme << "' is deprecated" + << " and will be removed in tensorflow 2.6 or higher." + << " Please switch to tensorflow-io" + << " (https://github.com/tensorflow/io) for file system" + << " support of '" << scheme << "'."; + } // TODO(b/32704451): Don't just ignore the ::tensorflow::Status object! env->RegisterFileSystem(scheme, []() -> FileSystem* { return new Factory; }) .IgnoreError(); @@ -647,16 +662,19 @@ struct Register { // Register a FileSystem implementation for a scheme. Files with names that have // "scheme://" prefixes are routed to use this implementation. -#define REGISTER_FILE_SYSTEM_ENV(env, scheme, factory) \ - REGISTER_FILE_SYSTEM_UNIQ_HELPER(__COUNTER__, env, scheme, factory) -#define REGISTER_FILE_SYSTEM_UNIQ_HELPER(ctr, env, scheme, factory) \ - REGISTER_FILE_SYSTEM_UNIQ(ctr, env, scheme, factory) -#define REGISTER_FILE_SYSTEM_UNIQ(ctr, env, scheme, factory) \ +#define REGISTER_FILE_SYSTEM_ENV(env, scheme, factory, legacy) \ + REGISTER_FILE_SYSTEM_UNIQ_HELPER(__COUNTER__, env, scheme, factory, legacy) +#define REGISTER_FILE_SYSTEM_UNIQ_HELPER(ctr, env, scheme, factory, legacy) \ + REGISTER_FILE_SYSTEM_UNIQ(ctr, env, scheme, factory, legacy) +#define REGISTER_FILE_SYSTEM_UNIQ(ctr, env, scheme, factory, legacy) \ static ::tensorflow::register_file_system::Register \ register_ff##ctr TF_ATTRIBUTE_UNUSED = \ - ::tensorflow::register_file_system::Register(env, scheme) + ::tensorflow::register_file_system::Register(env, scheme, legacy) #define REGISTER_FILE_SYSTEM(scheme, factory) \ - REGISTER_FILE_SYSTEM_ENV(::tensorflow::Env::Default(), scheme, factory); + REGISTER_FILE_SYSTEM_ENV(::tensorflow::Env::Default(), scheme, factory, false); + +#define REGISTER_LEGACY_FILE_SYSTEM(scheme, factory) \ + REGISTER_FILE_SYSTEM_ENV(::tensorflow::Env::Default(), scheme, factory, true); #endif // TENSORFLOW_CORE_PLATFORM_ENV_H_ diff --git a/tensorflow/core/platform/hadoop/hadoop_file_system.cc b/tensorflow/core/platform/hadoop/hadoop_file_system.cc index ed46a16d2e9..b7e7a8dbeaf 100644 --- a/tensorflow/core/platform/hadoop/hadoop_file_system.cc +++ b/tensorflow/core/platform/hadoop/hadoop_file_system.cc @@ -584,8 +584,8 @@ Status HadoopFileSystem::Stat(const string& fname, TransactionToken* token, return Status::OK(); } -REGISTER_FILE_SYSTEM("hdfs", HadoopFileSystem); -REGISTER_FILE_SYSTEM("viewfs", HadoopFileSystem); -REGISTER_FILE_SYSTEM("har", HadoopFileSystem); +REGISTER_LEGACY_FILE_SYSTEM("hdfs", HadoopFileSystem); +REGISTER_LEGACY_FILE_SYSTEM("viewfs", HadoopFileSystem); +REGISTER_LEGACY_FILE_SYSTEM("har", HadoopFileSystem); } // namespace tensorflow