experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Deprecate legacy hdfs file system

Browse Source 
This PR is part of the effort to switch to modular file system support
by deprecates hdfs file system and direct user to use modular file system
from tensorflow-io instead.

A `TF_ENABLE_LEGACY_FILESYSTEM=1` will allow the usage of legacy hdfs file
system the same way as before (a warning will be displayed).

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
This commit is contained in:
Yong Tang 2021-02-05 02:59:22 +00:00
parent b4bf78ffec
commit c16e21ae36
2 changed files with 29 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
tensorflow/core/platform/env.h
View File

@ -632,7 +632,22 @@ namespace register_file_system {
template <typename Factory>
struct Register {
Register(Env* env, const std::string& scheme) {
Register(Env* env, const std::string& scheme, bool legacy) {
// TODO(yongtang): Remove legacy file system registration for hdfs/s3/gcs
// after TF 2.6+.
if (legacy) {
const char* enable_legacy_env = getenv("TF_ENABLE_LEGACY_FILESYSTEM");
string enable_legacy =
enable_legacy_env ? absl::AsciiStrToLower(enable_legacy_env) : "";
if (!(enable_legacy == "true" || enable_legacy == "1")) {
return;
}
LOG(WARNING) << "Legacy file system for '" << scheme << "' is deprecated"
<< " and will be removed in tensorflow 2.6 or higher."
<< " Please switch to tensorflow-io"
<< " (https://github.com/tensorflow/io) for file system"
<< " support of '" << scheme << "'.";
}
// TODO(b/32704451): Don't just ignore the ::tensorflow::Status object!
env->RegisterFileSystem(scheme, []() -> FileSystem* { return new Factory; })
.IgnoreError();
@ -647,16 +662,19 @@ struct Register {
// Register a FileSystem implementation for a scheme. Files with names that have
// "scheme://" prefixes are routed to use this implementation.
#define REGISTER_FILE_SYSTEM_ENV(env, scheme, factory) \
REGISTER_FILE_SYSTEM_UNIQ_HELPER(__COUNTER__, env, scheme, factory)
#define REGISTER_FILE_SYSTEM_UNIQ_HELPER(ctr, env, scheme, factory) \
REGISTER_FILE_SYSTEM_UNIQ(ctr, env, scheme, factory)
#define REGISTER_FILE_SYSTEM_UNIQ(ctr, env, scheme, factory) \
#define REGISTER_FILE_SYSTEM_ENV(env, scheme, factory, legacy) \
REGISTER_FILE_SYSTEM_UNIQ_HELPER(__COUNTER__, env, scheme, factory, legacy)
#define REGISTER_FILE_SYSTEM_UNIQ_HELPER(ctr, env, scheme, factory, legacy) \
REGISTER_FILE_SYSTEM_UNIQ(ctr, env, scheme, factory, legacy)
#define REGISTER_FILE_SYSTEM_UNIQ(ctr, env, scheme, factory, legacy) \
static ::tensorflow::register_file_system::Register<factory> \
register_ff##ctr TF_ATTRIBUTE_UNUSED = \
::tensorflow::register_file_system::Register<factory>(env, scheme)
::tensorflow::register_file_system::Register<factory>(env, scheme, legacy)
#define REGISTER_FILE_SYSTEM(scheme, factory) \
REGISTER_FILE_SYSTEM_ENV(::tensorflow::Env::Default(), scheme, factory);
REGISTER_FILE_SYSTEM_ENV(::tensorflow::Env::Default(), scheme, factory, false);
#define REGISTER_LEGACY_FILE_SYSTEM(scheme, factory) \
REGISTER_FILE_SYSTEM_ENV(::tensorflow::Env::Default(), scheme, factory, true);
#endif // TENSORFLOW_CORE_PLATFORM_ENV_H_

6
tensorflow/core/platform/hadoop/hadoop_file_system.cc
View File

@ -584,8 +584,8 @@ Status HadoopFileSystem::Stat(const string& fname, TransactionToken* token,
return Status::OK();
}
REGISTER_FILE_SYSTEM("hdfs", HadoopFileSystem);
REGISTER_FILE_SYSTEM("viewfs", HadoopFileSystem);
REGISTER_FILE_SYSTEM("har", HadoopFileSystem);
REGISTER_LEGACY_FILE_SYSTEM("hdfs", HadoopFileSystem);
REGISTER_LEGACY_FILE_SYSTEM("viewfs", HadoopFileSystem);
REGISTER_LEGACY_FILE_SYSTEM("har", HadoopFileSystem);
} // namespace tensorflow