experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #46955 from yongtang:deprecate-hdfs

Browse Source 
PiperOrigin-RevId: 356378740
Change-Id: Iba3ed688bcbe56c9506fdf9e5842efefe948c4c0
This commit is contained in:
TensorFlower Gardener 2021-02-08 16:27:59 -08:00
commit e920f7a7ea
2 changed files with 34 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
tensorflow/core/platform/env.h
View File

@ -632,7 +632,22 @@ namespace register_file_system {
template <typename Factory>
struct Register {
Register(Env* env, const std::string& scheme) {
Register(Env* env, const std::string& scheme, bool legacy) {
// TODO(yongtang): Remove legacy file system registration for hdfs/s3/gcs
// after TF 2.6+.
if (legacy) {
const char* enable_legacy_env = getenv("TF_ENABLE_LEGACY_FILESYSTEM");
string enable_legacy =
enable_legacy_env ? absl::AsciiStrToLower(enable_legacy_env) : "";
if (!(enable_legacy == "true" || enable_legacy == "1")) {
return;
}
LOG(WARNING) << "Legacy file system for '" << scheme << "' is deprecated"
<< " and will be removed in tensorflow 2.6 or higher."
<< " Please switch to tensorflow-io"
<< " (https://github.com/tensorflow/io) for file system"
<< " support of '" << scheme << "'.";
}
// TODO(b/32704451): Don't just ignore the ::tensorflow::Status object!
env->RegisterFileSystem(scheme, []() -> FileSystem* { return new Factory; })
.IgnoreError();
@ -647,16 +662,21 @@ struct Register {
// Register a FileSystem implementation for a scheme. Files with names that have
// "scheme://" prefixes are routed to use this implementation.
#define REGISTER_FILE_SYSTEM_ENV(env, scheme, factory) \
REGISTER_FILE_SYSTEM_UNIQ_HELPER(__COUNTER__, env, scheme, factory)
#define REGISTER_FILE_SYSTEM_UNIQ_HELPER(ctr, env, scheme, factory) \
REGISTER_FILE_SYSTEM_UNIQ(ctr, env, scheme, factory)
#define REGISTER_FILE_SYSTEM_UNIQ(ctr, env, scheme, factory) \
static ::tensorflow::register_file_system::Register<factory> \
register_ff##ctr TF_ATTRIBUTE_UNUSED = \
::tensorflow::register_file_system::Register<factory>(env, scheme)
#define REGISTER_FILE_SYSTEM_ENV(env, scheme, factory, legacy) \
REGISTER_FILE_SYSTEM_UNIQ_HELPER(__COUNTER__, env, scheme, factory, legacy)
#define REGISTER_FILE_SYSTEM_UNIQ_HELPER(ctr, env, scheme, factory, legacy) \
REGISTER_FILE_SYSTEM_UNIQ(ctr, env, scheme, factory, legacy)
#define REGISTER_FILE_SYSTEM_UNIQ(ctr, env, scheme, factory, legacy) \
static ::tensorflow::register_file_system::Register<factory> \
register_ff##ctr TF_ATTRIBUTE_UNUSED = \
::tensorflow::register_file_system::Register<factory>(env, scheme, \
legacy)
#define REGISTER_FILE_SYSTEM(scheme, factory) \
REGISTER_FILE_SYSTEM_ENV(::tensorflow::Env::Default(), scheme, factory);
#define REGISTER_FILE_SYSTEM(scheme, factory) \
REGISTER_FILE_SYSTEM_ENV(::tensorflow::Env::Default(), scheme, factory, \
false);
#define REGISTER_LEGACY_FILE_SYSTEM(scheme, factory) \
REGISTER_FILE_SYSTEM_ENV(::tensorflow::Env::Default(), scheme, factory, true);
#endif // TENSORFLOW_CORE_PLATFORM_ENV_H_

6
tensorflow/core/platform/hadoop/hadoop_file_system.cc
View File

@ -584,8 +584,8 @@ Status HadoopFileSystem::Stat(const string& fname, TransactionToken* token,
return Status::OK();
}
REGISTER_FILE_SYSTEM("hdfs", HadoopFileSystem);
REGISTER_FILE_SYSTEM("viewfs", HadoopFileSystem);
REGISTER_FILE_SYSTEM("har", HadoopFileSystem);
REGISTER_LEGACY_FILE_SYSTEM("hdfs", HadoopFileSystem);
REGISTER_LEGACY_FILE_SYSTEM("viewfs", HadoopFileSystem);
REGISTER_LEGACY_FILE_SYSTEM("har", HadoopFileSystem);
} // namespace tensorflow