diff --git a/tensorflow/c/experimental/filesystem/plugins/hadoop/BUILD b/tensorflow/c/experimental/filesystem/plugins/hadoop/BUILD new file mode 100644 index 00000000000..1983159d377 --- /dev/null +++ b/tensorflow/c/experimental/filesystem/plugins/hadoop/BUILD @@ -0,0 +1,31 @@ +# Experimental hadoop filesystem plugin. +load("//tensorflow:tensorflow.bzl", "get_win_copts", "tf_cc_shared_object") + +package( + licenses = ["notice"], # Apache 2.0 +) + +# Filesystem implementation for HADOOP environments +tf_cc_shared_object( + name = "hadoop_filesystem", + framework_so = [], + linkstatic = False, + per_os_targets = 1, + visibility = ["//visibility:public"], + deps = [":hadoop_filesystem_impl"], +) + +# The real implementation of the filesystem. +cc_library( + name = "hadoop_filesystem_impl", + srcs = ["hadoop_filesystem.cc"], + hdrs = ["hadoop_filesystem.h"], + copts = select({ + "//conditions:default": [], + "//tensorflow:windows": get_win_copts(), + }), + deps = [ + "//tensorflow/c:tf_status", + "//tensorflow/c/experimental/filesystem:filesystem_interface", + ], +) diff --git a/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc b/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc new file mode 100644 index 00000000000..200d1b23adf --- /dev/null +++ b/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc @@ -0,0 +1,76 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.h" + +#include +#include + +#include "tensorflow/c/experimental/filesystem/filesystem_interface.h" +#include "tensorflow/c/tf_status.h" + +// Implementation of a filesystem for HADOOP environments. +// This filesystem will support `hdfs://`, `viewfs://` and `har://` URI schemes. + +static void* plugin_memory_allocate(size_t size) { return calloc(1, size); } +static void plugin_memory_free(void* ptr) { free(ptr); } + +// SECTION 1. Implementation for `TF_RandomAccessFile` +// ---------------------------------------------------------------------------- +namespace tf_random_access_file { + +// TODO(vnvo2409): Implement later + +} // namespace tf_random_access_file + +// SECTION 2. Implementation for `TF_WritableFile` +// ---------------------------------------------------------------------------- +namespace tf_writable_file { + +// TODO(vnvo2409): Implement later + +} // namespace tf_writable_file + +// SECTION 3. Implementation for `TF_ReadOnlyMemoryRegion` +// ---------------------------------------------------------------------------- +namespace tf_read_only_memory_region { + +// TODO(vnvo2409): Implement later + +} // namespace tf_read_only_memory_region + +// SECTION 4. Implementation for `TF_Filesystem`, the actual filesystem +// ---------------------------------------------------------------------------- +namespace tf_hadoop_filesystem { + +// TODO(vnvo2409): Implement later + +} // namespace tf_hadoop_filesystem + +static void ProvideFilesystemSupportFor(TF_FilesystemPluginOps* ops, + const char* uri) { + TF_SetFilesystemVersionMetadata(ops); + ops->scheme = strdup(uri); +} + +void TF_InitPlugin(TF_FilesystemPluginInfo* info) { + info->plugin_memory_allocate = plugin_memory_allocate; + info->plugin_memory_free = plugin_memory_free; + info->num_schemes = 3; + info->ops = static_cast( + plugin_memory_allocate(info->num_schemes * sizeof(info->ops[0]))); + ProvideFilesystemSupportFor(&info->ops[0], "hdfs"); + ProvideFilesystemSupportFor(&info->ops[1], "viewfs"); + ProvideFilesystemSupportFor(&info->ops[2], "har"); +} diff --git a/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.h b/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.h new file mode 100644 index 00000000000..850cefe0231 --- /dev/null +++ b/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.h @@ -0,0 +1,21 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_C_EXPERIMENTAL_FILESYSTEM_PLUGINS_HADOOP_HADOOP_FILESYSTEM_H_ +#define TENSORFLOW_C_EXPERIMENTAL_FILESYSTEM_PLUGINS_HADOOP_HADOOP_FILESYSTEM_H_ + +#include "tensorflow/c/experimental/filesystem/filesystem_interface.h" +#include "tensorflow/c/tf_status.h" + +#endif // TENSORFLOW_C_EXPERIMENTAL_FILESYSTEM_PLUGINS_HADOOP_HADOOP_FILESYSTEM_H_