From 7699824d80ba7e8f8b2653b965fbbec8ce65e789 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Wed, 30 Sep 2020 19:51:15 +0000 Subject: [PATCH 1/3] Expose TF_RegisterFilesystemPlugin C API This PR is part of the effort for modular file system. In modular file system `RegisterFilesystemPlugin` was a C++ API and there is no directly way to call this API through python. This PR exposes C API `TF_RegisterFilesystemPlugin`, so that it can be used in python bindings. Signed-off-by: Yong Tang --- tensorflow/c/BUILD | 1 + tensorflow/c/c_api.cc | 11 +++++++++++ tensorflow/c/c_api.h | 7 +++++++ 3 files changed, 19 insertions(+) diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD index 677ab3355ff..ba76e2f6705 100644 --- a/tensorflow/c/BUILD +++ b/tensorflow/c/BUILD @@ -217,6 +217,7 @@ tf_cuda_library( "//tensorflow/core:lib_internal", "//tensorflow/core/distributed_runtime:server_lib", "//tensorflow/core/kernels:logging_ops", + "//tensorflow/c/experimental/filesystem:modular_filesystem", ], }), alwayslink = 1, diff --git a/tensorflow/c/c_api.cc b/tensorflow/c/c_api.cc index a03e9227a75..9579efab94d 100644 --- a/tensorflow/c/c_api.cc +++ b/tensorflow/c/c_api.cc @@ -25,6 +25,7 @@ limitations under the License. #include "tensorflow/core/platform/platform.h" // NOLINT #if !defined(IS_MOBILE_PLATFORM) && !defined(IS_SLIM_BUILD) +#include "tensorflow/c/experimental/filesystem/modular_filesystem.h" #include "tensorflow/cc/framework/gradients.h" #include "tensorflow/cc/framework/ops.h" #include "tensorflow/cc/framework/scope_internal.h" @@ -2606,4 +2607,14 @@ void TF_RegisterLogListener(void (*listener)(const char*)) { #endif // !defined(IS_MOBILE_PLATFORM) && !defined(IS_SLIM_BUILD) } +void TF_RegisterFilesystemPlugin(const char* plugin_filename, + TF_Status* status) { +#if defined(IS_MOBILE_PLATFORM) || defined(IS_SLIM_BUILD) + status->status = tensorflow::errors::Unimplemented( + "FileSystem plugin functionality is not supported on mobile"); +#else + status->status = tensorflow::RegisterFilesystemPlugin(plugin_filename); +#endif // defined(IS_MOBILE_PLATFORM) || defined(IS_SLIM_BUILD) +} + } // end extern "C" diff --git a/tensorflow/c/c_api.h b/tensorflow/c/c_api.h index db5f8fd68f8..3f25f7ec10e 100644 --- a/tensorflow/c/c_api.h +++ b/tensorflow/c/c_api.h @@ -1577,6 +1577,13 @@ TF_CAPI_EXPORT extern void TF_DeleteServer(TF_Server* server); TF_CAPI_EXPORT extern void TF_RegisterLogListener( void (*listener)(const char*)); +// Register a FileSystem plugin from filename `plugin_filename`. +// +// On success, place OK in status. +// On failure, place an error status in status. +TF_CAPI_EXPORT extern void TF_RegisterFilesystemPlugin(const char* plugin_filename, + TF_Status* status); + #ifdef __cplusplus } /* end extern "C" */ #endif From bcf7e85b0c61b156f02594fff3c8041edda8d82a Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Wed, 30 Sep 2020 19:54:57 +0000 Subject: [PATCH 2/3] Expose Python binding of TF_RegisterFilesystemPlugin and tf.experimental.register_filesystem_plugin This PR is parf of modular file system to expose the python binding of C API `TF_RegisterFilesystemPlugin`. In addition, an experimental API `tf.experimental.register_filesystem_plugin` has been setup so that it is possible to register a modular file system plugin with: ``` tf.experimental.register_filesystem_plugin(plugin_path) ``` Signed-off-by: Yong Tang --- .../python/client/tf_session_wrapper.cc | 9 +++++++ tensorflow/python/framework/load_library.py | 25 +++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/tensorflow/python/client/tf_session_wrapper.cc b/tensorflow/python/client/tf_session_wrapper.cc index ac656d322c4..cfbe6cbe42a 100644 --- a/tensorflow/python/client/tf_session_wrapper.cc +++ b/tensorflow/python/client/tf_session_wrapper.cc @@ -1155,6 +1155,15 @@ PYBIND11_MODULE(_pywrap_tf_session, m) { return "TensorHandle"; }); + m.def( + "TF_RegisterFilesystemPlugin", + [](const char* plugin_filename) { + tensorflow::Safe_TF_StatusPtr status = + tensorflow::make_safe(TF_NewStatus()); + TF_RegisterFilesystemPlugin(plugin_filename, status.get()); + tensorflow::MaybeRaiseRegisteredFromTFStatus(status.get()); + }); + py::enum_(m, "TF_DataType") .value("TF_FLOAT", TF_FLOAT) .value("TF_DOUBLE", TF_DOUBLE) diff --git a/tensorflow/python/framework/load_library.py b/tensorflow/python/framework/load_library.py index f37b48e76c2..a8c7a4e4185 100644 --- a/tensorflow/python/framework/load_library.py +++ b/tensorflow/python/framework/load_library.py @@ -157,3 +157,28 @@ def load_library(library_location): errno.ENOENT, 'The file or folder to load kernel libraries from does not exist.', library_location) + + +@tf_export('experimental.register_filesystem_plugin') +def register_filesystem_plugin(plugin_location): + """Loads a TensorFlow FileSystem plugin. + + Args: + plugin_location: Path to the plugin. + Relative or absolute filesystem plugin path to a dynamic library file. + + Returns: + None + + Raises: + OSError: When the file to be loaded is not found. + RuntimeError: when unable to load the library. + """ + if os.path.exists(plugin_location): + py_tf.TF_RegisterFilesystemPlugin(plugin_location) + + else: + raise OSError( + errno.ENOENT, + 'The file to load file system plugin from does not exist.', + plugin_location) From cc30517761e97e7448d438d1ecd763af0a05929b Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sat, 10 Oct 2020 15:40:47 +0000 Subject: [PATCH 3/3] Update API golden to expose tf.experimental.register_filesystem_plugin Signed-off-by: Yong Tang --- tensorflow/tools/api/golden/v1/tensorflow.experimental.pbtxt | 4 ++++ tensorflow/tools/api/golden/v2/tensorflow.experimental.pbtxt | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/tensorflow/tools/api/golden/v1/tensorflow.experimental.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.experimental.pbtxt index c3a84b15dd6..dc60db610ff 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.experimental.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.experimental.pbtxt @@ -20,4 +20,8 @@ tf_module { name: "output_all_intermediates" argspec: "args=[\'state\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "register_filesystem_plugin" + argspec: "args=[\'plugin_location\'], varargs=None, keywords=None, defaults=None" + } } diff --git a/tensorflow/tools/api/golden/v2/tensorflow.experimental.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.experimental.pbtxt index 58384846276..33c28d715a5 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.experimental.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.experimental.pbtxt @@ -28,4 +28,8 @@ tf_module { name: "function_executor_type" argspec: "args=[\'executor_type\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "register_filesystem_plugin" + argspec: "args=[\'plugin_location\'], varargs=None, keywords=None, defaults=None" + } }