From fa3045baa03aa163d31a26273756efc636b91ba6 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Fri, 25 Sep 2020 11:17:33 +0000
Subject: [PATCH] Avoid using TF file system before TF is initialized with
plugins (avoid circular dependency)
While playing with modular file systems I noticed that
if modular file systems is enabled through bazel with linkage inclusion,
tensorflow will not be able to be imported:
```
>>> import tensorflow as tf
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/usr/local/lib/python3.8/dist-packages/tensorflow/__init__.py", line 434, in <module>
_ll.load_library(_main_dir)
File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/load_library.py", line 143, in load_library
if file_io.file_exists(library_location):
File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/lib/io/file_io.py", line 250, in file_exists
return file_exists_v2(filename)
File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/lib/io/file_io.py", line 268, in file_exists_v2
_pywrap_file_io.FileExists(compat.path_to_bytes(path))
tensorflow.python.framework.errors_impl.UnimplementedError: File system scheme '[local]' not implemented (file: '/usr/local/lib/python3.8/dist-packages/tensorflow/core/kernels')
```
The reason is that, at the very beginning of tensorflow, tf will try
to load libraries in .so file (see tensorflow/__init__.py).
However, at this time the plugins has not been initialized yet, so
TF's own `file_io.file_exists` is not working yet. But as
`tf.load_library` used `file_io.file_exists`, this causes a circular dependency.
This PR replaces `file_io.file_exists` with `os.path.exists` in or before `tf.load_library`.
This will not impact exisitng behavior: since `tf.load_library` relies on `dlopen`
(and `LoadLibrary` on Windows), a non-native file system (e.g., s3/gcs/etc) will
not work with `tf.load_library` anyway. As such there is no need to use gcs/s3 compatible
`file_io.file_exists`.
This PR is part of the effort to enable modular file system from external repos (e.g., tensorflow-io).
Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
tensorflow/api_template.__init__.py | 4 ++--
tensorflow/api_template_v1.__init__.py | 4 ++--
tensorflow/python/framework/load_library.py | 7 +++----
3 files changed, 7 insertions(+), 8 deletions(-)
diff --git a/tensorflow/api_template.__init__.py b/tensorflow/api_template.__init__.py
index 5932dda514d..99a278a14a4 100644
--- a/tensorflow/api_template.__init__.py
+++ b/tensorflow/api_template.__init__.py
@@ -138,12 +138,12 @@ if _running_from_pip_package():
for _s in _site_packages_dirs:
# Load first party dynamic kernels.
_main_dir = _os.path.join(_s, 'tensorflow/core/kernels')
- if _fi.file_exists(_main_dir):
+ if _os.path.exists(_main_dir):
_ll.load_library(_main_dir)
# Load third party dynamic kernels.
_plugin_dir = _os.path.join(_s, 'tensorflow-plugins')
- if _fi.file_exists(_plugin_dir):
+ if _os.path.exists(_plugin_dir):
_ll.load_library(_plugin_dir)
# Add module aliases
diff --git a/tensorflow/api_template_v1.__init__.py b/tensorflow/api_template_v1.__init__.py
index 0d1d2e56fae..ae82f7b4792 100644
--- a/tensorflow/api_template_v1.__init__.py
+++ b/tensorflow/api_template_v1.__init__.py
@@ -148,12 +148,12 @@ if _running_from_pip_package():
for _s in _site_packages_dirs:
# Load first party dynamic kernels.
_main_dir = _os.path.join(_s, 'tensorflow/core/kernels')
- if _fi.file_exists(_main_dir):
+ if _os.path.exists(_main_dir):
_ll.load_library(_main_dir)
# Load third party dynamic kernels.
_plugin_dir = _os.path.join(_s, 'tensorflow-plugins')
- if _fi.file_exists(_plugin_dir):
+ if _os.path.exists(_plugin_dir):
_ll.load_library(_plugin_dir)
# Delete modules that should be hidden from dir().
diff --git a/tensorflow/python/framework/load_library.py b/tensorflow/python/framework/load_library.py
index 2e7bade095b..f37b48e76c2 100644
--- a/tensorflow/python/framework/load_library.py
+++ b/tensorflow/python/framework/load_library.py
@@ -27,7 +27,6 @@ import sys
from tensorflow.python import _pywrap_python_op_gen
from tensorflow.python.client import pywrap_tf_session as py_tf
-from tensorflow.python.lib.io import file_io
from tensorflow.python.util import deprecation
from tensorflow.python.util.tf_export import tf_export
@@ -140,9 +139,9 @@ def load_library(library_location):
OSError: When the file to be loaded is not found.
RuntimeError: when unable to load the library.
"""
- if file_io.file_exists(library_location):
- if file_io.is_directory(library_location):
- directory_contents = file_io.list_directory(library_location)
+ if os.path.exists(library_location):
+ if os.path.isdir(library_location):
+ directory_contents = os.listdir(library_location)
kernel_libraries = [
os.path.join(library_location, f) for f in directory_contents