enable hdfs support for windows (#6987)
* enabled hdfs support for windows * change define to cost char *
This commit is contained in:
parent
f7d07f5d96
commit
3570b5de07
@ -21,6 +21,7 @@ option(tensorflow_VERBOSE "Enable for verbose output" OFF)
|
|||||||
option(tensorflow_ENABLE_GPU "Enable GPU support" OFF)
|
option(tensorflow_ENABLE_GPU "Enable GPU support" OFF)
|
||||||
option(tensorflow_ENABLE_SSL_SUPPORT "Enable boringssl support" OFF)
|
option(tensorflow_ENABLE_SSL_SUPPORT "Enable boringssl support" OFF)
|
||||||
option(tensorflow_ENABLE_GRPC_SUPPORT "Enable gRPC support" ON)
|
option(tensorflow_ENABLE_GRPC_SUPPORT "Enable gRPC support" ON)
|
||||||
|
option(tensorflow_ENABLE_HDFS_SUPPORT "Enable HDFS support" OFF)
|
||||||
option(tensorflow_BUILD_CC_EXAMPLE "Build the C++ tutorial example" ON)
|
option(tensorflow_BUILD_CC_EXAMPLE "Build the C++ tutorial example" ON)
|
||||||
option(tensorflow_BUILD_PYTHON_BINDINGS "Build the Python bindings" ON)
|
option(tensorflow_BUILD_PYTHON_BINDINGS "Build the Python bindings" ON)
|
||||||
option(tensorflow_BUILD_ALL_KERNELS "Build all OpKernels" ON)
|
option(tensorflow_BUILD_ALL_KERNELS "Build all OpKernels" ON)
|
||||||
|
@ -18,7 +18,7 @@ for instructions on how to install a pre-built TensorFlow package on Windows.
|
|||||||
|
|
||||||
### Current known limitations
|
### Current known limitations
|
||||||
* It is not possible to load a custom Op library.
|
* It is not possible to load a custom Op library.
|
||||||
* GCS and HDFS file systems are not supported.
|
* GCS file system is not supported.
|
||||||
* The following Ops are not currently implemented:
|
* The following Ops are not currently implemented:
|
||||||
- Dequantize
|
- Dequantize
|
||||||
- QuantizeAndDequantize
|
- QuantizeAndDequantize
|
||||||
|
@ -158,6 +158,14 @@ if(tensorflow_ENABLE_SSL_SUPPORT)
|
|||||||
list(APPEND tf_core_lib_srcs ${tf_core_platform_cloud_srcs})
|
list(APPEND tf_core_lib_srcs ${tf_core_platform_cloud_srcs})
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
|
if (tensorflow_ENABLE_HDFS_SUPPORT)
|
||||||
|
list(APPEND tf_core_platform_hdfs_srcs
|
||||||
|
"${tensorflow_source_dir}/tensorflow/core/platform/hadoop/hadoop_file_system.cc"
|
||||||
|
"${tensorflow_source_dir}/tensorflow/core/platform/hadoop/hadoop_file_system.h"
|
||||||
|
)
|
||||||
|
list(APPEND tf_core_lib_srcs ${tf_core_platform_hdfs_srcs})
|
||||||
|
endif()
|
||||||
|
|
||||||
file(GLOB_RECURSE tf_core_lib_test_srcs
|
file(GLOB_RECURSE tf_core_lib_test_srcs
|
||||||
"${tensorflow_source_dir}/tensorflow/core/lib/*test*.h"
|
"${tensorflow_source_dir}/tensorflow/core/lib/*test*.h"
|
||||||
"${tensorflow_source_dir}/tensorflow/core/lib/*test*.cc"
|
"${tensorflow_source_dir}/tensorflow/core/lib/*test*.cc"
|
||||||
|
@ -27,6 +27,7 @@ limitations under the License.
|
|||||||
#include "tensorflow/core/platform/posix/error.h"
|
#include "tensorflow/core/platform/posix/error.h"
|
||||||
#include "third_party/hadoop/hdfs.h"
|
#include "third_party/hadoop/hdfs.h"
|
||||||
|
|
||||||
|
|
||||||
namespace tensorflow {
|
namespace tensorflow {
|
||||||
|
|
||||||
template <typename R, typename... Args>
|
template <typename R, typename... Args>
|
||||||
@ -104,18 +105,23 @@ class LibHDFS {
|
|||||||
|
|
||||||
// libhdfs.so won't be in the standard locations. Use the path as specified
|
// libhdfs.so won't be in the standard locations. Use the path as specified
|
||||||
// in the libhdfs documentation.
|
// in the libhdfs documentation.
|
||||||
|
#if defined(PLATFORM_WINDOWS)
|
||||||
|
const char *kLibHdfsDso = "hdfs.dll";
|
||||||
|
#else
|
||||||
|
const char *kLibHdfsDso = "libhdfs.so";
|
||||||
|
#endif
|
||||||
char* hdfs_home = getenv("HADOOP_HDFS_HOME");
|
char* hdfs_home = getenv("HADOOP_HDFS_HOME");
|
||||||
if (hdfs_home == nullptr) {
|
if (hdfs_home == nullptr) {
|
||||||
status_ = errors::FailedPrecondition(
|
status_ = errors::FailedPrecondition(
|
||||||
"Environment variable HADOOP_HDFS_HOME not set");
|
"Environment variable HADOOP_HDFS_HOME not set");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
string path = io::JoinPath(hdfs_home, "lib", "native", "libhdfs.so");
|
string path = io::JoinPath(hdfs_home, "lib", "native", kLibHdfsDso);
|
||||||
status_ = TryLoadAndBind(path.c_str(), &handle_);
|
status_ = TryLoadAndBind(path.c_str(), &handle_);
|
||||||
if (!status_.ok()) {
|
if (!status_.ok()) {
|
||||||
// try load libhdfs.so using dynamic loader's search path in case libhdfs.so
|
// try load libhdfs.so using dynamic loader's search path in case libhdfs.so
|
||||||
// is installed in non-standard location
|
// is installed in non-standard location
|
||||||
status_ = TryLoadAndBind("libhdfs.so", &handle_);
|
status_ = TryLoadAndBind(kLibHdfsDso, &handle_);
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
Loading…
x
Reference in New Issue
Block a user