experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

enable hdfs support for windows (#6987)

Browse Source 
* enabled hdfs support for windows

* change define to cost char *
This commit is contained in:
guschmue 2017-01-24 09:23:52 -08:00 committed by Shanqing Cai
parent f7d07f5d96
commit 3570b5de07
4 changed files with 18 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
tensorflow/contrib/cmake/CMakeLists.txt
View File

@ -21,6 +21,7 @@ option(tensorflow_VERBOSE "Enable for verbose output" OFF)
option(tensorflow_ENABLE_GPU "Enable GPU support" OFF) option(tensorflow_ENABLE_GPU "Enable GPU support" OFF)
option(tensorflow_ENABLE_SSL_SUPPORT "Enable boringssl support" OFF) option(tensorflow_ENABLE_SSL_SUPPORT "Enable boringssl support" OFF)
option(tensorflow_ENABLE_GRPC_SUPPORT "Enable gRPC support" ON) option(tensorflow_ENABLE_GRPC_SUPPORT "Enable gRPC support" ON)
option(tensorflow_ENABLE_HDFS_SUPPORT "Enable HDFS support" OFF)
option(tensorflow_BUILD_CC_EXAMPLE "Build the C++ tutorial example" ON) option(tensorflow_BUILD_CC_EXAMPLE "Build the C++ tutorial example" ON)
option(tensorflow_BUILD_PYTHON_BINDINGS "Build the Python bindings" ON) option(tensorflow_BUILD_PYTHON_BINDINGS "Build the Python bindings" ON)
option(tensorflow_BUILD_ALL_KERNELS "Build all OpKernels" ON) option(tensorflow_BUILD_ALL_KERNELS "Build all OpKernels" ON)

2
tensorflow/contrib/cmake/README.md
View File

@ -18,7 +18,7 @@ for instructions on how to install a pre-built TensorFlow package on Windows.
### Current known limitations ### Current known limitations
* It is not possible to load a custom Op library. * It is not possible to load a custom Op library.
* GCS and HDFS file systems are not supported. * GCS file system is not supported.
* The following Ops are not currently implemented: * The following Ops are not currently implemented:
- Dequantize - Dequantize
- QuantizeAndDequantize - QuantizeAndDequantize

8
tensorflow/contrib/cmake/tf_core_framework.cmake
View File

@ -158,6 +158,14 @@ if(tensorflow_ENABLE_SSL_SUPPORT)
list(APPEND tf_core_lib_srcs ${tf_core_platform_cloud_srcs}) list(APPEND tf_core_lib_srcs ${tf_core_platform_cloud_srcs})
endif() endif()
if (tensorflow_ENABLE_HDFS_SUPPORT)
list(APPEND tf_core_platform_hdfs_srcs
"${tensorflow_source_dir}/tensorflow/core/platform/hadoop/hadoop_file_system.cc"
"${tensorflow_source_dir}/tensorflow/core/platform/hadoop/hadoop_file_system.h"
)
list(APPEND tf_core_lib_srcs ${tf_core_platform_hdfs_srcs})
endif()
file(GLOB_RECURSE tf_core_lib_test_srcs file(GLOB_RECURSE tf_core_lib_test_srcs
"${tensorflow_source_dir}/tensorflow/core/lib/*test*.h" "${tensorflow_source_dir}/tensorflow/core/lib/*test*.h"
"${tensorflow_source_dir}/tensorflow/core/lib/*test*.cc" "${tensorflow_source_dir}/tensorflow/core/lib/*test*.cc"

10
tensorflow/core/platform/hadoop/hadoop_file_system.cc
View File

@ -27,6 +27,7 @@ limitations under the License.
#include "tensorflow/core/platform/posix/error.h" #include "tensorflow/core/platform/posix/error.h"
#include "third_party/hadoop/hdfs.h" #include "third_party/hadoop/hdfs.h"
namespace tensorflow { namespace tensorflow {
template <typename R, typename... Args> template <typename R, typename... Args>
@ -104,18 +105,23 @@ class LibHDFS {
// libhdfs.so won't be in the standard locations. Use the path as specified // libhdfs.so won't be in the standard locations. Use the path as specified
// in the libhdfs documentation. // in the libhdfs documentation.
#if defined(PLATFORM_WINDOWS)
const char *kLibHdfsDso = "hdfs.dll";
#else
const char *kLibHdfsDso = "libhdfs.so";
#endif
char* hdfs_home = getenv("HADOOP_HDFS_HOME"); char* hdfs_home = getenv("HADOOP_HDFS_HOME");
if (hdfs_home == nullptr) { if (hdfs_home == nullptr) {
status_ = errors::FailedPrecondition( status_ = errors::FailedPrecondition(
"Environment variable HADOOP_HDFS_HOME not set"); "Environment variable HADOOP_HDFS_HOME not set");
return; return;
} }
string path = io::JoinPath(hdfs_home, "lib", "native", "libhdfs.so"); string path = io::JoinPath(hdfs_home, "lib", "native", kLibHdfsDso);
status_ = TryLoadAndBind(path.c_str(), &handle_); status_ = TryLoadAndBind(path.c_str(), &handle_);
if (!status_.ok()) { if (!status_.ok()) {
// try load libhdfs.so using dynamic loader's search path in case libhdfs.so // try load libhdfs.so using dynamic loader's search path in case libhdfs.so
// is installed in non-standard location // is installed in non-standard location
status_ = TryLoadAndBind("libhdfs.so", &handle_); status_ = TryLoadAndBind(kLibHdfsDso, &handle_);
} }
return; return;
} }