experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

enable hdfs support for windows (#6987)

Browse Source 
* enabled hdfs support for windows

* change define to cost char *
This commit is contained in:
guschmue 2017-01-24 09:23:52 -08:00 committed by Shanqing Cai
parent f7d07f5d96
commit 3570b5de07
4 changed files with 18 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
tensorflow/contrib/cmake/CMakeLists.txt
View File

@ -21,6 +21,7 @@ option(tensorflow_VERBOSE "Enable for verbose output" OFF)
option(tensorflow_ENABLE_GPU "Enable GPU support" OFF)
option(tensorflow_ENABLE_SSL_SUPPORT "Enable boringssl support" OFF)
option(tensorflow_ENABLE_GRPC_SUPPORT "Enable gRPC support" ON)
option(tensorflow_ENABLE_HDFS_SUPPORT "Enable HDFS support" OFF)
option(tensorflow_BUILD_CC_EXAMPLE "Build the C++ tutorial example" ON)
option(tensorflow_BUILD_PYTHON_BINDINGS "Build the Python bindings" ON)
option(tensorflow_BUILD_ALL_KERNELS "Build all OpKernels" ON)

2
tensorflow/contrib/cmake/README.md
View File

@ -18,7 +18,7 @@ for instructions on how to install a pre-built TensorFlow package on Windows.
### Current known limitations
* It is not possible to load a custom Op library.
* GCS and HDFS file systems are not supported.
* GCS file system is not supported.
* The following Ops are not currently implemented:
- Dequantize
- QuantizeAndDequantize

8
tensorflow/contrib/cmake/tf_core_framework.cmake
View File

@ -158,6 +158,14 @@ if(tensorflow_ENABLE_SSL_SUPPORT)
list(APPEND tf_core_lib_srcs ${tf_core_platform_cloud_srcs})
endif()
if (tensorflow_ENABLE_HDFS_SUPPORT)
list(APPEND tf_core_platform_hdfs_srcs
"${tensorflow_source_dir}/tensorflow/core/platform/hadoop/hadoop_file_system.cc"
"${tensorflow_source_dir}/tensorflow/core/platform/hadoop/hadoop_file_system.h"
)
list(APPEND tf_core_lib_srcs ${tf_core_platform_hdfs_srcs})
endif()
file(GLOB_RECURSE tf_core_lib_test_srcs
"${tensorflow_source_dir}/tensorflow/core/lib/*test*.h"
"${tensorflow_source_dir}/tensorflow/core/lib/*test*.cc"

10
tensorflow/core/platform/hadoop/hadoop_file_system.cc
View File

@ -27,6 +27,7 @@ limitations under the License.
#include "tensorflow/core/platform/posix/error.h"
#include "third_party/hadoop/hdfs.h"
namespace tensorflow {
template <typename R, typename... Args>
@ -104,18 +105,23 @@ class LibHDFS {
// libhdfs.so won't be in the standard locations. Use the path as specified
// in the libhdfs documentation.
#if defined(PLATFORM_WINDOWS)
const char *kLibHdfsDso = "hdfs.dll";
#else
const char *kLibHdfsDso = "libhdfs.so";
#endif
char* hdfs_home = getenv("HADOOP_HDFS_HOME");
if (hdfs_home == nullptr) {
status_ = errors::FailedPrecondition(
"Environment variable HADOOP_HDFS_HOME not set");
return;
}
string path = io::JoinPath(hdfs_home, "lib", "native", "libhdfs.so");
string path = io::JoinPath(hdfs_home, "lib", "native", kLibHdfsDso);
status_ = TryLoadAndBind(path.c_str(), &handle_);
if (!status_.ok()) {
// try load libhdfs.so using dynamic loader's search path in case libhdfs.so
// is installed in non-standard location
status_ = TryLoadAndBind("libhdfs.so", &handle_);
status_ = TryLoadAndBind(kLibHdfsDso, &handle_);
}
return;
}