From 33bf2e77badf4bb2bff76dd993c6a69d6c90a2b3 Mon Sep 17 00:00:00 2001 From: Vo Van Nghia Date: Sun, 3 May 2020 23:49:53 +0700 Subject: [PATCH] Add windows support for s3 --- .../core/platform/default/build_config.bzl | 4 +++- tensorflow/core/platform/s3/BUILD | 7 ++++++- tensorflow/core/platform/s3/s3_file_system.cc | 9 +++++++- third_party/aws/BUILD.bazel | 21 ++++++++++++++++++- third_party/aws/aws-c-common.bazel | 9 ++++++++ third_party/aws/aws-checksums.bazel | 7 ++++++- 6 files changed, 52 insertions(+), 5 deletions(-) diff --git a/tensorflow/core/platform/default/build_config.bzl b/tensorflow/core/platform/default/build_config.bzl index fa3bd466d8e..33586be9b65 100644 --- a/tensorflow/core/platform/default/build_config.bzl +++ b/tensorflow/core/platform/default/build_config.bzl @@ -658,7 +658,9 @@ def tf_additional_core_deps(): clean_dep("//tensorflow:android"): [], clean_dep("//tensorflow:ios"): [], clean_dep("//tensorflow:linux_s390x"): [], - clean_dep("//tensorflow:windows"): [], + clean_dep("//tensorflow:windows"): [ + clean_dep("//tensorflow/core/platform/s3:s3_file_system"), + ], clean_dep("//tensorflow:no_aws_support"): [], "//conditions:default": [ clean_dep("//tensorflow/core/platform/s3:s3_file_system"), diff --git a/tensorflow/core/platform/s3/BUILD b/tensorflow/core/platform/s3/BUILD index d174b108279..f99e5a0329e 100644 --- a/tensorflow/core/platform/s3/BUILD +++ b/tensorflow/core/platform/s3/BUILD @@ -81,7 +81,12 @@ cc_library( hdrs = [ "s3_file_system.h", ], - deps = [ + deps = select({ + "@org_tensorflow//tensorflow:windows": [ + "//tensorflow/core/platform:retrying_file_system" + ], + "//conditions:default": [], + }) + [ ":aws_crypto", ":aws_logging", "//tensorflow/core:lib", diff --git a/tensorflow/core/platform/s3/s3_file_system.cc b/tensorflow/core/platform/s3/s3_file_system.cc index 02658242ab7..ec03b64b733 100644 --- a/tensorflow/core/platform/s3/s3_file_system.cc +++ b/tensorflow/core/platform/s3/s3_file_system.cc @@ -48,6 +48,13 @@ limitations under the License. namespace tensorflow { namespace { +#ifdef PLATFORM_WINDOWS +// On Windows, `Aws::FileSystem::CreateTempFilePath()` return +// `C:\Users\username\AppData\Local\Temp\`. Adding suffix will cause an error. +static const char* kS3TempFileSuffix = NULL; +#else +static const char* kS3TempFileSuffix = "/tmp/s3_filesystem_XXXXXX"; +#endif static const char* kS3FileSystemAllocationTag = "S3FileSystemAllocation"; static const size_t kS3ReadAppendableFileBufferSize = 1024 * 1024; static const int64 kS3TimeoutMsec = 300000; // 5 min @@ -271,7 +278,7 @@ class S3WritableFile : public WritableFile { transfer_manager_(transfer_manager), sync_needed_(true), outfile_(Aws::MakeShared( - kS3FileSystemAllocationTag, "/tmp/s3_filesystem_XXXXXX", + kS3FileSystemAllocationTag, kS3TempFileSuffix, std::ios_base::binary | std::ios_base::trunc | std::ios_base::in | std::ios_base::out)) {} diff --git a/third_party/aws/BUILD.bazel b/third_party/aws/BUILD.bazel index fd355eeceb1..bb8903eb2ce 100644 --- a/third_party/aws/BUILD.bazel +++ b/third_party/aws/BUILD.bazel @@ -31,6 +31,14 @@ cc_library( "aws-cpp-sdk-core/source/platform/linux-shared/*.cpp", ]), "//conditions:default": [], + }) + select({ + "//conditions:default": glob([ + "aws-cpp-sdk-core/source/net/linux-shared/*.cpp", + ]), + "@org_tensorflow//tensorflow:windows": glob([ + "aws-cpp-sdk-core/source/platform/windows/*.cpp", + "aws-cpp-sdk-core/source/net/windows/*.cpp", + ]), }) + glob([ "aws-cpp-sdk-core/include/**/*.h", "aws-cpp-sdk-core/source/*.cpp", @@ -59,7 +67,6 @@ cc_library( "aws-cpp-sdk-transfer/include/**/*.h", "aws-cpp-sdk-transfer/source/**/*.cpp", "aws-cpp-sdk-core/source/monitoring/*.cpp", - "aws-cpp-sdk-core/source/net/linux-shared/*.cpp", "aws-cpp-sdk-core/source/utils/memory/*.cpp", "aws-cpp-sdk-core/source/utils/crypto/openssl/*.cpp", ]), @@ -94,6 +101,18 @@ cc_library( "ENABLE_CURL_CLIENT", "OPENSSL_IS_BORINGSSL", ], + "@org_tensorflow//tensorflow:windows": [ + "PLATFORM_WINDOWS", + "ENABLE_CURL_CLIENT", + "OPENSSL_IS_BORINGSSL", + ], + "//conditions:default": [], + }), + linkopts = select({ + "@org_tensorflow//tensorflow:windows": [ + "-DEFAULTLIB:Userenv.lib", + "-DEFAULTLIB:Version.lib", + ], "//conditions:default": [], }), includes = [ diff --git a/third_party/aws/aws-c-common.bazel b/third_party/aws/aws-c-common.bazel index f27f50a6eb3..a66fbcb1164 100644 --- a/third_party/aws/aws-c-common.bazel +++ b/third_party/aws/aws-c-common.bazel @@ -28,6 +28,9 @@ cc_library( "@org_tensorflow//tensorflow:raspberry_pi_armeabi": glob([ "source/posix/*.c", ]), + "@org_tensorflow//tensorflow:windows": glob([ + "source/windows/*.c", + ]), "//conditions:default": [], }) + glob([ "source/*.c", @@ -38,6 +41,12 @@ cc_library( "include/**/*.h", "include/aws/common/**/*.inl" ]), + linkopts = select({ + "@org_tensorflow//tensorflow:windows": [ + "-DEFAULTLIB:BCrypt.lib", + ], + "//conditions:default": [], + }), includes = [ "include/", ], diff --git a/third_party/aws/aws-checksums.bazel b/third_party/aws/aws-checksums.bazel index 5aa175795b8..759cb2e6fcf 100644 --- a/third_party/aws/aws-checksums.bazel +++ b/third_party/aws/aws-checksums.bazel @@ -9,7 +9,12 @@ exports_files(["LICENSE"]) cc_library( name = "aws-checksums", - srcs = glob([ + srcs = select({ + "@org_tensorflow//tensorflow:windows": glob([ + "source/visualc/*.c", + ]), + "//conditions:default": [], + }) + glob([ "source/intel/*.c", "source/*.c", ]),