experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #36478 from rahul003:aws_sdk

Browse Source 
PiperOrigin-RevId: 296004710
Change-Id: I3f2e9fd5107ccff43fb3e770d9bc607c2fa314a7
This commit is contained in:
TensorFlower Gardener 2020-02-19 10:53:01 -08:00
commit 64d98345d3
10 changed files with 188 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tensorflow/core/platform/s3/aws_logging.cc
View File

@ -69,6 +69,8 @@ void AWSLogSystem::LogMessage(Aws::Utils::Logging::LogLevel log_level,
}
}
void AWSLogSystem::Flush() { return; }
namespace {
// Taken from tensorflow/core/platform/default/logging.cc

14
tensorflow/core/platform/s3/aws_logging.h
View File

@ -34,7 +34,7 @@ class AWSLogSystem : public Aws::Utils::Logging::LogSystemInterface {
virtual ~AWSLogSystem() = default;
// Gets the currently configured log level.
virtual Aws::Utils::Logging::LogLevel GetLogLevel(void) const override {
Aws::Utils::Logging::LogLevel GetLogLevel(void) const override {
return log_level_;
}
@ -47,13 +47,15 @@ class AWSLogSystem : public Aws::Utils::Logging::LogSystemInterface {
// it's unsafe. See LogStream.
// Since non-static C++ methods have an implicit this argument,
// TF_PRINTF_ATTRIBUTE should be counted from two (vs. one).
virtual void Log(Aws::Utils::Logging::LogLevel log_level, const char* tag,
const char* format, ...) override TF_PRINTF_ATTRIBUTE(4, 5);
void Log(Aws::Utils::Logging::LogLevel log_level, const char* tag,
const char* format, ...) override TF_PRINTF_ATTRIBUTE(4, 5);
// Writes the stream to ProcessFormattedStatement.
virtual void LogStream(Aws::Utils::Logging::LogLevel log_level,
const char* tag,
const Aws::OStringStream& messageStream) override;
void LogStream(Aws::Utils::Logging::LogLevel log_level, const char* tag,
const Aws::OStringStream& messageStream) override;
// Flushes the buffered messages if the logger supports buffering
void Flush() override;
private:
void LogMessage(Aws::Utils::Logging::LogLevel log_level,

4
tensorflow/core/platform/s3/s3_file_system.cc
View File

@ -622,8 +622,8 @@ Status S3FileSystem::RenameFile(const string& src, const string& target) {
Aws::String src_key = object.GetKey();
Aws::String target_key = src_key;
target_key.replace(0, src_object.length(), target_object.c_str());
Aws::String source = Aws::String(src_bucket.c_str()) + "/" +
Aws::Utils::StringUtils::URLEncode(src_key.c_str());
Aws::String source =
Aws::String(src_bucket.c_str()) + "/" + src_key.c_str();
copyObjectRequest.SetBucket(target_bucket.c_str());
copyObjectRequest.SetKey(target_key);

6
tensorflow/tools/lib_package/BUILD
View File

@ -173,6 +173,9 @@ genrule(
"//tensorflow:no_aws_support": [],
"//conditions:default": [
"@aws//:LICENSE",
"@aws-checksums//:LICENSE",
"@aws-c-event-stream//:LICENSE",
"@aws-c-common//:LICENSE",
],
}) + select({
"//tensorflow:android": [],
@ -253,6 +256,9 @@ genrule(
"//tensorflow:no_aws_support": [],
"//conditions:default": [
"@aws//:LICENSE",
"@aws-checksums//:LICENSE",
"@aws-c-event-stream//:LICENSE",
"@aws-c-common//:LICENSE",
],
}) + select({
"//tensorflow:android": [],

5
tensorflow/tools/pip_package/BUILD
View File

@ -1,7 +1,7 @@
# Description:
# Tools for building the TensorFlow pip package.
load("//tensorflow:tensorflow.bzl", "if_windows", "if_xla_available", "transitive_hdrs")
load("//tensorflow:tensorflow.bzl", "if_windows", "transitive_hdrs")
load("//third_party/mkl:build_defs.bzl", "if_mkl", "if_mkl_ml")
load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda")
load("@local_config_syslibs//:build_defs.bzl", "if_not_system_lib")
@ -198,6 +198,9 @@ filegroup(
"//tensorflow:no_aws_support": [],
"//conditions:default": [
"@aws//:LICENSE",
"@aws-c-common//:LICENSE",
"@aws-c-event-stream//:LICENSE",
"@aws-checksums//:LICENSE",
],
}) + select({
"//tensorflow:android": [],

30
third_party/aws/BUILD.bazel vendored
View File

@ -1,5 +1,7 @@
# Description:
# AWS C++ SDK
# AWS C++ SDK
load("@org_tensorflow//third_party:common.bzl", "template_rule")
package(default_visibility = ["//visibility:public"])
@ -7,8 +9,6 @@ licenses(["notice"]) # Apache 2.0
exports_files(["LICENSE"])
load("@org_tensorflow//third_party:common.bzl", "template_rule")
cc_library(
name = "aws",
srcs = select({
@ -44,7 +44,9 @@ cc_library(
"aws-cpp-sdk-core/source/http/standard/**/*.cpp",
"aws-cpp-sdk-core/source/utils/*.cpp",
"aws-cpp-sdk-core/source/utils/base64/**/*.cpp",
"aws-cpp-sdk-core/source/utils/event/*.cpp",
"aws-cpp-sdk-core/source/utils/json/**/*.cpp",
"aws-cpp-sdk-core/source/utils/logging/*.cpp",
"aws-cpp-sdk-core/source/utils/logging/**/*.cpp",
"aws-cpp-sdk-core/source/utils/memory/**/*.cpp",
"aws-cpp-sdk-core/source/utils/stream/**/*.cpp",
@ -54,35 +56,41 @@ cc_library(
"aws-cpp-sdk-core/source/utils/crypto/factory/**/*.cpp",
"aws-cpp-sdk-s3/include/**/*.h",
"aws-cpp-sdk-s3/source/**/*.cpp",
"aws-cpp-sdk-core/source/monitoring/*.cpp",
"aws-cpp-sdk-core/source/net/linux-shared/*.cpp",
"aws-cpp-sdk-core/source/utils/memory/*.cpp",
"aws-cpp-sdk-core/source/utils/crypto/openssl/*.cpp",
]),
hdrs = [
"aws-cpp-sdk-core/include/aws/core/SDKConfig.h",
],
copts = [
"-DENABLE_OPENSSL_ENCRYPTION",
"-DAWS_SDK_VERSION_MAJOR=1",
"-DAWS_SDK_VERSION_MINOR=5",
"-DAWS_SDK_VERSION_PATCH=8",
"-DAWS_SDK_VERSION_MINOR=7",
"-DAWS_SDK_VERSION_PATCH=266",
"-DOPENSSL_IS_BORINGSSL",
],
defines = select({
"@org_tensorflow//tensorflow:linux_aarch64": [
"PLATFORM_LINUX",
"ENABLE_CURL_CLIENT",
"ENABLE_NO_ENCRYPTION",
"OPENSSL_IS_BORINGSSL",
],
"@org_tensorflow//tensorflow:linux_x86_64": [
"PLATFORM_LINUX",
"ENABLE_CURL_CLIENT",
"ENABLE_NO_ENCRYPTION",
"OPENSSL_IS_BORINGSSL",
],
"@org_tensorflow//tensorflow:macos": [
"PLATFORM_APPLE",
"ENABLE_CURL_CLIENT",
"ENABLE_NO_ENCRYPTION",
"OPENSSL_IS_BORINGSSL",
],
"@org_tensorflow//tensorflow:linux_ppc64le": [
"PLATFORM_LINUX",
"ENABLE_CURL_CLIENT",
"ENABLE_NO_ENCRYPTION",
"OPENSSL_IS_BORINGSSL",
],
"//conditions:default": [],
}),
@ -91,6 +99,10 @@ cc_library(
"aws-cpp-sdk-s3/include/",
],
deps = [
"@aws-c-common",
"@aws-c-event-stream",
"@aws-checksums",
"@boringssl//:crypto",
"@curl",
],
)

58
third_party/aws/aws-c-common.bazel vendored Normal file
View File

@ -0,0 +1,58 @@
# Description:
# AWS C++ SDK
package(default_visibility = ["//visibility:public"])
licenses(["notice"]) # Apache 2.0
exports_files(["LICENSE"])
load("@org_tensorflow//third_party:common.bzl", "template_rule")
cc_library(
name = "aws-c-common",
srcs = select({
"@org_tensorflow//tensorflow:linux_aarch64": glob([
"source/posix/*.c",
"source/arch/*.c"
]),
"@org_tensorflow//tensorflow:linux_x86_64": glob([
"source/posix/*.c",
]),
"@org_tensorflow//tensorflow:macos": glob([
"source/posix/*.c",
]),
"@org_tensorflow//tensorflow:linux_ppc64le": glob([
"source/posix/*.c",
]),
"@org_tensorflow//tensorflow:raspberry_pi_armeabi": glob([
"source/posix/*.c",
]),
"//conditions:default": [],
}) + glob([
"source/*.c",
]),
hdrs = [
"include/aws/common/config.h"
] + glob([
"include/**/*.h",
"include/aws/common/**/*.inl"
]),
includes = [
"include/",
],
deps = [
],
)
template_rule(
name = "config_h",
src = "include/aws/common/config.h.in",
out = "include/aws/common/config.h",
substitutions = {
"cmakedefine AWS_HAVE_GCC_OVERFLOW_MATH_EXTENSIONS": "undef AWS_HAVE_GCC_OVERFLOW_MATH_EXTENSIONS",
"cmakedefine AWS_HAVE_GCC_INLINE_ASM": "define AWS_HAVE_GCC_INLINE_ASM",
"cmakedefine AWS_HAVE_MSVC_MULX": "undef AWS_HAVE_MSVC_MULX",
"cmakedefine AWS_HAVE_EXECINFO": "define AWS_HAVE_EXECINFO",
},
)

25
third_party/aws/aws-c-event-stream.bazel vendored Normal file
View File

@ -0,0 +1,25 @@
# Description:
# AWS C++ SDK
package(default_visibility = ["//visibility:public"])
licenses(["notice"]) # Apache 2.0
exports_files(["LICENSE"])
cc_library(
name = "aws-c-event-stream",
srcs = glob([
"source/*.c",
]),
hdrs = glob([
"include/**/*.h"
]),
includes = [
"include/",
],
deps = [
"@aws-c-common",
"@aws-checksums",
],
)

25
third_party/aws/aws-checksums.bazel vendored Normal file
View File

@ -0,0 +1,25 @@
# Description:
# AWS C++ SDK
package(default_visibility = ["//visibility:public"])
licenses(["notice"]) # Apache 2.0
exports_files(["LICENSE"])
cc_library(
name = "aws-checksums",
srcs = glob([
"source/intel/*.c",
"source/*.c",
]),
hdrs = glob([
"include/**/*.h"
]),
includes = [
"include/",
],
deps = [
"@aws-c-common",
],
)

41
third_party/aws/workspace.bzl vendored
View File

@ -9,10 +9,43 @@ def repo():
third_party_http_archive(
name = "aws",
urls = [
"https://storage.googleapis.com/mirror.tensorflow.org/github.com/aws/aws-sdk-cpp/archive/1.5.8.tar.gz",
"https://github.com/aws/aws-sdk-cpp/archive/1.5.8.tar.gz",
"https://mirror.bazel.build/github.com/aws/aws-sdk-cpp/archive/1.7.266.tar.gz",
"https://github.com/aws/aws-sdk-cpp/archive/1.7.266.tar.gz",
],
sha256 = "89905075fe50aa13e0337ff905c2e8c1ce9caf77a3504484a7cda39179120ffc",
strip_prefix = "aws-sdk-cpp-1.5.8",
sha256 = "39fd8a2999260d2b8fcbc8187f1ed5299972c2b8bd14adb7850fd674fea67fb7",
strip_prefix = "aws-sdk-cpp-1.7.266",
build_file = "//third_party/aws:BUILD.bazel",
)
third_party_http_archive(
name = "aws-c-common",
urls = [
"https://mirror.tensorflow.org/github.com/awslabs/aws-c-common/archive/v0.4.29.tar.gz",
"https://github.com/awslabs/aws-c-common/archive/v0.4.29.tar.gz",
],
sha256 = "01c2a58553a37b3aa5914d9e0bf7bf14507ff4937bc5872a678892ca20fcae1f",
strip_prefix = "aws-c-common-0.4.29",
build_file = "//third_party/aws:aws-c-common.bazel",
)
third_party_http_archive(
name = "aws-c-event-stream",
urls = [
"https://mirror.tensorflow.org/github.com/awslabs/aws-c-event-stream/archive/v0.1.4.tar.gz",
"https://github.com/awslabs/aws-c-event-stream/archive/v0.1.4.tar.gz",
],
sha256 = "31d880d1c868d3f3df1e1f4b45e56ac73724a4dc3449d04d47fc0746f6f077b6",
strip_prefix = "aws-c-event-stream-0.1.4",
build_file = "//third_party/aws:aws-c-event-stream.bazel",
)
third_party_http_archive(
name = "aws-checksums",
urls = [
"https://mirror.tensorflow.org/github.com/awslabs/aws-checksums/archive/v0.1.5.tar.gz",
"https://github.com/awslabs/aws-checksums/archive/v0.1.5.tar.gz",
],
sha256 = "6e6bed6f75cf54006b6bafb01b3b96df19605572131a2260fddaf0e87949ced0",
strip_prefix = "aws-checksums-0.1.5",
build_file = "//third_party/aws:aws-checksums.bazel",
)