diff --git a/tensorflow/core/platform/cloud/gcs_file_system.cc b/tensorflow/core/platform/cloud/gcs_file_system.cc index 5d395f3d821..1bd4d86eef6 100644 --- a/tensorflow/core/platform/cloud/gcs_file_system.cc +++ b/tensorflow/core/platform/cloud/gcs_file_system.cc @@ -14,14 +14,18 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/core/platform/cloud/gcs_file_system.h" + #include #include + #include #include #include #include #include #include + +#include "tensorflow/core/platform/strcat.h" #ifdef _WIN32 #include // for _mktemp #endif @@ -128,6 +132,15 @@ constexpr char kAllowedBucketLocations[] = "GCS_ALLOWED_BUCKET_LOCATIONS"; // is running in and restricts to buckets in that region. constexpr char kDetectZoneSentinelValue[] = "auto"; +// How to upload new data when Flush() is called multiple times. +// By default the entire file is reuploaded. +constexpr char kAppendMode[] = "GCS_APPEND_MODE"; +// If GCS_APPEND_MODE=compose then instead the new data is uploaded to a +// temporary object and composed with the original object. This is disabled by +// default as the multiple API calls required add a risk of stranding temporary +// objects. +constexpr char kComposeAppend[] = "compose"; + Status GetTmpFilename(string* filename) { *filename = io::GetTempFilename(""); return Status::OK(); @@ -379,15 +392,18 @@ class GcsWritableFile : public WritableFile { GcsFileSystem* filesystem, GcsFileSystem::TimeoutConfig* timeouts, std::function file_cache_erase, - RetryConfig retry_config) + RetryConfig retry_config, bool compose_append) : bucket_(bucket), object_(object), filesystem_(filesystem), timeouts_(timeouts), file_cache_erase_(std::move(file_cache_erase)), sync_needed_(true), - retry_config_(retry_config) { + retry_config_(retry_config), + compose_append_(compose_append), + start_offset_(0) { // TODO: to make it safer, outfile_ should be constructed from an FD + VLOG(3) << "GcsWritableFile: " << GetGcsPath(); if (GetTmpFilename(&tmp_content_filename_).ok()) { outfile_.open(tmp_content_filename_, std::ofstream::binary | std::ofstream::app); @@ -403,14 +419,18 @@ class GcsWritableFile : public WritableFile { GcsFileSystem* filesystem, const string& tmp_content_filename, GcsFileSystem::TimeoutConfig* timeouts, std::function file_cache_erase, - RetryConfig retry_config) + RetryConfig retry_config, bool compose_append) : bucket_(bucket), object_(object), filesystem_(filesystem), timeouts_(timeouts), file_cache_erase_(std::move(file_cache_erase)), sync_needed_(true), - retry_config_(retry_config) { + retry_config_(retry_config), + compose_append_(compose_append), + start_offset_(0) { + VLOG(3) << "GcsWritableFile: " << GetGcsPath() << "with existing file " + << tmp_content_filename; tmp_content_filename_ = tmp_content_filename; outfile_.open(tmp_content_filename_, std::ofstream::binary | std::ofstream::app); @@ -423,6 +443,7 @@ class GcsWritableFile : public WritableFile { Status Append(StringPiece data) override { TF_RETURN_IF_ERROR(CheckWritable()); + VLOG(3) << "Append: " << GetGcsPath() << " size " << data.length(); sync_needed_ = true; outfile_ << data; if (!outfile_.good()) { @@ -433,6 +454,7 @@ class GcsWritableFile : public WritableFile { } Status Close() override { + VLOG(3) << "Close:" << GetGcsPath(); if (outfile_.is_open()) { Status sync_status = Sync(); if (sync_status.ok()) { @@ -443,18 +465,23 @@ class GcsWritableFile : public WritableFile { return Status::OK(); } - Status Flush() override { return Sync(); } + Status Flush() override { + VLOG(3) << "Flush:" << GetGcsPath(); + return Sync(); + } Status Name(StringPiece* result) const override { return errors::Unimplemented("GCSWritableFile does not support Name()"); } Status Sync() override { + VLOG(3) << "Sync started:" << GetGcsPath(); TF_RETURN_IF_ERROR(CheckWritable()); if (!sync_needed_) { return Status::OK(); } Status status = SyncImpl(); + VLOG(3) << "Sync finished " << GetGcsPath(); if (status.ok()) { sync_needed_ = false; } @@ -483,11 +510,26 @@ class GcsWritableFile : public WritableFile { "Could not write to the internal temporary file."); } string session_uri; - TF_RETURN_IF_ERROR(CreateNewUploadSession(&session_uri)); + uint64 start_offset = 0; + string object_to_upload = object_; + bool should_compose = false; + if (compose_append_) { + start_offset = start_offset_; + // Only compose if the object has already been uploaded to GCS + should_compose = start_offset > 0; + if (should_compose) { + object_to_upload = + strings::StrCat(io::Dirname(object_), "/.tmpcompose/", + io::Basename(object_), ".", start_offset_); + } + } + TF_RETURN_IF_ERROR( + CreateNewUploadSession(&session_uri, start_offset, object_to_upload)); uint64 already_uploaded = 0; bool first_attempt = true; const Status upload_status = RetryingUtils::CallWithRetries( - [&first_attempt, &already_uploaded, &session_uri, this]() { + [&first_attempt, &already_uploaded, &session_uri, &start_offset, + this]() { if (!first_attempt) { bool completed; TF_RETURN_IF_ERROR(RequestUploadSessionStatus( @@ -502,7 +544,7 @@ class GcsWritableFile : public WritableFile { } } first_attempt = false; - return UploadToSession(session_uri, already_uploaded); + return UploadToSession(session_uri, start_offset, already_uploaded); }, retry_config_); if (upload_status.code() == errors::Code::NOT_FOUND) { @@ -512,6 +554,12 @@ class GcsWritableFile : public WritableFile { strings::StrCat("Upload to gs://", bucket_, "/", object_, " failed, caused by: ", upload_status.ToString())); } + if (upload_status.ok()) { + if (should_compose) { + TF_RETURN_IF_ERROR(AppendObject(object_to_upload)); + } + TF_RETURN_IF_ERROR(GetCurrentFileSize(&start_offset_)); + } return upload_status; } @@ -534,7 +582,8 @@ class GcsWritableFile : public WritableFile { } /// Initiates a new resumable upload session. - Status CreateNewUploadSession(string* session_uri) { + Status CreateNewUploadSession(string* session_uri, uint64 start_offset, + string object_to_upload) { uint64 file_size; TF_RETURN_IF_ERROR(GetCurrentFileSize(&file_size)); @@ -542,10 +591,11 @@ class GcsWritableFile : public WritableFile { std::unique_ptr request; TF_RETURN_IF_ERROR(filesystem_->CreateHttpRequest(&request)); - request->SetUri(strings::StrCat( - kGcsUploadUriBase, "b/", bucket_, - "/o?uploadType=resumable&name=", request->EscapeString(object_))); - request->AddHeader("X-Upload-Content-Length", std::to_string(file_size)); + request->SetUri(strings::StrCat(kGcsUploadUriBase, "b/", bucket_, + "/o?uploadType=resumable&name=", + request->EscapeString(object_to_upload))); + request->AddHeader("X-Upload-Content-Length", + std::to_string(file_size - start_offset)); request->SetPostEmptyBody(); request->SetResultBuffer(&output_buffer); request->SetTimeouts(timeouts_->connect, timeouts_->idle, @@ -561,6 +611,37 @@ class GcsWritableFile : public WritableFile { return Status::OK(); } + /// Appends the data of append_object to the original object and deletes + /// append_object. + Status AppendObject(string append_object) { + VLOG(3) << "AppendObject: " << GetGcsPathWithObject(append_object) << " to " + << GetGcsPath(); + std::unique_ptr request; + TF_RETURN_IF_ERROR(filesystem_->CreateHttpRequest(&request)); + + request->SetUri(strings::StrCat(kGcsUriBase, "b/", bucket_, "/o/", + request->EscapeString(object_), + "/compose")); + + const string request_body = + strings::StrCat("{'sourceObjects': [{'name': '", object_, + "'},{'name': '", append_object, "'}]}"); + request->SetTimeouts(timeouts_->connect, timeouts_->idle, + timeouts_->metadata); + request->AddHeader("content-type", "application/json"); + request->SetPostFromBuffer(request_body.c_str(), request_body.size()); + return RetryingUtils::CallWithRetries( + [&request, &append_object, this]() { + TF_RETURN_WITH_CONTEXT_IF_ERROR(request->Send(), + " when composing to ", GetGcsPath()); + TF_RETURN_WITH_CONTEXT_IF_ERROR( + filesystem_->DeleteFile(GetGcsPathWithObject(append_object)), + " when cleaning up."); + return Status::OK(); + }, + retry_config_); + } + /// \brief Requests status of a previously initiated upload session. /// /// If the upload has already succeeded, sets 'completed' to true. @@ -628,7 +709,8 @@ class GcsWritableFile : public WritableFile { return Status::OK(); } - Status UploadToSession(const string& session_uri, uint64 start_offset) { + Status UploadToSession(const string& session_uri, uint64 start_offset, + uint64 already_uploaded) { uint64 file_size; TF_RETURN_IF_ERROR(GetCurrentFileSize(&file_size)); @@ -637,13 +719,14 @@ class GcsWritableFile : public WritableFile { request->SetUri(session_uri); if (file_size > 0) { request->AddHeader("Content-Range", - strings::StrCat("bytes ", start_offset, "-", - file_size - 1, "/", file_size)); + strings::StrCat("bytes ", already_uploaded, "-", + file_size - start_offset - 1, "/", + file_size - start_offset)); } request->SetTimeouts(timeouts_->connect, timeouts_->idle, timeouts_->write); - TF_RETURN_IF_ERROR( - request->SetPutFromFile(tmp_content_filename_, start_offset)); + TF_RETURN_IF_ERROR(request->SetPutFromFile( + tmp_content_filename_, start_offset + already_uploaded)); TF_RETURN_WITH_CONTEXT_IF_ERROR(request->Send(), " when uploading ", GetGcsPath()); // Erase the file from the file cache on every successful write. @@ -651,9 +734,10 @@ class GcsWritableFile : public WritableFile { return Status::OK(); } - string GetGcsPath() const { - return strings::StrCat("gs://", bucket_, "/", object_); + string GetGcsPathWithObject(string object) const { + return strings::StrCat("gs://", bucket_, "/", object); } + string GetGcsPath() const { return GetGcsPathWithObject(object_); } string bucket_; string object_; @@ -664,6 +748,8 @@ class GcsWritableFile : public WritableFile { std::function file_cache_erase_; bool sync_needed_; // whether there is buffered data that needs to be synced RetryConfig retry_config_; + bool compose_append_; + uint64 start_offset_; }; class GcsReadOnlyMemoryRegion : public ReadOnlyMemoryRegion { @@ -849,6 +935,14 @@ GcsFileSystem::GcsFileSystem(bool make_default_cache) { GetEnvVar(kAllowedBucketLocations, SplitByCommaToLowercaseSet, &allowed_locations_); + + StringPiece append_mode; + GetEnvVar(kAppendMode, StringPieceIdentity, &append_mode); + if (append_mode == kComposeAppend) { + compose_append_ = true; + } else { + compose_append_ = false; + } } GcsFileSystem::GcsFileSystem( @@ -859,7 +953,8 @@ GcsFileSystem::GcsFileSystem( size_t stat_cache_max_entries, uint64 matching_paths_cache_max_age, size_t matching_paths_cache_max_entries, RetryConfig retry_config, TimeoutConfig timeouts, const std::unordered_set& allowed_locations, - std::pair* additional_header) + std::pair* additional_header, + bool compose_append) : auth_provider_(std::move(auth_provider)), http_request_factory_(std::move(http_request_factory)), zone_provider_(std::move(zone_provider)), @@ -872,6 +967,7 @@ GcsFileSystem::GcsFileSystem( bucket_location_cache_(new BucketLocationCache( kCacheNeverExpire, kBucketLocationCacheMaxEntries)), allowed_locations_(allowed_locations), + compose_append_(compose_append), timeouts_(timeouts), retry_config_(retry_config), additional_header_(additional_header) {} @@ -1056,9 +1152,10 @@ Status GcsFileSystem::NewWritableFile(const string& fname, std::unique_ptr* result) { string bucket, object; TF_RETURN_IF_ERROR(ParseGcsPath(fname, false, &bucket, &object)); - result->reset(new GcsWritableFile(bucket, object, this, &timeouts_, - [this, fname]() { ClearFileCaches(fname); }, - retry_config_)); + result->reset(new GcsWritableFile( + bucket, object, this, &timeouts_, + [this, fname]() { ClearFileCaches(fname); }, retry_config_, + compose_append_)); return Status::OK(); } @@ -1098,7 +1195,8 @@ Status GcsFileSystem::NewAppendableFile(const string& fname, TF_RETURN_IF_ERROR(ParseGcsPath(fname, false, &bucket, &object)); result->reset(new GcsWritableFile( bucket, object, this, old_content_filename, &timeouts_, - [this, fname]() { ClearFileCaches(fname); }, retry_config_)); + [this, fname]() { ClearFileCaches(fname); }, retry_config_, + compose_append_)); return Status::OK(); } @@ -1629,6 +1727,7 @@ Status GcsFileSystem::RenameFile(const string& src, const string& target) { // Uses a GCS API command to copy the object and then deletes the old one. Status GcsFileSystem::RenameObject(const string& src, const string& target) { + VLOG(3) << "RenameObject: started gs://" << src << " to " << target; string src_bucket, src_object, target_bucket, target_object; TF_RETURN_IF_ERROR(ParseGcsPath(src, false, &src_bucket, &src_object)); TF_RETURN_IF_ERROR( @@ -1664,6 +1763,7 @@ Status GcsFileSystem::RenameObject(const string& src, const string& target) { "locations or storage classes is not supported."); } + VLOG(3) << "RenameObject: finished from: gs://" << src << " to " << target; // In case the delete API call failed, but the deletion actually happened // on the server side, we can't just retry the whole RenameFile operation // because the source object is already gone. diff --git a/tensorflow/core/platform/cloud/gcs_file_system.h b/tensorflow/core/platform/cloud/gcs_file_system.h index d1d8aed54d4..f066cc31eb4 100644 --- a/tensorflow/core/platform/cloud/gcs_file_system.h +++ b/tensorflow/core/platform/cloud/gcs_file_system.h @@ -122,7 +122,8 @@ class GcsFileSystem : public FileSystem { size_t matching_paths_cache_max_entries, RetryConfig retry_config, TimeoutConfig timeouts, const std::unordered_set& allowed_locations, - std::pair* additional_header); + std::pair* additional_header, + bool compose_append); Status NewRandomAccessFile( const string& fname, std::unique_ptr* result) override; @@ -187,6 +188,8 @@ class GcsFileSystem : public FileSystem { std::unordered_set allowed_locations() const { return allowed_locations_; } + + bool compose_append() const { return compose_append_; } string additional_header_name() const { return additional_header_ ? additional_header_->first : ""; } @@ -373,6 +376,7 @@ class GcsFileSystem : public FileSystem { using BucketLocationCache = ExpiringLRUCache; std::unique_ptr bucket_location_cache_; std::unordered_set allowed_locations_; + bool compose_append_; TimeoutConfig timeouts_; diff --git a/tensorflow/core/platform/cloud/gcs_file_system_test.cc b/tensorflow/core/platform/cloud/gcs_file_system_test.cc index 544ddc32043..6892bd7cc26 100644 --- a/tensorflow/core/platform/cloud/gcs_file_system_test.cc +++ b/tensorflow/core/platform/cloud/gcs_file_system_test.cc @@ -22,6 +22,7 @@ limitations under the License. #include "tensorflow/core/platform/cloud/http_request_fake.h" #include "tensorflow/core/platform/errors.h" #include "tensorflow/core/platform/str_util.h" +#include "tensorflow/core/platform/strcat.h" #include "tensorflow/core/platform/test.h" // Undef DeleteFile macro defined in wndows.h. @@ -73,16 +74,16 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_NoBlockCache) { "Range: 6-11\n" "Timeouts: 5 1 20\n", "6789")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); std::unique_ptr file; TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); @@ -129,7 +130,7 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_Buffered) { 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, kTestRetryConfig, kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + nullptr /* gcs additional header */, false /* compose append */); std::unique_ptr file; TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); @@ -177,7 +178,7 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_Buffered_Errors) { 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, kTestRetryConfig, kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + nullptr /* gcs additional header */, false /* compose append */); std::unique_ptr file; TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); @@ -224,7 +225,7 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_Buffered_ReadAtEOF) { 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, kTestRetryConfig, kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + nullptr /* gcs additional header */, false /* compose append */); std::unique_ptr file; TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); @@ -265,7 +266,7 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_Buffered_CachedOutOfRange) { 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, kTestRetryConfig, kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + nullptr /* gcs additional header */, false /* compose append */); std::unique_ptr file; TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); @@ -316,7 +317,7 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_Buffered_CachedNotSequential) { 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, kTestRetryConfig, kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + nullptr /* gcs additional header */, false /* compose append */); std::unique_ptr file; TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); @@ -357,7 +358,7 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_Buffered_Growing) { 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, kTestRetryConfig, kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + nullptr /* gcs additional header */, false /* compose append */); std::unique_ptr file; TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); @@ -404,7 +405,7 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_Buffered_ReadBackwards) { 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, kTestRetryConfig, kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + nullptr /* gcs additional header */, false /* compose append */); std::unique_ptr file; TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); @@ -437,16 +438,16 @@ TEST(GcsFileSystemTest, "location":"US-EAST1" })")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsAuto, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsAuto, + nullptr /* gcs additional header */, false /* compose append */); std::unique_ptr file; TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); @@ -479,16 +480,16 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_WithLocationConstraintCaching) { "location":"US-EAST1" })")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsAuto, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsAuto, + nullptr /* gcs additional header */, false /* compose append */); std::unique_ptr file; @@ -520,16 +521,16 @@ TEST(GcsFileSystemTest, "location":"BARFOO" })")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsAuto, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsAuto, + nullptr /* gcs additional header */, false /* compose append */); std::unique_ptr file; EXPECT_EQ(tensorflow::errors::FailedPrecondition( @@ -552,16 +553,16 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_NoBlockCache_DifferentN) { "Range: 3-12\n" "Timeouts: 5 1 20\n", "3456789")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); std::unique_ptr file; TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); @@ -621,7 +622,7 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_WithBlockCache) { 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, kTestRetryConfig, kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + nullptr /* gcs additional header */, false /* compose append */); char scratch[100]; StringPiece result; @@ -710,7 +711,7 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_WithBlockCache_Flush) { 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, kTestRetryConfig, kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + nullptr /* gcs additional header */, false /* compose append */); char scratch[100]; StringPiece result; @@ -750,17 +751,17 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_WithBlockCache_MaxStaleness) { "Range: 8-15\n" "Timeouts: 5 1 20\n", "89abcdef")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 8 /* block size */, 16 /* max bytes */, - 3600 /* max staleness */, 3600 /* stat cache max age */, - 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 8 /* block size */, + 16 /* max bytes */, 3600 /* max staleness */, + 3600 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); char scratch[100]; StringPiece result; // There should only be two HTTP requests issued to GCS even though we iterate @@ -830,7 +831,7 @@ TEST(GcsFileSystemTest, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, kTestRetryConfig, kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + nullptr /* gcs additional header */, false /* compose append */); std::unique_ptr file; TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); @@ -849,17 +850,17 @@ TEST(GcsFileSystemTest, TEST(GcsFileSystemTest, NewRandomAccessFile_NoObjectName) { std::vector requests; - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* read ahead bytes */, 0 /* max bytes */, - 0 /* max staleness */, 0 /* stat cache max age */, - 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* read ahead bytes */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); std::unique_ptr file; EXPECT_EQ(errors::Code::INVALID_ARGUMENT, @@ -883,16 +884,16 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_InconsistentRead) { "012")}); // Set stat_cache_max_age to 1000s so that StatCache could work. - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 1e3 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 1e3 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); // Stat the file first so that the file stats are cached. FileStatistics stat; @@ -959,7 +960,7 @@ TEST(GcsFileSystemTest, NewWritableFile) { 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, kTestRetryConfig, kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + nullptr /* gcs additional header */, false /* compose append */); // Read from the file first, to fill the block cache. std::unique_ptr rfile; @@ -1042,16 +1043,16 @@ TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadSucceeds) { "Timeouts: 5 1 30\n" "Put body: t2\n", "")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); std::unique_ptr file; TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable.txt", &file)); @@ -1112,17 +1113,17 @@ TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadSucceedsOnGetStatus) { "Range: 0-7\n" "Timeouts: 5 1 20\n", "01234567")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 8 /* block size */, 8 /* max bytes */, - 3600 /* max staleness */, 3600 /* stat cache max age */, - 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 8 /* block size */, + 8 /* max bytes */, 3600 /* max staleness */, + 3600 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); // Pull the file's first block into the cache. This will trigger the first // HTTP request to GCS. std::unique_ptr rfile; @@ -1208,7 +1209,8 @@ TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadAllAttemptsFail) { 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, RetryConfig(2 /* .init_delay_time_us */), kTestTimeoutConfig, - *kAllowedLocationsDefault, nullptr /* gcs additional header */); + *kAllowedLocationsDefault, nullptr /* gcs additional header */, + false /* compose append */); std::unique_ptr file; TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable.txt", &file)); @@ -1262,16 +1264,16 @@ TEST(GcsFileSystemTest, NewWritableFile_UploadReturns410) { "Timeouts: 5 1 30\n" "Put body: content1,content2\n", "")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); { std::unique_ptr file; @@ -1302,16 +1304,16 @@ TEST(GcsFileSystemTest, NewWritableFile_UploadReturns410) { TEST(GcsFileSystemTest, NewWritableFile_NoObjectName) { std::vector requests; - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); std::unique_ptr file; EXPECT_EQ(errors::Code::INVALID_ARGUMENT, @@ -1375,7 +1377,7 @@ TEST(GcsFileSystemTest, NewAppendableFile) { 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, kTestRetryConfig, kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + nullptr /* gcs additional header */, false /* compose append */); // Create an appendable file. This should read the file from GCS, and pull its // contents into the block cache. @@ -1401,16 +1403,16 @@ TEST(GcsFileSystemTest, NewAppendableFile) { TEST(GcsFileSystemTest, NewAppendableFile_NoObjectName) { std::vector requests; - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); std::unique_ptr file; EXPECT_EQ(errors::Code::INVALID_ARGUMENT, @@ -1435,16 +1437,16 @@ TEST(GcsFileSystemTest, NewReadOnlyMemoryRegionFromFile) { "Range: 0-", content.size() - 1, "\n", "Timeouts: 5 1 20\n"), content)}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); std::unique_ptr region; TF_EXPECT_OK(fs.NewReadOnlyMemoryRegionFromFile( @@ -1456,16 +1458,16 @@ TEST(GcsFileSystemTest, NewReadOnlyMemoryRegionFromFile) { TEST(GcsFileSystemTest, NewReadOnlyMemoryRegionFromFile_NoObjectName) { std::vector requests; - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); std::unique_ptr region; EXPECT_EQ(errors::Code::INVALID_ARGUMENT, @@ -1480,16 +1482,16 @@ TEST(GcsFileSystemTest, FileExists_YesAsObject) { "Timeouts: 5 1 10\n", strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\"," "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); TF_EXPECT_OK(fs.FileExists("gs://bucket/path/file1.txt")); } @@ -1510,16 +1512,16 @@ TEST(GcsFileSystemTest, FileExists_YesAsFolder) { "Timeouts: 5 1 10\n", "{\"items\": [ " " { \"name\": \"path/subfolder/\" }]}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); TF_EXPECT_OK(fs.FileExists("gs://bucket/path/subfolder")); } @@ -1536,16 +1538,16 @@ TEST(GcsFileSystemTest, FileExists_YesAsBucket) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "{\"size\": \"100\"}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); TF_EXPECT_OK(fs.FileExists("gs://bucket1")); TF_EXPECT_OK(fs.FileExists("gs://bucket1/")); @@ -1566,16 +1568,16 @@ TEST(GcsFileSystemTest, FileExists_NotAsObjectOrFolder) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "{\"items\": []}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); EXPECT_EQ(errors::Code::NOT_FOUND, fs.FileExists("gs://bucket/path/file1.txt").code()); @@ -1593,16 +1595,16 @@ TEST(GcsFileSystemTest, FileExists_NotAsBucket) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "", errors::NotFound("404"), 404)}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); EXPECT_EQ(errors::Code::INVALID_ARGUMENT, fs.FileExists("gs://bucket2/").code()); EXPECT_EQ(errors::Code::INVALID_ARGUMENT, @@ -1641,7 +1643,7 @@ TEST(GcsFileSystemTest, FileExists_StatCache) { 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, kTestRetryConfig, kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + nullptr /* gcs additional header */, false /* compose append */); // The stat cache will ensure that repeated lookups don't trigger additional // HTTP requests. @@ -1668,7 +1670,7 @@ TEST(GcsFileSystemTest, FileExists_DirectoryMark) { 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, kTestRetryConfig, kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + nullptr /* gcs additional header */, false /* compose append */); TF_EXPECT_OK(fs.FileExists("gs://bucket/dir/")); TF_EXPECT_OK(fs.IsDirectory("gs://bucket/dir/")); @@ -1682,16 +1684,16 @@ TEST(GcsFileSystemTest, GetChildren_NoItems) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "{\"prefixes\": [\"path/subpath/\"]}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); std::vector children; TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children)); @@ -1710,16 +1712,16 @@ TEST(GcsFileSystemTest, GetChildren_ThreeFiles) { " { \"name\": \"path/file1.txt\" }," " { \"name\": \"path/file3.txt\" }]," "\"prefixes\": [\"path/subpath/\"]}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); std::vector children; TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children)); @@ -1739,16 +1741,16 @@ TEST(GcsFileSystemTest, GetChildren_SelfDirectoryMarker) { " { \"name\": \"path/\" }," " { \"name\": \"path/file3.txt\" }]," "\"prefixes\": [\"path/subpath/\"]}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); std::vector children; TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children)); @@ -1767,16 +1769,16 @@ TEST(GcsFileSystemTest, GetChildren_ThreeFiles_NoSlash) { " { \"name\": \"path/file1.txt\" }," " { \"name\": \"path/file3.txt\" }]," "\"prefixes\": [\"path/subpath/\"]}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); std::vector children; TF_EXPECT_OK(fs.GetChildren("gs://bucket/path", &children)); @@ -1792,16 +1794,16 @@ TEST(GcsFileSystemTest, GetChildren_Root) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "{}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); std::vector children; TF_EXPECT_OK(fs.GetChildren("gs://bucket-a-b-c", &children)); @@ -1817,16 +1819,16 @@ TEST(GcsFileSystemTest, GetChildren_Empty) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "{}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); std::vector children; TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children)); @@ -1858,16 +1860,16 @@ TEST(GcsFileSystemTest, GetChildren_Pagination) { " { \"name\": \"path/file4.txt\" }," " { \"name\": \"path/file5.txt\" }]}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); std::vector children; TF_EXPECT_OK(fs.GetChildren("gs://bucket/path", &children)); @@ -1885,16 +1887,16 @@ TEST(GcsFileSystemTest, GetMatchingPaths_NoWildcard) { "Timeouts: 5 1 10\n", "{\"items\": [ " " { \"name\": \"path/subpath/file2.txt\" }]}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); std::vector result; TF_EXPECT_OK( @@ -1913,16 +1915,16 @@ TEST(GcsFileSystemTest, GetMatchingPaths_BucketAndWildcard) { " { \"name\": \"path/file1.txt\" }," " { \"name\": \"path/subpath/file2.txt\" }," " { \"name\": \"path/file3.txt\" }]}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); std::vector result; TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/*/*", &result)); @@ -1942,16 +1944,16 @@ TEST(GcsFileSystemTest, GetMatchingPaths_FolderAndWildcard_Matches) { " { \"name\": \"path/file1.txt\" }," " { \"name\": \"path/subpath/file2.txt\" }," " { \"name\": \"path/file3.txt\" }]}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); std::vector result; TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*/file2.txt", &result)); @@ -1968,16 +1970,16 @@ TEST(GcsFileSystemTest, GetMatchingPaths_SelfDirectoryMarker) { "{\"items\": [ " " { \"name\": \"path/\" }," " { \"name\": \"path/file3.txt\" }]}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); std::vector result; TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*", &result)); @@ -1993,16 +1995,16 @@ TEST(GcsFileSystemTest, GetMatchingPaths_SlashInObjectName) { "{\"items\": [ " " { \"name\": \"path/\" }," " { \"name\": \"path//foo.txt\" }]}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); std::vector result; TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*", &result)); @@ -2018,16 +2020,16 @@ TEST(GcsFileSystemTest, GetMatchingPaths_SlashInObjectNameEscaped) { "{\"items\": [ " " { \"name\": \"path/\" }," " { \"name\": \"path//foo.txt\" }]}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); std::vector result; TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/\\/*", &result)); @@ -2044,16 +2046,16 @@ TEST(GcsFileSystemTest, GetMatchingPaths_FolderAndWildcard_NoMatches) { " { \"name\": \"path/file1.txt\" }," " { \"name\": \"path/subpath/file2.txt\" }," " { \"name\": \"path/file3.txt\" }]}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); std::vector result; TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*/file3.txt", &result)); @@ -2062,16 +2064,16 @@ TEST(GcsFileSystemTest, GetMatchingPaths_FolderAndWildcard_NoMatches) { TEST(GcsFileSystemTest, GetMatchingPaths_OnlyWildcard) { std::vector requests; - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); std::vector result; EXPECT_EQ(errors::Code::INVALID_ARGUMENT, @@ -2096,16 +2098,16 @@ TEST(GcsFileSystemTest, GetMatchingPaths_Cache) { " { \"name\": \"path/file1.txt\" }," " { \"name\": \"path/subpath/file2.txt\" }," " { \"name\": \"path/file3.txt\" }]}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 3600 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 3600 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); // Repeated calls to fs.GetMatchingPaths on these patterns should not lead to // any additional HTTP requests to GCS. @@ -2139,16 +2141,16 @@ TEST(GcsFileSystemTest, GetMatchingPaths_Cache_Flush) { "Timeouts: 5 1 10\n", "{\"items\": [ " " { \"name\": \"path/subpath/file2.txt\" }]}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 3600 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 3600 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); // This loop should trigger the first HTTP request to GCS. for (int i = 0; i < 10; i++) { @@ -2212,7 +2214,7 @@ TEST(GcsFileSystemTest, DeleteFile) { 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, kTestRetryConfig, kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + nullptr /* gcs additional header */, false /* compose append */); // Do an initial read of the file to load its contents into the block cache. char scratch[100]; @@ -2231,16 +2233,16 @@ TEST(GcsFileSystemTest, DeleteFile) { TEST(GcsFileSystemTest, DeleteFile_NoObjectName) { std::vector requests; - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); EXPECT_EQ(errors::Code::INVALID_ARGUMENT, fs.DeleteFile("gs://bucket/").code()); @@ -2283,7 +2285,7 @@ TEST(GcsFileSystemTest, DeleteFile_StatCacheRemoved) { 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, kTestRetryConfig, kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + nullptr /* gcs additional header */, false /* compose append */); // Stats the file first so the stat is cached. FileStatistics stat_before_deletion; @@ -2304,16 +2306,16 @@ TEST(GcsFileSystemTest, DeleteDir_Empty) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "{}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); TF_EXPECT_OK(fs.DeleteDir("gs://bucket/path/")); } @@ -2333,16 +2335,16 @@ TEST(GcsFileSystemTest, DeleteDir_OnlyDirMarkerLeft) { "Timeouts: 5 1 10\n" "Delete: yes\n", "")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); TF_EXPECT_OK(fs.DeleteDir("gs://bucket/path/")); } @@ -2353,16 +2355,16 @@ TEST(GcsFileSystemTest, DeleteDir_BucketOnly) { "name%2CnextPageToken&maxResults=2\nAuth Token: fake_token\n" "Timeouts: 5 1 10\n", "{}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); TF_EXPECT_OK(fs.DeleteDir("gs://bucket")); } @@ -2375,16 +2377,16 @@ TEST(GcsFileSystemTest, DeleteDir_NonEmpty) { "Timeouts: 5 1 10\n", "{\"items\": [ " " { \"name\": \"path/file1.txt\" }]}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); EXPECT_EQ(error::Code::FAILED_PRECONDITION, fs.DeleteDir("gs://bucket/path/").code()); @@ -2398,16 +2400,16 @@ TEST(GcsFileSystemTest, GetFileSize) { "Timeouts: 5 1 10\n", strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\"," "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); uint64 size; TF_EXPECT_OK(fs.GetFileSize("gs://bucket/file.txt", &size)); @@ -2416,16 +2418,16 @@ TEST(GcsFileSystemTest, GetFileSize) { TEST(GcsFileSystemTest, GetFileSize_NoObjectName) { std::vector requests; - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); uint64 size; EXPECT_EQ(errors::Code::INVALID_ARGUMENT, @@ -2502,16 +2504,16 @@ TEST(GcsFileSystemTest, RenameFile_Folder) { "Timeouts: 5 1 10\n" "Delete: yes\n", "")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); TF_EXPECT_OK(fs.RenameFile("gs://bucket/path1", "gs://bucket/path2/")); } @@ -2603,7 +2605,7 @@ TEST(GcsFileSystemTest, RenameFile_Object) { 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, kTestRetryConfig, kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + nullptr /* gcs additional header */, false /* compose append */); // Do an initial read of the source and destination files to load their // contents into the block cache. char scratch[100]; @@ -2684,7 +2686,7 @@ TEST(GcsFileSystemTest, RenameFile_Object_FlushTargetStatCache) { 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, kTestRetryConfig, kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + nullptr /* gcs additional header */, false /* compose append */); // Do an initial stat of the destination file to load their contents into the // stat cache. FileStatistics stat_before_renaming; @@ -2742,16 +2744,16 @@ TEST(GcsFileSystemTest, RenameFile_Object_DeletionRetried) { "Timeouts: 5 1 10\n" "Delete: yes\n", "", errors::NotFound("404"), 404)}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); TF_EXPECT_OK( fs.RenameFile("gs://bucket/path/src.txt", "gs://bucket/path/dst.txt")); @@ -2784,16 +2786,16 @@ TEST(GcsFileSystemTest, RenameFile_Object_Incomplete) { "Post: yes\n" "Timeouts: 5 1 10\n", "{\"done\": false}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); EXPECT_EQ( errors::Code::UNIMPLEMENTED, @@ -2809,16 +2811,16 @@ TEST(GcsFileSystemTest, Stat_Object) { "Timeouts: 5 1 10\n", strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\"," "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); FileStatistics stat; TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", &stat)); @@ -2843,16 +2845,16 @@ TEST(GcsFileSystemTest, Stat_Folder) { "Timeouts: 5 1 10\n", "{\"items\": [ " " { \"name\": \"subfolder/\" }]}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); FileStatistics stat; TF_EXPECT_OK(fs.Stat("gs://bucket/subfolder", &stat)); @@ -2876,16 +2878,16 @@ TEST(GcsFileSystemTest, Stat_ObjectOrFolderNotFound) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "{}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); FileStatistics stat; EXPECT_EQ(error::Code::NOT_FOUND, fs.Stat("gs://bucket/path", &stat).code()); @@ -2897,16 +2899,16 @@ TEST(GcsFileSystemTest, Stat_Bucket) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "{}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); FileStatistics stat; TF_EXPECT_OK(fs.Stat("gs://bucket/", &stat)); @@ -2921,16 +2923,16 @@ TEST(GcsFileSystemTest, Stat_BucketNotFound) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "", errors::NotFound("404"), 404)}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); FileStatistics stat; EXPECT_EQ(error::Code::NOT_FOUND, fs.Stat("gs://bucket/", &stat).code()); @@ -2968,7 +2970,7 @@ TEST(GcsFileSystemTest, Stat_Cache) { 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, kTestRetryConfig, kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + nullptr /* gcs additional header */, false /* compose append */); // Repeated calls to fs.Stat on these paths should not lead to any additional // HTTP requests to GCS. @@ -3010,7 +3012,7 @@ TEST(GcsFileSystemTest, Stat_Cache_Flush) { 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, kTestRetryConfig, kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + nullptr /* gcs additional header */, false /* compose append */); // There should be a single HTTP request to GCS for fs.Stat in this loop. for (int i = 0; i < 10; i++) { FileStatistics stat; @@ -3038,16 +3040,16 @@ TEST(GcsFileSystemTest, Stat_FilenameEndingWithSlash) { "Timeouts: 5 1 10\n", strings::StrCat("{\"size\": \"5\",\"generation\": \"1\"," "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); FileStatistics stat; TF_EXPECT_OK(fs.Stat("gs://bucket/dir/", &stat)); @@ -3070,16 +3072,16 @@ TEST(GcsFileSystemTest, IsDirectory_NotFound) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "", errors::NotFound("404"), 404)}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); EXPECT_EQ(error::Code::NOT_FOUND, fs.IsDirectory("gs://bucket/file.txt").code()); @@ -3101,16 +3103,16 @@ TEST(GcsFileSystemTest, IsDirectory_NotDirectoryButObject) { "Timeouts: 5 1 10\n", strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\"," "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); EXPECT_EQ(error::Code::FAILED_PRECONDITION, fs.IsDirectory("gs://bucket/file.txt").code()); @@ -3132,16 +3134,16 @@ TEST(GcsFileSystemTest, IsDirectory_Yes) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "{\"items\": [{\"name\": \"subfolder/\"}]}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); TF_EXPECT_OK(fs.IsDirectory("gs://bucket/subfolder")); TF_EXPECT_OK(fs.IsDirectory("gs://bucket/subfolder/")); @@ -3159,16 +3161,16 @@ TEST(GcsFileSystemTest, IsDirectory_Bucket) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "{}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); TF_EXPECT_OK(fs.IsDirectory("gs://bucket")); TF_EXPECT_OK(fs.IsDirectory("gs://bucket/")); @@ -3180,16 +3182,16 @@ TEST(GcsFileSystemTest, IsDirectory_BucketNotFound) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "", errors::NotFound("404"), 404)}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); EXPECT_EQ(error::Code::NOT_FOUND, fs.IsDirectory("gs://bucket/").code()); } @@ -3222,16 +3224,16 @@ TEST(GcsFileSystemTest, CreateDir_Folder) { "Timeouts: 5 1 10\n", strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\"," "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); TF_EXPECT_OK(fs.CreateDir("gs://bucket/subpath")); EXPECT_EQ(errors::AlreadyExists("gs://bucket/subpath/"), @@ -3250,16 +3252,16 @@ TEST(GcsFileSystemTest, CreateDir_Bucket) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); TF_EXPECT_OK(fs.CreateDir("gs://bucket/")); TF_EXPECT_OK(fs.CreateDir("gs://bucket")); @@ -3322,16 +3324,16 @@ TEST(GcsFileSystemTest, DeleteRecursively_Ok) { "Timeouts: 5 1 10\n" "Delete: yes\n", "")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); int64 undeleted_files, undeleted_dirs; TF_EXPECT_OK(fs.DeleteRecursively("gs://bucket/path", &undeleted_files, @@ -3415,16 +3417,16 @@ TEST(GcsFileSystemTest, DeleteRecursively_DeletionErrors) { "Timeouts: 5 1 10\n", "", errors::NotFound("404"), 404)}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); int64 undeleted_files, undeleted_dirs; TF_EXPECT_OK(fs.DeleteRecursively("gs://bucket/path", &undeleted_files, @@ -3450,16 +3452,16 @@ TEST(GcsFileSystemTest, DeleteRecursively_NotAFolder) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "", errors::NotFound("404"), 404)}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); int64 undeleted_files, undeleted_dirs; EXPECT_EQ(error::Code::NOT_FOUND, @@ -3543,7 +3545,7 @@ TEST(GcsFileSystemTest, AdditionalRequestHeaderTest) { 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, kTestRetryConfig, kTestTimeoutConfig, *kAllowedLocationsDefault, - add_header /* gcs additional header */); + add_header /* gcs additional header */, false /* compose append */); std::unique_ptr request; TF_EXPECT_OK(fs7.CreateHttpRequest(&request)); @@ -3613,16 +3615,16 @@ TEST(GcsFileSystemTest, CreateHttpRequest) { "Auth Token: fake_token\n" "Header Hello: world\n", "{}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); std::unique_ptr request; TF_EXPECT_OK(fs.CreateHttpRequest(&request)); @@ -3676,16 +3678,16 @@ TEST(GcsFileSystemTest, Stat_StatsRecording) { "Timeouts: 5 1 10\n", strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\"," "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); TestGcsStats stats; fs.SetStats(&stats); @@ -3703,16 +3705,16 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_StatsRecording) { "Range: 0-5\n" "Timeouts: 5 1 20\n", "012345")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - std::unique_ptr(new FakeZoneProvider), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, kTestRetryConfig, - kTestTimeoutConfig, *kAllowedLocationsDefault, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); TestGcsStats stats; fs.SetStats(&stats); @@ -3732,5 +3734,253 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_StatsRecording) { EXPECT_EQ(6, stats.block_retrieved_bytes_transferred_); } +TEST(GcsFileSystemTest, NewAppendableFile_MultipleFlushesWithCompose) { + std::vector contents( + {"content0,", "content1,", "content2,", "content3,"}); + std::vector requests({ + // Fetch the file (stats and then content) + new FakeHttpRequest( + "Uri: " + "https://www.googleapis.com/storage/v1/b/bucket/o/" + "some%2Fpath%2Fappendable?fields=size%2Cgeneration%2Cupdated\n" + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", + strings::StrCat("{\"size\": \"8\",\"generation\": \"1\"," + "\"updated\": \"2016-04-29T23:15:24.896Z\"}")), + new FakeHttpRequest( + "Uri: " + "https://storage.googleapis.com/bucket/some%2Fpath%2Fappendable\n" + "Auth Token: fake_token\n" + "Range: 0-1048575\n" + "Timeouts: 5 1 20\n", + contents[0]), + // Upload entire file + new FakeHttpRequest( + "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?" + "uploadType=resumable&name=some%2Fpath%2Fappendable\n" + "Auth Token: fake_token\n" + "Header X-Upload-Content-Length: 18\n" + "Post: yes\n" + "Timeouts: 5 1 10\n", + "", {{"Location", "https://custom/upload/location"}}), + new FakeHttpRequest( + strings::StrCat("Uri: https://custom/upload/location\n" + "Auth Token: fake_token\n" + "Header Content-Range: bytes 0-17/18\n" + "Timeouts: 5 1 30\n" + "Put body: ", + contents[0], contents[1], "\n"), + ""), + // Upload new part to a temporary object + new FakeHttpRequest( + "Uri: " + "https://www.googleapis.com/upload/storage/v1/b/bucket/" + "o?uploadType=resumable&name=some%2Fpath%2F.tmpcompose%2Fappendable." + "18\n" + "Auth Token: fake_token\n" + "Header X-Upload-Content-Length: 9\n" + "Post: yes\n" + "Timeouts: 5 1 10\n", + "", + {{"Location", + "https://custom/upload/" + "location"}}), + new FakeHttpRequest( + strings::StrCat("Uri: https://custom/upload/location\n" + "Auth Token: fake_token\n" + "Header Content-Range: bytes 0-8/9\n" + "Timeouts: 5 1 30\n" + "Put body: ", + contents[2], "\n"), + ""), + // Compose the new part at the end of the original object. + new FakeHttpRequest("Uri: " + "https://www.googleapis.com/storage/v1/b/bucket/o/" + "some%2Fpath%2Fappendable/compose\n" + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n" + "Header content-type: application/json\n" + "Post body: {'sourceObjects': [{'name': " + "'some/path/appendable'},{'name': " + "'some/path/.tmpcompose/appendable.18'}]}\n", + ""), + // Delete the temporary object. + new FakeHttpRequest("Uri: " + "https://www.googleapis.com/storage/v1/b/bucket/o/" + "some%2Fpath%2F.tmpcompose%2Fappendable.18\n" + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n" + "Delete: yes\n", + ""), + new FakeHttpRequest( + "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?" + "uploadType=resumable&name=some%2Fpath%2F.tmpcompose%2Fappendable." + "27\n" + "Auth Token: fake_token\n" + "Header X-Upload-Content-Length: 9\n" + "Post: yes\n" + "Timeouts: 5 1 10\n", + "", {{"Location", "https://custom/upload/location"}}), + new FakeHttpRequest( + strings::StrCat("Uri: https://custom/upload/location\n" + "Auth Token: fake_token\n" + "Header Content-Range: bytes 0-8/9\n" + "Timeouts: 5 1 30\n" + "Put body: ", + contents[3], "\n"), + ""), + new FakeHttpRequest("Uri: " + "https://www.googleapis.com/storage/v1/b/bucket/o/" + "some%2Fpath%2Fappendable/compose\n" + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n" + "Header content-type: application/json\n" + "Post body: {'sourceObjects': [{'name': " + "'some/path/appendable'},{'name': " + "'some/path/.tmpcompose/appendable.27'}]}\n", + ""), + new FakeHttpRequest("Uri: " + "https://www.googleapis.com/storage/v1/b/bucket/o/" + "some%2Fpath%2F.tmpcompose%2Fappendable." + "27\n" + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n" + "Delete: yes\n", + ""), + }); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 32 /* block size */, + 32 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, true /* compose append */); + + // Create an appendable file. This should read the file from GCS, and pull its + // contents into the block cache. + std::unique_ptr wfile; + TF_EXPECT_OK( + fs.NewAppendableFile("gs://bucket/some/path/appendable", &wfile)); + TF_EXPECT_OK(wfile->Append(contents[1])); + TF_EXPECT_OK(wfile->Flush()); + TF_EXPECT_OK(wfile->Append(contents[2])); + TF_EXPECT_OK(wfile->Flush()); + TF_EXPECT_OK(wfile->Append(contents[3])); + TF_EXPECT_OK(wfile->Close()); +} + +TEST(GcsFileSystemTest, NewAppendableFile_MultipleFlushesWithoutCompose) { + std::vector contents( + {"content0,", "content1,", "content2,", "content3,"}); + std::vector requests({ + new FakeHttpRequest( + "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" + "path%2Fappendable?fields=size%2Cgeneration%2Cupdated\n" + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", + strings::StrCat("{\"size\": \"8\",\"generation\": \"1\"," + "\"updated\": \"2016-04-29T23:15:24.896Z\"}")), + new FakeHttpRequest( + "Uri: https://storage.googleapis.com/bucket/path%2Fappendable\n" + "Auth Token: fake_token\n" + "Range: 0-1048575\n" + "Timeouts: 5 1 20\n", + contents[0]), + new FakeHttpRequest( + "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?" + "uploadType=resumable&name=path%2Fappendable\n" + "Auth Token: fake_token\n" + "Header X-Upload-Content-Length: 18\n" + "Post: yes\n" + "Timeouts: 5 1 10\n", + "", {{"Location", "https://custom/upload/location"}}), + // Uploads entire file. + new FakeHttpRequest( + strings::StrCat("Uri: https://custom/upload/location\n" + "Auth Token: fake_token\n" + "Header Content-Range: bytes 0-17/18\n" + "Timeouts: 5 1 30\n" + "Put body: ", + contents[0], contents[1], "\n"), + ""), + new FakeHttpRequest("Uri: " + "https://www.googleapis.com/upload/storage/v1/b/" + "bucket/o?" + "uploadType=resumable&name=path%2Fappendable\n" + "Auth Token: fake_token\n" + "Header X-Upload-Content-Length: 27\n" + "Post: yes\n" + "Timeouts: 5 1 10\n", + "", + {{"Location", + "https://custom/upload/" + "location"}}), + // Uploads entire file again. + new FakeHttpRequest( + strings::StrCat("Uri: https://custom/upload/location\n" + "Auth Token: fake_token\n" + "Header Content-Range: bytes 0-26/27\n" + "Timeouts: 5 1 30\n" + "Put body: ", + contents[0], contents[1], contents[2], "\n"), + ""), + new FakeHttpRequest( + "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?" + "uploadType=resumable&name=path%2Fappendable\n" + "Auth Token: fake_token\n" + "Header X-Upload-Content-Length: 36\n" + "Post: yes\n" + "Timeouts: 5 1 10\n", + "", {{"Location", "https://custom/upload/location"}}), + // Uploads entire file again. + new FakeHttpRequest( + strings::StrCat("Uri: https://custom/upload/location\n" + "Auth Token: fake_token\n" + "Header Content-Range: bytes 0-35/36\n" + "Timeouts: 5 1 30\n" + "Put body: ", + contents[0], contents[1], contents[2], contents[3], + "\n"), + ""), + }); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 32 /* block size */, + 32 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, kTestRetryConfig, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */, false /* compose append */); + + // Create an appendable file. This should read the file from GCS, and pull its + // contents into the block cache. + std::unique_ptr wfile; + TF_EXPECT_OK(fs.NewAppendableFile("gs://bucket/path/appendable", &wfile)); + TF_EXPECT_OK(wfile->Append(contents[1])); + TF_EXPECT_OK(wfile->Flush()); + TF_EXPECT_OK(wfile->Append(contents[2])); + TF_EXPECT_OK(wfile->Flush()); + TF_EXPECT_OK(wfile->Append(contents[3])); + TF_EXPECT_OK(wfile->Close()); +} + +TEST(GcsFileSystemTest, AppendModeCompose) { + unsetenv("GCS_APPEND_MODE"); + setenv("GCS_APPEND_MODE", "compose", 1); + GcsFileSystem fs1; + EXPECT_EQ(true, fs1.compose_append()); +} + +TEST(GcsFileSystemTest, AppendModeDefault) { + unsetenv("GCS_APPEND_MODE"); + GcsFileSystem fs1; + EXPECT_EQ(false, fs1.compose_append()); +} + } // namespace } // namespace tensorflow