experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Make GCS filesystem/metadata lookup retries configurable

Browse Source 
PiperOrigin-RevId: 215243030
This commit is contained in:
Russell Power 2018-10-01 10:57:32 -07:00 committed by TensorFlower Gardener
parent 5fdf247424
commit ce1cdd52ed
13 changed files with 846 additions and 790 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
tensorflow/core/platform/cloud/compute_engine_metadata_client.cc
View File

@ -17,7 +17,6 @@ limitations under the License.
#include <utility>
#include "tensorflow/core/platform/cloud/curl_http_request.h"
#include "tensorflow/core/platform/cloud/retrying_utils.h"
namespace tensorflow {
@ -25,21 +24,14 @@ namespace {
// The URL to retrieve metadata when running in Google Compute Engine.
constexpr char kGceMetadataBaseUrl[] = "http://metadata/computeMetadata/v1/";
// The default initial delay between retries with exponential backoff.
constexpr int kInitialRetryDelayUsec = 500000; // 0.5 sec
} // namespace
ComputeEngineMetadataClient::ComputeEngineMetadataClient(
std::shared_ptr<HttpRequest::Factory> http_request_factory)
: ComputeEngineMetadataClient(std::move(http_request_factory),
kInitialRetryDelayUsec) {}
ComputeEngineMetadataClient::ComputeEngineMetadataClient(
std::shared_ptr<HttpRequest::Factory> http_request_factory,
int64 initial_retry_delay_usec)
const RetryConfig& config)
: http_request_factory_(std::move(http_request_factory)),
initial_retry_delay_usec_(initial_retry_delay_usec) {}
retry_config_(config) {}
Status ComputeEngineMetadataClient::GetMetadata(
const string& path, std::vector<char>* response_buffer) {
@ -52,8 +44,7 @@ Status ComputeEngineMetadataClient::GetMetadata(
return Status::OK();
};
return RetryingUtils::CallWithRetries(get_metadata_from_gce,
initial_retry_delay_usec_);
return RetryingUtils::CallWithRetries(get_metadata_from_gce, retry_config_);
}
} // namespace tensorflow

10
tensorflow/core/platform/cloud/compute_engine_metadata_client.h
View File

@ -18,6 +18,7 @@ limitations under the License.
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/platform/cloud/http_request.h"
#include "tensorflow/core/platform/cloud/retrying_utils.h"
namespace tensorflow {
@ -31,10 +32,11 @@ namespace tensorflow {
class ComputeEngineMetadataClient {
public:
explicit ComputeEngineMetadataClient(
std::shared_ptr<HttpRequest::Factory> http_request_factory);
ComputeEngineMetadataClient(
std::shared_ptr<HttpRequest::Factory> http_request_factory,
int64 initial_retry_delay_usec);
const RetryConfig& config = RetryConfig(
10000, /* init_delay_time_us = 1 ms */
1000000 /* max_delay_time_us = 1 s */
));
virtual ~ComputeEngineMetadataClient() {}
/// \brief Get the metadata value for a given attribute of the metadata
@ -54,7 +56,7 @@ class ComputeEngineMetadataClient {
private:
std::shared_ptr<HttpRequest::Factory> http_request_factory_;
const int64 initial_retry_delay_usec_;
const RetryConfig retry_config_;
TF_DISALLOW_COPY_AND_ASSIGN(ComputeEngineMetadataClient);
};

6
tensorflow/core/platform/cloud/compute_engine_metadata_client_test.cc
View File

@ -30,7 +30,8 @@ TEST(ComputeEngineMetadataClientTest, GetMetadata) {
std::shared_ptr<HttpRequest::Factory> http_factory =
std::make_shared<FakeHttpRequestFactory>(&requests);
ComputeEngineMetadataClient client(http_factory, 0);
ComputeEngineMetadataClient client(http_factory,
RetryConfig(0 /* init_delay_time_us */));
std::vector<char> result;
TF_EXPECT_OK(
@ -56,7 +57,8 @@ TEST(ComputeEngineMetadataClientTest, RetryOnFailure) {
std::shared_ptr<HttpRequest::Factory> http_factory =
std::make_shared<FakeHttpRequestFactory>(&requests);
ComputeEngineMetadataClient client(http_factory, 0);
ComputeEngineMetadataClient client(http_factory,
RetryConfig(0 /* init_delay_time_us */));
std::vector<char> result;
TF_EXPECT_OK(

8
tensorflow/core/platform/cloud/compute_engine_zone_provider_test.cc
View File

@ -34,8 +34,8 @@ TEST_F(ComputeEngineZoneProviderTest, GetZone) {
auto httpRequestFactory = std::make_shared<FakeHttpRequestFactory>(&requests);
auto metadata_client =
std::make_shared<ComputeEngineMetadataClient>(httpRequestFactory, 0);
auto metadata_client = std::make_shared<ComputeEngineMetadataClient>(
httpRequestFactory, RetryConfig(0 /* init_delay_time_us */));
ComputeEngineZoneProvider provider(metadata_client);
@ -55,8 +55,8 @@ TEST_F(ComputeEngineZoneProviderTest, InvalidZoneString) {
auto httpRequestFactory = std::make_shared<FakeHttpRequestFactory>(&requests);
auto metadata_client =
std::make_shared<ComputeEngineMetadataClient>(httpRequestFactory, 0);
auto metadata_client = std::make_shared<ComputeEngineMetadataClient>(
httpRequestFactory, RetryConfig(0 /* init_delay_time_us */));
ComputeEngineZoneProvider provider(metadata_client);

25
tensorflow/core/platform/cloud/gcs_file_system.cc
View File

@ -333,14 +333,14 @@ class GcsWritableFile : public WritableFile {
GcsFileSystem* filesystem,
GcsFileSystem::TimeoutConfig* timeouts,
std::function<void()> file_cache_erase,
int64 initial_retry_delay_usec)
RetryConfig retry_config)
: bucket_(bucket),
object_(object),
filesystem_(filesystem),
timeouts_(timeouts),
file_cache_erase_(std::move(file_cache_erase)),
sync_needed_(true),
initial_retry_delay_usec_(initial_retry_delay_usec) {
retry_config_(retry_config) {
// TODO: to make it safer, outfile_ should be constructed from an FD
if (GetTmpFilename(&tmp_content_filename_).ok()) {
outfile_.open(tmp_content_filename_,
@ -357,14 +357,14 @@ class GcsWritableFile : public WritableFile {
GcsFileSystem* filesystem, const string& tmp_content_filename,
GcsFileSystem::TimeoutConfig* timeouts,
std::function<void()> file_cache_erase,
int64 initial_retry_delay_usec)
RetryConfig retry_config)
: bucket_(bucket),
object_(object),
filesystem_(filesystem),
timeouts_(timeouts),
file_cache_erase_(std::move(file_cache_erase)),
sync_needed_(true),
initial_retry_delay_usec_(initial_retry_delay_usec) {
retry_config_(retry_config) {
tmp_content_filename_ = tmp_content_filename;
outfile_.open(tmp_content_filename_,
std::ofstream::binary | std::ofstream::app);
@ -441,7 +441,7 @@ class GcsWritableFile : public WritableFile {
first_attempt = false;
return UploadToSession(session_uri, already_uploaded);
},
initial_retry_delay_usec_);
retry_config_);
if (upload_status.code() == errors::Code::NOT_FOUND) {
// GCS docs recommend retrying the whole upload. We're relying on the
// RetryingFileSystem to retry the Sync() call.
@ -586,7 +586,7 @@ class GcsWritableFile : public WritableFile {
GcsFileSystem::TimeoutConfig* timeouts_;
std::function<void()> file_cache_erase_;
bool sync_needed_; // whether there is buffered data that needs to be synced
int64 initial_retry_delay_usec_;
RetryConfig retry_config_;
};
class GcsReadOnlyMemoryRegion : public ReadOnlyMemoryRegion {
@ -791,7 +791,7 @@ GcsFileSystem::GcsFileSystem(
std::unique_ptr<ZoneProvider> zone_provider, size_t block_size,
size_t max_bytes, uint64 max_staleness, uint64 stat_cache_max_age,
size_t stat_cache_max_entries, uint64 matching_paths_cache_max_age,
size_t matching_paths_cache_max_entries, int64 initial_retry_delay_usec,
size_t matching_paths_cache_max_entries, RetryConfig retry_config,
TimeoutConfig timeouts, const std::unordered_set<string>& allowed_locations,
std::pair<const string, const string>* additional_header)
: auth_provider_(std::move(auth_provider)),
@ -806,7 +806,7 @@ GcsFileSystem::GcsFileSystem(
kCacheNeverExpire, kBucketLocationCacheMaxEntries)),
allowed_locations_(allowed_locations),
timeouts_(timeouts),
initial_retry_delay_usec_(initial_retry_delay_usec),
retry_config_(retry_config),
additional_header_(additional_header) {}
Status GcsFileSystem::NewRandomAccessFile(
@ -941,7 +941,7 @@ Status GcsFileSystem::NewWritableFile(const string& fname,
TF_RETURN_IF_ERROR(ParseGcsPath(fname, false, &bucket, &object));
result->reset(new GcsWritableFile(bucket, object, this, &timeouts_,
[this, fname]() { ClearFileCaches(fname); },
initial_retry_delay_usec_));
retry_config_));
return Status::OK();
}
@ -981,7 +981,7 @@ Status GcsFileSystem::NewAppendableFile(const string& fname,
TF_RETURN_IF_ERROR(ParseGcsPath(fname, false, &bucket, &object));
result->reset(new GcsWritableFile(
bucket, object, this, old_content_filename, &timeouts_,
[this, fname]() { ClearFileCaches(fname); }, initial_retry_delay_usec_));
[this, fname]() { ClearFileCaches(fname); }, retry_config_));
return Status::OK();
}
@ -1534,7 +1534,7 @@ Status GcsFileSystem::RenameObject(const string& src, const string& target) {
// on the server side, we can't just retry the whole RenameFile operation
// because the source object is already gone.
return RetryingUtils::DeleteWithRetries(
[this, &src]() { return DeleteFile(src); }, initial_retry_delay_usec_);
[this, &src]() { return DeleteFile(src); }, retry_config_);
}
Status GcsFileSystem::IsDirectory(const string& fname) {
@ -1590,8 +1590,7 @@ Status GcsFileSystem::DeleteRecursively(const string& dirname,
// and therefore RetryingFileSystem won't pay attention to the failures,
// we need to make sure these failures are properly retried.
const auto& delete_file_status = RetryingUtils::DeleteWithRetries(
[this, &full_path]() { return DeleteFile(full_path); },
initial_retry_delay_usec_);
[this, &full_path]() { return DeleteFile(full_path); }, retry_config_);
if (!delete_file_status.ok()) {
if (IsDirectory(full_path).ok()) {
// The object is a directory marker.

7
tensorflow/core/platform/cloud/gcs_file_system.h
View File

@ -93,7 +93,7 @@ class GcsFileSystem : public FileSystem {
uint64 stat_cache_max_age, size_t stat_cache_max_entries,
uint64 matching_paths_cache_max_age,
size_t matching_paths_cache_max_entries,
int64 initial_retry_delay_usec, TimeoutConfig timeouts,
RetryConfig retry_config, TimeoutConfig timeouts,
const std::unordered_set<string>& allowed_locations,
std::pair<const string, const string>* additional_header);
@ -332,7 +332,7 @@ class GcsFileSystem : public FileSystem {
GcsStatsInterface* stats_ = nullptr; // Not owned.
/// The initial delay for exponential backoffs when retrying failed calls.
const int64 initial_retry_delay_usec_ = 1000000L;
RetryConfig retry_config_;
// Additional header material to be transmitted with all GCS requests
std::unique_ptr<std::pair<const string, const string>> additional_header_;
@ -344,7 +344,8 @@ class GcsFileSystem : public FileSystem {
class RetryingGcsFileSystem : public RetryingFileSystem<GcsFileSystem> {
public:
RetryingGcsFileSystem()
: RetryingFileSystem(std::unique_ptr<GcsFileSystem>(new GcsFileSystem)) {}
: RetryingFileSystem(std::unique_ptr<GcsFileSystem>(new GcsFileSystem),
RetryConfig(100000 /* init_delay_time_us */)) {}
};
} // namespace tensorflow

1280
tensorflow/core/platform/cloud/gcs_file_system_test.cc
View File

File diff suppressed because it is too large Load Diff

20
tensorflow/core/platform/cloud/google_auth_provider_test.cc
View File

@ -93,8 +93,8 @@ TEST_F(GoogleAuthProviderTest, EnvironmentVariable_Caching) {
std::shared_ptr<HttpRequest::Factory> fakeHttpRequestFactory =
std::make_shared<FakeHttpRequestFactory>(&requests);
auto metadataClient =
std::make_shared<ComputeEngineMetadataClient>(fakeHttpRequestFactory, 0);
auto metadataClient = std::make_shared<ComputeEngineMetadataClient>(
fakeHttpRequestFactory, RetryConfig(0 /* init_delay_time_us */));
GoogleAuthProvider provider(std::unique_ptr<OAuthClient>(oauth_client),
metadataClient, &env);
oauth_client->return_token = "fake-token";
@ -129,8 +129,8 @@ TEST_F(GoogleAuthProviderTest, GCloudRefreshToken) {
FakeEnv env;
std::shared_ptr<HttpRequest::Factory> fakeHttpRequestFactory =
std::make_shared<FakeHttpRequestFactory>(&requests);
auto metadataClient =
std::make_shared<ComputeEngineMetadataClient>(fakeHttpRequestFactory, 0);
auto metadataClient = std::make_shared<ComputeEngineMetadataClient>(
fakeHttpRequestFactory, RetryConfig(0 /* init_delay_time_us */));
GoogleAuthProvider provider(std::unique_ptr<OAuthClient>(oauth_client),
metadataClient, &env);
@ -178,8 +178,8 @@ TEST_F(GoogleAuthProviderTest, RunningOnGCE) {
FakeEnv env;
std::shared_ptr<HttpRequest::Factory> fakeHttpRequestFactory =
std::make_shared<FakeHttpRequestFactory>(&requests);
auto metadataClient =
std::make_shared<ComputeEngineMetadataClient>(fakeHttpRequestFactory, 0);
auto metadataClient = std::make_shared<ComputeEngineMetadataClient>(
fakeHttpRequestFactory, RetryConfig(0 /* init_delay_time_us */));
GoogleAuthProvider provider(std::unique_ptr<OAuthClient>(oauth_client),
metadataClient, &env);
@ -206,8 +206,8 @@ TEST_F(GoogleAuthProviderTest, OverrideForTesting) {
FakeEnv env;
std::shared_ptr<HttpRequest::Factory> fakeHttpRequestFactory =
std::make_shared<FakeHttpRequestFactory>(&empty_requests);
auto metadataClient =
std::make_shared<ComputeEngineMetadataClient>(fakeHttpRequestFactory, 0);
auto metadataClient = std::make_shared<ComputeEngineMetadataClient>(
fakeHttpRequestFactory, RetryConfig(0 /* init_delay_time_us */));
GoogleAuthProvider provider(std::unique_ptr<OAuthClient>(oauth_client),
metadataClient, &env);
@ -228,8 +228,8 @@ TEST_F(GoogleAuthProviderTest, NothingAvailable) {
FakeEnv env;
std::shared_ptr<HttpRequest::Factory> fakeHttpRequestFactory =
std::make_shared<FakeHttpRequestFactory>(&requests);
auto metadataClient =
std::make_shared<ComputeEngineMetadataClient>(fakeHttpRequestFactory, 0);
auto metadataClient = std::make_shared<ComputeEngineMetadataClient>(
fakeHttpRequestFactory, RetryConfig(0 /* init_delay_time_us */));
GoogleAuthProvider provider(std::unique_ptr<OAuthClient>(oauth_client),
metadataClient, &env);

67
tensorflow/core/platform/cloud/retrying_file_system.h
View File

@ -34,9 +34,9 @@ template <typename Underlying>
class RetryingFileSystem : public FileSystem {
public:
RetryingFileSystem(std::unique_ptr<Underlying> base_file_system,
int64 delay_microseconds = 1000000)
const RetryConfig& retry_config)
: base_file_system_(std::move(base_file_system)),
initial_delay_microseconds_(delay_microseconds) {}
retry_config_(retry_config) {}
Status NewRandomAccessFile(
const string& filename,
@ -55,7 +55,7 @@ class RetryingFileSystem : public FileSystem {
Status FileExists(const string& fname) override {
return RetryingUtils::CallWithRetries(
[this, &fname]() { return base_file_system_->FileExists(fname); },
initial_delay_microseconds_);
retry_config_);
}
Status GetChildren(const string& dir, std::vector<string>* result) override {
@ -63,7 +63,7 @@ class RetryingFileSystem : public FileSystem {
[this, &dir, result]() {
return base_file_system_->GetChildren(dir, result);
},
initial_delay_microseconds_);
retry_config_);
}
Status GetMatchingPaths(const string& pattern,
@ -72,31 +72,31 @@ class RetryingFileSystem : public FileSystem {
[this, &pattern, result]() {
return base_file_system_->GetMatchingPaths(pattern, result);
},
initial_delay_microseconds_);
retry_config_);
}
Status Stat(const string& fname, FileStatistics* stat) override {
return RetryingUtils::CallWithRetries(
[this, &fname, stat]() { return base_file_system_->Stat(fname, stat); },
initial_delay_microseconds_);
retry_config_);
}
Status DeleteFile(const string& fname) override {
return RetryingUtils::DeleteWithRetries(
[this, &fname]() { return base_file_system_->DeleteFile(fname); },
initial_delay_microseconds_);
retry_config_);
}
Status CreateDir(const string& dirname) override {
return RetryingUtils::CallWithRetries(
[this, &dirname]() { return base_file_system_->CreateDir(dirname); },
initial_delay_microseconds_);
retry_config_);
}
Status DeleteDir(const string& dirname) override {
return RetryingUtils::DeleteWithRetries(
[this, &dirname]() { return base_file_system_->DeleteDir(dirname); },
initial_delay_microseconds_);
retry_config_);
}
Status GetFileSize(const string& fname, uint64* file_size) override {
@ -104,7 +104,7 @@ class RetryingFileSystem : public FileSystem {
[this, &fname, file_size]() {
return base_file_system_->GetFileSize(fname, file_size);
},
initial_delay_microseconds_);
retry_config_);
}
Status RenameFile(const string& src, const string& target) override {
@ -112,13 +112,13 @@ class RetryingFileSystem : public FileSystem {
[this, &src, &target]() {
return base_file_system_->RenameFile(src, target);
},
initial_delay_microseconds_);
retry_config_);
}
Status IsDirectory(const string& dirname) override {
return RetryingUtils::CallWithRetries(
[this, &dirname]() { return base_file_system_->IsDirectory(dirname); },
initial_delay_microseconds_);
retry_config_);
}
Status DeleteRecursively(const string& dirname, int64* undeleted_files,
@ -128,7 +128,7 @@ class RetryingFileSystem : public FileSystem {
return base_file_system_->DeleteRecursively(dirname, undeleted_files,
undeleted_dirs);
},
initial_delay_microseconds_);
retry_config_);
}
void FlushCaches() override { base_file_system_->FlushCaches(); }
@ -137,7 +137,7 @@ class RetryingFileSystem : public FileSystem {
private:
std::unique_ptr<Underlying> base_file_system_;
const int64 initial_delay_microseconds_;
const RetryConfig retry_config_;
TF_DISALLOW_COPY_AND_ASSIGN(RetryingFileSystem);
};
@ -147,9 +147,8 @@ namespace retrying_internals {
class RetryingRandomAccessFile : public RandomAccessFile {
public:
RetryingRandomAccessFile(std::unique_ptr<RandomAccessFile> base_file,
int64 delay_microseconds)
: base_file_(std::move(base_file)),
initial_delay_microseconds_(delay_microseconds) {}
const RetryConfig& retry_config)
: base_file_(std::move(base_file)), retry_config_(retry_config) {}
Status Read(uint64 offset, size_t n, StringPiece* result,
char* scratch) const override {
@ -157,20 +156,19 @@ class RetryingRandomAccessFile : public RandomAccessFile {
[this, offset, n, result, scratch]() {
return base_file_->Read(offset, n, result, scratch);
},
initial_delay_microseconds_);
retry_config_);
}
private:
std::unique_ptr<RandomAccessFile> base_file_;
const int64 initial_delay_microseconds_;
const RetryConfig retry_config_;
};
class RetryingWritableFile : public WritableFile {
public:
RetryingWritableFile(std::unique_ptr<WritableFile> base_file,
int64 delay_microseconds)
: base_file_(std::move(base_file)),
initial_delay_microseconds_(delay_microseconds) {}
const RetryConfig& retry_config)
: base_file_(std::move(base_file)), retry_config_(retry_config) {}
~RetryingWritableFile() override {
// Makes sure the retrying version of Close() is called in the destructor.
@ -179,25 +177,24 @@ class RetryingWritableFile : public WritableFile {
Status Append(StringPiece data) override {
return RetryingUtils::CallWithRetries(
[this, &data]() { return base_file_->Append(data); },
initial_delay_microseconds_);
[this, &data]() { return base_file_->Append(data); }, retry_config_);
}
Status Close() override {
return RetryingUtils::CallWithRetries(
[this]() { return base_file_->Close(); }, initial_delay_microseconds_);
[this]() { return base_file_->Close(); }, retry_config_);
}
Status Flush() override {
return RetryingUtils::CallWithRetries(
[this]() { return base_file_->Flush(); }, initial_delay_microseconds_);
[this]() { return base_file_->Flush(); }, retry_config_);
}
Status Sync() override {
return RetryingUtils::CallWithRetries(
[this]() { return base_file_->Sync(); }, initial_delay_microseconds_);
[this]() { return base_file_->Sync(); }, retry_config_);
}
private:
std::unique_ptr<WritableFile> base_file_;
const int64 initial_delay_microseconds_;
const RetryConfig retry_config_;
};
} // namespace retrying_internals
@ -210,9 +207,9 @@ Status RetryingFileSystem<Underlying>::NewRandomAccessFile(
[this, &filename, &base_file]() {
return base_file_system_->NewRandomAccessFile(filename, &base_file);
},
initial_delay_microseconds_));
retry_config_));
result->reset(new retrying_internals::RetryingRandomAccessFile(
std::move(base_file), initial_delay_microseconds_));
std::move(base_file), retry_config_));
return Status::OK();
}
@ -224,9 +221,9 @@ Status RetryingFileSystem<Underlying>::NewWritableFile(
[this, &filename, &base_file]() {
return base_file_system_->NewWritableFile(filename, &base_file);
},
initial_delay_microseconds_));
retry_config_));
result->reset(new retrying_internals::RetryingWritableFile(
std::move(base_file), initial_delay_microseconds_));
std::move(base_file), retry_config_));
return Status::OK();
}
@ -238,9 +235,9 @@ Status RetryingFileSystem<Underlying>::NewAppendableFile(
[this, &filename, &base_file]() {
return base_file_system_->NewAppendableFile(filename, &base_file);
},
initial_delay_microseconds_));
retry_config_));
result->reset(new retrying_internals::RetryingWritableFile(
std::move(base_file), initial_delay_microseconds_));
std::move(base_file), retry_config_));
return Status::OK();
}
@ -252,7 +249,7 @@ Status RetryingFileSystem<Underlying>::NewReadOnlyMemoryRegionFromFile(
return base_file_system_->NewReadOnlyMemoryRegionFromFile(filename,
result);
},
initial_delay_microseconds_);
retry_config_);
}
} // namespace tensorflow

102
tensorflow/core/platform/cloud/retrying_file_system_test.cc
View File

@ -184,7 +184,8 @@ TEST(RetryingFileSystemTest, NewRandomAccessFile_ImmediateSuccess) {
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
base_fs->random_access_file_to_return = std::move(base_file);
RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
RetryingFileSystem<MockFileSystem> fs(
std::move(base_fs), RetryConfig(0 /* init_delay_time_us */));
// Retrieve the wrapped random access file.
std::unique_ptr<RandomAccessFile> random_access_file;
@ -211,7 +212,8 @@ TEST(RetryingFileSystemTest, NewRandomAccessFile_SuccessWith3rdTry) {
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
base_fs->random_access_file_to_return = std::move(base_file);
RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
RetryingFileSystem<MockFileSystem> fs(
std::move(base_fs), RetryConfig(0 /* init_delay_time_us */));
// Retrieve the wrapped random access file.
std::unique_ptr<RandomAccessFile> random_access_file;
@ -235,7 +237,8 @@ TEST(RetryingFileSystemTest, NewRandomAccessFile_AllRetriesFailed) {
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
base_fs->random_access_file_to_return = std::move(base_file);
RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
RetryingFileSystem<MockFileSystem> fs(
std::move(base_fs), RetryConfig(0 /* init_delay_time_us */));
// Retrieve the wrapped random access file.
std::unique_ptr<RandomAccessFile> random_access_file;
@ -265,7 +268,8 @@ TEST(RetryingFileSystemTest, NewRandomAccessFile_NoRetriesForSomeErrors) {
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
base_fs->random_access_file_to_return = std::move(base_file);
RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
RetryingFileSystem<MockFileSystem> fs(
std::move(base_fs), RetryConfig(0 /* init_delay_time_us */));
// Retrieve the wrapped random access file.
std::unique_ptr<RandomAccessFile> random_access_file;
@ -291,7 +295,8 @@ TEST(RetryingFileSystemTest, NewWritableFile_ImmediateSuccess) {
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
base_fs->writable_file_to_return = std::move(base_file);
RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
RetryingFileSystem<MockFileSystem> fs(
std::move(base_fs), RetryConfig(0 /* init_delay_time_us */));
// Retrieve the wrapped writable file.
std::unique_ptr<WritableFile> writable_file;
@ -317,7 +322,8 @@ TEST(RetryingFileSystemTest, NewWritableFile_SuccessWith3rdTry) {
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
base_fs->writable_file_to_return = std::move(base_file);
RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
RetryingFileSystem<MockFileSystem> fs(
std::move(base_fs), RetryConfig(0 /* init_delay_time_us */));
// Retrieve the wrapped writable file.
std::unique_ptr<WritableFile> writable_file;
@ -343,7 +349,8 @@ TEST(RetryingFileSystemTest, NewWritableFile_SuccessWith3rdTry_ViaDestructor) {
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
base_fs->writable_file_to_return = std::move(base_file);
RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
RetryingFileSystem<MockFileSystem> fs(
std::move(base_fs), RetryConfig(0 /* init_delay_time_us */));
// Retrieve the wrapped writable file.
std::unique_ptr<WritableFile> writable_file;
@ -368,7 +375,8 @@ TEST(RetryingFileSystemTest, NewAppendableFile_SuccessWith3rdTry) {
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
base_fs->writable_file_to_return = std::move(base_file);
RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
RetryingFileSystem<MockFileSystem> fs(
std::move(base_fs), RetryConfig(0 /* init_delay_time_us */));
// Retrieve the wrapped appendable file.
std::unique_ptr<WritableFile> writable_file;
@ -391,7 +399,8 @@ TEST(RetryingFileSystemTest, NewWritableFile_AllRetriesFailed) {
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
base_fs->writable_file_to_return = std::move(base_file);
RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
RetryingFileSystem<MockFileSystem> fs(
std::move(base_fs), RetryConfig(0 /* init_delay_time_us */));
// Retrieve the wrapped writable file.
std::unique_ptr<WritableFile> writable_file;
@ -412,7 +421,8 @@ TEST(RetryingFileSystemTest,
std::make_tuple("NewReadOnlyMemoryRegionFromFile", Status::OK())});
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
RetryingFileSystem<MockFileSystem> fs(
std::move(base_fs), RetryConfig(0 /* init_delay_time_us */));
std::unique_ptr<ReadOnlyMemoryRegion> result;
TF_EXPECT_OK(fs.NewReadOnlyMemoryRegionFromFile("filename.txt", &result));
@ -423,7 +433,8 @@ TEST(RetryingFileSystemTest, NewReadOnlyMemoryRegionFromFile_AllRetriesFailed) {
CreateRetriableErrors("NewReadOnlyMemoryRegionFromFile", 11);
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
RetryingFileSystem<MockFileSystem> fs(
std::move(base_fs), RetryConfig(0 /* init_delay_time_us */));
std::unique_ptr<ReadOnlyMemoryRegion> result;
const auto& status =
@ -440,7 +451,8 @@ TEST(RetryingFileSystemTest, GetChildren_SuccessWith2ndTry) {
std::make_tuple("GetChildren", Status::OK())});
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
RetryingFileSystem<MockFileSystem> fs(
std::move(base_fs), RetryConfig(0 /* init_delay_time_us */));
std::vector<string> result;
TF_EXPECT_OK(fs.GetChildren("gs://path", &result));
@ -450,7 +462,8 @@ TEST(RetryingFileSystemTest, GetChildren_AllRetriesFailed) {
ExpectedCalls expected_fs_calls = CreateRetriableErrors("GetChildren", 11);
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
RetryingFileSystem<MockFileSystem> fs(
std::move(base_fs), RetryConfig(0 /* init_delay_time_us */));
std::vector<string> result;
const auto& status = fs.GetChildren("gs://path", &result);
@ -466,7 +479,8 @@ TEST(RetryingFileSystemTest, GetMatchingPaths_SuccessWith2ndTry) {
std::make_tuple("GetMatchingPaths", Status::OK())});
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
RetryingFileSystem<MockFileSystem> fs(
std::move(base_fs), RetryConfig(0 /* init_delay_time_us */));
std::vector<string> result;
TF_EXPECT_OK(fs.GetMatchingPaths("gs://path/dir", &result));
@ -477,7 +491,8 @@ TEST(RetryingFileSystemTest, GetMatchingPaths_AllRetriesFailed) {
CreateRetriableErrors("GetMatchingPaths", 11);
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
RetryingFileSystem<MockFileSystem> fs(
std::move(base_fs), RetryConfig(0 /* init_delay_time_us */));
std::vector<string> result;
const auto& status = fs.GetMatchingPaths("gs://path/dir", &result);
@ -492,7 +507,8 @@ TEST(RetryingFileSystemTest, DeleteFile_SuccessWith2ndTry) {
std::make_tuple("DeleteFile", Status::OK())});
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
RetryingFileSystem<MockFileSystem> fs(
std::move(base_fs), RetryConfig(0 /* init_delay_time_us */));
std::vector<string> result;
TF_EXPECT_OK(fs.DeleteFile("gs://path/file.txt"));
@ -502,7 +518,8 @@ TEST(RetryingFileSystemTest, DeleteFile_AllRetriesFailed) {
ExpectedCalls expected_fs_calls = CreateRetriableErrors("DeleteFile", 11);
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
RetryingFileSystem<MockFileSystem> fs(
std::move(base_fs), RetryConfig(0 /* init_delay_time_us */));
std::vector<string> result;
const auto& status = fs.DeleteFile("gs://path/file.txt");
@ -517,7 +534,8 @@ TEST(RetryingFileSystemTest, CreateDir_SuccessWith2ndTry) {
std::make_tuple("CreateDir", Status::OK())});
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
RetryingFileSystem<MockFileSystem> fs(
std::move(base_fs), RetryConfig(0 /* init_delay_time_us */));
std::vector<string> result;
TF_EXPECT_OK(fs.CreateDir("gs://path/newdir"));
@ -527,7 +545,8 @@ TEST(RetryingFileSystemTest, CreateDir_AllRetriesFailed) {
ExpectedCalls expected_fs_calls = CreateRetriableErrors("CreateDir", 11);
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
RetryingFileSystem<MockFileSystem> fs(
std::move(base_fs), RetryConfig(0 /* init_delay_time_us */));
std::vector<string> result;
const auto& status = fs.CreateDir("gs://path/newdir");
@ -542,7 +561,8 @@ TEST(RetryingFileSystemTest, DeleteDir_SuccessWith2ndTry) {
std::make_tuple("DeleteDir", Status::OK())});
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
RetryingFileSystem<MockFileSystem> fs(
std::move(base_fs), RetryConfig(0 /* init_delay_time_us */));
std::vector<string> result;
TF_EXPECT_OK(fs.DeleteDir("gs://path/dir"));
@ -552,7 +572,8 @@ TEST(RetryingFileSystemTest, DeleteDir_AllRetriesFailed) {
ExpectedCalls expected_fs_calls = CreateRetriableErrors("DeleteDir", 11);
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
RetryingFileSystem<MockFileSystem> fs(
std::move(base_fs), RetryConfig(0 /* init_delay_time_us */));
std::vector<string> result;
const auto& status = fs.DeleteDir("gs://path/dir");
@ -568,7 +589,8 @@ TEST(RetryingFileSystemTest, GetFileSize_SuccessWith2ndTry) {
std::make_tuple("GetFileSize", Status::OK())});
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
RetryingFileSystem<MockFileSystem> fs(
std::move(base_fs), RetryConfig(0 /* init_delay_time_us */));
uint64 size;
TF_EXPECT_OK(fs.GetFileSize("gs://path/file.txt", &size));
@ -578,7 +600,8 @@ TEST(RetryingFileSystemTest, GetFileSize_AllRetriesFailed) {
ExpectedCalls expected_fs_calls = CreateRetriableErrors("GetFileSize", 11);
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
RetryingFileSystem<MockFileSystem> fs(
std::move(base_fs), RetryConfig(0 /* init_delay_time_us */));
uint64 size;
const auto& status = fs.GetFileSize("gs://path/file.txt", &size);
@ -593,7 +616,8 @@ TEST(RetryingFileSystemTest, RenameFile_SuccessWith2ndTry) {
std::make_tuple("RenameFile", Status::OK())});
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
RetryingFileSystem<MockFileSystem> fs(
std::move(base_fs), RetryConfig(0 /* init_delay_time_us */));
TF_EXPECT_OK(fs.RenameFile("old_name", "new_name"));
}
@ -602,7 +626,8 @@ TEST(RetryingFileSystemTest, RenameFile_AllRetriesFailed) {
ExpectedCalls expected_fs_calls = CreateRetriableErrors("RenameFile", 11);
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
RetryingFileSystem<MockFileSystem> fs(
std::move(base_fs), RetryConfig(0 /* init_delay_time_us */));
const auto& status = fs.RenameFile("old_name", "new_name");
EXPECT_TRUE(
@ -616,7 +641,8 @@ TEST(RetryingFileSystemTest, Stat_SuccessWith2ndTry) {
std::make_tuple("Stat", Status::OK())});
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
RetryingFileSystem<MockFileSystem> fs(
std::move(base_fs), RetryConfig(0 /* init_delay_time_us */));
FileStatistics stat;
TF_EXPECT_OK(fs.Stat("file_name", &stat));
@ -626,7 +652,8 @@ TEST(RetryingFileSystemTest, Stat_AllRetriesFailed) {
ExpectedCalls expected_fs_calls = CreateRetriableErrors("Stat", 11);
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
RetryingFileSystem<MockFileSystem> fs(
std::move(base_fs), RetryConfig(0 /* init_delay_time_us */));
FileStatistics stat;
const auto& status = fs.Stat("file_name", &stat);
@ -639,7 +666,8 @@ TEST(RetryingFileSystemTest, FileExists_AllRetriesFailed) {
ExpectedCalls expected_fs_calls = CreateRetriableErrors("FileExists", 11);
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
RetryingFileSystem<MockFileSystem> fs(
std::move(base_fs), RetryConfig(0 /* init_delay_time_us */));
const auto& status = fs.FileExists("file_name");
EXPECT_TRUE(
@ -653,7 +681,8 @@ TEST(RetryingFileSystemTest, FileExists_SuccessWith2ndTry) {
std::make_tuple("FileExists", Status::OK())});
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
RetryingFileSystem<MockFileSystem> fs(
std::move(base_fs), RetryConfig(0 /* init_delay_time_us */));
TF_EXPECT_OK(fs.FileExists("gs://path/dir"));
}
@ -665,7 +694,8 @@ TEST(RetryingFileSystemTest, IsDirectory_SuccessWith2ndTry) {
std::make_tuple("IsDirectory", Status::OK())});
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
RetryingFileSystem<MockFileSystem> fs(
std::move(base_fs), RetryConfig(0 /* init_delay_time_us */));
TF_EXPECT_OK(fs.IsDirectory("gs://path/dir"));
}
@ -674,7 +704,8 @@ TEST(RetryingFileSystemTest, IsDirectory_AllRetriesFailed) {
ExpectedCalls expected_fs_calls = CreateRetriableErrors("IsDirectory", 11);
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
RetryingFileSystem<MockFileSystem> fs(
std::move(base_fs), RetryConfig(0 /* init_delay_time_us */));
const auto& status = fs.IsDirectory("gs://path/dir");
EXPECT_TRUE(
@ -689,7 +720,8 @@ TEST(RetryingFileSystemTest, DeleteRecursively_SuccessWith2ndTry) {
std::make_tuple("DeleteRecursively", Status::OK())});
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
RetryingFileSystem<MockFileSystem> fs(
std::move(base_fs), RetryConfig(0 /* init_delay_time_us */));
int64 undeleted_files, undeleted_dirs;
TF_EXPECT_OK(
@ -701,7 +733,8 @@ TEST(RetryingFileSystemTest, DeleteRecursively_AllRetriesFailed) {
CreateRetriableErrors("DeleteRecursively", 11);
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
RetryingFileSystem<MockFileSystem> fs(
std::move(base_fs), RetryConfig(0 /* init_delay_time_us */));
int64 undeleted_files, undeleted_dirs;
const auto& status =
@ -715,7 +748,8 @@ TEST(RetryingFileSystemTest, FlushCaches) {
ExpectedCalls none;
bool flushed = false;
std::unique_ptr<MockFileSystem> base_fs(new MockFileSystem(none, &flushed));
RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
RetryingFileSystem<MockFileSystem> fs(
std::move(base_fs), RetryConfig(0 /* init_delay_time_us */));
fs.FlushCaches();
EXPECT_TRUE(flushed);
}

35
tensorflow/core/platform/cloud/retrying_utils.cc
View File

@ -23,11 +23,6 @@ namespace tensorflow {
namespace {
// In case of failure, every call will be retried kMaxRetries times.
constexpr int kMaxRetries = 10;
// Maximum backoff time in microseconds.
constexpr int64 kMaximumBackoffMicroseconds = 32000000; // 32 seconds.
bool IsRetriable(error::Code code) {
switch (code) {
case error::UNAVAILABLE:
@ -43,40 +38,41 @@ bool IsRetriable(error::Code code) {
} // namespace
Status RetryingUtils::CallWithRetries(const std::function<Status()>& f,
const int64 initial_delay_microseconds) {
return CallWithRetries(f, initial_delay_microseconds, [](int64 micros) {
return Env::Default()->SleepForMicroseconds(micros);
});
const RetryConfig& config) {
return CallWithRetries(
f,
[](int64 micros) { return Env::Default()->SleepForMicroseconds(micros); },
config);
}
Status RetryingUtils::CallWithRetries(
const std::function<Status()>& f, const int64 initial_delay_microseconds,
const std::function<void(int64)>& sleep_usec) {
const std::function<Status()>& f,
const std::function<void(int64)>& sleep_usec, const RetryConfig& config) {
int retries = 0;
while (true) {
auto status = f();
if (!IsRetriable(status.code())) {
return status;
}
if (retries >= kMaxRetries) {
if (retries >= config.max_retries) {
// Return AbortedError, so that it doesn't get retried again somewhere
// at a higher level.
return Status(
error::ABORTED,
strings::StrCat(
"All ", kMaxRetries,
"All ", config.max_retries,
" retry attempts failed. The last failure: ", status.ToString()));
}
int64 delay_micros = 0;
if (initial_delay_microseconds > 0) {
if (config.init_delay_time_us > 0) {
const int64 random_micros = random::New64() % 1000000;
delay_micros = std::min(initial_delay_microseconds << retries,
kMaximumBackoffMicroseconds) +
delay_micros = std::min(config.init_delay_time_us << retries,
config.max_delay_time_us) +
random_micros;
}
LOG(INFO) << "The operation failed and will be automatically retried in "
<< (delay_micros / 1000000.0) << " seconds (attempt "
<< (retries + 1) << " out of " << kMaxRetries
<< (retries + 1) << " out of " << config.max_retries
<< "), caused by: " << status.ToString();
sleep_usec(delay_micros);
retries++;
@ -84,8 +80,7 @@ Status RetryingUtils::CallWithRetries(
}
Status RetryingUtils::DeleteWithRetries(
const std::function<Status()>& delete_func,
const int64 initial_delay_microseconds) {
const std::function<Status()>& delete_func, const RetryConfig& config) {
bool is_retried = false;
return RetryingUtils::CallWithRetries(
[delete_func, &is_retried]() {
@ -96,7 +91,7 @@ Status RetryingUtils::DeleteWithRetries(
is_retried = true;
return status;
},
initial_delay_microseconds);
config);
}
} // namespace tensorflow

29
tensorflow/core/platform/cloud/retrying_utils.h
View File

@ -21,6 +21,26 @@ limitations under the License.
namespace tensorflow {
// Default time before reporting failure: ~100 seconds.
struct RetryConfig {
RetryConfig(int64 init_delay_time_us = 100 * 1000,
int64 max_delay_time_us = 32 * 1000 * 1000,
int max_retries = 10) {
this->init_delay_time_us = init_delay_time_us;
this->max_delay_time_us = max_delay_time_us;
this->max_retries = max_retries;
}
// In case of failure, every call will be retried max_retries times.
int max_retries;
// Initial backoff time
int64 init_delay_time_us;
// Maximum backoff time in microseconds.
int64 max_delay_time_us;
};
class RetryingUtils {
public:
/// \brief Retries the function in case of failure with exponential backoff.
@ -31,18 +51,19 @@ class RetryingUtils {
/// retries.
/// If all retries failed, returns the last error status.
static Status CallWithRetries(const std::function<Status()>& f,
const int64 initial_delay_microseconds);
const RetryConfig& config);
/// sleep_usec is a function that sleeps for the given number of microseconds.
static Status CallWithRetries(const std::function<Status()>& f,
const int64 initial_delay_microseconds,
const std::function<void(int64)>& sleep_usec);
const std::function<void(int64)>& sleep_usec,
const RetryConfig& config);
/// \brief A retrying wrapper for a function that deletes a resource.
///
/// The function takes care of the scenario when a delete operation
/// returns a failure but succeeds under the hood: if a retry returns
/// NOT_FOUND, the whole operation is considered a success.
static Status DeleteWithRetries(const std::function<Status()>& delete_func,
const int64 initial_delay_microseconds);
const RetryConfig& config);
};
} // namespace tensorflow

32
tensorflow/core/platform/cloud/retrying_utils_test.cc
View File

@ -30,7 +30,8 @@ TEST(RetryingUtilsTest, CallWithRetries_RetryDelays) {
};
std::function<Status()> f = []() { return errors::Unavailable("Failed."); };
const auto& status = RetryingUtils::CallWithRetries(f, 500000L, sleep);
const auto& status = RetryingUtils::CallWithRetries(
f, sleep, RetryConfig(500000 /* init_delay_time_us */));
EXPECT_EQ(errors::Code::ABORTED, status.code());
EXPECT_TRUE(str_util::StrContains(
status.error_message(),
@ -60,8 +61,10 @@ TEST(RetryingUtilsTest, CallWithRetries_NotFoundIsNotRetried) {
results.erase(results.begin());
return result;
};
EXPECT_EQ(errors::Code::NOT_FOUND,
RetryingUtils::CallWithRetries(f, 0).code());
EXPECT_EQ(
errors::Code::NOT_FOUND,
RetryingUtils::CallWithRetries(f, RetryConfig(0 /* init_delay_time_us */))
.code());
}
TEST(RetryingUtilsTest, CallWithRetries_ImmediateSuccess) {
@ -74,7 +77,8 @@ TEST(RetryingUtilsTest, CallWithRetries_ImmediateSuccess) {
results.erase(results.begin());
return result;
};
TF_EXPECT_OK(RetryingUtils::CallWithRetries(f, 1.0, sleep));
TF_EXPECT_OK(RetryingUtils::CallWithRetries(
f, sleep, RetryConfig(1L /* init_delay_time_us */)));
}
TEST(RetryingUtilsTest, CallWithRetries_EventualSuccess) {
@ -86,7 +90,8 @@ TEST(RetryingUtilsTest, CallWithRetries_EventualSuccess) {
results.erase(results.begin());
return result;
};
TF_EXPECT_OK(RetryingUtils::CallWithRetries(f, 0));
TF_EXPECT_OK(RetryingUtils::CallWithRetries(
f, RetryConfig(0 /* init_delay_time_us */)));
}
TEST(RetryingUtilsTest, DeleteWithRetries_ImmediateSuccess) {
@ -96,7 +101,8 @@ TEST(RetryingUtilsTest, DeleteWithRetries_ImmediateSuccess) {
delete_results.erase(delete_results.begin());
return result;
};
TF_EXPECT_OK(RetryingUtils::DeleteWithRetries(delete_func, 0));
TF_EXPECT_OK(RetryingUtils::DeleteWithRetries(
delete_func, RetryConfig(0 /* init_delay_time_us */)));
}
TEST(RetryingUtilsTest, DeleteWithRetries_EventualSuccess) {
@ -106,7 +112,8 @@ TEST(RetryingUtilsTest, DeleteWithRetries_EventualSuccess) {
delete_results.erase(delete_results.begin());
return result;
};
TF_EXPECT_OK(RetryingUtils::DeleteWithRetries(delete_func, 0));
TF_EXPECT_OK(RetryingUtils::DeleteWithRetries(
delete_func, RetryConfig(0 /* init_delay_time_us */)));
}
TEST(RetryingUtilsTest, DeleteWithRetries_PermissionDeniedNotRetried) {
@ -118,7 +125,9 @@ TEST(RetryingUtilsTest, DeleteWithRetries_PermissionDeniedNotRetried) {
return result;
};
EXPECT_EQ(errors::Code::PERMISSION_DENIED,
RetryingUtils::DeleteWithRetries(delete_func, 0).code());
RetryingUtils::DeleteWithRetries(
delete_func, RetryConfig(0 /* init_delay_time_us */))
.code());
}
TEST(RetryingUtilsTest, DeleteWithRetries_SuccessThroughFileNotFound) {
@ -129,7 +138,8 @@ TEST(RetryingUtilsTest, DeleteWithRetries_SuccessThroughFileNotFound) {
delete_results.erase(delete_results.begin());
return result;
};
TF_EXPECT_OK(RetryingUtils::DeleteWithRetries(delete_func, 0));
TF_EXPECT_OK(RetryingUtils::DeleteWithRetries(
delete_func, RetryConfig(0 /* init_delay_time_us */)));
}
TEST(RetryingUtilsTest, DeleteWithRetries_FirstNotFoundReturnedAsIs) {
@ -140,7 +150,9 @@ TEST(RetryingUtilsTest, DeleteWithRetries_FirstNotFoundReturnedAsIs) {
return result;
};
EXPECT_EQ(error::NOT_FOUND,
RetryingUtils::DeleteWithRetries(delete_func, 0).code());
RetryingUtils::DeleteWithRetries(
delete_func, RetryConfig(0 /* init_delay_time_us */))
.code());
}
} // namespace