experiments
/
STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Make ParseGcsPath protected so that it can be overriden by subclass file systems. 

PiperOrigin-RevId: 306866392
Change-Id: I4589e695585786c95c7be73698b37363dfc2bab4
This commit is contained in:
A. Unique TensorFlower 2020-04-16 09:54:34 -07:00 committed by TensorFlower Gardener
parent b54025732b
commit f303d000c6
2 changed files with 41 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
tensorflow/core/platform/cloud/gcs_file_system.cc
View File

@ -133,34 +133,6 @@ Status GetTmpFilename(string* filename) {
return Status::OK();
}
/// \brief Splits a GCS path to a bucket and an object.
///
/// For example, "gs://bucket-name/path/to/file.txt" gets split into
/// "bucket-name" and "path/to/file.txt".
/// If fname only contains the bucket and empty_object_ok = true, the returned
/// object is empty.
Status ParseGcsPath(StringPiece fname, bool empty_object_ok, string* bucket,
string* object) {
StringPiece scheme, bucketp, objectp;
io::ParseURI(fname, &scheme, &bucketp, &objectp);
if (scheme != "gs") {
return errors::InvalidArgument("GCS path doesn't start with 'gs://': ",
fname);
}
*bucket = string(bucketp);
if (bucket->empty() || *bucket == ".") {
return errors::InvalidArgument("GCS path doesn't contain a bucket name: ",
fname);
}
absl::ConsumePrefix(&objectp, "/");
*object = string(objectp);
if (!empty_object_ok && object->empty()) {
return errors::InvalidArgument("GCS path doesn't contain an object name: ",
fname);
}
return Status::OK();
}
/// Appends a trailing slash if the name doesn't already have one.
string MaybeAppendSlash(const string& name) {
if (name.empty()) {
@ -1034,6 +1006,34 @@ Status GcsFileSystem::LoadBufferFromGCS(const string& fname, size_t offset,
return Status::OK();
}
Status GcsFileSystem::ParseGcsPathForScheme(StringPiece fname, string scheme,
bool empty_object_ok,
string* bucket, string* object) {
StringPiece parsed_scheme, bucketp, objectp;
io::ParseURI(fname, &parsed_scheme, &bucketp, &objectp);
if (parsed_scheme != scheme) {
return errors::InvalidArgument("GCS path doesn't start with 'gs://': ",
fname);
}
*bucket = string(bucketp);
if (bucket->empty() || *bucket == ".") {
return errors::InvalidArgument("GCS path doesn't contain a bucket name: ",
fname);
}
absl::ConsumePrefix(&objectp, "/");
*object = string(objectp);
if (!empty_object_ok && object->empty()) {
return errors::InvalidArgument("GCS path doesn't contain an object name: ",
fname);
}
return Status::OK();
}
Status GcsFileSystem::ParseGcsPath(StringPiece fname, bool empty_object_ok,
string* bucket, string* object) {
return ParseGcsPathForScheme(fname, "gs", empty_object_ok, bucket, object);
}
void GcsFileSystem::ClearFileCaches(const string& fname) {
tf_shared_lock l(block_cache_lock_);
file_block_cache_->RemoveFile(fname);

13
tensorflow/core/platform/cloud/gcs_file_system.h
View File

@ -264,6 +264,19 @@ class GcsFileSystem : public FileSystem {
virtual Status LoadBufferFromGCS(const string& fname, size_t offset, size_t n,
char* buffer, size_t* bytes_transferred);
Status ParseGcsPathForScheme(StringPiece fname, string scheme,
bool empty_object_ok, string* bucket,
string* object);
/// \brief Splits a GCS path to a bucket and an object.
///
/// For example, "gs://bucket-name/path/to/file.txt" gets split into
/// "bucket-name" and "path/to/file.txt".
/// If fname only contains the bucket and empty_object_ok = true, the returned
/// object is empty.
virtual Status ParseGcsPath(StringPiece fname, bool empty_object_ok,
string* bucket, string* object);
std::shared_ptr<ComputeEngineMetadataClient> compute_engine_metadata_client_;
private: