From 5659465166daa218168c1f50f1d63c30f9f2bbd9 Mon Sep 17 00:00:00 2001 From: Gunhan Gulsoy Date: Mon, 10 Feb 2020 18:09:34 -0800 Subject: [PATCH] To handle cloud filesystem paths on windows correctly, provide a way to run fnmatch without fnmatch. PiperOrigin-RevId: 294349136 Change-Id: Iebc466c2b3974ae5f4bd7524a8d51624df4cb70c --- tensorflow/core/platform/BUILD | 1 + tensorflow/core/platform/default/BUILD | 1 + tensorflow/core/platform/default/build_config.bzl | 1 + tensorflow/core/platform/file_system.cc | 8 ++++++++ tensorflow/core/platform/file_system.h | 7 +++++++ tensorflow/core/platform/file_system_helper.cc | 2 +- tensorflow/core/platform/regexp.h | 3 ++- tensorflow/core/platform/windows/windows_file_system.cc | 6 ++++++ tensorflow/core/platform/windows/windows_file_system.h | 2 ++ 9 files changed, 29 insertions(+), 2 deletions(-) diff --git a/tensorflow/core/platform/BUILD b/tensorflow/core/platform/BUILD index c64e2e20fb7..5d00b0235a7 100644 --- a/tensorflow/core/platform/BUILD +++ b/tensorflow/core/platform/BUILD @@ -1396,6 +1396,7 @@ filegroup( "protobuf_util.cc", "raw_coding.h", "refcount.h", + "regexp.h", "scanner.cc", "scanner.h", "setround.cc", diff --git a/tensorflow/core/platform/default/BUILD b/tensorflow/core/platform/default/BUILD index 616c628a4c1..d11c92e3deb 100644 --- a/tensorflow/core/platform/default/BUILD +++ b/tensorflow/core/platform/default/BUILD @@ -113,6 +113,7 @@ cc_library( "//tensorflow/core/platform:path", "//tensorflow/core/platform:platform_port", "//tensorflow/core/platform:protobuf", + "//tensorflow/core/platform:regexp", "//tensorflow/core/platform:setround", "//tensorflow/core/platform:status", "//tensorflow/core/platform:str_util", diff --git a/tensorflow/core/platform/default/build_config.bzl b/tensorflow/core/platform/default/build_config.bzl index 1eb8be69643..b47eb23a3f1 100644 --- a/tensorflow/core/platform/default/build_config.bzl +++ b/tensorflow/core/platform/default/build_config.bzl @@ -762,6 +762,7 @@ def tf_portable_deps_no_runtime(use_lite_protos = False): "//tensorflow/core/util:stats_calculator_portable", "//tensorflow/core:mobile_additional_lib_deps", "//tensorflow/core:protos_all_cc_impl", + "@com_googlesource_code_re2//:re2", "@farmhash_archive//:farmhash", ] + tf_protobuf_deps(use_lite_protos) diff --git a/tensorflow/core/platform/file_system.cc b/tensorflow/core/platform/file_system.cc index 2a4c029d2ae..c7b573fc292 100644 --- a/tensorflow/core/platform/file_system.cc +++ b/tensorflow/core/platform/file_system.cc @@ -26,12 +26,20 @@ limitations under the License. #include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/errors.h" #include "tensorflow/core/platform/platform.h" +#include "tensorflow/core/platform/regexp.h" #include "tensorflow/core/platform/scanner.h" #include "tensorflow/core/platform/str_util.h" #include "tensorflow/core/platform/strcat.h" namespace tensorflow { +bool FileSystem::Match(const string& filename, const string& pattern) { + string regexp(pattern); + RE2::GlobalReplace(®exp, "\\*", "[^/]*"); + RE2::GlobalReplace(®exp, "\\?", "."); + return RE2::FullMatch(filename, regexp); +} + string FileSystem::TranslateName(const string& name) const { // If the name is empty, CleanPath returns "." which is incorrect and // we should return the empty path instead. diff --git a/tensorflow/core/platform/file_system.h b/tensorflow/core/platform/file_system.h index 23b43817f95..bc860865a53 100644 --- a/tensorflow/core/platform/file_system.h +++ b/tensorflow/core/platform/file_system.h @@ -146,6 +146,13 @@ class FileSystem { virtual tensorflow::Status GetMatchingPaths(const string& pattern, std::vector* results) = 0; + /// \brief Checks if the given filename matches the pattern. + /// + /// This function provides the equivalent of posix fnmatch, however it is + /// implemented without fnmatch to ensure that this can be used for cloud + /// filesystems on windows. For windows filesystems, it uses PathMatchSpec. + virtual bool Match(const string& filename, const string& pattern); + /// \brief Obtains statistics for the given path. virtual tensorflow::Status Stat(const string& fname, FileStatistics* stat) = 0; diff --git a/tensorflow/core/platform/file_system_helper.cc b/tensorflow/core/platform/file_system_helper.cc index da3acba7d1a..64b175c4d17 100644 --- a/tensorflow/core/platform/file_system_helper.cc +++ b/tensorflow/core/platform/file_system_helper.cc @@ -119,7 +119,7 @@ Status GetMatchingPaths(FileSystem* fs, Env* env, const string& pattern, // Match all obtained files to the input pattern. for (const auto& f : all_files) { - if (env->MatchPath(f, eval_pattern)) { + if (fs->Match(f, eval_pattern)) { results->push_back(f); } } diff --git a/tensorflow/core/platform/regexp.h b/tensorflow/core/platform/regexp.h index 361346a94b5..859822844d9 100644 --- a/tensorflow/core/platform/regexp.h +++ b/tensorflow/core/platform/regexp.h @@ -18,7 +18,8 @@ limitations under the License. #include "tensorflow/core/platform/platform.h" -#if defined(PLATFORM_GOOGLE) || defined(PLATFORM_GOOGLE_ANDROID) +#if defined(PLATFORM_GOOGLE) || defined(PLATFORM_GOOGLE_ANDROID) || \ + defined(PLATFORM_GOOGLE_IOS) || defined(PLATFORM_PORTABLE_GOOGLE) #include "third_party/re2/re2.h" #else #include "re2/re2.h" diff --git a/tensorflow/core/platform/windows/windows_file_system.cc b/tensorflow/core/platform/windows/windows_file_system.cc index ac6b9b01114..98945ebcccd 100644 --- a/tensorflow/core/platform/windows/windows_file_system.cc +++ b/tensorflow/core/platform/windows/windows_file_system.cc @@ -548,6 +548,12 @@ Status WindowsFileSystem::GetMatchingPaths(const string& pattern, return Status::OK(); } +bool WindowsFileSystem::Match(const string& filename, const string& pattern) { + std::wstring ws_path(Utf8ToWideChar(path)); + std::wstring ws_pattern(Utf8ToWideChar(pattern)); + return PathMatchSpecW(ws_path.c_str(), ws_pattern.c_str()) == TRUE; +} + Status WindowsFileSystem::Stat(const string& fname, FileStatistics* stat) { Status result; struct _stat sbuf; diff --git a/tensorflow/core/platform/windows/windows_file_system.h b/tensorflow/core/platform/windows/windows_file_system.h index 25612639c34..9bf8a018113 100644 --- a/tensorflow/core/platform/windows/windows_file_system.h +++ b/tensorflow/core/platform/windows/windows_file_system.h @@ -52,6 +52,8 @@ class WindowsFileSystem : public FileSystem { Status GetMatchingPaths(const string& pattern, std::vector* result) override; + bool Match(const string& filename, const string& pattern) override; + Status Stat(const string& fname, FileStatistics* stat) override; Status DeleteFile(const string& fname) override;