experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Workarounds for the compiler issues we face when moving tensorflow::Status from core/lib/core to core:platform.

Browse Source 
SWIG does not seem to correctly infer the namespaces. So the workaround is to explicitly mark the Status objects.

PiperOrigin-RevId: 277819981
Change-Id: If43340fcbfd68751c083445b1371d37ea73b0c3b
This commit is contained in:
Gunhan Gulsoy 2019-10-31 16:00:52 -07:00 committed by TensorFlower Gardener
parent ef9eb1cf64
commit fe1c95dee3
3 changed files with 57 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
tensorflow/c/tf_status_helper.h
View File

@ -22,10 +22,11 @@ limitations under the License.
namespace tensorflow { namespace tensorflow {
// Set the attribute of "tf_status" from the attributes of "status". // Set the attribute of "tf_status" from the attributes of "status".
void Set_TF_Status_from_Status(TF_Status* tf_status, const Status& status); void Set_TF_Status_from_Status(TF_Status* tf_status,
const tensorflow::Status& status);
// Returns a "status" from "tf_status". // Returns a "status" from "tf_status".
Status StatusFromTF_Status(const TF_Status* tf_status); tensorflow::Status StatusFromTF_Status(const TF_Status* tf_status);
} // namespace tensorflow } // namespace tensorflow

24
tensorflow/core/lib/io/buffered_inputstream.h
View File

@ -41,9 +41,9 @@ class BufferedInputStream : public InputStreamInterface {
~BufferedInputStream() override; ~BufferedInputStream() override;
Status ReadNBytes(int64 bytes_to_read, tstring* result) override; tensorflow::Status ReadNBytes(int64 bytes_to_read, tstring* result) override;
Status SkipNBytes(int64 bytes_to_skip) override; tensorflow::Status SkipNBytes(int64 bytes_to_skip) override;
int64 Tell() const override; int64 Tell() const override;
@ -56,7 +56,7 @@ class BufferedInputStream : public InputStreamInterface {
// Note: When seeking backwards in a stream, this implementation uses // Note: When seeking backwards in a stream, this implementation uses
// Reset() + SkipNBytes(), so its performance will be dependent // Reset() + SkipNBytes(), so its performance will be dependent
// largely on the performance of SkipNBytes(). // largely on the performance of SkipNBytes().
Status Seek(int64 position); tensorflow::Status Seek(int64 position);
// Read one text line of data into "*result" until end-of-file or a // Read one text line of data into "*result" until end-of-file or a
// \n is read. (The \n is not included in the result.) Overwrites // \n is read. (The \n is not included in the result.) Overwrites
@ -65,7 +65,7 @@ class BufferedInputStream : public InputStreamInterface {
// If successful, returns OK. If we are already at the end of the // If successful, returns OK. If we are already at the end of the
// file, we return an OUT_OF_RANGE error. Otherwise, we return // file, we return an OUT_OF_RANGE error. Otherwise, we return
// some other non-OK status. // some other non-OK status.
Status ReadLine(string* result); tensorflow::Status ReadLine(string* result);
// Returns one text line of data until end-of-file or a '\n' is read. The '\n' // Returns one text line of data until end-of-file or a '\n' is read. The '\n'
// is included in the result. // is included in the result.
@ -80,13 +80,13 @@ class BufferedInputStream : public InputStreamInterface {
// Note: the amount of memory used by this function call is unbounded, so only // Note: the amount of memory used by this function call is unbounded, so only
// use in ops that expect that behavior. // use in ops that expect that behavior.
template <typename T> template <typename T>
Status ReadAll(T* result); tensorflow::Status ReadAll(T* result);
Status Reset() override; tensorflow::Status Reset() override;
private: private:
Status FillBuffer(); tensorflow::Status FillBuffer();
Status ReadLineHelper(string* result, bool include_eol); tensorflow::Status ReadLineHelper(string* result, bool include_eol);
InputStreamInterface* input_stream_; // not owned. InputStreamInterface* input_stream_; // not owned.
size_t size_; // buffer size. size_t size_; // buffer size.
@ -97,16 +97,18 @@ class BufferedInputStream : public InputStreamInterface {
bool owns_input_stream_ = false; bool owns_input_stream_ = false;
// When EoF is reached, file_status_ contains the status to skip unnecessary // When EoF is reached, file_status_ contains the status to skip unnecessary
// buffer allocations. // buffer allocations.
Status file_status_ = Status::OK(); tensorflow::Status file_status_ = Status::OK();
TF_DISALLOW_COPY_AND_ASSIGN(BufferedInputStream); TF_DISALLOW_COPY_AND_ASSIGN(BufferedInputStream);
}; };
// Explicit instantiations defined in buffered_inputstream.cc. // Explicit instantiations defined in buffered_inputstream.cc.
#ifndef SWIG #ifndef SWIG
extern template Status BufferedInputStream::ReadAll<string>(string* result); extern template tensorflow::Status BufferedInputStream::ReadAll<string>(
string* result);
#ifdef USE_TSTRING #ifdef USE_TSTRING
extern template Status BufferedInputStream::ReadAll<tstring>(tstring* result); extern template tensorflow::Status BufferedInputStream::ReadAll<tstring>(
tstring* result);
#endif // USE_TSTRING #endif // USE_TSTRING
#endif // SWIG #endif // SWIG

78
tensorflow/core/platform/file_system.h
View File

@ -24,7 +24,6 @@ limitations under the License.
#include <vector> #include <vector>
#include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/platform/cord.h" #include "tensorflow/core/platform/cord.h"
#include "tensorflow/core/platform/file_statistics.h" #include "tensorflow/core/platform/file_statistics.h"
#include "tensorflow/core/platform/macros.h" #include "tensorflow/core/platform/macros.h"
@ -60,7 +59,7 @@ class FileSystem {
/// ///
/// The ownership of the returned RandomAccessFile is passed to the caller /// The ownership of the returned RandomAccessFile is passed to the caller
/// and the object should be deleted when is not used. /// and the object should be deleted when is not used.
virtual Status NewRandomAccessFile( virtual tensorflow::Status NewRandomAccessFile(
const string& fname, std::unique_ptr<RandomAccessFile>* result) = 0; const string& fname, std::unique_ptr<RandomAccessFile>* result) = 0;
/// \brief Creates an object that writes to a new file with the specified /// \brief Creates an object that writes to a new file with the specified
@ -75,8 +74,8 @@ class FileSystem {
/// ///
/// The ownership of the returned WritableFile is passed to the caller /// The ownership of the returned WritableFile is passed to the caller
/// and the object should be deleted when is not used. /// and the object should be deleted when is not used.
virtual Status NewWritableFile(const string& fname, virtual tensorflow::Status NewWritableFile(
std::unique_ptr<WritableFile>* result) = 0; const string& fname, std::unique_ptr<WritableFile>* result) = 0;
/// \brief Creates an object that either appends to an existing file, or /// \brief Creates an object that either appends to an existing file, or
/// writes to a new file (if the file does not exist to begin with). /// writes to a new file (if the file does not exist to begin with).
@ -89,8 +88,8 @@ class FileSystem {
/// ///
/// The ownership of the returned WritableFile is passed to the caller /// The ownership of the returned WritableFile is passed to the caller
/// and the object should be deleted when is not used. /// and the object should be deleted when is not used.
virtual Status NewAppendableFile(const string& fname, virtual tensorflow::Status NewAppendableFile(
std::unique_ptr<WritableFile>* result) = 0; const string& fname, std::unique_ptr<WritableFile>* result) = 0;
/// \brief Creates a readonly region of memory with the file context. /// \brief Creates a readonly region of memory with the file context.
/// ///
@ -102,11 +101,11 @@ class FileSystem {
/// ///
/// The ownership of the returned ReadOnlyMemoryRegion is passed to the caller /// The ownership of the returned ReadOnlyMemoryRegion is passed to the caller
/// and the object should be deleted when is not used. /// and the object should be deleted when is not used.
virtual Status NewReadOnlyMemoryRegionFromFile( virtual tensorflow::Status NewReadOnlyMemoryRegionFromFile(
const string& fname, std::unique_ptr<ReadOnlyMemoryRegion>* result) = 0; const string& fname, std::unique_ptr<ReadOnlyMemoryRegion>* result) = 0;
/// Returns OK if the named path exists and NOT_FOUND otherwise. /// Returns OK if the named path exists and NOT_FOUND otherwise.
virtual Status FileExists(const string& fname) = 0; virtual tensorflow::Status FileExists(const string& fname) = 0;
/// Returns true if all the listed files exist, false otherwise. /// Returns true if all the listed files exist, false otherwise.
/// if status is not null, populate the vector with a detailed status /// if status is not null, populate the vector with a detailed status
@ -117,8 +116,8 @@ class FileSystem {
/// \brief Returns the immediate children in the given directory. /// \brief Returns the immediate children in the given directory.
/// ///
/// The returned paths are relative to 'dir'. /// The returned paths are relative to 'dir'.
virtual Status GetChildren(const string& dir, virtual tensorflow::Status GetChildren(const string& dir,
std::vector<string>* result) = 0; std::vector<string>* result) = 0;
/// \brief Given a pattern, stores in *results the set of paths that matches /// \brief Given a pattern, stores in *results the set of paths that matches
/// that pattern. *results is cleared. /// that pattern. *results is cleared.
@ -142,21 +141,22 @@ class FileSystem {
/// * OK - no errors /// * OK - no errors
/// * UNIMPLEMENTED - Some underlying functions (like GetChildren) are not /// * UNIMPLEMENTED - Some underlying functions (like GetChildren) are not
/// implemented /// implemented
virtual Status GetMatchingPaths(const string& pattern, virtual tensorflow::Status GetMatchingPaths(const string& pattern,
std::vector<string>* results) = 0; std::vector<string>* results) = 0;
/// \brief Obtains statistics for the given path. /// \brief Obtains statistics for the given path.
virtual Status Stat(const string& fname, FileStatistics* stat) = 0; virtual tensorflow::Status Stat(const string& fname,
FileStatistics* stat) = 0;
/// \brief Deletes the named file. /// \brief Deletes the named file.
virtual Status DeleteFile(const string& fname) = 0; virtual tensorflow::Status DeleteFile(const string& fname) = 0;
/// \brief Creates the specified directory. /// \brief Creates the specified directory.
/// Typical return codes: /// Typical return codes:
/// * OK - successfully created the directory. /// * OK - successfully created the directory.
/// * ALREADY_EXISTS - directory with name dirname already exists. /// * ALREADY_EXISTS - directory with name dirname already exists.
/// * PERMISSION_DENIED - dirname is not writable. /// * PERMISSION_DENIED - dirname is not writable.
virtual Status CreateDir(const string& dirname) = 0; virtual tensorflow::Status CreateDir(const string& dirname) = 0;
/// \brief Creates the specified directory and all the necessary /// \brief Creates the specified directory and all the necessary
/// subdirectories. /// subdirectories.
@ -164,10 +164,10 @@ class FileSystem {
/// * OK - successfully created the directory and sub directories, even if /// * OK - successfully created the directory and sub directories, even if
/// they were already created. /// they were already created.
/// * PERMISSION_DENIED - dirname or some subdirectory is not writable. /// * PERMISSION_DENIED - dirname or some subdirectory is not writable.
virtual Status RecursivelyCreateDir(const string& dirname); virtual tensorflow::Status RecursivelyCreateDir(const string& dirname);
/// \brief Deletes the specified directory. /// \brief Deletes the specified directory.
virtual Status DeleteDir(const string& dirname) = 0; virtual tensorflow::Status DeleteDir(const string& dirname) = 0;
/// \brief Deletes the specified directory and all subdirectories and files /// \brief Deletes the specified directory and all subdirectories and files
/// underneath it. This is accomplished by traversing the directory tree /// underneath it. This is accomplished by traversing the directory tree
@ -193,18 +193,20 @@ class FileSystem {
/// * PERMISSION_DENIED - dirname or some descendant is not writable /// * PERMISSION_DENIED - dirname or some descendant is not writable
/// * UNIMPLEMENTED - Some underlying functions (like Delete) are not /// * UNIMPLEMENTED - Some underlying functions (like Delete) are not
/// implemented /// implemented
virtual Status DeleteRecursively(const string& dirname, virtual tensorflow::Status DeleteRecursively(const string& dirname,
int64* undeleted_files, int64* undeleted_files,
int64* undeleted_dirs); int64* undeleted_dirs);
/// \brief Stores the size of `fname` in `*file_size`. /// \brief Stores the size of `fname` in `*file_size`.
virtual Status GetFileSize(const string& fname, uint64* file_size) = 0; virtual tensorflow::Status GetFileSize(const string& fname,
uint64* file_size) = 0;
/// \brief Overwrites the target if it exists. /// \brief Overwrites the target if it exists.
virtual Status RenameFile(const string& src, const string& target) = 0; virtual tensorflow::Status RenameFile(const string& src,
const string& target) = 0;
/// \brief Copy the src to target. /// \brief Copy the src to target.
virtual Status CopyFile(const string& src, const string& target); virtual tensorflow::Status CopyFile(const string& src, const string& target);
/// \brief Translate an URI to a filename for the FileSystem implementation. /// \brief Translate an URI to a filename for the FileSystem implementation.
/// ///
@ -221,7 +223,7 @@ class FileSystem {
/// * NOT_FOUND - The path entry does not exist. /// * NOT_FOUND - The path entry does not exist.
/// * PERMISSION_DENIED - Insufficient permissions. /// * PERMISSION_DENIED - Insufficient permissions.
/// * UNIMPLEMENTED - The file factory doesn't support directories. /// * UNIMPLEMENTED - The file factory doesn't support directories.
virtual Status IsDirectory(const string& fname); virtual tensorflow::Status IsDirectory(const string& fname);
/// \brief Flushes any cached filesystem objects from memory. /// \brief Flushes any cached filesystem objects from memory.
virtual void FlushCaches(); virtual void FlushCaches();
@ -241,7 +243,7 @@ class RandomAccessFile {
/// ///
/// This is an optional operation that may not be implemented by every /// This is an optional operation that may not be implemented by every
/// filesystem. /// filesystem.
virtual Status Name(StringPiece* result) const { virtual tensorflow::Status Name(StringPiece* result) const {
return errors::Unimplemented("This filesystem does not support Name()"); return errors::Unimplemented("This filesystem does not support Name()");
} }
@ -260,13 +262,14 @@ class RandomAccessFile {
/// because of EOF. /// because of EOF.
/// ///
/// Safe for concurrent use by multiple threads. /// Safe for concurrent use by multiple threads.
virtual Status Read(uint64 offset, size_t n, StringPiece* result, virtual tensorflow::Status Read(uint64 offset, size_t n, StringPiece* result,
char* scratch) const = 0; char* scratch) const = 0;
// TODO(ebrevdo): Remove this ifdef when absl is updated. // TODO(ebrevdo): Remove this ifdef when absl is updated.
#if defined(PLATFORM_GOOGLE) #if defined(PLATFORM_GOOGLE)
/// \brief Read up to `n` bytes from the file starting at `offset`. /// \brief Read up to `n` bytes from the file starting at `offset`.
virtual Status Read(uint64 offset, size_t n, absl::Cord* cord) const { virtual tensorflow::Status Read(uint64 offset, size_t n,
absl::Cord* cord) const {
return errors::Unimplemented( return errors::Unimplemented(
"Read(uint64, size_t, absl::Cord*) is not " "Read(uint64, size_t, absl::Cord*) is not "
"implemented"); "implemented");
@ -287,12 +290,12 @@ class WritableFile {
virtual ~WritableFile(); virtual ~WritableFile();
/// \brief Append 'data' to the file. /// \brief Append 'data' to the file.
virtual Status Append(StringPiece data) = 0; virtual tensorflow::Status Append(StringPiece data) = 0;
// TODO(ebrevdo): Remove this ifdef when absl is updated. // TODO(ebrevdo): Remove this ifdef when absl is updated.
#if defined(PLATFORM_GOOGLE) #if defined(PLATFORM_GOOGLE)
// \brief Append 'data' to the file. // \brief Append 'data' to the file.
virtual Status Append(const absl::Cord& cord) { virtual tensorflow::Status Append(const absl::Cord& cord) {
return errors::Unimplemented("Append(absl::Cord) is not implemented"); return errors::Unimplemented("Append(absl::Cord) is not implemented");
} }
#endif #endif
@ -304,7 +307,7 @@ class WritableFile {
/// Typical return codes (not guaranteed to be exhaustive): /// Typical return codes (not guaranteed to be exhaustive):
/// * OK /// * OK
/// * Other codes, as returned from Flush() /// * Other codes, as returned from Flush()
virtual Status Close() = 0; virtual tensorflow::Status Close() = 0;
/// \brief Flushes the file and optionally syncs contents to filesystem. /// \brief Flushes the file and optionally syncs contents to filesystem.
/// ///
@ -316,13 +319,13 @@ class WritableFile {
/// eventually flush the contents. If the OS or machine crashes /// eventually flush the contents. If the OS or machine crashes
/// after a successful flush, the contents may or may not be /// after a successful flush, the contents may or may not be
/// persisted, depending on the implementation. /// persisted, depending on the implementation.
virtual Status Flush() = 0; virtual tensorflow::Status Flush() = 0;
// \brief Returns the name of the file. // \brief Returns the name of the file.
/// ///
/// This is an optional operation that may not be implemented by every /// This is an optional operation that may not be implemented by every
/// filesystem. /// filesystem.
virtual Status Name(StringPiece* result) const { virtual tensorflow::Status Name(StringPiece* result) const {
return errors::Unimplemented("This filesystem does not support Name()"); return errors::Unimplemented("This filesystem does not support Name()");
} }
@ -332,14 +335,14 @@ class WritableFile {
/// of the file have been persisted to the filesystem; if the OS /// of the file have been persisted to the filesystem; if the OS
/// or machine crashes after a successful Sync, the contents should /// or machine crashes after a successful Sync, the contents should
/// be properly saved. /// be properly saved.
virtual Status Sync() = 0; virtual tensorflow::Status Sync() = 0;
/// \brief Retrieves the current write position in the file, or -1 on /// \brief Retrieves the current write position in the file, or -1 on
/// error. /// error.
/// ///
/// This is an optional operation, subclasses may choose to return /// This is an optional operation, subclasses may choose to return
/// errors::Unimplemented. /// errors::Unimplemented.
virtual Status Tell(int64* position) { virtual tensorflow::Status Tell(int64* position) {
*position = -1; *position = -1;
return errors::Unimplemented("This filesystem does not support Tell()"); return errors::Unimplemented("This filesystem does not support Tell()");
} }
@ -375,9 +378,10 @@ class FileSystemRegistry {
typedef std::function<FileSystem*()> Factory; typedef std::function<FileSystem*()> Factory;
virtual ~FileSystemRegistry(); virtual ~FileSystemRegistry();
virtual Status Register(const string& scheme, Factory factory) = 0; virtual tensorflow::Status Register(const string& scheme,
Factory factory) = 0;
virtual FileSystem* Lookup(const string& scheme) = 0; virtual FileSystem* Lookup(const string& scheme) = 0;
virtual Status GetRegisteredFileSystemSchemes( virtual tensorflow::Status GetRegisteredFileSystemSchemes(
std::vector<string>* schemes) = 0; std::vector<string>* schemes) = 0;
}; };