experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Export the events_writer functions from C++ to Python with pybind11 instead of swig. This is part of a larger effort to deprecate swig and eventually with modularization break pywrap_tensorflow into smaller components. It will also make exporting C++ ops to Python significantly easier. XLA is using the pybind11 macros already. Please refer to https://github.com/tensorflow/community/blob/master/rfcs/20190208-pybind11.md for more information.

Browse Source 
We are slightly changing the behavior of EventsWriter. Rather than returning a bad status when writing to a deleted file which is often ignored by the user, we are removing the FileStillExists check.

TensorBoard has already been prepared for this change via: https://github.com/tensorflow/tensorboard/pull/2683

PiperOrigin-RevId: 270945727
This commit is contained in:
Amit Patankar 2019-09-24 11:14:39 -07:00 committed by TensorFlower Gardener
parent e9c8a604cf
commit 98f2265343
11 changed files with 87 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
tensorflow/core/util/events_writer.cc
View File

@ -131,18 +131,6 @@ Status EventsWriter::Flush() {
TF_RETURN_WITH_CONTEXT_IF_ERROR(recordio_file_->Sync(), "Failed to sync ",
num_outstanding_events_, " events to ",
filename_);
// The FileStillExists() condition is necessary because
// recordio_writer_->Sync() can return OK even if the underlying
// file has been deleted. EventWriter.FileDeletionBeforeWriting
// demonstrates this and will fail if the FileHasDisappeared()
// condition is removed.
// Also, we deliberately attempt to Sync() before checking for a
// disappearing file, in case for some file system File::Exists() is
// false after File::Open() but before File::Sync().
TF_RETURN_WITH_CONTEXT_IF_ERROR(FileStillExists(), "Failed to flush ",
num_outstanding_events_, " events to ",
filename_);
VLOG(1) << "Wrote " << num_outstanding_events_ << " events to disk.";
num_outstanding_events_ = 0;
return Status::OK();

8
tensorflow/core/util/events_writer_test.cc
View File

@ -142,9 +142,7 @@ TEST(EventWriter, FailFlush) {
WriteFile(&writer);
TF_EXPECT_OK(env()->FileExists(filename));
TF_ASSERT_OK(env()->DeleteFile(filename));
EXPECT_EQ(errors::Code::NOT_FOUND, env()->FileExists(filename).code());
EXPECT_FALSE(writer.Flush().ok());
EXPECT_EQ(errors::Code::NOT_FOUND, env()->FileExists(filename).code());
EXPECT_TRUE(writer.Flush().ok());
}
TEST(EventWriter, FailClose) {
@ -154,9 +152,7 @@ TEST(EventWriter, FailClose) {
WriteFile(&writer);
TF_EXPECT_OK(env()->FileExists(filename));
TF_ASSERT_OK(env()->DeleteFile(filename));
EXPECT_EQ(errors::Code::NOT_FOUND, env()->FileExists(filename).code());
EXPECT_FALSE(writer.Close().ok());
EXPECT_EQ(errors::Code::NOT_FOUND, env()->FileExists(filename).code());
EXPECT_TRUE(writer.Close().ok());
}
TEST(EventWriter, InitWriteClose) {

22
tensorflow/python/BUILD
View File

@ -98,6 +98,7 @@ py_library(
"//third_party/py/tensorflow_core:__subpackages__",
],
deps = [
":_pywrap_events_writer",
":_pywrap_stat_summarizer",
":_pywrap_tfprof",
":_pywrap_util_port",
@ -463,7 +464,7 @@ tf_python_pybind_extension(
],
)
pybind_extension(
tf_python_pybind_extension(
name = "_pywrap_util_port",
srcs = ["util/port_wrapper.cc"],
hdrs = ["//tensorflow/core:util_port_hdrs"],
@ -475,6 +476,23 @@ pybind_extension(
],
)
tf_python_pybind_extension(
name = "_pywrap_events_writer",
srcs = ["client/events_writer_wrapper.cc"],
module_name = "_pywrap_events_writer",
deps = [
":pybind11_absl",
":pybind11_proto",
":pybind11_status",
"//tensorflow/core:framework",
"//tensorflow/core:lib",
"//tensorflow/core:protos_all_cc",
"//third_party/python_runtime:headers",
"@com_google_absl//absl/strings",
"@pybind11",
],
)
cc_library(
name = "cpp_python_util",
srcs = ["util/util.cc"],
@ -784,6 +802,7 @@ py_library(
],
srcs_version = "PY2AND3",
deps = [
":_pywrap_events_writer",
":_pywrap_stat_summarizer",
":_pywrap_tfprof",
":_pywrap_util_port",
@ -4990,7 +5009,6 @@ tf_py_wrap_cc(
srcs = ["tensorflow.i"],
swig_includes = [
"client/device_lib.i",
"client/events_writer.i",
"client/tf_session.i",
"client/tf_sessionrun_wrapper.i",
"framework/python_op_gen.i",

1
tensorflow/python/__init__.py
View File

@ -49,6 +49,7 @@ import numpy as np
from tensorflow.python import pywrap_tensorflow
from tensorflow.python import _pywrap_utils
from tensorflow.python import _pywrap_tfprof
from tensorflow.python import _pywrap_events_writer
from tensorflow.python import _pywrap_util_port
from tensorflow.python import _pywrap_stat_summarizer

54
tensorflow/python/client/events_writer.i
View File

@ -1,54 +0,0 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
%include "tensorflow/python/lib/core/strings.i"
%include "tensorflow/python/platform/base.i"
%{
#include "tensorflow/core/util/events_writer.h"
#include "tensorflow/core/util/event.pb.h"
%}
%nodefaultctor EventsWriter;
%ignore tensorflow::Status::operator=;
%include "tensorflow/core/lib/core/status.h"
%ignoreall
%unignore tensorflow;
%unignore tensorflow::EventsWriter;
%unignore tensorflow::EventsWriter::EventsWriter;
%unignore tensorflow::EventsWriter::~EventsWriter;
%unignore tensorflow::EventsWriter::InitWithSuffix;
%unignore tensorflow::EventsWriter::FileName;
%rename("_WriteSerializedEvent") tensorflow::EventsWriter::WriteSerializedEvent;
%unignore tensorflow::EventsWriter::Flush;
%unignore tensorflow::EventsWriter::Close;
%include "tensorflow/core/util/events_writer.h"
%unignoreall
%newobject tensorflow::EventsWriter::EventsWriter;
%extend tensorflow::EventsWriter {
%insert("python") %{
def WriteEvent(self, event):
from tensorflow.core.util.event_pb2 import Event
if not isinstance(event, Event):
raise TypeError("Expected an event_pb2.Event proto, "
" but got %s" % type(event))
return self._WriteSerializedEvent(event.SerializeToString())
%}
}

6
tensorflow/python/client/events_writer_test.py
View File

@ -22,7 +22,7 @@ import os.path
from tensorflow.core.framework import summary_pb2
from tensorflow.core.util import event_pb2
from tensorflow.python import pywrap_tensorflow
from tensorflow.python import _pywrap_events_writer
from tensorflow.python.framework import errors
from tensorflow.python.framework import test_util
from tensorflow.python.lib.io import tf_record
@ -34,7 +34,7 @@ class PywrapeventsWriterTest(test_util.TensorFlowTestCase):
def testWriteEvents(self):
file_prefix = os.path.join(self.get_temp_dir(), "events")
writer = pywrap_tensorflow.EventsWriter(compat.as_bytes(file_prefix))
writer = _pywrap_events_writer.EventsWriter(compat.as_bytes(file_prefix))
filename = compat.as_text(writer.FileName())
event_written = event_pb2.Event(
wall_time=123.45,
@ -74,7 +74,7 @@ class PywrapeventsWriterTest(test_util.TensorFlowTestCase):
return "Invalid"
with self.assertRaisesRegexp(TypeError, "Invalid"):
pywrap_tensorflow.EventsWriter(b"foo").WriteEvent(_Invalid())
_pywrap_events_writer.EventsWriter(b"foo").WriteEvent(_Invalid())
if __name__ == "__main__":

49
tensorflow/python/client/events_writer_wrapper.cc Normal file
View File

@ -0,0 +1,49 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "absl/strings/string_view.h"
#include "include/pybind11/pybind11.h"
#include "include/pybind11/pytypes.h"
#include "tensorflow/core/util/events_writer.h"
#include "tensorflow/python/lib/core/pybind11_absl.h"
#include "tensorflow/python/lib/core/pybind11_proto.h"
#include "tensorflow/python/lib/core/pybind11_status.h"
namespace py = pybind11;
PYBIND11_MODULE(_pywrap_events_writer, m) {
py::class_<tensorflow::EventsWriter> events_writer_class(m, "EventsWriter");
events_writer_class.def(py::init<const std::string&>())
.def("InitWithSuffix",
[](tensorflow::EventsWriter& self, const std::string& suffix) {
return self.InitWithSuffix(suffix);
})
.def("FileName",
[](tensorflow::EventsWriter& self) { return self.FileName(); })
.def("_WriteSerializedEvent",
[](tensorflow::EventsWriter& self,
const absl::string_view event_str) {
self.WriteSerializedEvent(event_str);
})
.def("Flush", [](tensorflow::EventsWriter& self) { return self.Flush(); })
.def("Close", [](tensorflow::EventsWriter& self) { return self.Close(); })
.def("WriteEvent",
[](tensorflow::EventsWriter& self, const py::object obj) {
// Verify the proto type is an event prior to writing.
tensorflow::CheckProtoType(obj, "tensorflow.Event");
self.WriteSerializedEvent(
obj.attr("SerializeToString")().cast<std::string>());
});
};

3
tensorflow/python/eager/profiler.py
View File

@ -38,6 +38,7 @@ import datetime
import os
import threading
from tensorflow.python import _pywrap_events_writer
from tensorflow.python import pywrap_tensorflow
from tensorflow.python.eager import context
from tensorflow.python.framework import c_api_util
@ -122,7 +123,7 @@ def maybe_create_event_file(logdir):
if file_name.endswith(_EVENT_FILE_SUFFIX):
return
# TODO(b/127330388): Use summary_ops_v2.create_file_writer instead.
event_writer = pywrap_tensorflow.EventsWriter(
event_writer = _pywrap_events_writer.EventsWriter(
compat.as_bytes(os.path.join(logdir, 'events')))
event_writer.InitWithSuffix(compat.as_bytes(_EVENT_FILE_SUFFIX))

4
tensorflow/python/summary/writer/event_file_writer.py
View File

@ -25,7 +25,7 @@ import time
import six
from tensorflow.core.util import event_pb2
from tensorflow.python import pywrap_tensorflow
from tensorflow.python import _pywrap_events_writer
from tensorflow.python.platform import gfile
from tensorflow.python.util import compat
@ -66,7 +66,7 @@ class EventFileWriter(object):
if not gfile.IsDirectory(self._logdir):
gfile.MakeDirs(self._logdir)
self._event_queue = six.moves.queue.Queue(max_queue)
self._ev_writer = pywrap_tensorflow.EventsWriter(
self._ev_writer = _pywrap_events_writer.EventsWriter(
compat.as_bytes(os.path.join(self._logdir, "events")))
self._flush_secs = flush_secs
self._sentinel_event = self._get_sentinel_event()

1
tensorflow/python/tensorflow.i
View File

@ -26,7 +26,6 @@ limitations under the License.
%include "tensorflow/python/lib/io/py_record_reader.i"
%include "tensorflow/python/lib/io/py_record_writer.i"
%include "tensorflow/python/client/events_writer.i"
%include "tensorflow/python/client/tf_session.i"
%include "tensorflow/python/client/device_lib.i"

8
tensorflow/tools/def_file_filter/symbols_pybind.txt
View File

@ -44,3 +44,11 @@ tensorflow::grappler::graph_analyzer::GraphAnalyzerTool
[bfloat16_lib] # bfloat16
tensorflow::RegisterNumpyBfloat16
tensorflow::Bfloat16PyType
[events_writer] # events_writer
tensorflow::EventsWriter::Init
tensorflow::EventsWriter::InitWithSuffix
tensorflow::EventsWriter::WriteSerializedEvent
tensorflow::EventsWriter::Flush
tensorflow::EventsWriter::Close