experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Don't load events until all runs have been added.

Browse Source 
This means that the runs will show up in TensorBoard even though they won't have
events yet (since we still only load from one event at a time). This requires us
to disable the activation checking logic in EventAccumulator.
Change: 123252077
This commit is contained in:
A. Unique TensorFlower 2016-05-25 13:13:37 -08:00 committed by TensorFlower Gardener
parent 5c145f0e3c
commit 22791b144c
4 changed files with 4 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
tensorflow/python/summary/event_accumulator.py
View File

@ -114,8 +114,7 @@ class EventAccumulator(object):
`Accumulator.Scalars(tag)`) allow for the retrieval of all data
associated with that tag.
Before usage, the `EventAccumulator` must be activated via `Reload()`. This
method synchronosly loads all of the data written so far.
The `Reload()` method synchronously loads all of the data written so far.
Histograms, audio, and images are very large, so storing all of them is not
recommended.
@ -175,7 +174,6 @@ class EventAccumulator(object):
self._compression_bps = compression_bps
self.purge_orphaned_data = purge_orphaned_data
self._activated = False
self.most_recent_step = -1
self.most_recent_wall_time = -1
self.file_version = None
@ -188,12 +186,10 @@ class EventAccumulator(object):
"""Loads all events added since the last call to `Reload`.
If `Reload` was never called, loads all events in the file.
Calling `Reload` activates the `EventAccumulator`.
Returns:
The `EventAccumulator`.
"""
self._activated = True
with self._generator_mutex:
for event in self._generator.Load():
if event.HasField('file_version'):
@ -232,13 +228,9 @@ class EventAccumulator(object):
def Tags(self):
"""Return all tags found in the value stream.
Raises:
RuntimeError: If the `EventAccumulator` has not been activated.
Returns:
A `{tagType: ['list', 'of', 'tags']}` dictionary.
"""
self._VerifyActivated()
return {IMAGES: self._images.Keys(),
AUDIO: self._audio.Keys(),
HISTOGRAMS: self._histograms.Keys(),
@ -255,12 +247,10 @@ class EventAccumulator(object):
Raises:
KeyError: If the tag is not found.
RuntimeError: If the `EventAccumulator` has not been activated.
Returns:
An array of `ScalarEvent`s.
"""
self._VerifyActivated()
return self._scalars.Items(tag)
def Graph(self):
@ -268,12 +258,10 @@ class EventAccumulator(object):
Raises:
ValueError: If there is no graph for this run.
RuntimeError: If the `EventAccumulator` has not been activated.
Returns:
The `graph_def` proto.
"""
self._VerifyActivated()
if self._graph is None:
raise ValueError('There is no graph in this EventAccumulator')
graph = graph_pb2.GraphDef()
@ -288,12 +276,10 @@ class EventAccumulator(object):
Raises:
ValueError: If the tag is not found.
RuntimeError: If the `EventAccumulator` has not been activated.
Returns:
The metadata in form of `RunMetadata` proto.
"""
self._VerifyActivated()
if tag not in self._tagged_metadata:
raise ValueError('There is no run metadata with this tag name')
@ -309,12 +295,10 @@ class EventAccumulator(object):
Raises:
KeyError: If the tag is not found.
RuntimeError: If the `EventAccumulator` has not been activated.
Returns:
An array of `HistogramEvent`s.
"""
self._VerifyActivated()
return self._histograms.Items(tag)
def CompressedHistograms(self, tag):
@ -325,12 +309,10 @@ class EventAccumulator(object):
Raises:
KeyError: If the tag is not found.
RuntimeError: If the `EventAccumulator` has not been activated.
Returns:
An array of `CompressedHistogramEvent`s.
"""
self._VerifyActivated()
return self._compressed_histograms.Items(tag)
def Images(self, tag):
@ -341,12 +323,10 @@ class EventAccumulator(object):
Raises:
KeyError: If the tag is not found.
RuntimeError: If the `EventAccumulator` has not been activated.
Returns:
An array of `ImageEvent`s.
"""
self._VerifyActivated()
return self._images.Items(tag)
def Audio(self, tag):
@ -357,12 +337,10 @@ class EventAccumulator(object):
Raises:
KeyError: If the tag is not found.
RuntimeError: If the `EventAccumulator` has not been activated.
Returns:
An array of `AudioEvent`s.
"""
self._VerifyActivated()
return self._audio.Items(tag)
def _MaybePurgeOrphanedData(self, event):
@ -599,10 +577,6 @@ class EventAccumulator(object):
event.wall_time, *expired_per_type)
logging.warn(purge_msg)
def _VerifyActivated(self):
if not self._activated:
raise RuntimeError('Accumulator must be activated before it may be used.')
def _GetPurgeMessage(most_recent_step, most_recent_wall_time, event_step,
event_wall_time, num_expired_scalars, num_expired_histos,

12
tensorflow/python/summary/event_accumulator_test.py
View File

@ -456,18 +456,6 @@ class MockingEventAccumulatorTest(EventAccumulatorTest):
self.assertEqual(acc.Audio('snd1'), [snd1])
self.assertEqual(acc.Audio('snd2'), [snd2])
def testActivation(self):
gen = _EventGenerator()
acc = ea.EventAccumulator(gen)
self.assertFalse(acc._activated)
with self.assertRaises(RuntimeError):
acc.Tags()
with self.assertRaises(RuntimeError):
acc.Scalars('s1')
acc.Reload()
self.assertTrue(acc._activated)
acc._activated = False
def testKeyError(self):
gen = _EventGenerator()
acc = ea.EventAccumulator(gen)

10
tensorflow/python/summary/event_multiplexer.py
View File

@ -113,8 +113,7 @@ class EventMultiplexer(object):
accumulator.
If `Reload` has been called, it will `Reload` the newly created
accumulators. This maintains the invariant that once the Multiplexer was
activated, all of its accumulators are active.
accumulators.
Args:
path: Path to the event files (or event directory) for given run.
@ -199,7 +198,6 @@ class EventMultiplexer(object):
Raises:
KeyError: If the run is not found, or the tag is not available for
the given run.
RuntimeError: If the run's `EventAccumulator` has not been activated.
Returns:
An array of `event_accumulator.ScalarEvents`.
@ -216,7 +214,6 @@ class EventMultiplexer(object):
Raises:
KeyError: If the run is not found.
ValueError: If the run does not have an associated graph.
RuntimeError: If the run's EventAccumulator has not been activated.
Returns:
The `graph_def` protobuf data structure.
@ -234,7 +231,6 @@ class EventMultiplexer(object):
Raises:
KeyError: If the run is not found, or the tag is not available for the
given run.
RuntimeError: If the run's EventAccumulator has not been activated.
Returns:
The metadata in the form of `RunMetadata` protobuf data structure.
@ -252,7 +248,6 @@ class EventMultiplexer(object):
Raises:
KeyError: If the run is not found, or the tag is not available for
the given run.
RuntimeError: If the run's `EventAccumulator` has not been activated.
Returns:
An array of `event_accumulator.HistogramEvents`.
@ -270,7 +265,6 @@ class EventMultiplexer(object):
Raises:
KeyError: If the run is not found, or the tag is not available for
the given run.
RuntimeError: If the run's EventAccumulator has not been activated.
Returns:
An array of `event_accumulator.CompressedHistogramEvents`.
@ -288,7 +282,6 @@ class EventMultiplexer(object):
Raises:
KeyError: If the run is not found, or the tag is not available for
the given run.
RuntimeError: If the run's `EventAccumulator` has not been activated.
Returns:
An array of `event_accumulator.ImageEvents`.
@ -306,7 +299,6 @@ class EventMultiplexer(object):
Raises:
KeyError: If the run is not found, or the tag is not available for
the given run.
RuntimeError: If the run's `EventAccumulator` has not been activated.
Returns:
An array of `event_accumulator.AudioEvents`.

7
tensorflow/tensorboard/backend/server.py
View File

@ -120,12 +120,9 @@ def StartMultiplexerReloadingThread(multiplexer, path_to_run, load_interval):
Returns:
A started `threading.Thread` that reloads the multiplexer.
"""
# Ensure the Multiplexer initializes in a loaded state before it adds runs
# So it can handle HTTP requests while runs are loading
multiplexer.Reload()
# We don't call multiplexer.Reload() here because that would make
# AddRunsFromDirectory block until the runs have all loaded.
for path in path_to_run.keys():
if gcs.IsGCSPath(path):
gcs.CheckIsSupported()