From 22791b144c9727de2bc14ff0326918caa28b64ad Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 25 May 2016 13:13:37 -0800 Subject: [PATCH] Don't load events until all runs have been added. This means that the runs will show up in TensorBoard even though they won't have events yet (since we still only load from one event at a time). This requires us to disable the activation checking logic in EventAccumulator. Change: 123252077 --- .../python/summary/event_accumulator.py | 28 +------------------ .../python/summary/event_accumulator_test.py | 12 -------- .../python/summary/event_multiplexer.py | 10 +------ tensorflow/tensorboard/backend/server.py | 7 ++--- 4 files changed, 4 insertions(+), 53 deletions(-) diff --git a/tensorflow/python/summary/event_accumulator.py b/tensorflow/python/summary/event_accumulator.py index 2ee8a369f76..204ed009129 100644 --- a/tensorflow/python/summary/event_accumulator.py +++ b/tensorflow/python/summary/event_accumulator.py @@ -114,8 +114,7 @@ class EventAccumulator(object): `Accumulator.Scalars(tag)`) allow for the retrieval of all data associated with that tag. - Before usage, the `EventAccumulator` must be activated via `Reload()`. This - method synchronosly loads all of the data written so far. + The `Reload()` method synchronously loads all of the data written so far. Histograms, audio, and images are very large, so storing all of them is not recommended. @@ -175,7 +174,6 @@ class EventAccumulator(object): self._compression_bps = compression_bps self.purge_orphaned_data = purge_orphaned_data - self._activated = False self.most_recent_step = -1 self.most_recent_wall_time = -1 self.file_version = None @@ -188,12 +186,10 @@ class EventAccumulator(object): """Loads all events added since the last call to `Reload`. If `Reload` was never called, loads all events in the file. - Calling `Reload` activates the `EventAccumulator`. Returns: The `EventAccumulator`. """ - self._activated = True with self._generator_mutex: for event in self._generator.Load(): if event.HasField('file_version'): @@ -232,13 +228,9 @@ class EventAccumulator(object): def Tags(self): """Return all tags found in the value stream. - Raises: - RuntimeError: If the `EventAccumulator` has not been activated. - Returns: A `{tagType: ['list', 'of', 'tags']}` dictionary. """ - self._VerifyActivated() return {IMAGES: self._images.Keys(), AUDIO: self._audio.Keys(), HISTOGRAMS: self._histograms.Keys(), @@ -255,12 +247,10 @@ class EventAccumulator(object): Raises: KeyError: If the tag is not found. - RuntimeError: If the `EventAccumulator` has not been activated. Returns: An array of `ScalarEvent`s. """ - self._VerifyActivated() return self._scalars.Items(tag) def Graph(self): @@ -268,12 +258,10 @@ class EventAccumulator(object): Raises: ValueError: If there is no graph for this run. - RuntimeError: If the `EventAccumulator` has not been activated. Returns: The `graph_def` proto. """ - self._VerifyActivated() if self._graph is None: raise ValueError('There is no graph in this EventAccumulator') graph = graph_pb2.GraphDef() @@ -288,12 +276,10 @@ class EventAccumulator(object): Raises: ValueError: If the tag is not found. - RuntimeError: If the `EventAccumulator` has not been activated. Returns: The metadata in form of `RunMetadata` proto. """ - self._VerifyActivated() if tag not in self._tagged_metadata: raise ValueError('There is no run metadata with this tag name') @@ -309,12 +295,10 @@ class EventAccumulator(object): Raises: KeyError: If the tag is not found. - RuntimeError: If the `EventAccumulator` has not been activated. Returns: An array of `HistogramEvent`s. """ - self._VerifyActivated() return self._histograms.Items(tag) def CompressedHistograms(self, tag): @@ -325,12 +309,10 @@ class EventAccumulator(object): Raises: KeyError: If the tag is not found. - RuntimeError: If the `EventAccumulator` has not been activated. Returns: An array of `CompressedHistogramEvent`s. """ - self._VerifyActivated() return self._compressed_histograms.Items(tag) def Images(self, tag): @@ -341,12 +323,10 @@ class EventAccumulator(object): Raises: KeyError: If the tag is not found. - RuntimeError: If the `EventAccumulator` has not been activated. Returns: An array of `ImageEvent`s. """ - self._VerifyActivated() return self._images.Items(tag) def Audio(self, tag): @@ -357,12 +337,10 @@ class EventAccumulator(object): Raises: KeyError: If the tag is not found. - RuntimeError: If the `EventAccumulator` has not been activated. Returns: An array of `AudioEvent`s. """ - self._VerifyActivated() return self._audio.Items(tag) def _MaybePurgeOrphanedData(self, event): @@ -599,10 +577,6 @@ class EventAccumulator(object): event.wall_time, *expired_per_type) logging.warn(purge_msg) - def _VerifyActivated(self): - if not self._activated: - raise RuntimeError('Accumulator must be activated before it may be used.') - def _GetPurgeMessage(most_recent_step, most_recent_wall_time, event_step, event_wall_time, num_expired_scalars, num_expired_histos, diff --git a/tensorflow/python/summary/event_accumulator_test.py b/tensorflow/python/summary/event_accumulator_test.py index f6b60b91db9..b154d853322 100644 --- a/tensorflow/python/summary/event_accumulator_test.py +++ b/tensorflow/python/summary/event_accumulator_test.py @@ -456,18 +456,6 @@ class MockingEventAccumulatorTest(EventAccumulatorTest): self.assertEqual(acc.Audio('snd1'), [snd1]) self.assertEqual(acc.Audio('snd2'), [snd2]) - def testActivation(self): - gen = _EventGenerator() - acc = ea.EventAccumulator(gen) - self.assertFalse(acc._activated) - with self.assertRaises(RuntimeError): - acc.Tags() - with self.assertRaises(RuntimeError): - acc.Scalars('s1') - acc.Reload() - self.assertTrue(acc._activated) - acc._activated = False - def testKeyError(self): gen = _EventGenerator() acc = ea.EventAccumulator(gen) diff --git a/tensorflow/python/summary/event_multiplexer.py b/tensorflow/python/summary/event_multiplexer.py index a0f4ef402f3..00eab3d215d 100644 --- a/tensorflow/python/summary/event_multiplexer.py +++ b/tensorflow/python/summary/event_multiplexer.py @@ -113,8 +113,7 @@ class EventMultiplexer(object): accumulator. If `Reload` has been called, it will `Reload` the newly created - accumulators. This maintains the invariant that once the Multiplexer was - activated, all of its accumulators are active. + accumulators. Args: path: Path to the event files (or event directory) for given run. @@ -199,7 +198,6 @@ class EventMultiplexer(object): Raises: KeyError: If the run is not found, or the tag is not available for the given run. - RuntimeError: If the run's `EventAccumulator` has not been activated. Returns: An array of `event_accumulator.ScalarEvents`. @@ -216,7 +214,6 @@ class EventMultiplexer(object): Raises: KeyError: If the run is not found. ValueError: If the run does not have an associated graph. - RuntimeError: If the run's EventAccumulator has not been activated. Returns: The `graph_def` protobuf data structure. @@ -234,7 +231,6 @@ class EventMultiplexer(object): Raises: KeyError: If the run is not found, or the tag is not available for the given run. - RuntimeError: If the run's EventAccumulator has not been activated. Returns: The metadata in the form of `RunMetadata` protobuf data structure. @@ -252,7 +248,6 @@ class EventMultiplexer(object): Raises: KeyError: If the run is not found, or the tag is not available for the given run. - RuntimeError: If the run's `EventAccumulator` has not been activated. Returns: An array of `event_accumulator.HistogramEvents`. @@ -270,7 +265,6 @@ class EventMultiplexer(object): Raises: KeyError: If the run is not found, or the tag is not available for the given run. - RuntimeError: If the run's EventAccumulator has not been activated. Returns: An array of `event_accumulator.CompressedHistogramEvents`. @@ -288,7 +282,6 @@ class EventMultiplexer(object): Raises: KeyError: If the run is not found, or the tag is not available for the given run. - RuntimeError: If the run's `EventAccumulator` has not been activated. Returns: An array of `event_accumulator.ImageEvents`. @@ -306,7 +299,6 @@ class EventMultiplexer(object): Raises: KeyError: If the run is not found, or the tag is not available for the given run. - RuntimeError: If the run's `EventAccumulator` has not been activated. Returns: An array of `event_accumulator.AudioEvents`. diff --git a/tensorflow/tensorboard/backend/server.py b/tensorflow/tensorboard/backend/server.py index cfdd6c56543..b025a2f5b9f 100644 --- a/tensorflow/tensorboard/backend/server.py +++ b/tensorflow/tensorboard/backend/server.py @@ -120,12 +120,9 @@ def StartMultiplexerReloadingThread(multiplexer, path_to_run, load_interval): Returns: A started `threading.Thread` that reloads the multiplexer. - """ - # Ensure the Multiplexer initializes in a loaded state before it adds runs - # So it can handle HTTP requests while runs are loading - multiplexer.Reload() - + # We don't call multiplexer.Reload() here because that would make + # AddRunsFromDirectory block until the runs have all loaded. for path in path_to_run.keys(): if gcs.IsGCSPath(path): gcs.CheckIsSupported()