From 1e8f618d2085c9a4f140df6525dce84eb227ef90 Mon Sep 17 00:00:00 2001 From: Dan Smilkov Date: Wed, 2 Nov 2016 09:24:55 -0800 Subject: [PATCH 01/54] Set initial t-sne perplexity based on dataset size. Set it to P = sqrt(N) / 4, where N is the number of data points. Change: 137961121 --- .../vz_projector/vz-projector-projections-panel.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tensorflow/tensorboard/components/vz_projector/vz-projector-projections-panel.ts b/tensorflow/tensorboard/components/vz_projector/vz-projector-projections-panel.ts index b319cffc29f..a2267494a63 100644 --- a/tensorflow/tensorboard/components/vz_projector/vz-projector-projections-panel.ts +++ b/tensorflow/tensorboard/components/vz_projector/vz-projector-projections-panel.ts @@ -133,7 +133,7 @@ export class ProjectionsPanel extends ProjectionsPanelPolymer { this.polymerChangesTriggerReprojection = true; } - private updateTSNEPerplexityFromUIChange() { + private updateTSNEPerplexityFromSliderChange() { if (this.perplexitySlider) { this.perplexity = +this.perplexitySlider.value; } @@ -161,8 +161,8 @@ export class ProjectionsPanel extends ProjectionsPanelPolymer { this.perplexitySlider.value = this.perplexity.toString(); this.perplexitySlider.addEventListener( - 'change', () => this.updateTSNEPerplexityFromUIChange()); - this.updateTSNEPerplexityFromUIChange(); + 'change', () => this.updateTSNEPerplexityFromSliderChange()); + this.updateTSNEPerplexityFromSliderChange(); this.learningRateInput.addEventListener( 'change', () => this.updateTSNELearningRateFromUIChange()); @@ -217,7 +217,7 @@ export class ProjectionsPanel extends ProjectionsPanelPolymer { this.computeAllCentroids(); this.setZDropdownEnabled(this.pcaIs3d); - this.updateTSNEPerplexityFromUIChange(); + this.updateTSNEPerplexityFromSliderChange(); this.updateTSNELearningRateFromUIChange(); if (this.iterationLabel) { this.iterationLabel.text(bookmark.tSNEIteration.toString()); @@ -284,6 +284,9 @@ export class ProjectionsPanel extends ProjectionsPanelPolymer { this.dataSet = dataSet; this.originalDataSet = originalDataSet; this.dim = dim; + let perplexity = Math.ceil(Math.sqrt(dataSet.points.length) / 4); + this.perplexitySlider.value = perplexity.toString(); + this.updateTSNEPerplexityFromSliderChange(); this.clearCentroids(); this.dom.select('#tsne-sampling') From 7040f4dfcfcf8173c26dbc5f8edb176449511a4d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 2 Nov 2016 09:27:52 -0800 Subject: [PATCH 02/54] Change v2 session bundle compression method to match v1. Change: 137961557 --- tensorflow/core/util/tensor_bundle/tensor_bundle.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tensorflow/core/util/tensor_bundle/tensor_bundle.cc b/tensorflow/core/util/tensor_bundle/tensor_bundle.cc index 61a69a3840f..4b1a01277c8 100644 --- a/tensorflow/core/util/tensor_bundle/tensor_bundle.cc +++ b/tensorflow/core/util/tensor_bundle/tensor_bundle.cc @@ -343,7 +343,11 @@ Status BundleWriter::Finish() { status_ = env_->NewWritableFile(MetaFilename(prefix_), &file); if (!status_.ok()) return status_; { - table::TableBuilder builder(table::Options(), file.get()); + // N.B.: the default use of Snappy compression may not be supported on all + // platforms (e.g. Android). The metadata file is small, so this is fine. + table::Options options; + options.compression = table::kNoCompression; + table::TableBuilder builder(options, file.get()); // Header entry. BundleHeaderProto header; header.set_num_shards(1); From bdd76154a1aa389d48b930085b7e8a11d7d9a4bf Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 2 Nov 2016 09:28:40 -0800 Subject: [PATCH 03/54] Delete old Eigen threadpool and associated code. Get rid of EIGEN_USE_NONBLOCKING_THREAD_POOL symbol. Change: 137961673 --- tensorflow/core/lib/core/threadpool.cc | 4 ---- tensorflow/core/lib/core/threadpool_test.cc | 2 -- tensorflow/core/util/work_sharder.cc | 2 -- 3 files changed, 8 deletions(-) diff --git a/tensorflow/core/lib/core/threadpool.cc b/tensorflow/core/lib/core/threadpool.cc index c3704da0b12..534ef902fb9 100644 --- a/tensorflow/core/lib/core/threadpool.cc +++ b/tensorflow/core/lib/core/threadpool.cc @@ -88,16 +88,12 @@ struct ThreadPool::Impl : Eigen::ThreadPoolTempl { void ParallelFor(int64 total, int64 cost_per_unit, std::function fn) { -#ifdef EIGEN_USE_NONBLOCKING_THREAD_POOL CHECK_GE(total, 0); CHECK_EQ(total, (int64)(Eigen::Index)total); Eigen::ThreadPoolDevice device(this, this->NumThreads()); device.parallelFor( total, Eigen::TensorOpCost(0, 0, cost_per_unit), [&fn](Eigen::Index first, Eigen::Index last) { fn(first, last); }); -#else - CHECK(0); // should not be used with the old thread pool -#endif } }; diff --git a/tensorflow/core/lib/core/threadpool_test.cc b/tensorflow/core/lib/core/threadpool_test.cc index cf8926b54d1..c7d8db51364 100644 --- a/tensorflow/core/lib/core/threadpool_test.cc +++ b/tensorflow/core/lib/core/threadpool_test.cc @@ -57,7 +57,6 @@ TEST(ThreadPool, DoWork) { } } -#ifdef EIGEN_USE_NONBLOCKING_THREAD_POOL TEST(ThreadPool, ParallelFor) { // Make ParallelFor use as many threads as possible. int64 kHugeCost = 1 << 30; @@ -80,7 +79,6 @@ TEST(ThreadPool, ParallelFor) { } } } -#endif static void BM_Sequential(int iters) { ThreadPool pool(Env::Default(), "test", kNumThreads); diff --git a/tensorflow/core/util/work_sharder.cc b/tensorflow/core/util/work_sharder.cc index 6cede8d461e..7922fc9224e 100644 --- a/tensorflow/core/util/work_sharder.cc +++ b/tensorflow/core/util/work_sharder.cc @@ -31,12 +31,10 @@ void Shard(int max_parallelism, thread::ThreadPool* workers, int64 total, work(0, total); return; } -#ifdef EIGEN_USE_NONBLOCKING_THREAD_POOL if (max_parallelism >= workers->NumThreads()) { workers->ParallelFor(total, cost_per_unit, work); return; } -#endif cost_per_unit = std::max(1LL, cost_per_unit); // We shard [0, total) into "num_shards" shards. // 1 <= num_shards <= num worker threads From 87ef75cfb4f4d22b8e2e59b42c30d825f302d0b7 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 2 Nov 2016 09:30:46 -0800 Subject: [PATCH 04/54] Docstring example and formatting updates Change: 137961957 --- .../learn/python/learn/estimators/linear.py | 26 +++++++------------ .../g3doc/api_docs/python/contrib.learn.md | 26 +++++++------------ .../tf.contrib.learn.LinearRegressor.md | 11 +++----- .../tf.contrib.learn.LinearClassifier.md | 15 +++++------ 4 files changed, 30 insertions(+), 48 deletions(-) diff --git a/tensorflow/contrib/learn/python/learn/estimators/linear.py b/tensorflow/contrib/learn/python/learn/estimators/linear.py index d960d6857de..b09373f8e87 100644 --- a/tensorflow/contrib/learn/python/learn/estimators/linear.py +++ b/tensorflow/contrib/learn/python/learn/estimators/linear.py @@ -267,21 +267,18 @@ class LinearClassifier(evaluable.Evaluable, trainable.Trainable): Example: ```python - education = sparse_column_with_hash_bucket(column_name="education", - hash_bucket_size=1000) - occupation = sparse_column_with_hash_bucket(column_name="occupation", - hash_bucket_size=1000) + sparse_column_a = sparse_column_with_hash_bucket(...) + sparse_column_b = sparse_column_with_hash_bucket(...) - education_x_occupation = crossed_column(columns=[education, occupation], - hash_bucket_size=10000) + sparse_feature_a_x_sparse_feature_b = crossed_column(...) # Estimator using the default optimizer. estimator = LinearClassifier( - feature_columns=[occupation, education_x_occupation]) + feature_columns=[sparse_column_a, sparse_feature_a_x_sparse_feature_b]) # Or estimator using the FTRL optimizer with regularization. estimator = LinearClassifier( - feature_columns=[occupation, education_x_occupation], + feature_columns=[sparse_column_a, sparse_feature_a_x_sparse_feature_b], optimizer=tf.train.FtrlOptimizer( learning_rate=0.1, l1_regularization_strength=0.001 @@ -289,7 +286,7 @@ class LinearClassifier(evaluable.Evaluable, trainable.Trainable): # Or estimator using the SDCAOptimizer. estimator = LinearClassifier( - feature_columns=[occupation, education_x_occupation], + feature_columns=[sparse_column_a, sparse_feature_a_x_sparse_feature_b], optimizer=tf.contrib.linear_optimizer.SDCAOptimizer( example_id_column='example_id', num_loss_partitions=..., @@ -561,16 +558,13 @@ class LinearRegressor(evaluable.Evaluable, trainable.Trainable): Example: ```python - education = sparse_column_with_hash_bucket(column_name="education", - hash_bucket_size=1000) - occupation = sparse_column_with_hash_bucket(column_name="occupation", - hash_bucket_size=1000) + sparse_column_a = sparse_column_with_hash_bucket(...) + sparse_column_b = sparse_column_with_hash_bucket(...) - education_x_occupation = crossed_column(columns=[education, occupation], - hash_bucket_size=10000) + sparse_feature_a_x_sparse_feature_b = crossed_column(...) estimator = LinearRegressor( - feature_columns=[occupation, education_x_occupation]) + feature_columns=[sparse_column_a, sparse_feature_a_x_sparse_feature_b]) # Input builders def input_fn_train: # returns x, y diff --git a/tensorflow/g3doc/api_docs/python/contrib.learn.md b/tensorflow/g3doc/api_docs/python/contrib.learn.md index 8d55f807aed..30f6bbee6d4 100644 --- a/tensorflow/g3doc/api_docs/python/contrib.learn.md +++ b/tensorflow/g3doc/api_docs/python/contrib.learn.md @@ -1265,21 +1265,18 @@ classes. When number of possible classes is 2, this is binary classification. Example: ```python -education = sparse_column_with_hash_bucket(column_name="education", - hash_bucket_size=1000) -occupation = sparse_column_with_hash_bucket(column_name="occupation", - hash_bucket_size=1000) +sparse_column_a = sparse_column_with_hash_bucket(...) +sparse_column_b = sparse_column_with_hash_bucket(...) -education_x_occupation = crossed_column(columns=[education, occupation], - hash_bucket_size=10000) +sparse_feature_a_x_sparse_feature_b = crossed_column(...) # Estimator using the default optimizer. estimator = LinearClassifier( - feature_columns=[occupation, education_x_occupation]) + feature_columns=[sparse_column_a, sparse_feature_a_x_sparse_feature_b]) # Or estimator using the FTRL optimizer with regularization. estimator = LinearClassifier( - feature_columns=[occupation, education_x_occupation], + feature_columns=[sparse_column_a, sparse_feature_a_x_sparse_feature_b], optimizer=tf.train.FtrlOptimizer( learning_rate=0.1, l1_regularization_strength=0.001 @@ -1287,7 +1284,7 @@ estimator = LinearClassifier( # Or estimator using the SDCAOptimizer. estimator = LinearClassifier( - feature_columns=[occupation, education_x_occupation], + feature_columns=[sparse_column_a, sparse_feature_a_x_sparse_feature_b], optimizer=tf.contrib.linear_optimizer.SDCAOptimizer( example_id_column='example_id', num_loss_partitions=..., @@ -1483,16 +1480,13 @@ feature values. Example: ```python -education = sparse_column_with_hash_bucket(column_name="education", - hash_bucket_size=1000) -occupation = sparse_column_with_hash_bucket(column_name="occupation", - hash_bucket_size=1000) +sparse_column_a = sparse_column_with_hash_bucket(...) +sparse_column_b = sparse_column_with_hash_bucket(...) -education_x_occupation = crossed_column(columns=[education, occupation], - hash_bucket_size=10000) +sparse_feature_a_x_sparse_feature_b = crossed_column(...) estimator = LinearRegressor( - feature_columns=[occupation, education_x_occupation]) + feature_columns=[sparse_column_a, sparse_feature_a_x_sparse_feature_b]) # Input builders def input_fn_train: # returns x, y diff --git a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard0/tf.contrib.learn.LinearRegressor.md b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard0/tf.contrib.learn.LinearRegressor.md index bdeab9de13c..2352b13897a 100644 --- a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard0/tf.contrib.learn.LinearRegressor.md +++ b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard0/tf.contrib.learn.LinearRegressor.md @@ -6,16 +6,13 @@ feature values. Example: ```python -education = sparse_column_with_hash_bucket(column_name="education", - hash_bucket_size=1000) -occupation = sparse_column_with_hash_bucket(column_name="occupation", - hash_bucket_size=1000) +sparse_column_a = sparse_column_with_hash_bucket(...) +sparse_column_b = sparse_column_with_hash_bucket(...) -education_x_occupation = crossed_column(columns=[education, occupation], - hash_bucket_size=10000) +sparse_feature_a_x_sparse_feature_b = crossed_column(...) estimator = LinearRegressor( - feature_columns=[occupation, education_x_occupation]) + feature_columns=[sparse_column_a, sparse_feature_a_x_sparse_feature_b]) # Input builders def input_fn_train: # returns x, y diff --git a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard1/tf.contrib.learn.LinearClassifier.md b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard1/tf.contrib.learn.LinearClassifier.md index 9327ccc2c1f..3f6584c1f82 100644 --- a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard1/tf.contrib.learn.LinearClassifier.md +++ b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard1/tf.contrib.learn.LinearClassifier.md @@ -6,21 +6,18 @@ classes. When number of possible classes is 2, this is binary classification. Example: ```python -education = sparse_column_with_hash_bucket(column_name="education", - hash_bucket_size=1000) -occupation = sparse_column_with_hash_bucket(column_name="occupation", - hash_bucket_size=1000) +sparse_column_a = sparse_column_with_hash_bucket(...) +sparse_column_b = sparse_column_with_hash_bucket(...) -education_x_occupation = crossed_column(columns=[education, occupation], - hash_bucket_size=10000) +sparse_feature_a_x_sparse_feature_b = crossed_column(...) # Estimator using the default optimizer. estimator = LinearClassifier( - feature_columns=[occupation, education_x_occupation]) + feature_columns=[sparse_column_a, sparse_feature_a_x_sparse_feature_b]) # Or estimator using the FTRL optimizer with regularization. estimator = LinearClassifier( - feature_columns=[occupation, education_x_occupation], + feature_columns=[sparse_column_a, sparse_feature_a_x_sparse_feature_b], optimizer=tf.train.FtrlOptimizer( learning_rate=0.1, l1_regularization_strength=0.001 @@ -28,7 +25,7 @@ estimator = LinearClassifier( # Or estimator using the SDCAOptimizer. estimator = LinearClassifier( - feature_columns=[occupation, education_x_occupation], + feature_columns=[sparse_column_a, sparse_feature_a_x_sparse_feature_b], optimizer=tf.contrib.linear_optimizer.SDCAOptimizer( example_id_column='example_id', num_loss_partitions=..., From 350fd444ce1592c91c150c180dba6a4f2c9de136 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 2 Nov 2016 09:45:30 -0800 Subject: [PATCH 05/54] Initialize resources in the prediction path before loading from a checkpoint. Reload all saveable objects from the graph. Change: 137964298 --- tensorflow/contrib/learn/BUILD | 2 ++ .../contrib/learn/python/learn/graph_actions.py | 8 ++++++-- .../learn/python/learn/graph_actions_test.py | 15 +++++++++++++++ tensorflow/python/BUILD | 1 + 4 files changed, 24 insertions(+), 2 deletions(-) diff --git a/tensorflow/contrib/learn/BUILD b/tensorflow/contrib/learn/BUILD index 62d7bb77c98..b93089c9cb7 100644 --- a/tensorflow/contrib/learn/BUILD +++ b/tensorflow/contrib/learn/BUILD @@ -291,7 +291,9 @@ py_test( deps = [ ":learn", "//tensorflow:tensorflow_py", + "//tensorflow/python:extra_py_tests_deps", "//tensorflow/python:framework_test_lib", + "//tensorflow/python:test_ops", ], ) diff --git a/tensorflow/contrib/learn/python/learn/graph_actions.py b/tensorflow/contrib/learn/python/learn/graph_actions.py index 0c5152b553f..baee707a5f6 100644 --- a/tensorflow/contrib/learn/python/learn/graph_actions.py +++ b/tensorflow/contrib/learn/python/learn/graph_actions.py @@ -40,6 +40,7 @@ from tensorflow.python.framework import ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import data_flow_ops from tensorflow.python.ops import logging_ops +from tensorflow.python.ops import resources from tensorflow.python.ops import variables from tensorflow.python.platform import tf_logging as logging from tensorflow.python.training import basic_session_run_hooks @@ -77,7 +78,8 @@ def get_summary_writer(logdir): def _make_saver(graph, keep_checkpoint_max=5): - vars_to_save = graph.get_collection(ops.GraphKeys.VARIABLES) + vars_to_save = (graph.get_collection(ops.GraphKeys.VARIABLES) + + graph.get_collection(ops.GraphKeys.SAVEABLE_OBJECTS)) if vars_to_save: return tf_saver.Saver(vars_to_save, sharded=True, @@ -846,9 +848,11 @@ def run_feeds_iter(output_dict, feed_dicts, restore_checkpoint_path=None): raise ValueError('feed_dicts is invalid: %s.' % feed_dicts) graph = contrib_ops.get_graph_from_inputs(output_dict.values()) - with graph.as_default() as g: with tf_session.Session('') as session: + session.run( + resources.initialize_resources(resources.shared_resources() + + resources.local_resources())) if restore_checkpoint_path: _restore_from_checkpoint(session, g, restore_checkpoint_path) else: diff --git a/tensorflow/contrib/learn/python/learn/graph_actions_test.py b/tensorflow/contrib/learn/python/learn/graph_actions_test.py index 9a7306ad4ad..c8c73d5de52 100644 --- a/tensorflow/contrib/learn/python/learn/graph_actions_test.py +++ b/tensorflow/contrib/learn/python/learn/graph_actions_test.py @@ -28,6 +28,8 @@ from tensorflow.contrib.learn.python import learn from tensorflow.contrib.learn.python.learn.monitors import BaseMonitor from tensorflow.python.framework import meta_graph from tensorflow.python.framework import ops +from tensorflow.python.framework import test_ops +from tensorflow.python.ops import resources from tensorflow.python.ops import variables @@ -194,6 +196,19 @@ class GraphActionsTest(tf.test.TestCase): pass self.assertTrue(request_stop.called) + def test_run_feeds_iter_calls_resources_init(self): + with tf.Graph().as_default() as g: + in0, _, _ = self._build_inference_graph() + handle = test_ops.stub_resource_handle_op(container='a', shared_name='b') + resources.register_resource( + handle=handle, + create_op=test_ops.resource_create_op(handle), + is_initialized_op=test_ops.resource_initialized_op(handle)) + + for _ in learn.graph_actions.run_feeds_iter({'in0': in0}, + feed_dicts=[{}]): + self.assertTrue(test_ops.resource_initialized_op(handle).eval()) + def test_infer_different_default_graph(self): with self.test_session(): self._assert_ckpt(self._output_dir, False) diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index a32c76273d6..1701aaf5cbd 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -41,6 +41,7 @@ py_library( ":summary", ":training", ":ops", + ":test_ops", "//tensorflow/python/debug:debug_py", ] + if_not_windows([ "//tensorflow/contrib:contrib_py", From 064c3bbc3801874f88ba8fde9bedef620158c7b7 Mon Sep 17 00:00:00 2001 From: Nikhil Thorat Date: Wed, 2 Nov 2016 09:46:50 -0800 Subject: [PATCH 06/54] Make the demo datasets provider fetch a projector config JSON file. This makes "Smart Reply 5K" the first in the list of tensors, and we now choose the first tensor in the embeddings array as the default tensor. Change: 137964537 --- .../vz_projector/data-provider-demo.ts | 94 +++++-------------- .../vz_projector/vz-projector-app.html | 4 +- .../components/vz_projector/vz-projector.ts | 7 +- 3 files changed, 31 insertions(+), 74 deletions(-) diff --git a/tensorflow/tensorboard/components/vz_projector/data-provider-demo.ts b/tensorflow/tensorboard/components/vz_projector/data-provider-demo.ts index a839ad4a0b3..643862c797a 100644 --- a/tensorflow/tensorboard/components/vz_projector/data-provider-demo.ts +++ b/tensorflow/tensorboard/components/vz_projector/data-provider-demo.ts @@ -20,69 +20,15 @@ import * as logging from './logging'; /** Data provider that loads data from a demo folder. */ export class DemoDataProvider implements DataProvider { - /** List of demo datasets for showing the capabilities of the tool. */ - private DEMO_CONFIG: ProjectorConfig = { - embeddings: [ - { - tensorName: 'Word2Vec 5K', - tensorShape: [5000, 200], - tensorPath: 'word2vec_5000_200d_tensors.tsv', - metadataPath: 'word2vec_5000_200d_labels.tsv' - }, - { - tensorName: 'Word2Vec 10K', - tensorShape: [10000, 200], - tensorPath: 'word2vec_10000_200d_tensors.tsv', - metadataPath: 'word2vec_10000_200d_labels.tsv' - }, - { - tensorName: 'Word2Vec All', - tensorShape: [71291, 200], - tensorPath: 'word2vec_full_200d_tensors.tsv', - metadataPath: 'word2vec_full_200d_labels.tsv' - }, - { - tensorName: 'SmartReply 5K', - tensorShape: [5000, 256], - tensorPath: 'smartreply_5000_256d_tensors.tsv', - metadataPath: 'smartreply_5000_256d_labels.tsv' - }, - { - tensorName: 'SmartReply All', - tensorShape: [35860, 256], - tensorPath: 'smartreply_full_256d_tensors.tsv', - metadataPath: 'smartreply_full_256d_labels.tsv' - }, - { - tensorName: 'Mnist with images 10K', - tensorShape: [10000, 784], - tensorPath: 'mnist_10k_784d_tensors.tsv', - metadataPath: 'mnist_10k_784d_labels.tsv', - sprite: { - imagePath: 'mnist_10k_sprite.png', - singleImageDim: [28, 28] - } - }, - { - tensorName: 'Iris', - tensorShape: [150, 4], - tensorPath: 'iris_tensors.tsv', - metadataPath: 'iris_labels.tsv' - }, - { - tensorName: 'Unit Cube', - tensorShape: [8, 3], - tensorPath: 'cube_tensors.tsv', - metadataPath: 'cube_metadata.tsv' - } - ], - modelCheckpointPath: 'Demo datasets' - }; - /** Name of the folder where the demo datasets are stored. */ - private DEMO_FOLDER = 'data'; + private projectorConfigPath: string; + private projectorConfig: ProjectorConfig; + + constructor(projectorConfigPath: string) { + this.projectorConfigPath = projectorConfigPath; + } private getEmbeddingInfo(tensorName: string): EmbeddingInfo { - let embeddings = this.DEMO_CONFIG.embeddings; + let embeddings = this.projectorConfig.embeddings; for (let i = 0; i < embeddings.length; i++) { let embedding = embeddings[i]; if (embedding.tensorName === tensorName) { @@ -98,18 +44,28 @@ export class DemoDataProvider implements DataProvider { retrieveProjectorConfig(run: string, callback: (d: ProjectorConfig) => void) : void { - callback(this.DEMO_CONFIG); + let msgId = logging.setModalMessage('Fetching projector config...'); + d3.json(this.projectorConfigPath, (err, projectorConfig) => { + if (err) { + logging.setModalMessage('Error: ' + err.responseText); + return; + } + logging.setModalMessage(null, msgId); + this.projectorConfig = projectorConfig; + callback(projectorConfig); + }); } getDefaultTensor(run: string, callback: (tensorName: string) => void) { - callback('SmartReply 5K'); + // Return the first tensor as the default tensor. + callback(this.projectorConfig.embeddings[0].tensorName); } retrieveTensor(run: string, tensorName: string, callback: (ds: DataSet) => void) { let embedding = this.getEmbeddingInfo(tensorName); let separator = embedding.tensorPath.substr(-3) === 'tsv' ? '\t' : ' '; - let url = `${this.DEMO_FOLDER}/${embedding.tensorPath}`; + let url = `${embedding.tensorPath}`; logging.setModalMessage('Fetching tensors...', TENSORS_MSG_ID); d3.text(url, (error: any, dataString: string) => { if (error) { @@ -125,16 +81,12 @@ export class DemoDataProvider implements DataProvider { retrieveSpriteAndMetadata(run: string, tensorName: string, callback: (r: SpriteAndMetadataInfo) => void) { let embedding = this.getEmbeddingInfo(tensorName); - let metadataPath = null; - if (embedding.metadataPath) { - metadataPath = `${this.DEMO_FOLDER}/${embedding.metadataPath}`; - } let spriteImagePath = null; if (embedding.sprite && embedding.sprite.imagePath) { - spriteImagePath = `${this.DEMO_FOLDER}/${embedding.sprite.imagePath}`; + spriteImagePath = embedding.sprite.imagePath; } - dataProvider.retrieveSpriteAndMetadataInfo(metadataPath, spriteImagePath, - embedding.sprite, callback); + dataProvider.retrieveSpriteAndMetadataInfo( + embedding.metadataPath, spriteImagePath, embedding.sprite, callback); } getBookmarks( diff --git a/tensorflow/tensorboard/components/vz_projector/vz-projector-app.html b/tensorflow/tensorboard/components/vz_projector/vz-projector-app.html index f2a9fcad137..6221485c2d1 100644 --- a/tensorflow/tensorboard/components/vz_projector/vz-projector-app.html +++ b/tensorflow/tensorboard/components/vz_projector/vz-projector-app.html @@ -76,7 +76,9 @@ vz-projector { - + From 63f7b65ad281cc9c3d348d7e46f5c6299e668896 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 2 Nov 2016 22:15:02 -0800 Subject: [PATCH 37/54] Subscribe syntax for allowing side effects when tensors are executed. Change: 138037407 --- tensorflow/python/BUILD | 14 ++ tensorflow/python/framework/framework_lib.py | 1 + tensorflow/python/framework/subscribe.py | 144 ++++++++++++++++++ tensorflow/python/framework/subscribe_test.py | 59 +++++++ 4 files changed, 218 insertions(+) create mode 100644 tensorflow/python/framework/subscribe.py create mode 100644 tensorflow/python/framework/subscribe_test.py diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index 1701aaf5cbd..2511cfee373 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -283,6 +283,7 @@ py_library( "framework/meta_graph.py", "framework/random_seed.py", "framework/sparse_tensor.py", + "framework/subscribe.py", "framework/tensor_util.py", ], srcs_version = "PY2AND3", @@ -352,6 +353,19 @@ py_test( ], ) +py_test( + name = "framework_subscribe_test", + size = "small", + srcs = ["framework/subscribe_test.py"], + main = "framework/subscribe_test.py", + srcs_version = "PY2AND3", + deps = [ + ":framework_test_lib", + ":platform_test", + "//tensorflow:tensorflow_py", + ], +) + py_test( name = "contrib_test", size = "small", diff --git a/tensorflow/python/framework/framework_lib.py b/tensorflow/python/framework/framework_lib.py index 16b9f347e49..4f44041df73 100644 --- a/tensorflow/python/framework/framework_lib.py +++ b/tensorflow/python/framework/framework_lib.py @@ -95,6 +95,7 @@ from tensorflow.python.framework.ops import convert_to_tensor from tensorflow.python.framework.ops import convert_to_tensor_or_indexed_slices from tensorflow.python.framework.random_seed import get_seed from tensorflow.python.framework.random_seed import set_random_seed +from tensorflow.python.framework.subscribe import subscribe from tensorflow.python.framework.importer import import_graph_def # Needed when you defined a new Op in C++. diff --git a/tensorflow/python/framework/subscribe.py b/tensorflow/python/framework/subscribe.py new file mode 100644 index 00000000000..53d299a976c --- /dev/null +++ b/tensorflow/python/framework/subscribe.py @@ -0,0 +1,144 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Subscribe function.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops + + +def _recursive_apply(tensors, apply_fn): + """Helper method to recursively apply a function to structure of tensors. + + The structure of the tensors should take the form similar to fetches in + `tf.Session` and includes single `Tensor`, `list`, nested `list`, `tuple`, + `namedtuple`, or `dict`. + + Args: + tensors: Single `Tensor`, `list`, nested `list, `tuple`, + `namedtuple`, or `dict`. + apply_fn: Function to apply to each `Tensor` and should return a `Tensor`. + Returns: + Returns the modified tensors with the same structure. + Raises: + `TypeError` if undefined type in the tensors structure. + """ + tensors_type = type(tensors) + if tensors_type is ops.Tensor: + return apply_fn(tensors) + elif isinstance(tensors, (list, tuple)): + tensors = [_recursive_apply(t, apply_fn) for t in tensors] + if tensors_type is list: + return list(tensors) + elif tensors_type is tuple: + return tuple(tensors) + return tensors_type(*tensors) # collections.namedtuple + elif tensors_type is dict: + return dict([(k, _recursive_apply(v, apply_fn)) + for k, v in tensors.iteritems()]) + else: + raise TypeError('_recursive_apply argument %r has invalid type %r' % + (tensors, tensors_type)) + + +def _control_outputs(op): + """Returns the control_input consumers for the supplied `Operation`. + + Args: + op: The `Operation` to find consumers of. + Yields: + A list of ops that have op as a control dependency. + """ + for o in op.graph.get_operations(): + if op in o.control_inputs: + yield o + + +def _subscribe(tensor, side_effects): + """Helper method that subscribes a single tensor to a list of side_effects. + + Args: + tensor: `tf.Tensor` + side_effects: List of side_effect functions see subscribe for details. + Returns: + The modified replacement to the passed in tensor which triggers the side + effects. + """ + update_input = [] + for consumer_op in list(tensor.consumers()): # explicit copy + update_input.append((consumer_op, list(consumer_op.inputs).index(tensor))) + + update_control_input = list(_control_outputs(tensor.op)) + + # Trailing slash on name scope to replace the scope. + name_scope = tensor.op.name + '/subscription/' + with ops.name_scope(name_scope): + outs = [] + for s in side_effects: + outs += s(tensor) + + with ops.control_dependencies(outs): + out = array_ops.identity(tensor) + + for consumer_op, index in update_input: + consumer_op._update_input(index, out) # pylint: disable=protected-access + + for consumer_op in update_control_input: + consumer_op._control_inputs.remove(tensor.op) # pylint: disable=protected-access + consumer_op._control_inputs.append(out.op) # pylint: disable=protected-access + consumer_op._recompute_node_def() # pylint: disable=protected-access + + return out + + +def subscribe(tensors, side_effects): + """Subscribe to a tensor. + + This method will attach side effect graphs to a given set + of tensors. Set of tensors follows from session.run and supports + single `Tensor`, `list`, nested `list`, `tuple`, `namedtuple`, or `dict`. It + returns the tensors in the same passed in structure, but as clones with + side effects applied. The supplied side effect graphs are specified + as a constructor function which takes the target tensor and + constructs a side effect graph and returns a list of ops that should + be control dependencies on fetching the tensor. It will append + 'subscription' to the name scope of the tensor for every node in + the side effect graph. These control dependencies are what trigger + the side effects. Subscribe will construct the additions to your + graph and return the created identity tensor downstream of the control + dependencies. Use these tensors as you would normally in the rest of + your tensorflow code. + + Args: + tensors: `Tensor` or set of tensors to subscribe to. Set of tensors format + follows from `Session.run` and supports single `Tensor`, `list`, nested + `list`, `tuple`, `namedtuple`, or `dict`. + side_effects: Function(s) that takes a `Tensor`, construct a subgraph, and + return a nonempty list of control dependencies. This can be a single + function or list of functions. + Returns: + Subscribed tensors, which are identity copies of the passed in tensors + in the same passed in structure, but the graph has been modified + such that these are downstream of the control dependencies for + the side effect graphs. Use these functionally equivelant tensors + instead of the passed in tensors for further construction or running. + """ + if not hasattr(side_effects, '__iter__'): + side_effects = [side_effects] + return _recursive_apply(tensors, lambda t: _subscribe(t, side_effects)) diff --git a/tensorflow/python/framework/subscribe_test.py b/tensorflow/python/framework/subscribe_test.py new file mode 100644 index 00000000000..8371c2cfc43 --- /dev/null +++ b/tensorflow/python/framework/subscribe_test.py @@ -0,0 +1,59 @@ +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Tests for tf.subscribe.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + +from tensorflow.python.framework import subscribe +from tensorflow.python.framework import test_util +from tensorflow.python.platform import googletest + + +class SubscribeTest(test_util.TensorFlowTestCase): + + def testSideEffect(self): + a = tf.constant(1) + b = tf.constant(1) + c = tf.add(a, b) + with tf.control_dependencies([c]): + d = tf.constant(42) + n = tf.neg(c) + + shared = [] + + def sub(t): + shared.append(t) + return t + + c = subscribe.subscribe(c, lambda t: tf.py_func(sub, [t], [t.dtype])) + + with self.test_session() as sess: + c_out = sess.run([c]) + n_out = sess.run([n]) + d_out = sess.run([d]) + + self.assertEquals(n_out, [-2]) + self.assertEquals(c_out, [2]) + self.assertEquals(d_out, [42]) + self.assertEquals(shared, [2, 2, 2]) + + +if __name__ == '__main__': + googletest.main() From 81743e74f45f0c846c86c34b2a00e48a687b0640 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 3 Nov 2016 07:39:29 -0800 Subject: [PATCH 38/54] Delete obsolete comment. a) EIGEN_USE_NONBLOCKING_THREAD_POOL no longer exist. The non-blocking threadpool is the default (It can be disabled by the symbol EIGEN_USE_SIMPLE_THREAD_POOL). b) An eigen_initialize.cc file is no longer needed due to recent fixes for thread safety in Eigen. Change: 138074328 --- eigen.BUILD | 2 -- 1 file changed, 2 deletions(-) diff --git a/eigen.BUILD b/eigen.BUILD index 8a699f6aa84..8ce28ac0766 100644 --- a/eigen.BUILD +++ b/eigen.BUILD @@ -62,8 +62,6 @@ cc_library( # This define (mostly) guarantees we don't link any problematic # code. We use it, but we do not rely on it, as evidenced above. "EIGEN_MPL2_ONLY", - # TODO(jart): Use EIGEN_USE_NONBLOCKING_THREAD_POOL but first add an - # eigen_initialize.cc file and alwayslink=1. ], includes = ["."], visibility = ["//visibility:public"], From f19dc6c5e28694bf067d0bbc9698b57ec4068b4e Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 3 Nov 2016 07:42:29 -0800 Subject: [PATCH 39/54] Add a GPU implementation of the ParameterizedTruncatedNormalOp kernel. Benchmarks: Benchmark Time(ns) CPU(ns) Iterations -------------------------------------------------------------------------- BM_PTruncatedNormal_gpu_1000_1000 4632369 5175938 100 184.251M items/s BM_PTruncatedNormal_2SD_gpu_10000_100 2849437 3368804 206 283.090M items/s BM_PTruncatedNormal_OneTail_gpu_10000_100 3300317 3905713 179 244.174M items/s Change: 138074670 --- .../parameterized_truncated_normal_op.cc | 48 ++-- .../parameterized_truncated_normal_op.h | 23 +- ...arameterized_truncated_normal_op_gpu.cu.cc | 214 ++++++++++++++++++ .../parameterized_truncated_normal_op_test.cc | 3 + tensorflow/python/kernel_tests/BUILD | 2 +- .../parameterized_truncated_normal_op_test.py | 58 +++-- 6 files changed, 299 insertions(+), 49 deletions(-) create mode 100644 tensorflow/core/kernels/parameterized_truncated_normal_op_gpu.cu.cc diff --git a/tensorflow/core/kernels/parameterized_truncated_normal_op.cc b/tensorflow/core/kernels/parameterized_truncated_normal_op.cc index 4d31edbb1a9..77c4b7a7299 100644 --- a/tensorflow/core/kernels/parameterized_truncated_normal_op.cc +++ b/tensorflow/core/kernels/parameterized_truncated_normal_op.cc @@ -46,25 +46,6 @@ namespace functor { using random::PhiloxRandom; using random::SingleSampleAdapter; -// Sample a truncated normal random variable, with mean, stddev, minval, and -// maxval parameters for each batch. Uses two rejection sampling algorithms -// described in http://rd.springer.com/article/10.1007/BF00143942. -// -// Either minval may be -infinity, or maxval may be +infinity. If the interval -// (minval, maxval) is empty, the result is NaN. Large intervals which include -// both tails may have reduced accuracy. -template -struct TruncatedNormalFunctor { - void operator()(OpKernelContext* ctx, const Device& d, int64 num_batches, - int64 samples_per_batch, int64 num_elements, - typename TTypes::ConstFlat means, - typename TTypes::ConstFlat stddevs, - typename TTypes::ConstFlat minvals, - typename TTypes::ConstFlat maxvals, - const random::PhiloxRandom& gen, - typename TTypes::Flat output); -}; - template struct TruncatedNormalFunctor { static const int kMaxIterations = 100; @@ -96,8 +77,8 @@ struct TruncatedNormalFunctor { // Vectorized intermediate calculations for uniform rejection sampling. // We always generate at most 4 samples. - tensorflow::random::Array z; - tensorflow::random::Array g; + Eigen::array z; + Eigen::array g; for (int64 b = start_batch; b < limit_batch; ++b) { // We are passed a flat array for each of the parameter tensors. @@ -145,13 +126,7 @@ struct TruncatedNormalFunctor { if (diff < cutoff) { // Sample from a uniform distribution on [normMin, normMax]. - T plusFactor; - if (normMin < T(0)) { - // normMax > 0 because it is flipped otherwise. - plusFactor = T(0); - } else { - plusFactor = normMin * normMin; - } + const T plusFactor = (normMin < T(0)) ? T(0) : normMin * normMin; while (sample < limit_sample) { const auto rand = dist(&gen_copy); @@ -395,4 +370,21 @@ TF_CALL_double(REGISTER); #undef REGISTER +#if GOOGLE_CUDA + +#define REGISTER(TYPE) \ + REGISTER_KERNEL_BUILDER(Name("ParameterizedTruncatedNormal") \ + .Device(DEVICE_GPU) \ + .HostMemory("shape") \ + .TypeConstraint("dtype"), \ + ParameterizedTruncatedNormalOp) + +TF_CALL_half(REGISTER); +TF_CALL_float(REGISTER); +TF_CALL_double(REGISTER); + +#undef REGISTER + +#endif // GOOGLE_CUDA + } // end namespace tensorflow diff --git a/tensorflow/core/kernels/parameterized_truncated_normal_op.h b/tensorflow/core/kernels/parameterized_truncated_normal_op.h index a46bb1c9fa6..cc801eb8109 100644 --- a/tensorflow/core/kernels/parameterized_truncated_normal_op.h +++ b/tensorflow/core/kernels/parameterized_truncated_normal_op.h @@ -16,14 +16,35 @@ limitations under the License. #ifndef TENSORFLOW_KERNELS_PARAMETERIZED_TRUNCATED_NORMAL_OP_H_ #define TENSORFLOW_KERNELS_PARAMETERIZED_TRUNCATED_NORMAL_OP_H_ +#include "tensorflow/core/framework/tensor_types.h" +#include "tensorflow/core/lib/random/random_distributions.h" + namespace tensorflow { class OpKernelContext; namespace functor { +// Sample a truncated normal random variable, with mean, stddev, minval, and +// maxval parameters for each batch. Uses two rejection sampling algorithms +// described in http://rd.springer.com/article/10.1007/BF00143942. +// +// Either minval may be -infinity, or maxval may be +infinity. If the interval +// (minval, maxval) is empty, the result is NaN. Large intervals which include +// both tails may have reduced accuracy. template -struct TruncatedNormalFunctor; +struct TruncatedNormalFunctor { + void operator()(OpKernelContext* ctx, const Device& d, int64 num_batches, + int64 samples_per_batch, int64 num_elements, + typename TTypes::ConstFlat means, + typename TTypes::ConstFlat stddevs, + typename TTypes::ConstFlat minvals, + typename TTypes::ConstFlat maxvals, + const random::PhiloxRandom& gen, + typename TTypes::Flat output); + + static const int kMaxIterations = 100; +}; } // namespace functor } // namespace tensorflow diff --git a/tensorflow/core/kernels/parameterized_truncated_normal_op_gpu.cu.cc b/tensorflow/core/kernels/parameterized_truncated_normal_op_gpu.cu.cc new file mode 100644 index 00000000000..42d47440690 --- /dev/null +++ b/tensorflow/core/kernels/parameterized_truncated_normal_op_gpu.cu.cc @@ -0,0 +1,214 @@ +/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#if GOOGLE_CUDA + +#define EIGEN_USE_GPU + +#include "tensorflow/core/kernels/parameterized_truncated_normal_op.h" + +#include +#include +#include + +#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" +#include "tensorflow/core/framework/tensor_types.h" +#include "tensorflow/core/lib/random/philox_random.h" +#include "tensorflow/core/lib/random/random_distributions.h" +#include "tensorflow/core/util/cuda_kernel_helper.h" + +#define UNROLL _Pragma("unroll") + +namespace tensorflow { + +class OpKernelContext; + +namespace functor { + +typedef Eigen::GpuDevice GPUDevice; + +template +__global__ void __launch_bounds__(1024) + TruncatedNormalKernel(random::PhiloxRandom gen, T* data, int64 num_batches, + int64 samples_per_batch, int64 num_elements, + const T* means, bool single_mean, const T* stddevs, + bool single_stddev, const T* minvals, + bool single_minval, const T* maxvals, + bool single_maxval, int64 kMaxIterations) { + const int32 max_samples_per_item = 2 * kMaxIterations; + // Initial offset as given by CUDA_1D_KERNEL_LOOP. + const int32 initial_offset = blockIdx.x * blockDim.x + threadIdx.x; + gen.Skip(max_samples_per_item * initial_offset); + typedef random::UniformDistribution Uniform; + Uniform dist; + const int kDistSize = Uniform::kResultElementCount; + const T quietNaN = Eigen::NumTraits::quiet_NaN(); + + // We skip the total number of threads to get to the next element. To produce + // deterministic results between devices, each element in the output array + // skips max_samples_per_item in the generator. Then after generating this + // item, we need to skip the samples for one element for every thread to get + // to the next element that we actually process. + const int32 samples_between_processed_elements = + max_samples_per_item * (gridDim.x * blockDim.x); + + CUDA_1D_KERNEL_LOOP(offset, num_elements) { + // Track how many more samples we need to skip before we process the next + // element. + int32 remaining_samples = samples_between_processed_elements; + + const int64 batch_id = offset / samples_per_batch; + T mean = means[single_mean ? 0 : batch_id]; + const T input_stddev = stddevs[single_stddev ? 0 : batch_id]; + T minval = minvals[single_minval ? 0 : batch_id]; + T maxval = maxvals[single_maxval ? 0 : batch_id]; + + // Flip the distribution if we can make the lower bound positive. + T stddev; + if (Eigen::numext::isinf(minval) || maxval < mean) { + // Reverse all calculations. normMin and normMax will be flipped. + // std::swap is a host function (not available in CUDA). + T temp = minval; + minval = maxval; + maxval = temp; + stddev = -input_stddev; + } else { + stddev = input_stddev; + } + + // Calculate normalized samples, then scale them. + const T normMin = (minval - mean) / stddev; + const T normMax = (maxval - mean) / stddev; + + // Determine the method to use. + const T sqrtFactor = Eigen::numext::sqrt((normMin * normMin) + T(4)); + const T cutoff = + T(2) * + Eigen::numext::exp(T(0.5) + (normMin * (normMin - sqrtFactor)) / T(4)) / + (normMin + sqrtFactor); + const T diff = normMax - normMin; + + // Validate the normalized min and max, because the originals may have been + // flipped already. + if (!(input_stddev > T(0) && normMin < normMax && + (Eigen::numext::isfinite(normMin) || + Eigen::numext::isfinite(normMax)))) { + data[offset] = quietNaN; + } else if (diff < cutoff) { + // Sample from a uniform distribution on [normMin, normMax]. + + // Vectorized intermediate calculations for uniform rejection sampling. + // We always generate at most 4 samples. + Eigen::array z; + Eigen::array g; + + const T plusFactor = (normMin < T(0)) ? T(0) : normMin * normMin; + + int numIterations = 0; + while (numIterations < kMaxIterations) { + const auto rand = dist(&gen); + remaining_samples -= gen.kResultElementCount; + UNROLL for (int i = 0; i < kDistSize; i++) { + z[i] = rand[i] * diff + normMin; + } + UNROLL for (int i = 0; i < kDistSize; i++) { + g[i] = (plusFactor - z[i] * z[i]) / 2.0; + } + + const auto u = dist(&gen); + remaining_samples -= gen.kResultElementCount; + UNROLL for (int i = 0; i < kDistSize; i++) { + if (u[i] <= Eigen::numext::exp(g[i]) || + numIterations + 1 >= kMaxIterations) { + // Accept the sample z. + // If we run out of iterations, just use the current uniform + // sample. Emperically, the probability of accepting each sample + // is at least 50% for typical inputs, so we will always accept + // by 100 iterations. + // This introduces a slight inaccuracy when at least one bound + // is large, minval is negative and maxval is positive. + data[offset] = z[i] * stddev + mean; + // Break out of the nested loop by updating numIterations. + numIterations = kMaxIterations; + break; + } else { + numIterations++; + } + } + } + } else { + // Sample from an exponential distribution with alpha maximizing + // acceptance probability, offset by normMin from the origin. + // Accept only if less than normMax. + const T alpha = + (normMin + Eigen::numext::sqrt((normMin * normMin) + T(4))) / T(2); + int numIterations = 0; + while (numIterations < kMaxIterations) { + auto rand = dist(&gen); + remaining_samples -= gen.kResultElementCount; + UNROLL for (int i = 0; i < kDistSize; i += 2) { + const T z = -Eigen::numext::log(rand[i]) / alpha + normMin; + const T x = normMin < alpha ? alpha - z : normMin - alpha; + const T g = Eigen::numext::exp(-x * x / 2.0); + const T u = rand[i + 1]; + if ((u <= g && z < normMax) || numIterations + 1 >= kMaxIterations) { + data[offset] = z * stddev + mean; + // Break out of the nested loop by updating numIterations. + numIterations = kMaxIterations; + break; + } else { + numIterations++; + } + } + } + } + + gen.Skip(remaining_samples); + } +} + +// Partial specialization for GPU +template +struct TruncatedNormalFunctor { + static const int kMaxIterations = 100; + + void operator()(OpKernelContext* ctx, const GPUDevice& d, int64 num_batches, + int64 samples_per_batch, int64 num_elements, + typename TTypes::ConstFlat means, + typename TTypes::ConstFlat stddevs, + typename TTypes::ConstFlat minvals, + typename TTypes::ConstFlat maxvals, + const random::PhiloxRandom& gen, + typename TTypes::Flat output) { + const auto config = GetCudaLaunchConfig(num_elements, d); + + TruncatedNormalKernel< + T><<>>( + gen, output.data(), num_batches, samples_per_batch, num_elements, + means.data(), means.dimension(0) == 1, stddevs.data(), + stddevs.dimension(0) == 1, minvals.data(), minvals.dimension(0) == 1, + maxvals.data(), maxvals.dimension(0) == 1, kMaxIterations); + }; +}; + +// Explicit instantiation of the GPU distributions functors +template struct TruncatedNormalFunctor; +template struct TruncatedNormalFunctor; +template struct TruncatedNormalFunctor; + +} // namespace functor +} // namespace tensorflow + +#endif // GOOGLE_CUDA diff --git a/tensorflow/core/kernels/parameterized_truncated_normal_op_test.cc b/tensorflow/core/kernels/parameterized_truncated_normal_op_test.cc index 13d1187f926..07f2f75ca5a 100644 --- a/tensorflow/core/kernels/parameterized_truncated_normal_op_test.cc +++ b/tensorflow/core/kernels/parameterized_truncated_normal_op_test.cc @@ -131,5 +131,8 @@ static Graph* PTruncatedNormalOneTail(int num_batches, int samples_per_batch) { BM_PTruncatedNormalDev(cpu, 1000, 1000); BM_PTruncatedNormalDev_2SD(cpu, 10000, 100); BM_PTruncatedNormalDev_OneTail(cpu, 10000, 100); +BM_PTruncatedNormalDev(gpu, 1000, 1000); +BM_PTruncatedNormalDev_2SD(gpu, 10000, 100); +BM_PTruncatedNormalDev_OneTail(gpu, 10000, 100); } // namespace tensorflow diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD index 24a1d8e7c97..0100e6b3268 100644 --- a/tensorflow/python/kernel_tests/BUILD +++ b/tensorflow/python/kernel_tests/BUILD @@ -220,7 +220,7 @@ cuda_py_test( additional_deps = ["//tensorflow:tensorflow_py"], ) -tf_py_test( +cuda_py_test( name = "parameterized_truncated_normal_op_test", size = "small", srcs = ["parameterized_truncated_normal_op_test.py"], diff --git a/tensorflow/python/kernel_tests/parameterized_truncated_normal_op_test.py b/tensorflow/python/kernel_tests/parameterized_truncated_normal_op_test.py index 8d41029c0b5..1c09949598a 100644 --- a/tensorflow/python/kernel_tests/parameterized_truncated_normal_op_test.py +++ b/tensorflow/python/kernel_tests/parameterized_truncated_normal_op_test.py @@ -97,10 +97,10 @@ def z_test(real, expected, i, num_samples): class ParameterizedTruncatedNormalTest(tf.test.TestCase): - use_gpu = False + _use_gpu = False z_limit = 6.0 - # Stop at moment 20 to avoid numerical errors in the theoretical moments. + # Stop at moment 10 to avoid numerical errors in the theoretical moments. max_moment = 10 def validateMoments(self, shape, mean, stddev, minval, maxval, seed=1618): @@ -109,9 +109,11 @@ class ParameterizedTruncatedNormalTest(tf.test.TestCase): # Give up early if we are unable to import it. import scipy.stats # pylint: disable=g-import-not-at-top,unused-variable tf.set_random_seed(seed) - with self.test_session(use_gpu=self.use_gpu): - samples = random_ops.parameterized_truncated_normal( - shape, mean, stddev, minval, maxval).eval() + with self.test_session(use_gpu=self._use_gpu): + samples = random_ops.parameterized_truncated_normal(shape, mean, stddev, + minval, + maxval).eval() + assert (~np.isnan(samples)).all() moments = calculate_moments(samples, self.max_moment) expected_moments = TruncatedNormalMoments(mean, stddev, minval, maxval) num_samples = functools.reduce(lambda x, y: x * y, shape, 1) @@ -131,9 +133,11 @@ class ParameterizedTruncatedNormalTest(tf.test.TestCase): try: import scipy.stats # pylint: disable=g-import-not-at-top tf.set_random_seed(seed) - with self.test_session(use_gpu=self.use_gpu): - samples = random_ops.parameterized_truncated_normal( - shape, mean, stddev, minval, maxval).eval() + with self.test_session(use_gpu=self._use_gpu): + samples = random_ops.parameterized_truncated_normal(shape, mean, stddev, + minval, + maxval).eval() + assert (~np.isnan(samples)).all() minval = max(mean - stddev * 10, minval) maxval = min(mean + stddev * 10, maxval) dist = scipy.stats.norm(loc=mean, scale=stddev) @@ -173,8 +177,12 @@ class ParameterizedTruncatedNormalTest(tf.test.TestCase): self.validateKolmogorovSmirnov([10**5], 0.0, 0.1, 0.05, 0.10) +class ParameterizedTruncatedNormalGpuTest(ParameterizedTruncatedNormalTest): + _use_gpu = True + + # Benchmarking code -def parameterized_vs_naive(shape, num_iters): +def parameterized_vs_naive(shape, num_iters, use_gpu=False): np.random.seed(1618) # Make it reproducible. # No CSE/CF. @@ -183,17 +191,29 @@ def parameterized_vs_naive(shape, num_iters): graph_options=tf.GraphOptions(optimizer_options=optimizer_options)) with tf.Session(config=config) as sess: - param_op = tf.group(random_ops.parameterized_truncated_normal(shape)) - naive_op = tf.group(random_ops.truncated_normal(shape)) + with tf.device("/cpu:0" if not use_gpu else None): + param_op = tf.group(random_ops.parameterized_truncated_normal(shape)) + naive_op = tf.group(random_ops.truncated_normal(shape)) + # Burn-in to avoid session setup costs in the timing. + sess.run(param_op) + sess.run(param_op) param_dt = timeit.timeit(lambda: sess.run(param_op), number=num_iters) + sess.run(naive_op) + sess.run(naive_op) naive_dt = timeit.timeit(lambda: sess.run(naive_op), number=num_iters) return param_dt, naive_dt class TruncatedNormalBenchmark(tf.test.Benchmark): - def benchmarkParameterizedOpVsNaiveOp(self): + def benchmarkParameterizedOpVsNaiveOpCpu(self): + self._benchmarkParameterizedOpVsNaiveOp(False) + + def benchmarkParameterizedOpVsNaiveOpGpu(self): + self._benchmarkParameterizedOpVsNaiveOp(True) + + def _benchmarkParameterizedOpVsNaiveOp(self, use_gpu): num_iters = 50 print(("Composition of new ParameterizedTruncatedNormalOp vs. " "naive TruncatedNormalOp [%d iters]") % num_iters) @@ -201,16 +221,16 @@ class TruncatedNormalBenchmark(tf.test.Benchmark): for shape in [[10000, 100], [1000, 1000], [1000000], [100, 100, 100], [20, 20, 20, 20]]: - p_dt, n_dt = parameterized_vs_naive(shape, num_iters) + p_dt, n_dt = parameterized_vs_naive(shape, num_iters, use_gpu) print("%s\t%.3f\t%.3f\t%.2f" % (shape, p_dt, n_dt, p_dt / n_dt)) shape_str = "-".join(map(str, shape)) - self.report_benchmark(name="parameterized_shape" + shape_str, - iters=num_iters, - wall_time=p_dt) - self.report_benchmark(name="naive_shape" + shape_str, - iters=num_iters, - wall_time=n_dt) + self.report_benchmark( + name="parameterized_shape" + shape_str, + iters=num_iters, + wall_time=p_dt) + self.report_benchmark( + name="naive_shape" + shape_str, iters=num_iters, wall_time=n_dt) if __name__ == "__main__": From 1a7c7144669657d90ce5339b1a48cee0f8f95439 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 3 Nov 2016 08:15:27 -0800 Subject: [PATCH 40/54] C++ Gradients: Adds gradient functions for some nn ops. Adds functionality to gradient checker to enable explicit initial values for some tensors. Change: 138078095 --- tensorflow/cc/BUILD | 30 +++++++ tensorflow/cc/framework/gradient_checker.cc | 50 ++++++++--- tensorflow/cc/framework/gradient_checker.h | 6 ++ tensorflow/cc/gradients/nn_grad.cc | 77 +++++++++++++++++ tensorflow/cc/gradients/nn_grad_test.cc | 91 +++++++++++++++++++++ 5 files changed, 241 insertions(+), 13 deletions(-) create mode 100644 tensorflow/cc/gradients/nn_grad.cc create mode 100644 tensorflow/cc/gradients/nn_grad_test.cc diff --git a/tensorflow/cc/BUILD b/tensorflow/cc/BUILD index a804955a36e..39d519a707c 100644 --- a/tensorflow/cc/BUILD +++ b/tensorflow/cc/BUILD @@ -264,6 +264,36 @@ tf_cc_test( ], ) +cc_library( + name = "nn_grad", + srcs = ["gradients/nn_grad.cc"], + deps = [ + ":cc_ops", + ":grad_op_registry", + ":ops", + ":scope", + "//tensorflow/core:core_cpu", + "//tensorflow/core:framework", + ], +) + +tf_cc_test( + name = "gradients_nn_grad_test", + srcs = ["gradients/nn_grad_test.cc"], + deps = [ + ":cc_ops", + ":grad_op_registry", + ":grad_testutil", + ":gradient_checker", + ":nn_grad", + ":testutil", + "//tensorflow/core:lib_internal", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "//tensorflow/core:testlib", + ], +) + tf_gen_op_wrappers_cc( name = "cc_ops", op_lib_names = [ diff --git a/tensorflow/cc/framework/gradient_checker.cc b/tensorflow/cc/framework/gradient_checker.cc index a729bdd24d3..57b955454e8 100644 --- a/tensorflow/cc/framework/gradient_checker.cc +++ b/tensorflow/cc/framework/gradient_checker.cc @@ -110,20 +110,15 @@ Status ComputeNumericJacobianTranspose(const Scope& scope, const ops::Output& x, return Status::OK(); } -} // namespace - template -Status ComputeGradientError(const Scope& scope, const ops::Output& x, - const TensorShape& x_shape, const ops::Output& y, - const TensorShape& y_shape, T* max_error) { +Status ComputeGradientErrorInternal(const Scope& scope, const ops::Output& x, + const TensorShape& x_shape, + const ops::Output& y, + const TensorShape& y_shape, Tensor* x_data, + T* max_error) { const int64 x_size = x_shape.num_elements(); const int64 y_size = y_shape.num_elements(); - // Initialize 'x_data' to random values. - Tensor x_data(x.type(), x_shape); - auto x_data_flat = x_data.flat(); - x_data_flat.setRandom(); - // Initialize theoretical Jacobian to zeros. Tensor jacobian_t(x.type(), {x_size, y_size}); auto jacobian_t_flat = jacobian_t.flat(); @@ -131,7 +126,7 @@ Status ComputeGradientError(const Scope& scope, const ops::Output& x, // Compute theoretical Jacobian. TF_RETURN_IF_ERROR(ComputeTheoreticalJacobianTranspose( - scope, x, x_shape, x_data, y, y_shape, &jacobian_t)); + scope, x, x_shape, *x_data, y, y_shape, &jacobian_t)); // Initialize numeric Jacobian to zeros. Tensor jacobian_n(x.type(), {x_size, y_size}); @@ -140,7 +135,7 @@ Status ComputeGradientError(const Scope& scope, const ops::Output& x, // Compute numeric Jacobian. TF_RETURN_IF_ERROR(ComputeNumericJacobianTranspose( - scope, x, x_shape, y, y_shape, 1e-3, &x_data, &jacobian_n)); + scope, x, x_shape, y, y_shape, 1e-3, x_data, &jacobian_n)); // Compute the maximum error between theoretical and numeric Jacobians. *max_error = 0.0; @@ -154,10 +149,39 @@ Status ComputeGradientError(const Scope& scope, const ops::Output& x, return Status::OK(); } +} // namespace + +template +Status ComputeGradientError(const Scope& scope, const ops::Output& x, + const TensorShape& x_shape, const ops::Output& y, + const TensorShape& y_shape, T* max_error) { + // Initialize 'x_data' to random values. + Tensor x_data(x.type(), x_shape); + auto x_data_flat = x_data.flat(); + x_data_flat.setRandom(); + // Compute gradient error. + return ComputeGradientErrorInternal(scope, x, x_shape, y, y_shape, &x_data, + max_error); +} + +template +Status ComputeGradientError(const Scope& scope, const ops::Output& x, + const Tensor& x_init_value, const ops::Output& y, + const TensorShape& y_shape, T* max_error) { + // Initialize 'x_data' from 'x_init_value'. + Tensor x_data(x_init_value); + // Compute gradient error. + return ComputeGradientErrorInternal(scope, x, x_data.shape(), y, y_shape, + &x_data, max_error); +} + #define INSTANTIATE_GRAD_ERR_TYPE(T) \ template Status ComputeGradientError( \ const Scope& scope, const ops::Output& x, const TensorShape& x_shape, \ - const ops::Output& y, const TensorShape& y_shape, T* max_error) + const ops::Output& y, const TensorShape& y_shape, T* max_error); \ + template Status ComputeGradientError( \ + const Scope& scope, const ops::Output& x, const Tensor& x_init_value, \ + const ops::Output& y, const TensorShape& y_shape, T* max_error); INSTANTIATE_GRAD_ERR_TYPE(float); INSTANTIATE_GRAD_ERR_TYPE(double); diff --git a/tensorflow/cc/framework/gradient_checker.h b/tensorflow/cc/framework/gradient_checker.h index 57e2154b68a..80876afe5c7 100644 --- a/tensorflow/cc/framework/gradient_checker.h +++ b/tensorflow/cc/framework/gradient_checker.h @@ -30,6 +30,12 @@ Status ComputeGradientError(const Scope& scope, const ops::Output& x, const TensorShape& x_shape, const ops::Output& y, const TensorShape& y_shape, T* max_error); +// Overload of ComputeGradientError which takes an initial value for 'x'. +template +Status ComputeGradientError(const Scope& scope, const ops::Output& x, + const Tensor& x_init_value, const ops::Output& y, + const TensorShape& y_shape, T* max_error); + } // namespace tensorflow #endif // THIRD_PARTY_TENSORFLOW_CC_FRAMEWORK_GRADIENT_CHECKER_H_ diff --git a/tensorflow/cc/gradients/nn_grad.cc b/tensorflow/cc/gradients/nn_grad.cc new file mode 100644 index 00000000000..657585e36fc --- /dev/null +++ b/tensorflow/cc/gradients/nn_grad.cc @@ -0,0 +1,77 @@ +/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/cc/ops/nn_ops.h" +#include "tensorflow/cc/ops/standard_ops.h" + +#include "tensorflow/cc/framework/grad_op_registry.h" + +namespace tensorflow { +namespace ops { +namespace { + +Status SoftmaxGrad(const Scope& scope, const Operation& op, + const std::vector& grad_inputs, + std::vector* grad_outputs) { + // Softmax gradient function. + // p = softmax(x) maps from [batch, n] to [batch, m] + // dp/dx = [dp0/dx0 ... dp0/dxn-1 ] + // [ ... ... ] + // [dpm-1/dx0 ... dpm-1/dxn-1] + // dL/dx = dp/dx * dL/dy + // + // Using alternative formula: + // dL/dx = dL/dy * y - sum(dL/dy * y) * y + // = (dL/dy - sum(dL/dy * y)) * y + auto y = op.output(0); + auto dyy = Mul(scope, grad_inputs[0], y); + auto sum = Reshape(scope, Sum(scope, dyy, {1}), {-1, 1}); + auto sub = Sub(scope, grad_inputs[0], sum); + auto dx = Mul(scope, sub, y); + grad_outputs->push_back(dx); + return scope.status(); +} +REGISTER_GRADIENT_OP("Softmax", SoftmaxGrad); + +Status ReluGradHelper(const Scope& scope, const Operation& op, + const std::vector& grad_inputs, + std::vector* grad_outputs) { + auto dx = ReluGrad(scope, grad_inputs[0], op.input(0)); + grad_outputs->push_back(dx); + return scope.status(); +} +REGISTER_GRADIENT_OP("Relu", ReluGradHelper); + +Status Relu6GradHelper(const Scope& scope, const Operation& op, + const std::vector& grad_inputs, + std::vector* grad_outputs) { + auto dx = Relu6Grad(scope, grad_inputs[0], op.input(0)); + grad_outputs->push_back(dx); + return scope.status(); +} +REGISTER_GRADIENT_OP("Relu6", Relu6GradHelper); + +Status EluGradHelper(const Scope& scope, const Operation& op, + const std::vector& grad_inputs, + std::vector* grad_outputs) { + auto dx = EluGrad(scope, grad_inputs[0], op.output(0)); + grad_outputs->push_back(dx); + return scope.status(); +} +REGISTER_GRADIENT_OP("Elu", EluGradHelper); + +} // anonymous namespace +} // namespace ops +} // namespace tensorflow diff --git a/tensorflow/cc/gradients/nn_grad_test.cc b/tensorflow/cc/gradients/nn_grad_test.cc new file mode 100644 index 00000000000..ef0a2f9626b --- /dev/null +++ b/tensorflow/cc/gradients/nn_grad_test.cc @@ -0,0 +1,91 @@ +/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/cc/framework/grad_op_registry.h" +#include "tensorflow/cc/framework/gradient_checker.h" +#include "tensorflow/cc/framework/testutil.h" +#include "tensorflow/cc/gradients/grad_testutil.h" +#include "tensorflow/cc/ops/standard_ops.h" +#include "tensorflow/core/framework/tensor_testutil.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/lib/random/random.h" + +namespace tensorflow { +using namespace ops; // NOLINT(build/namespaces) + +namespace { + +class NNGradTest : public ::testing::Test { + protected: + NNGradTest() : scope_(Scope::NewRootScope()) {} + + void RunTest(const Output& x, const TensorShape& x_shape, const Output& y, + const TensorShape& y_shape) { + float max_error; + TF_ASSERT_OK( + ComputeGradientError(scope_, x, x_shape, y, y_shape, &max_error)); + EXPECT_LT(max_error, 1e-4); + } + + void RunTest(const Output& x, const Tensor& x_init_value, const Output& y, + const TensorShape& y_shape) { + float max_error; + TF_ASSERT_OK( + ComputeGradientError(scope_, x, x_init_value, y, y_shape, &max_error)); + EXPECT_LT(max_error, 1e-4); + } + + Scope scope_; +}; + +TEST_F(NNGradTest, SoftmaxGrad) { + TensorShape shape({32, 10}); + auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(shape)); + auto y = Softmax(scope_, x); + RunTest(x, shape, y, shape); +} + +TEST_F(NNGradTest, ReluGrad) { + TensorShape shape({5, 2}); + auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(shape)); + auto y = Relu(scope_, x); + // Avoid input values where ReLU gradient is not well defined (around zero). + Tensor x_init_value = test::AsTensor( + {-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9}, {5, 2}); + RunTest(x, x_init_value, y, shape); +} + +TEST_F(NNGradTest, Relu6Grad) { + TensorShape shape({5, 2}); + auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(shape)); + auto y = Relu6(scope_, x); + // Avoid input values where ReLU gradient is not well defined (around zero + // and six). + Tensor x_init_value = test::AsTensor( + {-0.9, -0.7, -0.5, -0.3, -0.1, 6.1, 6.3, 6.5, 6.7, 6.9}, {5, 2}); + RunTest(x, x_init_value, y, shape); +} + +TEST_F(NNGradTest, EluGrad) { + TensorShape shape({5, 2}); + auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(shape)); + auto y = Elu(scope_, x); + Tensor x_init_value = test::AsTensor( + {-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9}, {5, 2}); + RunTest(x, x_init_value, y, shape); +} + +} // namespace +} // namespace tensorflow From 9c21427540b332e7fb06c5e9620b0e99119fd96d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 3 Nov 2016 08:18:12 -0800 Subject: [PATCH 41/54] Replace os.walk and glob.glob with corresponding gfile methods. Change: 138078360 --- tensorflow/examples/image_retraining/retrain.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tensorflow/examples/image_retraining/retrain.py b/tensorflow/examples/image_retraining/retrain.py index 392f0176d37..74c1de8fd7d 100644 --- a/tensorflow/examples/image_retraining/retrain.py +++ b/tensorflow/examples/image_retraining/retrain.py @@ -66,7 +66,6 @@ from __future__ import print_function import argparse from datetime import datetime -import glob import hashlib import os.path import random @@ -131,7 +130,7 @@ def create_image_lists(image_dir, testing_percentage, validation_percentage): print("Image directory '" + image_dir + "' not found.") return None result = {} - sub_dirs = [x[0] for x in os.walk(image_dir)] + sub_dirs = [x[0] for x in gfile.Walk(image_dir)] # The root directory comes first, so skip it. is_root_dir = True for sub_dir in sub_dirs: @@ -146,7 +145,7 @@ def create_image_lists(image_dir, testing_percentage, validation_percentage): print("Looking for images in '" + dir_name + "'") for extension in extensions: file_glob = os.path.join(image_dir, dir_name, '*.' + extension) - file_list.extend(glob.glob(file_glob)) + file_list.extend(gfile.Glob(file_glob)) if not file_list: print('No files found') continue From 4cbdead95f22de74bcbc72a68c9a38d465202db9 Mon Sep 17 00:00:00 2001 From: Illia Polosukhin Date: Thu, 3 Nov 2016 08:24:37 -0800 Subject: [PATCH 42/54] Renamed: * tf.all_variables to tf.global_variables * tf.VARIABLES -> tf.GLOBAL_VARIABLES * tf.initialize_all_variables -> tf.global_variable_initializers Change: 138078998 --- .../framework/python/ops/variables_test.py | 38 ++--- tensorflow/python/framework/ops.py | 37 +++-- .../python/kernel_tests/variables_test.py | 28 ++-- tensorflow/python/ops/state_ops.py | 17 ++- tensorflow/python/ops/variables.py | 143 +++++++++++------- tensorflow/python/util/decorator_utils.py | 22 +++ 6 files changed, 186 insertions(+), 99 deletions(-) diff --git a/tensorflow/contrib/framework/python/ops/variables_test.py b/tensorflow/contrib/framework/python/ops/variables_test.py index eb0a2c2d8eb..49683faf90f 100644 --- a/tensorflow/contrib/framework/python/ops/variables_test.py +++ b/tensorflow/contrib/framework/python/ops/variables_test.py @@ -36,7 +36,7 @@ class LocalVariableTest(tf.test.TestCase): variables = tf.local_variables() self.assertEquals(2, len(variables)) self.assertRaises(tf.OpError, sess.run, variables) - tf.initialize_variables(variables).run() + tf.variables_initializer(variables).run() self.assertAllEqual(set([value0, value1]), set(sess.run(variables))) def testLocalVariableNameAndShape(self): @@ -51,7 +51,7 @@ class LocalVariableTest(tf.test.TestCase): with self.test_session(): with tf.variable_scope('A'): a = tf.contrib.framework.local_variable(0) - self.assertFalse(a in tf.all_variables()) + self.assertFalse(a in tf.global_variables()) self.assertTrue(a in tf.local_variables()) def testLocalVariableNotInVariablesToRestore(self): @@ -82,7 +82,7 @@ class LocalVariableTest(tf.test.TestCase): def testInitializedVariableValue(self): with self.test_session() as sess: a = tf.contrib.framework.local_variable([0, 0, 0, 0, 0], name='a') - sess.run(tf.initialize_local_variables()) + sess.run(tf.local_variables_initializer()) self.assertAllEqual(a.eval(), [0]*5) @@ -439,7 +439,7 @@ class ModelVariablesTest(tf.test.TestCase): with self.test_session(): with tf.variable_scope('A'): a = tf.contrib.framework.model_variable('a', [5]) - self.assertTrue(a in tf.all_variables()) + self.assertTrue(a in tf.global_variables()) self.assertTrue(a in tf.get_collection(tf.GraphKeys.MODEL_VARIABLES)) self.assertFalse(a in tf.local_variables()) @@ -474,7 +474,7 @@ class ModelVariablesTest(tf.test.TestCase): with self.test_session() as sess: a = tf.contrib.framework.model_variable( 'a', [5], initializer=tf.ones_initializer) - sess.run(tf.initialize_all_variables()) + sess.run(tf.global_variables_initializer()) self.assertAllEqual(a.eval(), [1]*5) def testDeviceFn(self): @@ -667,7 +667,7 @@ class AssignFromValuesTest(tf.test.TestCase): var_names_to_values) # Initialize the variables. - sess.run(tf.initialize_all_variables()) + sess.run(tf.global_variables_initializer()) # Perform the assignment. sess.run(assign_op, feed_dict) @@ -697,7 +697,7 @@ class AssignFromValuesTest(tf.test.TestCase): var_names_to_values) # Initialize the variables. - sess.run(tf.initialize_all_variables()) + sess.run(tf.global_variables_initializer()) # Perform the assignment. sess.run(assign_op, feed_dict) @@ -725,7 +725,7 @@ class AssignFromValuesFnTest(tf.test.TestCase): init_fn = tf.contrib.framework.assign_from_values_fn(var_names_to_values) # Initialize the variables. - sess.run(tf.initialize_all_variables()) + sess.run(tf.global_variables_initializer()) # Perform the assignment. init_fn(sess) @@ -754,7 +754,7 @@ class AssignFromValuesFnTest(tf.test.TestCase): init_fn = tf.contrib.framework.assign_from_values_fn(var_names_to_values) # Initialize the variables. - sess.run(tf.initialize_all_variables()) + sess.run(tf.global_variables_initializer()) # Perform the assignment. init_fn(sess) @@ -786,7 +786,7 @@ class AssignFromCheckpointTest(tf.test.TestCase): var_value = var_names_to_values[var_name] var_list.append(tf.Variable(var_value, name=var_name)) saver = tf.train.Saver(var_list) - init_op = tf.initialize_variables(var_list) + init_op = tf.variables_initializer(var_list) sess.run(init_op) # Save the initialized values in the file at 'checkpoint_dir' return saver.save(sess, checkpoint_dir, global_step=global_step) @@ -808,7 +808,7 @@ class AssignFromCheckpointTest(tf.test.TestCase): model_path, vars_to_restore) # Initialize the variables. - sess.run(tf.initialize_all_variables()) + sess.run(tf.global_variables_initializer()) # Perform the assignment. sess.run(op, feed_dict) @@ -859,7 +859,7 @@ class AssignFromCheckpointTest(tf.test.TestCase): vars_to_restore) # Initialize the variables. - sess.run(tf.initialize_all_variables()) + sess.run(tf.global_variables_initializer()) # Perform the assignment. sess.run(op, feed_dict) @@ -890,7 +890,7 @@ class AssignFromCheckpointFnTest(tf.test.TestCase): var_value = var_names_to_values[var_name] var_list.append(tf.Variable(var_value, name=var_name)) saver = tf.train.Saver(var_list) - init_op = tf.initialize_variables(var_list) + init_op = tf.variables_initializer(var_list) sess.run(init_op) # Save the initialized values in the file at 'checkpoint_dir' return saver.save(sess, checkpoint_dir, global_step=global_step) @@ -912,7 +912,7 @@ class AssignFromCheckpointFnTest(tf.test.TestCase): model_path, vars_to_restore) # Initialize the variables. - sess.run(tf.initialize_all_variables()) + sess.run(tf.global_variables_initializer()) # Perform the assignment. init_fn(sess) @@ -938,7 +938,7 @@ class AssignFromCheckpointFnTest(tf.test.TestCase): model_path, vars_to_restore) # Initialize the variables. - sess.run(tf.initialize_all_variables()) + sess.run(tf.global_variables_initializer()) # Perform the assignment. with self.assertRaises(tf.errors.InvalidArgumentError): @@ -961,7 +961,7 @@ class AssignFromCheckpointFnTest(tf.test.TestCase): model_path, vars_to_restore, reshape_variables=True) # Initialize the variables. - sess.run(tf.initialize_all_variables()) + sess.run(tf.global_variables_initializer()) # Perform the assignment. init_fn(sess) @@ -989,7 +989,7 @@ class AssignFromCheckpointFnTest(tf.test.TestCase): vars_to_restore) # Initialize the variables. - sess.run(tf.initialize_all_variables()) + sess.run(tf.global_variables_initializer()) # Perform the assignment. with self.assertRaises(tf.errors.NotFoundError): @@ -1015,7 +1015,7 @@ class AssignFromCheckpointFnTest(tf.test.TestCase): ignore_missing_vars=True) # Initialize the variables. - sess.run(tf.initialize_all_variables()) + sess.run(tf.global_variables_initializer()) # Perform the assignment. init_fn(sess) @@ -1044,7 +1044,7 @@ class AssignFromCheckpointFnTest(tf.test.TestCase): ignore_missing_vars=True) # Initialize the variables. - sess.run(tf.initialize_all_variables()) + sess.run(tf.global_variables_initializer()) # Perform the assignment. init_fn(sess) diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index 1f8aeefba27..56018f735a4 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -43,6 +43,7 @@ from tensorflow.python.framework import tensor_shape from tensorflow.python.framework import versions from tensorflow.python.platform import tf_logging as logging from tensorflow.python.util import compat +from tensorflow.python.util import decorator_utils def _override_helper(clazz_object, operator, func): @@ -3823,10 +3824,18 @@ class GraphKeys(object): The following standard keys are defined: - * `VARIABLES`: the `Variable` objects that comprise a model, and - must be saved and restored together. See - [`tf.all_variables()`](../../api_docs/python/state_ops.md#all_variables) + * `GLOBAL_VARIABLES`: the default collection of `Variable` objects, shared + across distributed environment (model variables are subset of these). See + [`tf.global_variables()`](../../api_docs/python/state_ops.md#global_variables) for more details. + Commonly, all `TRAINABLE_VARIABLES` variables will be in `MODEL_VARIABLES`, + and all `MODEL_VARIABLES` variables will be in `GLOBAL_VARIABLES`. + * `LOCAL_VARIABLES`: the subset of `Variable` objects that are local to each + machine. Usually used for temporarily variables, like counters. + Note: use `tf.contrib.framework.local_variable` to add to this collection. + * `MODEL_VARIABLES`: the subset of `Variable` objects that are used in the + model for inference (feed forward). Note: use + `tf.contrib.framework.model_variable` to add to this collection. * `TRAINABLE_VARIABLES`: the subset of `Variable` objects that will be trained by an optimizer. See [`tf.trainable_variables()`](../../api_docs/python/state_ops.md#trainable_variables) @@ -3850,16 +3859,17 @@ class GraphKeys(object): * `ACTIVATIONS`: activations of neural network layers """ - # Key to collect Variable objects that must be saved and restored - # by the model. - VARIABLES = "variables" - # Key to collect Variable objects that will be trained by the - # optimizers. - TRAINABLE_VARIABLES = "trainable_variables" - # Key to collect local variables that are not saved/restored. + # Key to collect Variable objects that are global (shared across machines). + # Default collection for all variables, except local ones. + GLOBAL_VARIABLES = "variables" + # Key to collect local variables that are local to the machine and are not + # saved/restored. LOCAL_VARIABLES = "local_variables" # Key to collect model variables defined by layers. MODEL_VARIABLES = "model_variables" + # Key to collect Variable objects that will be trained by the + # optimizers. + TRAINABLE_VARIABLES = "trainable_variables" # Key to collect summaries. SUMMARIES = "summaries" # Key to collect QueueRunners. @@ -3909,6 +3919,13 @@ class GraphKeys(object): COND_CONTEXT = "cond_context" WHILE_CONTEXT = "while_context" + @decorator_utils.classproperty + def VARIABLES(cls): # pylint: disable=no-self-argument + logging.warning("VARIABLES collection name is deprecated, " + "please use GLOBAL_VARIABLES instead.\n" + "VARIABLES will be removed after 2017-03-02.") + return cls.GLOBAL_VARIABLES + def add_to_collection(name, value): """Wrapper for `Graph.add_to_collection()` using the default graph. diff --git a/tensorflow/python/kernel_tests/variables_test.py b/tensorflow/python/kernel_tests/variables_test.py index 1af848589f5..507a3b9f32d 100644 --- a/tensorflow/python/kernel_tests/variables_test.py +++ b/tensorflow/python/kernel_tests/variables_test.py @@ -49,7 +49,7 @@ class VariablesTestCase(tf.test.TestCase): with self.assertRaisesOpError("Attempting to use uninitialized value"): var1.eval() - tf.initialize_all_variables().run() + tf.global_variables_initializer().run() self.assertAllClose(0.0, var0.eval()) self.assertAllClose(1.1, var1.eval()) @@ -75,7 +75,7 @@ class VariablesTestCase(tf.test.TestCase): self.assertEqual([3, 6], depdep.get_shape()) self.assertEqual([3, 6], depdep.get_shape()) - tf.initialize_all_variables().run() + tf.global_variables_initializer().run() self.assertAllClose(rnd.eval(), dep.eval()) self.assertAllClose(rnd.eval() + dep.eval() + 2.0, @@ -95,7 +95,7 @@ class VariablesTestCase(tf.test.TestCase): plus_one = var.assign_add(1.0) minus_one = var.assign_sub(2.0) four = var.assign(4.0) - tf.initialize_all_variables().run() + tf.global_variables_initializer().run() self.assertAllClose(0.0, var.eval()) self.assertAllClose(1.0, plus_one.eval()) @@ -113,7 +113,7 @@ class VariablesTestCase(tf.test.TestCase): var = tf.Variable(zero) count_up_to = var.count_up_to(3) - tf.initialize_all_variables().run() + tf.global_variables_initializer().run() self.assertEqual(0, var.eval()) self.assertEqual(0, count_up_to.eval()) @@ -193,7 +193,7 @@ class VariablesTestCase(tf.test.TestCase): with self.test_session(): var_x = tf.Variable(2.0) var_y = tf.Variable(3.0) - tf.initialize_all_variables().run() + tf.global_variables_initializer().run() self.assertAllClose(2.0, var_x.eval()) self.assertAllClose(3.0, var_y.eval()) self.assertAllClose(5.0, tf.add(var_x, var_y).eval()) @@ -204,7 +204,7 @@ class VariablesTestCase(tf.test.TestCase): zero_size_const = tf.ones([2, 0]) variable_mul = tf.matmul(zero_size_const, zero_size_var) const_mul = tf.matmul(zero_size_const, zero_size_const, transpose_b=True) - tf.initialize_all_variables().run() + tf.global_variables_initializer().run() variable_output = variable_mul.eval() self.assertAllClose(const_mul.eval(), variable_output) self.assertAllClose([[0., 0.], [0., 0.]], variable_output) @@ -230,7 +230,7 @@ class VariablesTestCase(tf.test.TestCase): 2.0, trainable=True, collections=[tf.GraphKeys.TRAINABLE_VARIABLES, tf.GraphKeys.VARIABLES]) - self.assertEqual([var_x, var_y, var_z, var_t], tf.all_variables()) + self.assertEqual([var_x, var_y, var_z, var_t], tf.global_variables()) self.assertEqual([var_x, var_z, var_t], tf.trainable_variables()) def testOperators(self): @@ -269,7 +269,7 @@ class VariablesTestCase(tf.test.TestCase): var_t = tf.Variable(rnd) slice_v = var_t[2, 0:0] - tf.initialize_all_variables().run() + tf.global_variables_initializer().run() self.assertAllClose([2.0], add.eval()) self.assertAllClose([3.0], radd.eval()) self.assertAllClose([1.0], sub.eval()) @@ -302,7 +302,7 @@ class VariablesTestCase(tf.test.TestCase): def testSession(self): with self.test_session() as sess: var = tf.Variable([1, 12]) - tf.initialize_all_variables().run() + tf.global_variables_initializer().run() self.assertAllClose([1, 12], sess.run(var)) def testDevicePlacement(self): @@ -310,7 +310,7 @@ class VariablesTestCase(tf.test.TestCase): with tf.device("/cpu:0"): var = tf.Variable([1, 12]) init_value = var.initialized_value() - init_op = tf.initialize_all_variables() + init_op = tf.global_variables_initializer() self.assertEqual(var.op.device, init_value.device) self.assertEqual(var.op.device, init_op.device) sess.run(init_op) @@ -348,7 +348,7 @@ class VariablesTestCase(tf.test.TestCase): with self.assertRaises(tf.errors.FailedPreconditionError): v2.eval() - tf.initialize_all_variables().run() + tf.global_variables_initializer().run() self.assertAllClose(np.negative(value), v2.eval()) def testInitializerFunctionDevicePlacement(self): @@ -385,7 +385,7 @@ class IsInitializedTest(tf.test.TestCase): _ = v, w uninited = tf.report_uninitialized_variables() self.assertAllEqual(np.array([b"v", b"w"]), sess.run(uninited)) - tf.initialize_all_variables().run() + tf.global_variables_initializer().run() self.assertEqual(0, sess.run(uninited).size) def testVariableList(self): @@ -411,7 +411,7 @@ class IsInitializedTest(tf.test.TestCase): a = tf.Variable(tf.zeros([0, 2])) b = tf.Variable(tf.ones([2, 2])) objective = tf.reduce_sum(b + tf.matmul(a, a, transpose_a=True)) - tf.initialize_all_variables().run() + tf.global_variables_initializer().run() do_opt = tf.train.GradientDescentOptimizer(0.1).minimize(objective) sess.run([do_opt]) self.assertAllClose([[0.9, 0.9], [0.9, 0.9]], b.eval()) @@ -431,7 +431,7 @@ class ObsoleteIsInitializedTest(tf.test.TestCase): inited = tf.assert_variables_initialized() with self.assertRaisesOpError("Attempting to use uninitialized value"): sess.run(inited) - tf.initialize_all_variables().run() + tf.global_variables_initializer().run() sess.run(inited) def testVariableList(self): diff --git a/tensorflow/python/ops/state_ops.py b/tensorflow/python/ops/state_ops.py index 94c80227219..2c12865df06 100644 --- a/tensorflow/python/ops/state_ops.py +++ b/tensorflow/python/ops/state_ops.py @@ -22,15 +22,15 @@ TensorFlow provides a set of functions to help manage the set of variables collected in the graph. -@@all_variables -@@trainable_variables +@@global_variables @@local_variables @@model_variables +@@trainable_variables @@moving_average_variables -@@initialize_all_variables -@@initialize_variables -@@initialize_local_variables +@@global_variables_initializer +@@local_variables_initializer +@@variables_initializer @@is_variable_initialized @@report_uninitialized_variables @@assert_variables_initialized @@ -113,6 +113,13 @@ automatically by the optimizers in most cases. @@export_meta_graph @@import_meta_graph +# Deprecated functions (removed after 2017-03-02). Please don't use them. + +@@all_variables +@@initialize_all_variables +@@initialize_local_variables +@@initialize_variables + """ from __future__ import absolute_import diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py index 05f780ccaac..4478359064f 100644 --- a/tensorflow/python/ops/variables.py +++ b/tensorflow/python/ops/variables.py @@ -25,6 +25,7 @@ from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import state_ops +from tensorflow.python.util.deprecation import deprecated class Variable(object): @@ -81,16 +82,16 @@ class Variable(object): ``` The most common initialization pattern is to use the convenience function - `initialize_all_variables()` to add an Op to the graph that initializes + `global_variable_initializers()` to add an Op to the graph that initializes all the variables. You then run that Op after launching the graph. ```python - # Add an Op to initialize all variables. - init_op = tf.initialize_all_variables() + # Add an Op to initialize global variables. + init_op = tf.global_variable_initializers() # Launch the graph in a session. with tf.Session() as sess: - # Run the Op that initializes all variables. + # Run the Op that initializes global variables. sess.run(init_op) # ...you can now run any Op that uses variable values... ``` @@ -101,8 +102,8 @@ class Variable(object): All variables are automatically collected in the graph where they are created. By default, the constructor adds the new variable to the graph - collection `GraphKeys.VARIABLES`. The convenience function - `all_variables()` returns the contents of that collection. + collection `GraphKeys.GLOBAL_VARIABLES`. The convenience function + `global_variables()` returns the contents of that collection. When building a machine learning model it is often convenient to distinguish between variables holding the trainable model parameters and other variables @@ -158,7 +159,7 @@ class Variable(object): """Creates a new variable with value `initial_value`. The new variable is added to the graph collections listed in `collections`, - which defaults to `[GraphKeys.VARIABLES]`. + which defaults to `[GraphKeys.GLOBAL_VARIABLES]`. If `trainable` is `True` the variable is also added to the graph collection `GraphKeys.TRAINABLE_VARIABLES`. @@ -177,7 +178,7 @@ class Variable(object): collection `GraphKeys.TRAINABLE_VARIABLES`. This collection is used as the default list of variables to use by the `Optimizer` classes. collections: List of graph collections keys. The new variable is added to - these collections. Defaults to `[GraphKeys.VARIABLES]`. + these collections. Defaults to `[GraphKeys.GLOBAL_VARIABLES]`. validate_shape: If `False`, allows the variable to be initialized with a value of unknown shape. If `True`, the default, the shape of `initial_value` must be known. @@ -244,7 +245,7 @@ class Variable(object): collection `GraphKeys.TRAINABLE_VARIABLES`. This collection is used as the default list of variables to use by the `Optimizer` classes. collections: List of graph collections keys. The new variable is added to - these collections. Defaults to `[GraphKeys.VARIABLES]`. + these collections. Defaults to `[GraphKeys.GLOBAL_VARIABLES]`. validate_shape: If `False`, allows the variable to be initialized with a value of unknown shape. If `True`, the default, the shape of `initial_value` must be known. @@ -274,7 +275,7 @@ class Variable(object): "dtype must also be specified when initial_value is callable.") if collections is None: - collections = [ops.GraphKeys.VARIABLES] + collections = [ops.GraphKeys.GLOBAL_VARIABLES] if not isinstance(collections, (list, tuple, set)): raise ValueError( "collections argument to Variable constructor must be a list, tuple, " @@ -470,7 +471,7 @@ class Variable(object): ```python v = tf.Variable([1, 2]) - init = tf.initialize_all_variables() + init = tf.global_variable_initializers() with tf.Session() as sess: sess.run(init) @@ -1028,17 +1029,28 @@ class PartitionedVariable(object): "assign() has not been implemented for PartitionedVariable.") -def all_variables(): - """Returns all variables that must be saved/restored. +def global_variables(): + """Returns global variables. - The `Variable()` constructor automatically adds new variables to the graph - collection `GraphKeys.VARIABLES`. This convenience function returns the - contents of that collection. + Global variables are variables that are shared across machines in a + distributed environment. The `Variable()` constructor or `get_variable()` + automatically adds new variables to the graph collection + `GraphKeys.GLOBAL_VARIABLES`. + This convenience function returns the contents of that collection. + + An alternative to global variables are local variables. See + [`tf.local_variables()`](../../api_docs/python/state_ops.md#local_variables) Returns: A list of `Variable` objects. """ - return ops.get_collection(ops.GraphKeys.VARIABLES) + return ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) + + +@deprecated("2016-03-02", "Please use tf.global_variables instead.") +def all_variables(): + """See `tf.global_variables`.""" + return global_variables() def _all_saveable_objects(): @@ -1048,8 +1060,37 @@ def _all_saveable_objects(): A list of `Variable` and `SaveableObject` to be checkpointed """ # TODO(andreasst): make this function public once things are settled. - return ops.get_collection(ops.GraphKeys.VARIABLES) + ops.get_collection( - ops.GraphKeys.SAVEABLE_OBJECTS) + return (ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) + + ops.get_collection(ops.GraphKeys.SAVEABLE_OBJECTS)) + + +def local_variables(): + """Returns local variables. + + Local variables - per process variables, usually not saved/restored to + checkpoint and used for temporary or intermediate values. + For example, they can be used as counters for metrics computation or + number of epochs this machine has read data. + The `local_variable()` automatically adds new variable to + `GraphKeys.LOCAL_VARIABLES`. + This convenience function returns the contents of that collection. + + An alternative to local variables are global variables. See + [`tf.global_variables()`](../../api_docs/python/state_ops.md#global_variables) + + Returns: + A list of local `Variable` objects. + """ + return ops.get_collection(ops.GraphKeys.LOCAL_VARIABLES) + + +def model_variables(): + """Returns all variables in the MODEL_VARIABLES collection. + + Returns: + A list of local Variable objects. + """ + return ops.get_collection(ops.GraphKeys.MODEL_VARIABLES) def trainable_variables(): @@ -1066,24 +1107,6 @@ def trainable_variables(): return ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES) -def local_variables(): - """Returns all variables created with collection=[LOCAL_VARIABLES]. - - Returns: - A list of local Variable objects. - """ - return ops.get_collection(ops.GraphKeys.LOCAL_VARIABLES) - - -def model_variables(): - """Returns all variables in the MODEL_VARIABLES collection. - - Returns: - A list of local Variable objects. - """ - return ops.get_collection(ops.GraphKeys.MODEL_VARIABLES) - - def moving_average_variables(): """Returns all variables that maintain their moving averages. @@ -1098,7 +1121,7 @@ def moving_average_variables(): return ops.get_collection(ops.GraphKeys.MOVING_AVERAGE_VARIABLES) -def initialize_variables(var_list, name="init"): +def variables_initializer(var_list, name="init"): """Returns an Op that initializes a list of variables. After you launch the graph in a session, you can run the returned Op to @@ -1123,26 +1146,44 @@ def initialize_variables(var_list, name="init"): return control_flow_ops.no_op(name=name) -def initialize_all_variables(): - """Returns an Op that initializes all variables. +@deprecated("2017-03-02", "Use `tf.variables_initializer` instead.") +def initialize_variables(var_list, name="init"): + """See `tf.variables_initializer`.""" + return variables_initializer(var_list, name=name) - This is just a shortcut for `initialize_variables(all_variables())` + +def global_variables_initializer(): + """Returns an Op that initializes global variables. + + This is just a shortcut for `variable_initializers(global_variables())` Returns: - An Op that initializes all variables in the graph. + An Op that initializes global variables in the graph. """ - return initialize_variables(all_variables()) + return variables_initializer(global_variables()) -def initialize_local_variables(): +@deprecated("2017-03-02", "Use `tf.global_variables_initializer` instead.") +def initialize_all_variables(): + """See `tf.global_variables_initializer`.""" + return global_variables_initializer() + + +def local_variables_initializer(): """Returns an Op that initializes all local variables. - This is just a shortcut for `initialize_variables(local_variables())` + This is just a shortcut for `variable_initializers(local_variables())` Returns: An Op that initializes all local variables in the graph. """ - return initialize_variables(local_variables()) + return variables_initializer(local_variables()) + + +@deprecated("2017-03-02", "Use `tf.local_variables_initializer` instead.") +def initialize_local_variables(): + """See `tf.local_variables_initializer`.""" + return local_variables_initializer() def is_variable_initialized(variable): @@ -1173,13 +1214,13 @@ def assert_variables_initialized(var_list=None): Args: var_list: List of `Variable` objects to check. Defaults to the - value of `all_variables().` + value of `global_variables().` Returns: An Op, or None if there are no variables. """ if var_list is None: - var_list = all_variables() + local_variables() + var_list = global_variables() + local_variables() # Backwards compatibility for old-style variables. TODO(touts): remove. if not var_list: var_list = [] @@ -1208,7 +1249,7 @@ def report_uninitialized_variables(var_list=None, Args: var_list: List of `Variable` objects to check. Defaults to the - value of `all_variables() + local_variables()` + value of `global_variables() + local_variables()` name: Optional name of the `Operation`. Returns: @@ -1216,7 +1257,7 @@ def report_uninitialized_variables(var_list=None, 1-D tensor if there are no variables or no uninitialized variables. """ if var_list is None: - var_list = all_variables() + local_variables() + var_list = global_variables() + local_variables() # Backwards compatibility for old-style variables. TODO(touts): remove. if not var_list: var_list = [] @@ -1248,7 +1289,7 @@ ops.register_tensor_conversion_function( ops.register_dense_tensor_like_type(Variable) ops.register_proto_function( - ops.GraphKeys.VARIABLES, + ops.GraphKeys.GLOBAL_VARIABLES, proto_type=variable_pb2.VariableDef, to_proto=Variable.to_proto, from_proto=Variable.from_proto) diff --git a/tensorflow/python/util/decorator_utils.py b/tensorflow/python/util/decorator_utils.py index 155003498ce..c4b033d59ae 100644 --- a/tensorflow/python/util/decorator_utils.py +++ b/tensorflow/python/util/decorator_utils.py @@ -60,3 +60,25 @@ def validate_callable(func, decorator_name): ' @property appears before @%s in your source code:' '\n\n@property\n@%s\ndef method(...)' % ( func, decorator_name, decorator_name)) + + +class classproperty(object): # pylint: disable=invalid-name + """Class property decorator. + + Example usage: + + class MyClass(object): + + @classproperty + def value(cls): + return '123' + + > print MyClass.value + 123 + """ + + def __init__(self, func): + self._func = func + + def __get__(self, owner_self, owner_cls): + return self._func(owner_cls) From 10b0eb648e3d57fab48da6aef4f875c61f4d2099 Mon Sep 17 00:00:00 2001 From: Benoit Steiner Date: Thu, 3 Nov 2016 08:35:07 -0800 Subject: [PATCH 43/54] Added missing override qualifier for the FormatLibraryFileName method Change: 138079985 --- tensorflow/core/platform/posix/env.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/core/platform/posix/env.cc b/tensorflow/core/platform/posix/env.cc index 2f9c8e4b2f0..f353fb1c924 100644 --- a/tensorflow/core/platform/posix/env.cc +++ b/tensorflow/core/platform/posix/env.cc @@ -120,7 +120,8 @@ class PosixEnv : public Env { symbol); } - string FormatLibraryFileName(const string& name, const string& version) { + string FormatLibraryFileName(const string& name, + const string& version) override { return tensorflow::internal::FormatLibraryFileName(name, version); } }; From 80ea8adfcf8207e79b17000095d241d6b4feee6e Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 3 Nov 2016 09:08:59 -0800 Subject: [PATCH 44/54] Update generated Python Op docs. Change: 138083939 --- tensorflow/g3doc/api_docs/python/framework.md | 14 +- .../shard1/tf.all_variables.md | 14 +- .../shard2/tf.global_variables_initializer.md | 10 + .../shard2/tf.initialize_all_variables.md | 12 +- .../shard2/tf.initialize_local_variables.md | 12 +- .../shard2/tf.local_variables_initializer.md | 10 + .../tf.report_uninitialized_variables.md | 2 +- .../shard2/tf.variables_initializer.md | 24 +++ .../shard3/tf.global_variables.md | 17 ++ .../shard3/tf.local_variables.md | 15 +- .../shard4/tf.initialize_variables.md | 26 +-- .../shard7/tf.assert_variables_initialized.md | 2 +- .../shard8/tf.GraphKeys.md | 14 +- .../shard8/tf.Variable.md | 18 +- tensorflow/g3doc/api_docs/python/index.md | 4 + tensorflow/g3doc/api_docs/python/state_ops.md | 175 ++++++++++++------ 16 files changed, 250 insertions(+), 119 deletions(-) create mode 100644 tensorflow/g3doc/api_docs/python/functions_and_classes/shard2/tf.global_variables_initializer.md create mode 100644 tensorflow/g3doc/api_docs/python/functions_and_classes/shard2/tf.local_variables_initializer.md create mode 100644 tensorflow/g3doc/api_docs/python/functions_and_classes/shard2/tf.variables_initializer.md create mode 100644 tensorflow/g3doc/api_docs/python/functions_and_classes/shard3/tf.global_variables.md diff --git a/tensorflow/g3doc/api_docs/python/framework.md b/tensorflow/g3doc/api_docs/python/framework.md index 498e7d999f3..86de039e7bd 100644 --- a/tensorflow/g3doc/api_docs/python/framework.md +++ b/tensorflow/g3doc/api_docs/python/framework.md @@ -2845,10 +2845,18 @@ variables. The following standard keys are defined: -* `VARIABLES`: the `Variable` objects that comprise a model, and - must be saved and restored together. See - [`tf.all_variables()`](../../api_docs/python/state_ops.md#all_variables) +* `GLOBAL_VARIABLES`: the default collection of `Variable` objects, shared + across distributed environment (model variables are subset of these). See + [`tf.global_variables()`](../../api_docs/python/state_ops.md#global_variables) for more details. + Commonly, all `TRAINABLE_VARIABLES` variables will be in `MODEL_VARIABLES`, + and all `MODEL_VARIABLES` variables will be in `GLOBAL_VARIABLES`. +* `LOCAL_VARIABLES`: the subset of `Variable` objects that are local to each + machine. Usually used for temporarily variables, like counters. + Note: use `tf.contrib.framework.local_variable` to add to this collection. +* `MODEL_VARIABLES`: the subset of `Variable` objects that are used in the + model for inference (feed forward). Note: use + `tf.contrib.framework.model_variable` to add to this collection. * `TRAINABLE_VARIABLES`: the subset of `Variable` objects that will be trained by an optimizer. See [`tf.trainable_variables()`](../../api_docs/python/state_ops.md#trainable_variables) diff --git a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard1/tf.all_variables.md b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard1/tf.all_variables.md index 904b99f321a..a64640478ff 100644 --- a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard1/tf.all_variables.md +++ b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard1/tf.all_variables.md @@ -1,12 +1,8 @@ -### `tf.all_variables()` {#all_variables} +### `tf.all_variables(*args, **kwargs)` {#all_variables} -Returns all variables that must be saved/restored. +See `tf.global_variables`. (deprecated) -The `Variable()` constructor automatically adds new variables to the graph -collection `GraphKeys.VARIABLES`. This convenience function returns the -contents of that collection. - -##### Returns: - - A list of `Variable` objects. +THIS FUNCTION IS DEPRECATED. It will be removed after 2016-03-02. +Instructions for updating: +Please use tf.global_variables instead. diff --git a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard2/tf.global_variables_initializer.md b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard2/tf.global_variables_initializer.md new file mode 100644 index 00000000000..b1ebdcc3270 --- /dev/null +++ b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard2/tf.global_variables_initializer.md @@ -0,0 +1,10 @@ +### `tf.global_variables_initializer()` {#global_variables_initializer} + +Returns an Op that initializes global variables. + +This is just a shortcut for `variable_initializers(global_variables())` + +##### Returns: + + An Op that initializes global variables in the graph. + diff --git a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard2/tf.initialize_all_variables.md b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard2/tf.initialize_all_variables.md index 9a0e5d8261b..ec240fc6088 100644 --- a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard2/tf.initialize_all_variables.md +++ b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard2/tf.initialize_all_variables.md @@ -1,10 +1,8 @@ -### `tf.initialize_all_variables()` {#initialize_all_variables} +### `tf.initialize_all_variables(*args, **kwargs)` {#initialize_all_variables} -Returns an Op that initializes all variables. +See `tf.global_variables_initializer`. (deprecated) -This is just a shortcut for `initialize_variables(all_variables())` - -##### Returns: - - An Op that initializes all variables in the graph. +THIS FUNCTION IS DEPRECATED. It will be removed after 2017-03-02. +Instructions for updating: +Use `tf.global_variables_initializer` instead. diff --git a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard2/tf.initialize_local_variables.md b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard2/tf.initialize_local_variables.md index 2a56dbb9d69..a6c1395e918 100644 --- a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard2/tf.initialize_local_variables.md +++ b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard2/tf.initialize_local_variables.md @@ -1,10 +1,8 @@ -### `tf.initialize_local_variables()` {#initialize_local_variables} +### `tf.initialize_local_variables(*args, **kwargs)` {#initialize_local_variables} -Returns an Op that initializes all local variables. +See `tf.local_variables_initializer`. (deprecated) -This is just a shortcut for `initialize_variables(local_variables())` - -##### Returns: - - An Op that initializes all local variables in the graph. +THIS FUNCTION IS DEPRECATED. It will be removed after 2017-03-02. +Instructions for updating: +Use `tf.local_variables_initializer` instead. diff --git a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard2/tf.local_variables_initializer.md b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard2/tf.local_variables_initializer.md new file mode 100644 index 00000000000..3f726bdf7ad --- /dev/null +++ b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard2/tf.local_variables_initializer.md @@ -0,0 +1,10 @@ +### `tf.local_variables_initializer()` {#local_variables_initializer} + +Returns an Op that initializes all local variables. + +This is just a shortcut for `variable_initializers(local_variables())` + +##### Returns: + + An Op that initializes all local variables in the graph. + diff --git a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard2/tf.report_uninitialized_variables.md b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard2/tf.report_uninitialized_variables.md index 59c1394a4aa..e3ecdf7733b 100644 --- a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard2/tf.report_uninitialized_variables.md +++ b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard2/tf.report_uninitialized_variables.md @@ -9,7 +9,7 @@ variables if there are any, or an empty array if there are none. * `var_list`: List of `Variable` objects to check. Defaults to the - value of `all_variables() + local_variables()` + value of `global_variables() + local_variables()` * `name`: Optional name of the `Operation`. ##### Returns: diff --git a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard2/tf.variables_initializer.md b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard2/tf.variables_initializer.md new file mode 100644 index 00000000000..ec779e79f66 --- /dev/null +++ b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard2/tf.variables_initializer.md @@ -0,0 +1,24 @@ +### `tf.variables_initializer(var_list, name='init')` {#variables_initializer} + +Returns an Op that initializes a list of variables. + +After you launch the graph in a session, you can run the returned Op to +initialize all the variables in `var_list`. This Op runs all the +initializers of the variables in `var_list` in parallel. + +Calling `initialize_variables()` is equivalent to passing the list of +initializers to `Group()`. + +If `var_list` is empty, however, the function still returns an Op that can +be run. That Op just has no effect. + +##### Args: + + +* `var_list`: List of `Variable` objects to initialize. +* `name`: Optional name for the returned operation. + +##### Returns: + + An Op that run the initializers of all the specified variables. + diff --git a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard3/tf.global_variables.md b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard3/tf.global_variables.md new file mode 100644 index 00000000000..1939f422248 --- /dev/null +++ b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard3/tf.global_variables.md @@ -0,0 +1,17 @@ +### `tf.global_variables()` {#global_variables} + +Returns global variables. + +Global variables are variables that are shared across machines in a +distributed environment. The `Variable()` constructor or `get_variable()` +automatically adds new variables to the graph collection +`GraphKeys.GLOBAL_VARIABLES`. +This convenience function returns the contents of that collection. + +An alternative to global variables are local variables. See +[`tf.local_variables()`](../../api_docs/python/state_ops.md#local_variables) + +##### Returns: + + A list of `Variable` objects. + diff --git a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard3/tf.local_variables.md b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard3/tf.local_variables.md index b3612c7cbf3..26b4d127af5 100644 --- a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard3/tf.local_variables.md +++ b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard3/tf.local_variables.md @@ -1,8 +1,19 @@ ### `tf.local_variables()` {#local_variables} -Returns all variables created with collection=[LOCAL_VARIABLES]. +Returns local variables. + +Local variables - per process variables, usually not saved/restored to +checkpoint and used for temporary or intermediate values. +For example, they can be used as counters for metrics computation or +number of epochs this machine has read data. +The `local_variable()` automatically adds new variable to +`GraphKeys.LOCAL_VARIABLES`. +This convenience function returns the contents of that collection. + +An alternative to local variables are global variables. See +[`tf.global_variables()`](../../api_docs/python/state_ops.md#global_variables) ##### Returns: - A list of local Variable objects. + A list of local `Variable` objects. diff --git a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard4/tf.initialize_variables.md b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard4/tf.initialize_variables.md index 8941ab48535..3ab51c4b3c6 100644 --- a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard4/tf.initialize_variables.md +++ b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard4/tf.initialize_variables.md @@ -1,24 +1,8 @@ -### `tf.initialize_variables(var_list, name='init')` {#initialize_variables} +### `tf.initialize_variables(*args, **kwargs)` {#initialize_variables} -Returns an Op that initializes a list of variables. +See `tf.variables_initializer`. (deprecated) -After you launch the graph in a session, you can run the returned Op to -initialize all the variables in `var_list`. This Op runs all the -initializers of the variables in `var_list` in parallel. - -Calling `initialize_variables()` is equivalent to passing the list of -initializers to `Group()`. - -If `var_list` is empty, however, the function still returns an Op that can -be run. That Op just has no effect. - -##### Args: - - -* `var_list`: List of `Variable` objects to initialize. -* `name`: Optional name for the returned operation. - -##### Returns: - - An Op that run the initializers of all the specified variables. +THIS FUNCTION IS DEPRECATED. It will be removed after 2017-03-02. +Instructions for updating: +Use `tf.variables_initializer` instead. diff --git a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard7/tf.assert_variables_initialized.md b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard7/tf.assert_variables_initialized.md index ef61848aa87..ac8604579de 100644 --- a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard7/tf.assert_variables_initialized.md +++ b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard7/tf.assert_variables_initialized.md @@ -16,7 +16,7 @@ logged by the C++ runtime. This is expected. * `var_list`: List of `Variable` objects to check. Defaults to the - value of `all_variables().` + value of `global_variables().` ##### Returns: diff --git a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard8/tf.GraphKeys.md b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard8/tf.GraphKeys.md index 1d656f40180..965097f2b00 100644 --- a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard8/tf.GraphKeys.md +++ b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard8/tf.GraphKeys.md @@ -9,10 +9,18 @@ variables. The following standard keys are defined: -* `VARIABLES`: the `Variable` objects that comprise a model, and - must be saved and restored together. See - [`tf.all_variables()`](../../api_docs/python/state_ops.md#all_variables) +* `GLOBAL_VARIABLES`: the default collection of `Variable` objects, shared + across distributed environment (model variables are subset of these). See + [`tf.global_variables()`](../../api_docs/python/state_ops.md#global_variables) for more details. + Commonly, all `TRAINABLE_VARIABLES` variables will be in `MODEL_VARIABLES`, + and all `MODEL_VARIABLES` variables will be in `GLOBAL_VARIABLES`. +* `LOCAL_VARIABLES`: the subset of `Variable` objects that are local to each + machine. Usually used for temporarily variables, like counters. + Note: use `tf.contrib.framework.local_variable` to add to this collection. +* `MODEL_VARIABLES`: the subset of `Variable` objects that are used in the + model for inference (feed forward). Note: use + `tf.contrib.framework.model_variable` to add to this collection. * `TRAINABLE_VARIABLES`: the subset of `Variable` objects that will be trained by an optimizer. See [`tf.trainable_variables()`](../../api_docs/python/state_ops.md#trainable_variables) diff --git a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard8/tf.Variable.md b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard8/tf.Variable.md index 44fac742c80..777d56be50d 100644 --- a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard8/tf.Variable.md +++ b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard8/tf.Variable.md @@ -51,16 +51,16 @@ with tf.Session() as sess: ``` The most common initialization pattern is to use the convenience function -`initialize_all_variables()` to add an Op to the graph that initializes +`global_variable_initializers()` to add an Op to the graph that initializes all the variables. You then run that Op after launching the graph. ```python -# Add an Op to initialize all variables. -init_op = tf.initialize_all_variables() +# Add an Op to initialize global variables. +init_op = tf.global_variable_initializers() # Launch the graph in a session. with tf.Session() as sess: - # Run the Op that initializes all variables. + # Run the Op that initializes global variables. sess.run(init_op) # ...you can now run any Op that uses variable values... ``` @@ -71,8 +71,8 @@ variables are initialized in the right order. All variables are automatically collected in the graph where they are created. By default, the constructor adds the new variable to the graph -collection `GraphKeys.VARIABLES`. The convenience function -`all_variables()` returns the contents of that collection. +collection `GraphKeys.GLOBAL_VARIABLES`. The convenience function +`global_variables()` returns the contents of that collection. When building a machine learning model it is often convenient to distinguish between variables holding the trainable model parameters and other variables @@ -94,7 +94,7 @@ Creating a variable. Creates a new variable with value `initial_value`. The new variable is added to the graph collections listed in `collections`, -which defaults to `[GraphKeys.VARIABLES]`. +which defaults to `[GraphKeys.GLOBAL_VARIABLES]`. If `trainable` is `True` the variable is also added to the graph collection `GraphKeys.TRAINABLE_VARIABLES`. @@ -115,7 +115,7 @@ variable to its initial value. collection `GraphKeys.TRAINABLE_VARIABLES`. This collection is used as the default list of variables to use by the `Optimizer` classes. * `collections`: List of graph collections keys. The new variable is added to - these collections. Defaults to `[GraphKeys.VARIABLES]`. + these collections. Defaults to `[GraphKeys.GLOBAL_VARIABLES]`. * `validate_shape`: If `False`, allows the variable to be initialized with a value of unknown shape. If `True`, the default, the shape of `initial_value` must be known. @@ -301,7 +301,7 @@ more information on launching a graph and on sessions. ```python v = tf.Variable([1, 2]) -init = tf.initialize_all_variables() +init = tf.global_variable_initializers() with tf.Session() as sess: sess.run(init) diff --git a/tensorflow/g3doc/api_docs/python/index.md b/tensorflow/g3doc/api_docs/python/index.md index a0cd95857c6..5ed97804440 100644 --- a/tensorflow/g3doc/api_docs/python/index.md +++ b/tensorflow/g3doc/api_docs/python/index.md @@ -85,6 +85,8 @@ * [`get_checkpoint_state`](../../api_docs/python/state_ops.md#get_checkpoint_state) * [`get_variable`](../../api_docs/python/state_ops.md#get_variable) * [`get_variable_scope`](../../api_docs/python/state_ops.md#get_variable_scope) + * [`global_variables`](../../api_docs/python/state_ops.md#global_variables) + * [`global_variables_initializer`](../../api_docs/python/state_ops.md#global_variables_initializer) * [`import_meta_graph`](../../api_docs/python/state_ops.md#import_meta_graph) * [`IndexedSlices`](../../api_docs/python/state_ops.md#IndexedSlices) * [`initialize_all_tables`](../../api_docs/python/state_ops.md#initialize_all_tables) @@ -94,6 +96,7 @@ * [`is_variable_initialized`](../../api_docs/python/state_ops.md#is_variable_initialized) * [`latest_checkpoint`](../../api_docs/python/state_ops.md#latest_checkpoint) * [`local_variables`](../../api_docs/python/state_ops.md#local_variables) + * [`local_variables_initializer`](../../api_docs/python/state_ops.md#local_variables_initializer) * [`make_template`](../../api_docs/python/state_ops.md#make_template) * [`min_max_variable_partitioner`](../../api_docs/python/state_ops.md#min_max_variable_partitioner) * [`model_variables`](../../api_docs/python/state_ops.md#model_variables) @@ -123,6 +126,7 @@ * [`variable_axis_size_partitioner`](../../api_docs/python/state_ops.md#variable_axis_size_partitioner) * [`variable_op_scope`](../../api_docs/python/state_ops.md#variable_op_scope) * [`variable_scope`](../../api_docs/python/state_ops.md#variable_scope) + * [`variables_initializer`](../../api_docs/python/state_ops.md#variables_initializer) * [`VariableScope`](../../api_docs/python/state_ops.md#VariableScope) * [`zeros_initializer`](../../api_docs/python/state_ops.md#zeros_initializer) diff --git a/tensorflow/g3doc/api_docs/python/state_ops.md b/tensorflow/g3doc/api_docs/python/state_ops.md index 0a97c991534..29765e7c2f6 100644 --- a/tensorflow/g3doc/api_docs/python/state_ops.md +++ b/tensorflow/g3doc/api_docs/python/state_ops.md @@ -66,16 +66,16 @@ with tf.Session() as sess: ``` The most common initialization pattern is to use the convenience function -`initialize_all_variables()` to add an Op to the graph that initializes +`global_variable_initializers()` to add an Op to the graph that initializes all the variables. You then run that Op after launching the graph. ```python -# Add an Op to initialize all variables. -init_op = tf.initialize_all_variables() +# Add an Op to initialize global variables. +init_op = tf.global_variable_initializers() # Launch the graph in a session. with tf.Session() as sess: - # Run the Op that initializes all variables. + # Run the Op that initializes global variables. sess.run(init_op) # ...you can now run any Op that uses variable values... ``` @@ -86,8 +86,8 @@ variables are initialized in the right order. All variables are automatically collected in the graph where they are created. By default, the constructor adds the new variable to the graph -collection `GraphKeys.VARIABLES`. The convenience function -`all_variables()` returns the contents of that collection. +collection `GraphKeys.GLOBAL_VARIABLES`. The convenience function +`global_variables()` returns the contents of that collection. When building a machine learning model it is often convenient to distinguish between variables holding the trainable model parameters and other variables @@ -109,7 +109,7 @@ Creating a variable. Creates a new variable with value `initial_value`. The new variable is added to the graph collections listed in `collections`, -which defaults to `[GraphKeys.VARIABLES]`. +which defaults to `[GraphKeys.GLOBAL_VARIABLES]`. If `trainable` is `True` the variable is also added to the graph collection `GraphKeys.TRAINABLE_VARIABLES`. @@ -130,7 +130,7 @@ variable to its initial value. collection `GraphKeys.TRAINABLE_VARIABLES`. This collection is used as the default list of variables to use by the `Optimizer` classes. * `collections`: List of graph collections keys. The new variable is added to - these collections. Defaults to `[GraphKeys.VARIABLES]`. + these collections. Defaults to `[GraphKeys.GLOBAL_VARIABLES]`. * `validate_shape`: If `False`, allows the variable to be initialized with a value of unknown shape. If `True`, the default, the shape of `initial_value` must be known. @@ -316,7 +316,7 @@ more information on launching a graph and on sessions. ```python v = tf.Variable([1, 2]) -init = tf.initialize_all_variables() +init = tf.global_variable_initializers() with tf.Session() as sess: sess.run(init) @@ -1178,19 +1178,57 @@ collected in the graph. - - - -### `tf.all_variables()` {#all_variables} +### `tf.global_variables()` {#global_variables} -Returns all variables that must be saved/restored. +Returns global variables. -The `Variable()` constructor automatically adds new variables to the graph -collection `GraphKeys.VARIABLES`. This convenience function returns the -contents of that collection. +Global variables are variables that are shared across machines in a +distributed environment. The `Variable()` constructor or `get_variable()` +automatically adds new variables to the graph collection +`GraphKeys.GLOBAL_VARIABLES`. +This convenience function returns the contents of that collection. + +An alternative to global variables are local variables. See +[`tf.local_variables()`](../../api_docs/python/state_ops.md#local_variables) ##### Returns: A list of `Variable` objects. +- - - + +### `tf.local_variables()` {#local_variables} + +Returns local variables. + +Local variables - per process variables, usually not saved/restored to +checkpoint and used for temporary or intermediate values. +For example, they can be used as counters for metrics computation or +number of epochs this machine has read data. +The `local_variable()` automatically adds new variable to +`GraphKeys.LOCAL_VARIABLES`. +This convenience function returns the contents of that collection. + +An alternative to local variables are global variables. See +[`tf.global_variables()`](../../api_docs/python/state_ops.md#global_variables) + +##### Returns: + + A list of local `Variable` objects. + + +- - - + +### `tf.model_variables()` {#model_variables} + +Returns all variables in the MODEL_VARIABLES collection. + +##### Returns: + + A list of local Variable objects. + + - - - ### `tf.trainable_variables()` {#trainable_variables} @@ -1207,28 +1245,6 @@ contents of that collection. A list of Variable objects. -- - - - -### `tf.local_variables()` {#local_variables} - -Returns all variables created with collection=[LOCAL_VARIABLES]. - -##### Returns: - - A list of local Variable objects. - - -- - - - -### `tf.model_variables()` {#model_variables} - -Returns all variables in the MODEL_VARIABLES collection. - -##### Returns: - - A list of local Variable objects. - - - - - ### `tf.moving_average_variables()` {#moving_average_variables} @@ -1248,20 +1264,33 @@ This convenience function returns the contents of that collection. - - - -### `tf.initialize_all_variables()` {#initialize_all_variables} +### `tf.global_variables_initializer()` {#global_variables_initializer} -Returns an Op that initializes all variables. +Returns an Op that initializes global variables. -This is just a shortcut for `initialize_variables(all_variables())` +This is just a shortcut for `variable_initializers(global_variables())` ##### Returns: - An Op that initializes all variables in the graph. + An Op that initializes global variables in the graph. - - - -### `tf.initialize_variables(var_list, name='init')` {#initialize_variables} +### `tf.local_variables_initializer()` {#local_variables_initializer} + +Returns an Op that initializes all local variables. + +This is just a shortcut for `variable_initializers(local_variables())` + +##### Returns: + + An Op that initializes all local variables in the graph. + + +- - - + +### `tf.variables_initializer(var_list, name='init')` {#variables_initializer} Returns an Op that initializes a list of variables. @@ -1286,19 +1315,6 @@ be run. That Op just has no effect. An Op that run the initializers of all the specified variables. -- - - - -### `tf.initialize_local_variables()` {#initialize_local_variables} - -Returns an Op that initializes all local variables. - -This is just a shortcut for `initialize_variables(local_variables())` - -##### Returns: - - An Op that initializes all local variables in the graph. - - - - - ### `tf.is_variable_initialized(variable)` {#is_variable_initialized} @@ -1329,7 +1345,7 @@ variables if there are any, or an empty array if there are none. * `var_list`: List of `Variable` objects to check. Defaults to the - value of `all_variables() + local_variables()` + value of `global_variables() + local_variables()` * `name`: Optional name of the `Operation`. ##### Returns: @@ -1358,7 +1374,7 @@ logged by the C++ runtime. This is expected. * `var_list`: List of `Variable` objects to check. Defaults to the - value of `all_variables().` + value of `global_variables().` ##### Returns: @@ -3507,3 +3523,50 @@ device assignments have not changed. (i.e., there are no variables to restore). + +# Deprecated functions (removed after 2017-03-02). Please don't use them. + +- - - + +### `tf.all_variables(*args, **kwargs)` {#all_variables} + +See `tf.global_variables`. (deprecated) + +THIS FUNCTION IS DEPRECATED. It will be removed after 2016-03-02. +Instructions for updating: +Please use tf.global_variables instead. + + +- - - + +### `tf.initialize_all_variables(*args, **kwargs)` {#initialize_all_variables} + +See `tf.global_variables_initializer`. (deprecated) + +THIS FUNCTION IS DEPRECATED. It will be removed after 2017-03-02. +Instructions for updating: +Use `tf.global_variables_initializer` instead. + + +- - - + +### `tf.initialize_local_variables(*args, **kwargs)` {#initialize_local_variables} + +See `tf.local_variables_initializer`. (deprecated) + +THIS FUNCTION IS DEPRECATED. It will be removed after 2017-03-02. +Instructions for updating: +Use `tf.local_variables_initializer` instead. + + +- - - + +### `tf.initialize_variables(*args, **kwargs)` {#initialize_variables} + +See `tf.variables_initializer`. (deprecated) + +THIS FUNCTION IS DEPRECATED. It will be removed after 2017-03-02. +Instructions for updating: +Use `tf.variables_initializer` instead. + + From 7e8135cc3a238bf86e82ed7ea13671533e7f1290 Mon Sep 17 00:00:00 2001 From: Vijay Vasudevan Date: Thu, 3 Nov 2016 09:36:16 -0800 Subject: [PATCH 45/54] Expose fake quantization operators. Add shape function for FakeQuant gradient ops. Add a test for one of the more complicated ones. Change: 138087659 --- tensorflow/core/ops/array_ops.cc | 34 +++++++++++++++++++++++++++ tensorflow/core/ops/array_ops_test.cc | 19 +++++++++++++++ tensorflow/python/ops/array_ops.py | 15 ++++++++++++ 3 files changed, 68 insertions(+) diff --git a/tensorflow/core/ops/array_ops.cc b/tensorflow/core/ops/array_ops.cc index f7364b7411f..ce1f76503c8 100644 --- a/tensorflow/core/ops/array_ops.cc +++ b/tensorflow/core/ops/array_ops.cc @@ -4486,6 +4486,7 @@ REGISTER_OP("FakeQuantWithMinMaxArgsGradient") .Input("gradients: float") .Input("inputs: float") .Output("backprops: float") + .SetShapeFn(shape_inference::UnchangedShape) .Doc(R"doc( Compute gradients for a FakeQuantWithMinMaxArgs operation. @@ -4527,6 +4528,21 @@ REGISTER_OP("FakeQuantWithMinMaxVarsGradient") .Output("backprops_wrt_input: float") .Output("backprop_wrt_min: float") .Output("backprop_wrt_max: float") + .SetShapeFn([](InferenceContext* c) { + // gradients and inputs are same size. + ShapeHandle inputs; + TF_RETURN_IF_ERROR(c->Merge(c->input(0), c->input(1), &inputs)); + + // min and max are scalars + ShapeHandle min_max; + TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &min_max)); + TF_RETURN_IF_ERROR(c->Merge(min_max, c->input(3), &min_max)); + + c->set_output(0, inputs); + c->set_output(1, min_max); + c->set_output(2, min_max); + return Status::OK(); + }) .Doc(R"doc( Compute gradients for a FakeQuantWithMinMaxVars operation. @@ -4580,6 +4596,24 @@ REGISTER_OP("FakeQuantWithMinMaxVarsPerChannelGradient") .Output("backprops_wrt_input: float") .Output("backprop_wrt_min: float") .Output("backprop_wrt_max: float") + .SetShapeFn([](InferenceContext* c) { + ShapeHandle inputs; + TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(0), 1, &inputs)); + TF_RETURN_IF_ERROR(c->WithRankAtMost(inputs, 4, &inputs)); + TF_RETURN_IF_ERROR(c->Merge(inputs, c->input(1), &inputs)); + + ShapeHandle last_dim = c->Vector(c->Dim(inputs, -1)); + + ShapeHandle min_max; + TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 1, &min_max)); + TF_RETURN_IF_ERROR(c->Merge(min_max, last_dim, &min_max)); + TF_RETURN_IF_ERROR(c->Merge(c->input(3), min_max, &min_max)); + + c->set_output(0, inputs); + c->set_output(1, min_max); + c->set_output(2, min_max); + return Status::OK(); + }) .Doc(R"doc( Compute gradients for a FakeQuantWithMinMaxVarsPerChannel operation. diff --git a/tensorflow/core/ops/array_ops_test.cc b/tensorflow/core/ops/array_ops_test.cc index 4e10d72816e..691380fd265 100644 --- a/tensorflow/core/ops/array_ops_test.cc +++ b/tensorflow/core/ops/array_ops_test.cc @@ -1533,4 +1533,23 @@ TEST(ArrayOpsTest, FakeQuantWithMinMaxVarsPerChannel) { INFER_ERROR("must be equal", op, "[5];[4];[?]"); } +TEST(ArrayOpsTest, FakeQuantWithMinMaxVarsPerChannelGradient) { + ShapeInferenceTestOp op("FakeQuantWithMinMaxVarsPerChannelGradient"); + + INFER_OK(op, "?;?;?;?", "?;[?];[?]"); + INFER_OK(op, "[3];[3];[3];[3]", "in0;in3;in3"); + INFER_OK(op, "[1,3];[1,3];[3];[3]", "in0;in3;in3"); + INFER_OK(op, "[1,2,3,4];[1,2,3,4];[4];[4]", "in0;in3;in3"); + + // Rank check vectors. + INFER_ERROR("be equal rank", op, "[1,?,3];[1,?,3];[3];[]"); + INFER_ERROR("be rank 1", op, "[1,?,3];[1,?,3];[];[3]"); + INFER_ERROR("be at least rank 1", op, "[];[];[1];[1]"); + INFER_ERROR("be at most rank 4", op, "[1,2,3,4,5];[1,2,3,4,5];[1];[1]"); + + // Vectors must match each other, and match last dim of input. + INFER_ERROR("must be equal", op, "[1,3];[1,3];[2];[3]"); + INFER_ERROR("must be equal", op, "[1,3];[1,3];[3];[2]"); +} + } // end namespace tensorflow diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py index c3172b1873f..9206002f5ef 100644 --- a/tensorflow/python/ops/array_ops.py +++ b/tensorflow/python/ops/array_ops.py @@ -82,6 +82,15 @@ or join multiple tensors together. @@quantized_concat @@setdiff1d +## Fake quantization +Operations used to help train for better quantization accuracy. + +@@fake_quant_with_min_max_args +@@fake_quant_with_min_max_args_gradient +@@fake_quant_with_min_max_vars +@@fake_quant_with_min_max_vars_gradient +@@fake_quant_with_min_max_vars_per_channel +@@fake_quant_with_min_max_vars_per_channel_gradient """ from __future__ import absolute_import from __future__ import division @@ -2044,9 +2053,15 @@ def _FakeQuantWithMinMaxArgsGradient(op, grad): ops.RegisterShape("FakeQuantWithMinMaxArgs")(common_shapes.call_cpp_shape_fn) +ops.RegisterShape("FakeQuantWithMinMaxArgsGradient")( + common_shapes.call_cpp_shape_fn) ops.RegisterShape("FakeQuantWithMinMaxVars")(common_shapes.call_cpp_shape_fn) +ops.RegisterShape("FakeQuantWithMinMaxVarsGradient")( + common_shapes.call_cpp_shape_fn) ops.RegisterShape("FakeQuantWithMinMaxVarsPerChannel")( common_shapes.call_cpp_shape_fn) +ops.RegisterShape("FakeQuantWithMinMaxVarsPerChannelGradient")( + common_shapes.call_cpp_shape_fn) @ops.RegisterGradient("FakeQuantWithMinMaxVars") From 9974262689a54fda488dfb8f70c444b8ea719f7a Mon Sep 17 00:00:00 2001 From: Andrew Harp Date: Thu, 3 Nov 2016 09:51:08 -0800 Subject: [PATCH 46/54] Allow Android to use legacy scalars for parity in model compatibility. Change: 138089751 --- tensorflow/core/framework/op_kernel.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/framework/op_kernel.h b/tensorflow/core/framework/op_kernel.h index 4c14918ea70..145b8c53154 100644 --- a/tensorflow/core/framework/op_kernel.h +++ b/tensorflow/core/framework/op_kernel.h @@ -131,7 +131,7 @@ class OpKernel { // We allow legacy scalars within Google up until GraphDef version 6. // TODO(irving): Remove when we can drop support for GraphDef version 5. bool allow_legacy_scalars() const { -#if defined(PLATFORM_GOOGLE) +#if defined(PLATFORM_GOOGLE) || defined(PLATFORM_GOOGLE_ANDROID) return graph_def_version_ < 6; #else return false; From 5fc46c58774fa1bf8cc0f350373988df82f35024 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 3 Nov 2016 09:53:02 -0800 Subject: [PATCH 47/54] Update generated Python Op docs. Change: 138090031 --- tensorflow/g3doc/api_docs/python/array_ops.md | 172 ++++++++++++++++++ ...f.fake_quant_with_min_max_vars_gradient.md | 27 +++ ..._with_min_max_vars_per_channel_gradient.md | 30 +++ ...f.fake_quant_with_min_max_args_gradient.md | 21 +++ .../shard3/tf.fake_quant_with_min_max_vars.md | 25 +++ ...ake_quant_with_min_max_vars_per_channel.md | 25 +++ .../shard6/tf.fake_quant_with_min_max_args.md | 22 +++ tensorflow/g3doc/api_docs/python/index.md | 6 + 8 files changed, 328 insertions(+) create mode 100644 tensorflow/g3doc/api_docs/python/functions_and_classes/shard0/tf.fake_quant_with_min_max_vars_gradient.md create mode 100644 tensorflow/g3doc/api_docs/python/functions_and_classes/shard0/tf.fake_quant_with_min_max_vars_per_channel_gradient.md create mode 100644 tensorflow/g3doc/api_docs/python/functions_and_classes/shard1/tf.fake_quant_with_min_max_args_gradient.md create mode 100644 tensorflow/g3doc/api_docs/python/functions_and_classes/shard3/tf.fake_quant_with_min_max_vars.md create mode 100644 tensorflow/g3doc/api_docs/python/functions_and_classes/shard3/tf.fake_quant_with_min_max_vars_per_channel.md create mode 100644 tensorflow/g3doc/api_docs/python/functions_and_classes/shard6/tf.fake_quant_with_min_max_args.md diff --git a/tensorflow/g3doc/api_docs/python/array_ops.md b/tensorflow/g3doc/api_docs/python/array_ops.md index c2422e5f74f..83176a7d193 100644 --- a/tensorflow/g3doc/api_docs/python/array_ops.md +++ b/tensorflow/g3doc/api_docs/python/array_ops.md @@ -2844,3 +2844,175 @@ Returns the difference between the `x` and `y` treated as sets. A `Tensor` that is of type `index_dtype` representing indices from . + +## Fake quantization +Operations used to help train for better quantization accuracy. + +- - - + +### `tf.fake_quant_with_min_max_args(inputs, min=None, max=None, name=None)` {#fake_quant_with_min_max_args} + +Fake-quantize the 'inputs' tensor, type float to 'outputs' tensor of same type. + +Attributes [min; max] define the clamping range for the 'inputs' data. Op +divides this range into 255 steps (total of 256 values), then replaces each +'inputs' value with the closest of the quantized step values. + +Quantization is called fake since the output is still in floating point. + +##### Args: + + +* `inputs`: A `Tensor` of type `float32`. +* `min`: An optional `float`. Defaults to `-6`. +* `max`: An optional `float`. Defaults to `6`. +* `name`: A name for the operation (optional). + +##### Returns: + + A `Tensor` of type `float32`. + + +- - - + +### `tf.fake_quant_with_min_max_args_gradient(gradients, inputs, min=None, max=None, name=None)` {#fake_quant_with_min_max_args_gradient} + +Compute gradients for a FakeQuantWithMinMaxArgs operation. + +##### Args: + + +* `gradients`: A `Tensor` of type `float32`. + Backpropagated gradients above the FakeQuantWithMinMaxArgs operation. +* `inputs`: A `Tensor` of type `float32`. + Values passed as inputs to the FakeQuantWithMinMaxArgs operation. +* `min`: An optional `float`. Defaults to `-6`. +* `max`: An optional `float`. Defaults to `6`. +* `name`: A name for the operation (optional). + +##### Returns: + + A `Tensor` of type `float32`. + Backpropagated gradients below the FakeQuantWithMinMaxArgs operation: + `gradients * (inputs >= min && inputs <= max)`. + + +- - - + +### `tf.fake_quant_with_min_max_vars(inputs, min, max, name=None)` {#fake_quant_with_min_max_vars} + +Fake-quantize the 'inputs' tensor of type float and shape `[b, h, w, d]` via + +global float scalars `min` and `max` to 'outputs' tensor of same shape as +`inputs`. + +[min; max] is the clamping range for the 'inputs' data. Op divides this range +into 255 steps (total of 256 values), then replaces each 'inputs' value with the +closest of the quantized step values. + +This operation has a gradient and thus allows for training `min` and `max` values. + +##### Args: + + +* `inputs`: A `Tensor` of type `float32`. +* `min`: A `Tensor` of type `float32`. +* `max`: A `Tensor` of type `float32`. +* `name`: A name for the operation (optional). + +##### Returns: + + A `Tensor` of type `float32`. + + +- - - + +### `tf.fake_quant_with_min_max_vars_gradient(gradients, inputs, min, max, name=None)` {#fake_quant_with_min_max_vars_gradient} + +Compute gradients for a FakeQuantWithMinMaxVars operation. + +##### Args: + + +* `gradients`: A `Tensor` of type `float32`. + Backpropagated gradients above the FakeQuantWithMinMaxVars operation. +* `inputs`: A `Tensor` of type `float32`. + Values passed as inputs to the FakeQuantWithMinMaxVars operation. + min, max: Quantization interval, scalar floats. +* `min`: A `Tensor` of type `float32`. +* `max`: A `Tensor` of type `float32`. +* `name`: A name for the operation (optional). + +##### Returns: + + A tuple of `Tensor` objects (backprops_wrt_input, backprop_wrt_min, backprop_wrt_max). + +* `backprops_wrt_input`: A `Tensor` of type `float32`. Backpropagated gradients w.r.t. inputs: + `gradients * (inputs >= min && inputs <= max)`. +* `backprop_wrt_min`: A `Tensor` of type `float32`. Backpropagated gradients w.r.t. min parameter: + `sum(gradients * (inputs < min))`. +* `backprop_wrt_max`: A `Tensor` of type `float32`. Backpropagated gradients w.r.t. max parameter: + `sum(gradients * (inputs > max))`. + + +- - - + +### `tf.fake_quant_with_min_max_vars_per_channel(inputs, min, max, name=None)` {#fake_quant_with_min_max_vars_per_channel} + +Fake-quantize the 'inputs' tensor of type float and one of the shapes: `[d]`, + +`[b, d]` `[b, h, w, d]` via per-channel floats `min` and `max` of shape `[d]` +to 'outputs' tensor of same shape as `inputs`. + +[min; max] is the clamping range for the 'inputs' data in the corresponding +depth channel. Op divides this range into 255 steps (total of 256 values), then +replaces each 'inputs' value with the closest of the quantized step values. + +This operation has a gradient and thus allows for training `min` and `max` values. + +##### Args: + + +* `inputs`: A `Tensor` of type `float32`. +* `min`: A `Tensor` of type `float32`. +* `max`: A `Tensor` of type `float32`. +* `name`: A name for the operation (optional). + +##### Returns: + + A `Tensor` of type `float32`. + + +- - - + +### `tf.fake_quant_with_min_max_vars_per_channel_gradient(gradients, inputs, min, max, name=None)` {#fake_quant_with_min_max_vars_per_channel_gradient} + +Compute gradients for a FakeQuantWithMinMaxVarsPerChannel operation. + +##### Args: + + +* `gradients`: A `Tensor` of type `float32`. + Backpropagated gradients above the FakeQuantWithMinMaxVars operation, + shape one of: `[d]`, `[b, d]`, `[b, h, w, d]`. +* `inputs`: A `Tensor` of type `float32`. + Values passed as inputs to the FakeQuantWithMinMaxVars operation, shape + same as `gradients`. + min, max: Quantization interval, floats of shape `[d]`. +* `min`: A `Tensor` of type `float32`. +* `max`: A `Tensor` of type `float32`. +* `name`: A name for the operation (optional). + +##### Returns: + + A tuple of `Tensor` objects (backprops_wrt_input, backprop_wrt_min, backprop_wrt_max). + +* `backprops_wrt_input`: A `Tensor` of type `float32`. Backpropagated gradients w.r.t. inputs, shape same as + `inputs`: + `gradients * (inputs >= min && inputs <= max)`. +* `backprop_wrt_min`: A `Tensor` of type `float32`. Backpropagated gradients w.r.t. min parameter, shape `[d]`: + `sum_per_d(gradients * (inputs < min))`. +* `backprop_wrt_max`: A `Tensor` of type `float32`. Backpropagated gradients w.r.t. max parameter, shape `[d]`: + `sum_per_d(gradients * (inputs > max))`. + + diff --git a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard0/tf.fake_quant_with_min_max_vars_gradient.md b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard0/tf.fake_quant_with_min_max_vars_gradient.md new file mode 100644 index 00000000000..b363afe7cef --- /dev/null +++ b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard0/tf.fake_quant_with_min_max_vars_gradient.md @@ -0,0 +1,27 @@ +### `tf.fake_quant_with_min_max_vars_gradient(gradients, inputs, min, max, name=None)` {#fake_quant_with_min_max_vars_gradient} + +Compute gradients for a FakeQuantWithMinMaxVars operation. + +##### Args: + + +* `gradients`: A `Tensor` of type `float32`. + Backpropagated gradients above the FakeQuantWithMinMaxVars operation. +* `inputs`: A `Tensor` of type `float32`. + Values passed as inputs to the FakeQuantWithMinMaxVars operation. + min, max: Quantization interval, scalar floats. +* `min`: A `Tensor` of type `float32`. +* `max`: A `Tensor` of type `float32`. +* `name`: A name for the operation (optional). + +##### Returns: + + A tuple of `Tensor` objects (backprops_wrt_input, backprop_wrt_min, backprop_wrt_max). + +* `backprops_wrt_input`: A `Tensor` of type `float32`. Backpropagated gradients w.r.t. inputs: + `gradients * (inputs >= min && inputs <= max)`. +* `backprop_wrt_min`: A `Tensor` of type `float32`. Backpropagated gradients w.r.t. min parameter: + `sum(gradients * (inputs < min))`. +* `backprop_wrt_max`: A `Tensor` of type `float32`. Backpropagated gradients w.r.t. max parameter: + `sum(gradients * (inputs > max))`. + diff --git a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard0/tf.fake_quant_with_min_max_vars_per_channel_gradient.md b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard0/tf.fake_quant_with_min_max_vars_per_channel_gradient.md new file mode 100644 index 00000000000..a7a62e29b31 --- /dev/null +++ b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard0/tf.fake_quant_with_min_max_vars_per_channel_gradient.md @@ -0,0 +1,30 @@ +### `tf.fake_quant_with_min_max_vars_per_channel_gradient(gradients, inputs, min, max, name=None)` {#fake_quant_with_min_max_vars_per_channel_gradient} + +Compute gradients for a FakeQuantWithMinMaxVarsPerChannel operation. + +##### Args: + + +* `gradients`: A `Tensor` of type `float32`. + Backpropagated gradients above the FakeQuantWithMinMaxVars operation, + shape one of: `[d]`, `[b, d]`, `[b, h, w, d]`. +* `inputs`: A `Tensor` of type `float32`. + Values passed as inputs to the FakeQuantWithMinMaxVars operation, shape + same as `gradients`. + min, max: Quantization interval, floats of shape `[d]`. +* `min`: A `Tensor` of type `float32`. +* `max`: A `Tensor` of type `float32`. +* `name`: A name for the operation (optional). + +##### Returns: + + A tuple of `Tensor` objects (backprops_wrt_input, backprop_wrt_min, backprop_wrt_max). + +* `backprops_wrt_input`: A `Tensor` of type `float32`. Backpropagated gradients w.r.t. inputs, shape same as + `inputs`: + `gradients * (inputs >= min && inputs <= max)`. +* `backprop_wrt_min`: A `Tensor` of type `float32`. Backpropagated gradients w.r.t. min parameter, shape `[d]`: + `sum_per_d(gradients * (inputs < min))`. +* `backprop_wrt_max`: A `Tensor` of type `float32`. Backpropagated gradients w.r.t. max parameter, shape `[d]`: + `sum_per_d(gradients * (inputs > max))`. + diff --git a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard1/tf.fake_quant_with_min_max_args_gradient.md b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard1/tf.fake_quant_with_min_max_args_gradient.md new file mode 100644 index 00000000000..5c93c3e0468 --- /dev/null +++ b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard1/tf.fake_quant_with_min_max_args_gradient.md @@ -0,0 +1,21 @@ +### `tf.fake_quant_with_min_max_args_gradient(gradients, inputs, min=None, max=None, name=None)` {#fake_quant_with_min_max_args_gradient} + +Compute gradients for a FakeQuantWithMinMaxArgs operation. + +##### Args: + + +* `gradients`: A `Tensor` of type `float32`. + Backpropagated gradients above the FakeQuantWithMinMaxArgs operation. +* `inputs`: A `Tensor` of type `float32`. + Values passed as inputs to the FakeQuantWithMinMaxArgs operation. +* `min`: An optional `float`. Defaults to `-6`. +* `max`: An optional `float`. Defaults to `6`. +* `name`: A name for the operation (optional). + +##### Returns: + + A `Tensor` of type `float32`. + Backpropagated gradients below the FakeQuantWithMinMaxArgs operation: + `gradients * (inputs >= min && inputs <= max)`. + diff --git a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard3/tf.fake_quant_with_min_max_vars.md b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard3/tf.fake_quant_with_min_max_vars.md new file mode 100644 index 00000000000..d7815f04146 --- /dev/null +++ b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard3/tf.fake_quant_with_min_max_vars.md @@ -0,0 +1,25 @@ +### `tf.fake_quant_with_min_max_vars(inputs, min, max, name=None)` {#fake_quant_with_min_max_vars} + +Fake-quantize the 'inputs' tensor of type float and shape `[b, h, w, d]` via + +global float scalars `min` and `max` to 'outputs' tensor of same shape as +`inputs`. + +[min; max] is the clamping range for the 'inputs' data. Op divides this range +into 255 steps (total of 256 values), then replaces each 'inputs' value with the +closest of the quantized step values. + +This operation has a gradient and thus allows for training `min` and `max` values. + +##### Args: + + +* `inputs`: A `Tensor` of type `float32`. +* `min`: A `Tensor` of type `float32`. +* `max`: A `Tensor` of type `float32`. +* `name`: A name for the operation (optional). + +##### Returns: + + A `Tensor` of type `float32`. + diff --git a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard3/tf.fake_quant_with_min_max_vars_per_channel.md b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard3/tf.fake_quant_with_min_max_vars_per_channel.md new file mode 100644 index 00000000000..bc39cf9570a --- /dev/null +++ b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard3/tf.fake_quant_with_min_max_vars_per_channel.md @@ -0,0 +1,25 @@ +### `tf.fake_quant_with_min_max_vars_per_channel(inputs, min, max, name=None)` {#fake_quant_with_min_max_vars_per_channel} + +Fake-quantize the 'inputs' tensor of type float and one of the shapes: `[d]`, + +`[b, d]` `[b, h, w, d]` via per-channel floats `min` and `max` of shape `[d]` +to 'outputs' tensor of same shape as `inputs`. + +[min; max] is the clamping range for the 'inputs' data in the corresponding +depth channel. Op divides this range into 255 steps (total of 256 values), then +replaces each 'inputs' value with the closest of the quantized step values. + +This operation has a gradient and thus allows for training `min` and `max` values. + +##### Args: + + +* `inputs`: A `Tensor` of type `float32`. +* `min`: A `Tensor` of type `float32`. +* `max`: A `Tensor` of type `float32`. +* `name`: A name for the operation (optional). + +##### Returns: + + A `Tensor` of type `float32`. + diff --git a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard6/tf.fake_quant_with_min_max_args.md b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard6/tf.fake_quant_with_min_max_args.md new file mode 100644 index 00000000000..fcad8cb5001 --- /dev/null +++ b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard6/tf.fake_quant_with_min_max_args.md @@ -0,0 +1,22 @@ +### `tf.fake_quant_with_min_max_args(inputs, min=None, max=None, name=None)` {#fake_quant_with_min_max_args} + +Fake-quantize the 'inputs' tensor, type float to 'outputs' tensor of same type. + +Attributes [min; max] define the clamping range for the 'inputs' data. Op +divides this range into 255 steps (total of 256 values), then replaces each +'inputs' value with the closest of the quantized step values. + +Quantization is called fake since the output is still in floating point. + +##### Args: + + +* `inputs`: A `Tensor` of type `float32`. +* `min`: An optional `float`. Defaults to `-6`. +* `max`: An optional `float`. Defaults to `6`. +* `name`: A name for the operation (optional). + +##### Returns: + + A `Tensor` of type `float32`. + diff --git a/tensorflow/g3doc/api_docs/python/index.md b/tensorflow/g3doc/api_docs/python/index.md index 5ed97804440..3e5f2807658 100644 --- a/tensorflow/g3doc/api_docs/python/index.md +++ b/tensorflow/g3doc/api_docs/python/index.md @@ -143,6 +143,12 @@ * [`dynamic_stitch`](../../api_docs/python/array_ops.md#dynamic_stitch) * [`expand_dims`](../../api_docs/python/array_ops.md#expand_dims) * [`extract_image_patches`](../../api_docs/python/array_ops.md#extract_image_patches) + * [`fake_quant_with_min_max_args`](../../api_docs/python/array_ops.md#fake_quant_with_min_max_args) + * [`fake_quant_with_min_max_args_gradient`](../../api_docs/python/array_ops.md#fake_quant_with_min_max_args_gradient) + * [`fake_quant_with_min_max_vars`](../../api_docs/python/array_ops.md#fake_quant_with_min_max_vars) + * [`fake_quant_with_min_max_vars_gradient`](../../api_docs/python/array_ops.md#fake_quant_with_min_max_vars_gradient) + * [`fake_quant_with_min_max_vars_per_channel`](../../api_docs/python/array_ops.md#fake_quant_with_min_max_vars_per_channel) + * [`fake_quant_with_min_max_vars_per_channel_gradient`](../../api_docs/python/array_ops.md#fake_quant_with_min_max_vars_per_channel_gradient) * [`gather`](../../api_docs/python/array_ops.md#gather) * [`gather_nd`](../../api_docs/python/array_ops.md#gather_nd) * [`meshgrid`](../../api_docs/python/array_ops.md#meshgrid) From afa129793b748968ba0ca8f92693ba7d19b1298a Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 3 Nov 2016 09:54:41 -0800 Subject: [PATCH 48/54] Fix instructions for installing cuDNN to preserve symlinks. Change: 138090265 --- tensorflow/g3doc/get_started/os_setup.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/g3doc/get_started/os_setup.md b/tensorflow/g3doc/get_started/os_setup.md index fbef504ca34..8eb2ef146fe 100644 --- a/tensorflow/g3doc/get_started/os_setup.md +++ b/tensorflow/g3doc/get_started/os_setup.md @@ -610,8 +610,8 @@ to reflect the cuDNN version you downloaded): ``` bash tar xvzf cudnn-7.5-linux-x64-v5.1-ga.tgz -sudo cp cuda/include/cudnn.h /usr/local/cuda/include -sudo cp cuda/lib64/libcudnn* /usr/local/cuda/lib64 +sudo cp -P cuda/include/cudnn.h /usr/local/cuda/include +sudo cp -P cuda/lib64/libcudnn* /usr/local/cuda/lib64 sudo chmod a+r /usr/local/cuda/include/cudnn.h /usr/local/cuda/lib64/libcudnn* ``` From cbc9e25583046e0eac12f12ec24f0312909a9b4d Mon Sep 17 00:00:00 2001 From: Andrew Harp Date: Thu, 3 Nov 2016 10:04:54 -0800 Subject: [PATCH 49/54] Refactor Android demo to put activity specific code into ClassifierActivity to simplify adding new demos. Also add some utility classes and debug rendering (tap to enable). Change: 138091760 --- .../camera_connection_fragment.xml | 34 --- .../res/layout/camera_connection_fragment.xml | 10 +- .../org/tensorflow/demo/CameraActivity.java | 90 ++++++- .../demo/CameraConnectionFragment.java | 75 +++--- .../tensorflow/demo/ClassifierActivity.java | 224 +++++++++++++++++- .../src/org/tensorflow/demo/OverlayView.java | 94 ++++++++ .../org/tensorflow/demo/env/BorderedText.java | 119 ++++++++++ .../org/tensorflow/demo/env/ImageUtils.java | 76 +++++- .../src/org/tensorflow/demo/env/Size.java | 143 +++++++++++ 9 files changed, 772 insertions(+), 93 deletions(-) delete mode 100644 tensorflow/examples/android/res/layout-land/camera_connection_fragment.xml create mode 100644 tensorflow/examples/android/src/org/tensorflow/demo/OverlayView.java create mode 100644 tensorflow/examples/android/src/org/tensorflow/demo/env/BorderedText.java create mode 100644 tensorflow/examples/android/src/org/tensorflow/demo/env/Size.java diff --git a/tensorflow/examples/android/res/layout-land/camera_connection_fragment.xml b/tensorflow/examples/android/res/layout-land/camera_connection_fragment.xml deleted file mode 100644 index 543e5358e68..00000000000 --- a/tensorflow/examples/android/res/layout-land/camera_connection_fragment.xml +++ /dev/null @@ -1,34 +0,0 @@ - - - - - - - - diff --git a/tensorflow/examples/android/res/layout/camera_connection_fragment.xml b/tensorflow/examples/android/res/layout/camera_connection_fragment.xml index fcf08bf8835..420b69b5e3d 100644 --- a/tensorflow/examples/android/res/layout/camera_connection_fragment.xml +++ b/tensorflow/examples/android/res/layout/camera_connection_fragment.xml @@ -22,11 +22,17 @@ android:layout_width="wrap_content" android:layout_height="wrap_content" android:layout_alignParentBottom="true" /> - + - + + + diff --git a/tensorflow/examples/android/src/org/tensorflow/demo/CameraActivity.java b/tensorflow/examples/android/src/org/tensorflow/demo/CameraActivity.java index ede3af1467f..e8bbb999a7f 100644 --- a/tensorflow/examples/android/src/org/tensorflow/demo/CameraActivity.java +++ b/tensorflow/examples/android/src/org/tensorflow/demo/CameraActivity.java @@ -20,32 +20,72 @@ import android.Manifest; import android.app.Activity; import android.app.Fragment; import android.content.pm.PackageManager; +import android.media.Image.Plane; +import android.media.ImageReader.OnImageAvailableListener; import android.os.Build; import android.os.Bundle; +import android.os.Handler; +import android.os.HandlerThread; +import android.util.Size; +import android.view.MotionEvent; import android.view.WindowManager; import android.widget.Toast; +import java.nio.ByteBuffer; +import org.tensorflow.demo.env.Logger; + +public abstract class CameraActivity extends Activity implements OnImageAvailableListener { + private static final Logger LOGGER = new Logger(); -public abstract class CameraActivity extends Activity { private static final int PERMISSIONS_REQUEST = 1; private static final String PERMISSION_CAMERA = Manifest.permission.CAMERA; private static final String PERMISSION_STORAGE = Manifest.permission.WRITE_EXTERNAL_STORAGE; + private boolean debug = false; + + private Handler handler; + private HandlerThread handlerThread; + @Override protected void onCreate(final Bundle savedInstanceState) { - super.onCreate(savedInstanceState); + super.onCreate(null); getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON); setContentView(R.layout.activity_camera); if (hasPermission()) { - if (null == savedInstanceState) { - setFragment(); - } + setFragment(); } else { requestPermission(); } + } + @Override + public synchronized void onResume() { + super.onResume(); + + handlerThread = new HandlerThread("inference"); + handlerThread.start(); + handler = new Handler(handlerThread.getLooper()); + } + + @Override + public synchronized void onPause() { + super.onPause(); + handlerThread.quitSafely(); + try { + handlerThread.join(); + handlerThread = null; + handler = null; + } catch (final InterruptedException e) { + LOGGER.e(e, "Exception!"); + } + } + + protected synchronized void runInBackground(final Runnable r) { + if (handler != null) { + handler.post(r); + } } @Override @@ -82,11 +122,47 @@ public abstract class CameraActivity extends Activity { } protected void setFragment() { + final Fragment fragment = CameraConnectionFragment.newInstance( + new CameraConnectionFragment.ConnectionCallback(){ + @Override + public void onPreviewSizeChosen(final Size size, final int rotation) { + CameraActivity.this.onPreviewSizeChosen(size, rotation); + } + }, + this, getLayoutId(), getDesiredPreviewFrameSize()); + getFragmentManager() .beginTransaction() - .replace(R.id.container, createFragment()) + .replace(R.id.container, fragment) .commit(); } - protected abstract Fragment createFragment(); + protected void fillBytes(final Plane[] planes, final byte[][] yuvBytes) { + // Because of the variable row stride it's not possible to know in + // advance the actual necessary dimensions of the yuv planes. + for (int i = 0; i < planes.length; ++i) { + final ByteBuffer buffer = planes[i].getBuffer(); + if (yuvBytes[i] == null) { + LOGGER.i("Initializing buffer %d at size %d", i, buffer.capacity()); + yuvBytes[i] = new byte[buffer.capacity()]; + } + buffer.get(yuvBytes[i]); + } + } + + @Override + public boolean onTouchEvent(final MotionEvent event) { + if (event.getAction() == MotionEvent.ACTION_DOWN) { + debug = !debug; + } + return false; + } + + public boolean isDebug() { + return debug; + } + + protected abstract void onPreviewSizeChosen(final Size size, final int rotation); + protected abstract int getLayoutId(); + protected abstract int getDesiredPreviewFrameSize(); } diff --git a/tensorflow/examples/android/src/org/tensorflow/demo/CameraConnectionFragment.java b/tensorflow/examples/android/src/org/tensorflow/demo/CameraConnectionFragment.java index 0bd963b39ef..2e09e78b8a4 100644 --- a/tensorflow/examples/android/src/org/tensorflow/demo/CameraConnectionFragment.java +++ b/tensorflow/examples/android/src/org/tensorflow/demo/CameraConnectionFragment.java @@ -38,6 +38,7 @@ import android.hardware.camera2.CaptureResult; import android.hardware.camera2.TotalCaptureResult; import android.hardware.camera2.params.StreamConfigurationMap; import android.media.ImageReader; +import android.media.ImageReader.OnImageAvailableListener; import android.os.Bundle; import android.os.Handler; import android.os.HandlerThread; @@ -49,9 +50,6 @@ import android.view.TextureView; import android.view.View; import android.view.ViewGroup; import android.widget.Toast; - -import org.tensorflow.demo.env.Logger; - import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -59,6 +57,7 @@ import java.util.Comparator; import java.util.List; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; +import org.tensorflow.demo.env.Logger; public class CameraConnectionFragment extends Fragment { private static final Logger LOGGER = new Logger(); @@ -69,8 +68,6 @@ public class CameraConnectionFragment extends Fragment { */ private static final int MINIMUM_PREVIEW_SIZE = 320; - private ResultsView resultsView; - /** * Conversion from screen rotation to JPEG orientation. */ @@ -111,6 +108,14 @@ public class CameraConnectionFragment extends Fragment { public void onSurfaceTextureUpdated(final SurfaceTexture texture) {} }; + /** + * Callback for Activities to use to initialize their data once the + * selected preview size is known. + */ + public interface ConnectionCallback { + void onPreviewSizeChosen(Size size, int cameraRotation); + } + /** * ID of the current {@link CameraDevice}. */ @@ -184,16 +189,6 @@ public class CameraConnectionFragment extends Fragment { */ private Handler backgroundHandler; - /** - * An additional thread for running inference so as not to block the camera. - */ - private HandlerThread inferenceThread; - - /** - * A {@link Handler} for running tasks in the background. - */ - private Handler inferenceHandler; - /** * An {@link ImageReader} that handles preview frame capture. */ @@ -215,9 +210,10 @@ public class CameraConnectionFragment extends Fragment { private final Semaphore cameraOpenCloseLock = new Semaphore(1); /** - * A {@link Classifier} object wrapping TensorFlow to pass frames to. + * A {@link OnImageAvailableListener} to receive frames as they are available. */ - private final Classifier classifier; + private final OnImageAvailableListener imageListener; + /** * The input size in pixels desired by TensorFlow (width and height of a square bitmap). */ @@ -228,9 +224,15 @@ public class CameraConnectionFragment extends Fragment { */ private final int layout; + + private final ConnectionCallback cameraConnectionCallback; + private CameraConnectionFragment( - final Classifier classifier, final int layout, final int inputSize) { - this.classifier = classifier; + final ConnectionCallback connectionCallback, + final OnImageAvailableListener imageListener, + final int layout, final int inputSize) { + this.cameraConnectionCallback = connectionCallback; + this.imageListener = imageListener; this.layout = layout; this.inputSize = inputSize; } @@ -268,8 +270,12 @@ public class CameraConnectionFragment extends Fragment { final Size[] choices, final int width, final int height, final Size aspectRatio) { // Collect the supported resolutions that are at least as big as the preview Surface final List bigEnough = new ArrayList(); + + final int minWidth = Math.max(width, MINIMUM_PREVIEW_SIZE); + final int minHeight = Math.max(height, MINIMUM_PREVIEW_SIZE); + for (final Size option : choices) { - if (option.getHeight() >= MINIMUM_PREVIEW_SIZE && option.getWidth() >= MINIMUM_PREVIEW_SIZE) { + if (option.getHeight() >= minHeight && option.getWidth() >= minWidth) { LOGGER.i("Adding size: " + option.getWidth() + "x" + option.getHeight()); bigEnough.add(option); } else { @@ -289,8 +295,9 @@ public class CameraConnectionFragment extends Fragment { } public static CameraConnectionFragment newInstance( - final Classifier classifier, final int layout, final int inputSize) { - return new CameraConnectionFragment(classifier, layout, inputSize); + final ConnectionCallback callback, + final OnImageAvailableListener imageListener, final int layout, final int inputSize) { + return new CameraConnectionFragment(callback, imageListener, layout, inputSize); } @Override @@ -302,7 +309,6 @@ public class CameraConnectionFragment extends Fragment { @Override public void onViewCreated(final View view, final Bundle savedInstanceState) { textureView = (AutoFitTextureView) view.findViewById(R.id.texture); - resultsView = (ResultsView) view.findViewById(R.id.results); } @Override @@ -371,7 +377,8 @@ public class CameraConnectionFragment extends Fragment { // bus' bandwidth limitation, resulting in gorgeous previews but the storage of // garbage capture data. previewSize = - chooseOptimalSize(map.getOutputSizes(SurfaceTexture.class), width, height, largest); + chooseOptimalSize(map.getOutputSizes(SurfaceTexture.class), + inputSize, inputSize, largest); // We fit the aspect ratio of TextureView to the size of preview we picked. final int orientation = getResources().getConfiguration().orientation; @@ -382,6 +389,8 @@ public class CameraConnectionFragment extends Fragment { } CameraConnectionFragment.this.cameraId = cameraId; + + cameraConnectionCallback.onPreviewSizeChosen(previewSize, sensorOrientation); return; } } catch (final CameraAccessException e) { @@ -446,10 +455,6 @@ public class CameraConnectionFragment extends Fragment { backgroundThread = new HandlerThread("ImageListener"); backgroundThread.start(); backgroundHandler = new Handler(backgroundThread.getLooper()); - - inferenceThread = new HandlerThread("InferenceThread"); - inferenceThread.start(); - inferenceHandler = new Handler(inferenceThread.getLooper()); } /** @@ -457,22 +462,15 @@ public class CameraConnectionFragment extends Fragment { */ private void stopBackgroundThread() { backgroundThread.quitSafely(); - inferenceThread.quitSafely(); try { backgroundThread.join(); backgroundThread = null; backgroundHandler = null; - - inferenceThread.join(); - inferenceThread = null; - inferenceThread = null; } catch (final InterruptedException e) { LOGGER.e(e, "Exception!"); } } - private final TensorFlowImageListener tfPreviewListener = new TensorFlowImageListener(); - private final CameraCaptureSession.CaptureCallback captureCallback = new CameraCaptureSession.CaptureCallback() { @Override @@ -513,7 +511,7 @@ public class CameraConnectionFragment extends Fragment { ImageReader.newInstance( previewSize.getWidth(), previewSize.getHeight(), ImageFormat.YUV_420_888, 2); - previewReader.setOnImageAvailableListener(tfPreviewListener, backgroundHandler); + previewReader.setOnImageAvailableListener(imageListener, backgroundHandler); previewRequestBuilder.addTarget(previewReader.getSurface()); // Here, we create a CameraCaptureSession for camera preview. @@ -557,11 +555,6 @@ public class CameraConnectionFragment extends Fragment { } catch (final CameraAccessException e) { LOGGER.e(e, "Exception!"); } - - LOGGER.i("Getting assets."); - tfPreviewListener.initialize( - classifier, resultsView, inputSize, inferenceHandler, sensorOrientation); - LOGGER.i("TensorFlow initialized."); } /** diff --git a/tensorflow/examples/android/src/org/tensorflow/demo/ClassifierActivity.java b/tensorflow/examples/android/src/org/tensorflow/demo/ClassifierActivity.java index 104ffbbd088..6f695dd7667 100644 --- a/tensorflow/examples/android/src/org/tensorflow/demo/ClassifierActivity.java +++ b/tensorflow/examples/android/src/org/tensorflow/demo/ClassifierActivity.java @@ -16,12 +16,29 @@ package org.tensorflow.demo; +import android.graphics.Bitmap; +import android.graphics.Bitmap.Config; +import android.graphics.Canvas; +import android.graphics.Matrix; +import android.graphics.Paint; +import android.media.Image; +import android.media.Image.Plane; +import android.media.ImageReader; +import android.media.ImageReader.OnImageAvailableListener; +import android.os.SystemClock; +import android.os.Trace; +import android.util.Size; +import android.util.TypedValue; +import android.view.Display; import java.io.IOException; - -import android.app.Fragment; +import java.util.List; +import java.util.Vector; +import org.tensorflow.demo.OverlayView.DrawCallback; +import org.tensorflow.demo.env.BorderedText; +import org.tensorflow.demo.env.ImageUtils; import org.tensorflow.demo.env.Logger; -public class ClassifierActivity extends CameraActivity { +public class ClassifierActivity extends CameraActivity implements OnImageAvailableListener { private static final Logger LOGGER = new Logger(); // These are the settings for the original v1 Inception model. If you want to @@ -41,9 +58,58 @@ public class ClassifierActivity extends CameraActivity { private static final String LABEL_FILE = "file:///android_asset/imagenet_comp_graph_label_strings.txt"; + private static final boolean SAVE_PREVIEW_BITMAP = false; + + private static final boolean MAINTAIN_ASPECT = true; + + private TensorFlowImageClassifier classifier; + + private Integer sensorOrientation; + + private int previewWidth = 0; + private int previewHeight = 0; + private byte[][] yuvBytes; + private int[] rgbBytes = null; + private Bitmap rgbFrameBitmap = null; + private Bitmap croppedBitmap = null; + + private Bitmap cropCopyBitmap; + + private boolean computing = false; + + private long timestamp = 0; + + private Matrix frameToCropTransform; + private Matrix cropToFrameTransform; + + private ResultsView resultsView; + + private OverlayView overlayView; + + private BorderedText borderedText; + + private long lastProcessingTimeMs; + @Override - protected Fragment createFragment() { - final TensorFlowImageClassifier classifier = new TensorFlowImageClassifier(); + protected int getLayoutId() { + return R.layout.camera_connection_fragment; + } + + @Override + protected int getDesiredPreviewFrameSize() { + return INPUT_SIZE; + } + + private static final float TEXT_SIZE_DIP = 18; + + @Override + public void onPreviewSizeChosen(final Size size, final int rotation) { + final float textSizePx = TypedValue.applyDimension( + TypedValue.COMPLEX_UNIT_DIP, TEXT_SIZE_DIP, + getResources().getDisplayMetrics()); + borderedText = new BorderedText(textSizePx); + + classifier = new TensorFlowImageClassifier(); try { classifier.initializeTensorFlow( getAssets(), MODEL_FILE, LABEL_FILE, NUM_CLASSES, INPUT_SIZE, IMAGE_MEAN, IMAGE_STD, @@ -52,7 +118,151 @@ public class ClassifierActivity extends CameraActivity { LOGGER.e(e, "Exception!"); } - return CameraConnectionFragment.newInstance( - classifier, R.layout.camera_connection_fragment, INPUT_SIZE); + overlayView = (OverlayView) findViewById(R.id.overlay); + resultsView = (ResultsView) findViewById(R.id.results); + previewWidth = size.getWidth(); + previewHeight = size.getHeight(); + + final Display display = getWindowManager().getDefaultDisplay(); + final int screenOrientation = display.getRotation(); + + LOGGER.i("Sensor orientation: %d, Screen orientation: %d", + rotation, screenOrientation); + + sensorOrientation = rotation + screenOrientation; + + if (sensorOrientation % 180 == 90) { + overlayView.setAspectRatio(size.getHeight(), size.getWidth()); + } else { + overlayView.setAspectRatio(size.getWidth(), size.getHeight()); + } + + LOGGER.i("Initializing at size %dx%d", previewWidth, previewHeight); + rgbBytes = new int[previewWidth * previewHeight]; + rgbFrameBitmap = Bitmap.createBitmap(previewWidth, previewHeight, Config.ARGB_8888); + croppedBitmap = Bitmap.createBitmap(INPUT_SIZE, INPUT_SIZE, Config.ARGB_8888); + + frameToCropTransform = ImageUtils.getTransformationMatrix( + previewWidth, previewHeight, + INPUT_SIZE, INPUT_SIZE, + sensorOrientation, MAINTAIN_ASPECT); + + cropToFrameTransform = new Matrix(); + frameToCropTransform.invert(cropToFrameTransform); + + yuvBytes = new byte[3][]; + + overlayView.addCallback(new DrawCallback() { + @Override + public void drawCallback(final Canvas canvas) { + renderDebug(canvas); + } + }); + } + + @Override + public void onImageAvailable(final ImageReader reader) { + Image image = null; + + ++timestamp; + + try { + image = reader.acquireLatestImage(); + + if (image == null) { + return; + } + + if (computing) { + image.close(); + return; + } + computing = true; + + Trace.beginSection("imageAvailable"); + + final Plane[] planes = image.getPlanes(); + fillBytes(planes, yuvBytes); + + final int yRowStride = planes[0].getRowStride(); + final int uvRowStride = planes[1].getRowStride(); + final int uvPixelStride = planes[1].getPixelStride(); + ImageUtils.convertYUV420ToARGB8888( + yuvBytes[0], + yuvBytes[1], + yuvBytes[2], + rgbBytes, + previewWidth, + previewHeight, + yRowStride, + uvRowStride, + uvPixelStride, + false); + + image.close(); + } catch (final Exception e) { + if (image != null) { + image.close(); + } + LOGGER.e(e, "Exception!"); + Trace.endSection(); + return; + } + + rgbFrameBitmap.setPixels(rgbBytes, 0, previewWidth, 0, 0, previewWidth, previewHeight); + final Canvas canvas = new Canvas(croppedBitmap); + canvas.drawBitmap(rgbFrameBitmap, frameToCropTransform, null); + + // For examining the actual TF input. + if (SAVE_PREVIEW_BITMAP) { + ImageUtils.saveBitmap(croppedBitmap); + } + + runInBackground( + new Runnable() { + @Override + public void run() { + final long startTime = SystemClock.uptimeMillis(); + final List results = classifier.recognizeImage(croppedBitmap); + lastProcessingTimeMs = SystemClock.uptimeMillis() - startTime; + + cropCopyBitmap = Bitmap.createBitmap(croppedBitmap); + resultsView.setResults(results); + overlayView.postInvalidate(); + computing = false; + } + }); + + Trace.endSection(); + } + + private void renderDebug(Canvas canvas) { + if (!isDebug()) { + return; + } + final Bitmap copy = cropCopyBitmap; + if (copy != null) { + final Matrix matrix = new Matrix(); + final float scaleFactor = 2; + matrix.postScale(scaleFactor, scaleFactor); + matrix.postTranslate( + canvas.getWidth() - copy.getWidth() * scaleFactor, + canvas.getHeight() - copy.getHeight() * scaleFactor); + canvas.drawBitmap(copy, matrix, new Paint()); + + final Vector lines = new Vector(); + lines.add("Frame: " + previewWidth + "x" + previewHeight); + lines.add("Crop: " + copy.getWidth() + "x" + copy.getHeight()); + lines.add("View: " + canvas.getWidth() + "x" + canvas.getHeight()); + lines.add("Rotation: " + sensorOrientation); + lines.add("Inference time: " + lastProcessingTimeMs + "ms"); + + int lineNum = 0; + for (final String line : lines) { + borderedText.drawText(canvas, 10, + canvas.getHeight() - 10 - borderedText.getTextSize() * lineNum, line); + ++lineNum; + } + } } } diff --git a/tensorflow/examples/android/src/org/tensorflow/demo/OverlayView.java b/tensorflow/examples/android/src/org/tensorflow/demo/OverlayView.java new file mode 100644 index 00000000000..b874bb07380 --- /dev/null +++ b/tensorflow/examples/android/src/org/tensorflow/demo/OverlayView.java @@ -0,0 +1,94 @@ +/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +package org.tensorflow.demo; + +import android.content.Context; +import android.graphics.Canvas; +import android.util.AttributeSet; +import android.view.MotionEvent; +import android.view.View; +import android.view.View.MeasureSpec; +import java.util.LinkedList; +import java.util.List; + +/** + * A simple View providing a render callback to other classes. + */ +public class OverlayView extends View { + public OverlayView(final Context context, final AttributeSet attrs) { + super(context, attrs); + } + + /** + * Interface defining the callback for client classes. + */ + public interface DrawCallback { + public void drawCallback(final Canvas canvas); + } + + private int ratioWidth; + private int ratioHeight; + + private boolean debug; + + private final List callbacks = new LinkedList(); + + @Override + public boolean onTouchEvent(final MotionEvent e) { + super.onTouchEvent(e); + if (e.getAction() == MotionEvent.ACTION_DOWN) { + debug = !debug; + } + return false; + } + + public void addCallback(final DrawCallback callback) { + callbacks.add(callback); + } + + @Override + public synchronized void draw(final Canvas canvas) { + for (final DrawCallback callback : callbacks) { + callback.drawCallback(canvas); + } + } + + public void setAspectRatio(final int width, final int height) { + if (width < 0 || height < 0) { + throw new IllegalArgumentException("Size cannot be negative."); + } + ratioWidth = width; + ratioHeight = height; + requestLayout(); + } + + @Override + protected void onMeasure(final int widthMeasureSpec, final int heightMeasureSpec) { + super.onMeasure(widthMeasureSpec, heightMeasureSpec); + final int width = MeasureSpec.getSize(widthMeasureSpec); + final int height = MeasureSpec.getSize(heightMeasureSpec); + if (0 == ratioWidth || 0 == ratioHeight) { + setMeasuredDimension(width, height); + } else { + if (width < height * ratioWidth / ratioHeight) { + setMeasuredDimension(width, width * ratioHeight / ratioWidth); + } else { + setMeasuredDimension(height * ratioWidth / ratioHeight, height); + } + } + } + +} diff --git a/tensorflow/examples/android/src/org/tensorflow/demo/env/BorderedText.java b/tensorflow/examples/android/src/org/tensorflow/demo/env/BorderedText.java new file mode 100644 index 00000000000..e4b13bb7abf --- /dev/null +++ b/tensorflow/examples/android/src/org/tensorflow/demo/env/BorderedText.java @@ -0,0 +1,119 @@ +/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +package org.tensorflow.demo.env; + +import android.graphics.Canvas; +import android.graphics.Color; +import android.graphics.Paint; +import android.graphics.Paint.Align; +import android.graphics.Paint.Style; +import android.graphics.Rect; + +/** + * A class that encapsulates the tedious bits of rendering legible, bordered text onto a canvas. + */ +public class BorderedText { + private final Paint interiorPaint; + private final Paint exteriorPaint; + + private final float textSize; + + /** + * Creates a left-aligned bordered text object with a white interior, and a black exterior with + * the specified text size. + * + * @param textSize text size in pixels + */ + public BorderedText(final float textSize) { + this(Color.WHITE, Color.BLACK, textSize); + } + + /** + * Create a bordered text object with the specified interior and exterior colors, text size and + * alignment. + * + * @param interiorColor the interior text color + * @param exteriorColor the exterior text color + * @param textSize text size in pixels + */ + public BorderedText(final int interiorColor, final int exteriorColor, final float textSize) { + interiorPaint = new Paint(); + interiorPaint.setTextSize(textSize); + interiorPaint.setColor(interiorColor); + interiorPaint.setStyle(Style.FILL); + interiorPaint.setAntiAlias(false); + interiorPaint.setAlpha(255); + + exteriorPaint = new Paint(); + exteriorPaint.setTextSize(textSize); + exteriorPaint.setColor(exteriorColor); + exteriorPaint.setStyle(Style.FILL_AND_STROKE); + exteriorPaint.setStrokeWidth(textSize / 8); + exteriorPaint.setAntiAlias(false); + exteriorPaint.setAlpha(255); + + this.textSize = textSize; + } + + public void drawText(final Canvas canvas, final float posX, final float posY, final String text) { + /* + if (widths == null || widths.length < text.length()) { + widths = new float[text.length()]; + positions = new float[text.length() * 2]; + } + + exteriorPaint.getTextWidths(text, widths); + float lastPosX = posX; + for (int i = 0; i < widths.length; ++i) { + positions[i * 2] = lastPosX; + positions[i * 2 + 1] = posY; + lastPosX += widths[i]; + } + */ + + //canvas.drawPosText(text, positions, exteriorPaint); + //canvas.drawPosText(text, positions, exteriorPaint); + canvas.drawText(text, posX, posY, exteriorPaint); + canvas.drawText(text, posX, posY, interiorPaint); + } + + public void setInteriorColor(final int color) { + interiorPaint.setColor(color); + } + + public void setExteriorColor(final int color) { + exteriorPaint.setColor(color); + } + + public float getTextSize() { + return textSize; + } + + public void setAlpha(final int alpha) { + interiorPaint.setAlpha(alpha); + exteriorPaint.setAlpha(alpha); + } + + public void getTextBounds( + final String line, final int index, final int count, final Rect lineBounds) { + interiorPaint.getTextBounds(line, index, count, lineBounds); + } + + public void setTextAlign(final Align align) { + interiorPaint.setTextAlign(align); + exteriorPaint.setTextAlign(align); + } +} diff --git a/tensorflow/examples/android/src/org/tensorflow/demo/env/ImageUtils.java b/tensorflow/examples/android/src/org/tensorflow/demo/env/ImageUtils.java index a6a8c583190..6f957d1abd5 100644 --- a/tensorflow/examples/android/src/org/tensorflow/demo/env/ImageUtils.java +++ b/tensorflow/examples/android/src/org/tensorflow/demo/env/ImageUtils.java @@ -16,8 +16,8 @@ limitations under the License. package org.tensorflow.demo.env; import android.graphics.Bitmap; +import android.graphics.Matrix; import android.os.Environment; - import java.io.File; import java.io.FileOutputStream; @@ -49,6 +49,16 @@ public class ImageUtils { * @param bitmap The bitmap to save. */ public static void saveBitmap(final Bitmap bitmap) { + saveBitmap(bitmap, "preview.png"); + } + + /** + * Saves a Bitmap object to disk for analysis. + * + * @param bitmap The bitmap to save. + * @param filename The location to save the bitmap to. + */ + public static void saveBitmap(final Bitmap bitmap, final String filename) { final String root = Environment.getExternalStorageDirectory().getAbsolutePath() + File.separator + "tensorflow"; LOGGER.i("Saving %dx%d bitmap to %s.", bitmap.getWidth(), bitmap.getHeight(), root); @@ -58,7 +68,7 @@ public class ImageUtils { LOGGER.i("Make dir failed"); } - final String fname = "preview.png"; + final String fname = filename; final File file = new File(myDir, fname); if (file.exists()) { file.delete(); @@ -151,4 +161,66 @@ public class ImageUtils { */ public static native void convertRGB565ToYUV420SP( byte[] input, byte[] output, int width, int height); + + /** + * Returns a transformation matrix from one reference frame into another. + * Handles cropping (if maintaining aspect ratio is desired) and rotation. + * + * @param srcWidth Width of source frame. + * @param srcHeight Height of source frame. + * @param dstWidth Width of destination frame. + * @param dstHeight Height of destination frame. + * @param applyRotation Amount of rotation to apply from one frame to another. + * Must be a multiple of 90. + * @param maintainAspectRatio If true, will ensure that scaling in x and y remains constant, + * cropping the image if necessary. + * @return The transformation fulfilling the desired requirements. + */ + public static Matrix getTransformationMatrix( + final int srcWidth, + final int srcHeight, + final int dstWidth, + final int dstHeight, + final int applyRotation, + final boolean maintainAspectRatio) { + final Matrix matrix = new Matrix(); + + if (applyRotation != 0) { + // Translate so center of image is at origin. + matrix.postTranslate(-srcWidth / 2.0f, -srcHeight / 2.0f); + + // Rotate around origin. + matrix.postRotate(applyRotation); + } + + // Account for the already applied rotation, if any, and then determine how + // much scaling is needed for each axis. + final boolean transpose = (Math.abs(applyRotation) + 90) % 180 == 0; + + final int inWidth = transpose ? srcHeight : srcWidth; + final int inHeight = transpose ? srcWidth : srcHeight; + + // Apply scaling if necessary. + if (inWidth != dstWidth || inHeight != dstHeight) { + final float scaleFactorX = dstWidth / (float) inWidth; + final float scaleFactorY = dstHeight / (float) inHeight; + + if (maintainAspectRatio) { + // Scale by minimum factor so that dst is filled completely while + // maintaining the aspect ratio. Some image may fall off the edge. + final float scaleFactor = Math.max(scaleFactorX, scaleFactorX); + matrix.postScale(scaleFactor, scaleFactor); + } else { + // Scale exactly to fill dst from src. + matrix.postScale(scaleFactorX, scaleFactorY); + } + } + + if (applyRotation != 0) { + // Translate back from origin centered reference to destination frame. + matrix.postTranslate(dstWidth / 2.0f, dstHeight / 2.0f); + } + + return matrix; + } } diff --git a/tensorflow/examples/android/src/org/tensorflow/demo/env/Size.java b/tensorflow/examples/android/src/org/tensorflow/demo/env/Size.java new file mode 100644 index 00000000000..ef15d14daa8 --- /dev/null +++ b/tensorflow/examples/android/src/org/tensorflow/demo/env/Size.java @@ -0,0 +1,143 @@ +/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +package org.tensorflow.demo.env; + +import android.graphics.Bitmap; +import android.text.TextUtils; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +/** + * Size class independent of a Camera object. + */ +public class Size implements Comparable, Serializable { + + // 1.4 went out with this UID so we'll need to maintain it to preserve pending queries when + // upgrading. + public static final long serialVersionUID = 7689808733290872361L; + + public final int width; + public final int height; + + public Size(final int width, final int height) { + this.width = width; + this.height = height; + } + + public Size(final Bitmap bmp) { + this.width = bmp.getWidth(); + this.height = bmp.getHeight(); + } + + /** + * Rotate a size by the given number of degrees. + * @param size Size to rotate. + * @param rotation Degrees {0, 90, 180, 270} to rotate the size. + * @return Rotated size. + */ + public static Size getRotatedSize(final Size size, final int rotation) { + if (rotation % 180 != 0) { + // The phone is portrait, therefore the camera is sideways and frame should be rotated. + return new Size(size.height, size.width); + } + return size; + } + + public static Size parseFromString(String sizeString) { + if (TextUtils.isEmpty(sizeString)) { + return null; + } + + sizeString = sizeString.trim(); + + // The expected format is "x". + final String[] components = sizeString.split("x"); + if (components.length == 2) { + try { + final int width = Integer.parseInt(components[0]); + final int height = Integer.parseInt(components[1]); + return new Size(width, height); + } catch (final NumberFormatException e) { + return null; + } + } else { + return null; + } + } + + public static List sizeStringToList(final String sizes) { + final List sizeList = new ArrayList(); + if (sizes != null) { + final String[] pairs = sizes.split(","); + for (final String pair : pairs) { + final Size size = Size.parseFromString(pair); + if (size != null) { + sizeList.add(size); + } + } + } + return sizeList; + } + + public static String sizeListToString(final List sizes) { + String sizesString = ""; + if (sizes != null && sizes.size() > 0) { + sizesString = sizes.get(0).toString(); + for (int i = 1; i < sizes.size(); i++) { + sizesString += "," + sizes.get(i).toString(); + } + } + return sizesString; + } + + public final float aspectRatio() { + return (float) width / (float) height; + } + + @Override + public int compareTo(final Size other) { + return width * height - other.width * other.height; + } + + @Override + public boolean equals(final Object other) { + if (other == null) { + return false; + } + + if (!(other instanceof Size)) { + return false; + } + + final Size otherSize = (Size) other; + return (width == otherSize.width && height == otherSize.height); + } + + @Override + public int hashCode() { + return width * 32713 + height; + } + + @Override + public String toString() { + return dimensionsAsString(width, height); + } + + public static final String dimensionsAsString(final int width, final int height) { + return width + "x" + height; + } +} From 21f438cce3e723f9984463eb423ff6368ef5f260 Mon Sep 17 00:00:00 2001 From: Patrick Nguyen Date: Thu, 3 Nov 2016 10:13:29 -0800 Subject: [PATCH 50/54] Automated rollback of change 138001077 Change: 138093178 --- tensorflow/python/ops/standard_ops.py | 199 +------------------------- 1 file changed, 1 insertion(+), 198 deletions(-) diff --git a/tensorflow/python/ops/standard_ops.py b/tensorflow/python/ops/standard_ops.py index f3c987c8e7a..847d1b99c83 100644 --- a/tensorflow/python/ops/standard_ops.py +++ b/tensorflow/python/ops/standard_ops.py @@ -16,21 +16,17 @@ # pylint: disable=unused-import """Import names of Tensor Flow standard Ops.""" +# Imports the following modules so that @RegisterGradient get executed. from __future__ import absolute_import from __future__ import division from __future__ import print_function -import sys as _sys - -# Imports the following modules so that @RegisterGradient get executed. from tensorflow.python.ops import array_grad from tensorflow.python.ops import data_flow_grad from tensorflow.python.ops import math_grad from tensorflow.python.ops import sparse_grad from tensorflow.python.ops import state_grad from tensorflow.python.ops import tensor_array_grad -from tensorflow.python.util.all_util import remove_undocumented - # go/tf-wildcard-import # pylint: disable=wildcard-import @@ -82,196 +78,3 @@ from tensorflow.python.ops.tensor_array_ops import * from tensorflow.python.ops.variable_scope import * from tensorflow.python.ops.variables import * # pylint: enable=wildcard-import - -#### For use in remove_undocumented below: -from tensorflow.python.framework import constant_op as _constant_op -from tensorflow.python.ops import array_ops as _array_ops -from tensorflow.python.ops import check_ops as _check_ops -from tensorflow.python.ops import clip_ops as _clip_ops -from tensorflow.python.ops import control_flow_ops as _control_flow_ops -from tensorflow.python.ops import data_flow_ops as _data_flow_ops -from tensorflow.python.ops import functional_ops as _functional_ops -from tensorflow.python.ops import gradients as _gradients -from tensorflow.python.ops import histogram_ops as _histogram_ops -from tensorflow.python.ops import init_ops as _init_ops -from tensorflow.python.ops import io_ops as _io_ops -from tensorflow.python.ops import linalg_ops as _linalg_ops -from tensorflow.python.ops import logging_ops as _logging_ops -from tensorflow.python.ops import math_ops as _math_ops -from tensorflow.python.ops import numerics as _numerics -from tensorflow.python.ops import parsing_ops as _parsing_ops -from tensorflow.python.ops import partitioned_variables as _partitioned_variables -from tensorflow.python.ops import random_ops as _random_ops -from tensorflow.python.ops import script_ops as _script_ops -from tensorflow.python.ops import session_ops as _session_ops -from tensorflow.python.ops import sparse_ops as _sparse_ops -from tensorflow.python.ops import special_math_ops as _special_math_ops -from tensorflow.python.ops import state_ops as _state_ops -from tensorflow.python.ops import string_ops as _string_ops -from tensorflow.python.ops import template as _template -from tensorflow.python.ops import tensor_array_ops as _tensor_array_ops -from tensorflow.python.ops import variable_scope as _variable_scope -from tensorflow.python.ops import variables as _variables - - -_allowed_symbols_math_ops = [ - # TODO(drpng): decide if we want to reference these in the documentation. - "reduced_shape", - "sparse_segment_mean_grad", - "sparse_segment_sqrt_n_grad", - - # Legacy: will be removed. - "arg_max", - "arg_min", - "lin_space", - "sparse_matmul", # Use tf.matmul. - # Deprecated (see versions.h): - "batch_fft", - "batch_fft2d", - "batch_fft3d", - "batch_ifft", - "batch_ifft2d", - "batch_ifft3d", - - # These are documented in nn. - # We are are not importing nn because it would create a circular dependency. - "sigmoid", - "tanh", -] - -_allowed_symbols_array_ops = [ - # TODO(drpng): make sure they are documented. as _they - # Scalars: - "NEW_AXIS", - "SHRINK_AXIS", - "newaxis", - - # Documented in training.py. - # I do not import train, to avoid circular dependencies. - # TODO(drpng): this is defined in gen_array_ops, clearly not the right - # place. - "stop_gradient", - - # See gen_docs_combined for tf.copy documentation. - "copy", - - ## TODO(drpng): make them inaccessible directly. - ## TODO(drpng): Below, to-doc means that we need to find an appropriate - ## documentation section to reference. - ## For re-exporting to tf.*: - "constant", - "edit_distance", # to-doc - # From gen_array_ops: - "copy_host", # to-doc - "immutable_const", # to-doc - "invert_permutation", # to-doc - "quantize_and_dequantize", # to-doc - - # TODO(drpng): legacy symbols to be removed. - "list_diff", # Use tf.listdiff instead. - "batch_matrix_diag", - "batch_matrix_band_part", - "batch_matrix_diag_part", - "batch_matrix_set_diag", -] - -_allowed_symbols_partitioned_variables = [ - "PartitionedVariable", # Requires doc link. - # Legacy. - "create_partitioned_variables", - "variable_axis_size_partitioner", - "min_max_variable_partitioner", - "fixed_size_partitioner", -] - -_allowed_symbols_control_flow_ops = [ - # TODO(drpng): Find a place in the documentation to reference these or - # remove. - "control_trigger", - "loop_cond", - "merge", - "switch", -] - -_allowed_symbols_functional_ops = [ - "nest", # Used by legacy code. -] - -_allowed_symbols_gradients = [ - # Documented in training.py: - # Not importing training.py to avoid complex graph dependencies. - "AggregationMethod", - "gradients", # tf.gradients = gradients.gradients -] - -_allowed_symbols_clip_ops = [ - # Documented in training.py: - # Not importing training.py to avoid complex graph dependencies. - "clip_by_average_norm", - "clip_by_global_norm", - "clip_by_norm", - "clip_by_value", - "global_norm", -] - -_allowed_symbols_image_ops = [ - # Documented in training.py. - # We are not importing training.py to avoid complex dependencies. - "audio_summary", - "histogram_summary", - "image_summary", - "merge_all_summaries", - "merge_summary", - "scalar_summary", - - # TODO(drpng): link in training.py if it should be documented. - "get_summary_op", -] - -_allowed_symbols_misc = [ - "deserialize_many_sparse", - "parse_single_sequence_example", - "serialize_many_sparse", - "serialize_sparse", -] - -_allowed_symbols = (_allowed_symbols_array_ops + - _allowed_symbols_clip_ops + - _allowed_symbols_control_flow_ops + - _allowed_symbols_functional_ops + - _allowed_symbols_image_ops + - _allowed_symbols_gradients + - _allowed_symbols_math_ops + - _allowed_symbols_misc + - _allowed_symbols_partitioned_variables) - -remove_undocumented(__name__, _allowed_symbols, - [_sys.modules[__name__], - _array_ops, - _check_ops, - _clip_ops, - _control_flow_ops, - _constant_op, - _data_flow_ops, - _functional_ops, - _gradients, - _histogram_ops, - _init_ops, - _io_ops, - _linalg_ops, - _logging_ops, - _math_ops, - _numerics, - _parsing_ops, - _partitioned_variables, - _random_ops, - _script_ops, - _session_ops, - _sparse_ops, - _special_math_ops, - _state_ops, - _string_ops, - _template, - _tensor_array_ops, - _variable_scope, - _variables,]) From 36621ac3d54eb5dc6397db8a2d117e88ac90f4a7 Mon Sep 17 00:00:00 2001 From: Patrick Nguyen Date: Thu, 3 Nov 2016 10:50:37 -0800 Subject: [PATCH 51/54] Seal gradients interface. Change: 138098284 --- tensorflow/python/BUILD | 5 +- .../python/ops/candidate_sampling_ops.py | 2 +- tensorflow/python/ops/gradients.py | 823 +---------------- tensorflow/python/ops/gradients_impl.py | 831 ++++++++++++++++++ tensorflow/python/ops/gradients_test.py | 7 +- 5 files changed, 852 insertions(+), 816 deletions(-) create mode 100644 tensorflow/python/ops/gradients_impl.py diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index 2511cfee373..4dc1bcec172 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -870,7 +870,10 @@ py_library( py_library( name = "gradients", - srcs = ["ops/gradients.py"], + srcs = [ + "ops/gradients.py", + "ops/gradients_impl.py", + ], srcs_version = "PY2AND3", deps = [ ":array_grad", diff --git a/tensorflow/python/ops/candidate_sampling_ops.py b/tensorflow/python/ops/candidate_sampling_ops.py index a1cd8d803ef..b27167df542 100644 --- a/tensorflow/python/ops/candidate_sampling_ops.py +++ b/tensorflow/python/ops/candidate_sampling_ops.py @@ -13,7 +13,7 @@ # limitations under the License. # ============================================================================== -"""Wrappers for primitive Neural Net (NN) Operations.""" +"""Wrappers for candidate sampling operations.""" from __future__ import absolute_import from __future__ import division diff --git a/tensorflow/python/ops/gradients.py b/tensorflow/python/ops/gradients.py index 870fed3e188..692acc344c4 100644 --- a/tensorflow/python/ops/gradients.py +++ b/tensorflow/python/ops/gradients.py @@ -18,814 +18,15 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import collections -import contextlib -import warnings - -import numpy as np -import six -from six.moves import xrange # pylint: disable=redefined-builtin - -from tensorflow.core.framework import attr_value_pb2 -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.framework import tensor_util -from tensorflow.python.ops import array_grad # pylint: disable=unused-import -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_grad # pylint: disable=unused-import -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import image_grad # pylint: disable=unused-import -from tensorflow.python.ops import logging_ops # pylint: disable=unused-import -from tensorflow.python.ops import linalg_grad # pylint: disable=unused-import -from tensorflow.python.ops import math_grad # pylint: disable=unused-import -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import linalg_ops -from tensorflow.python.ops import functional_ops - -from tensorflow.python.platform import tf_logging as logging - -# Warn the user if we convert a sparse representation to dense with at -# least this number of elements. -_LARGE_SPARSE_NUM_ELEMENTS = 100000000 - - -def _IndexedSlicesToTensor(value, dtype=None, name=None, as_ref=False): - """Converts an IndexedSlices object `value` to a Tensor. - - NOTE(mrry): This function is potentially expensive. - - Args: - value: An ops.IndexedSlices object. - dtype: The dtype of the Tensor to be returned. - name: Optional name to use for the returned Tensor. - as_ref: True if a ref is requested. - - Returns: - A dense Tensor representing the values in the given IndexedSlices. - - Raises: - ValueError: If the IndexedSlices does not have the same dtype. - """ - _ = as_ref - if dtype and not dtype.is_compatible_with(value.dtype): - raise ValueError( - "Tensor conversion requested dtype %s for IndexedSlices with dtype %s" % - (dtype.name, value.dtype.name)) - if value.dense_shape is None: - raise ValueError( - "Tensor conversion requested for IndexedSlices without dense_shape: %s" - % str(value)) - # TODO(mrry): Consider adding static shape information to - # IndexedSlices, to avoid using numpy here. - dense_shape_value = tensor_util.constant_value(value.dense_shape) - if dense_shape_value is not None: - num_elements = np.prod(dense_shape_value) - if num_elements >= _LARGE_SPARSE_NUM_ELEMENTS: - warnings.warn( - "Converting sparse IndexedSlices to a dense Tensor with %d elements. " - "This may consume a large amount of memory." % num_elements) - else: - warnings.warn( - "Converting sparse IndexedSlices to a dense Tensor of unknown shape. " - "This may consume a large amount of memory.") - return math_ops.unsorted_segment_sum( - value.values, value.indices, value.dense_shape[0], name=name) - - -ops.register_tensor_conversion_function(ops.IndexedSlices, - _IndexedSlicesToTensor) - - -def _MarkReachedOps(from_ops, reached_ops): - """Mark all ops reached from "from_ops". - - Args: - from_ops: list of Operations. - reached_ops: list of booleans, indexed by operation id. - """ - queue = collections.deque() - queue.extend(from_ops) - while queue: - op = queue.popleft() - if not reached_ops[op._id]: - reached_ops[op._id] = True - for output in op.outputs: - queue.extend(output.consumers()) - - -def _GatherInputs(to_ops, reached_ops): - """List all inputs of to_ops that are in reached_ops. - - Args: - to_ops: list of Operations. - reached_ops: list of booleans, indexed by operation id. - - Returns: - The list of all inputs of to_ops that are in reached_ops. - That list includes all elements of to_ops. - """ - inputs = [] - queue = collections.deque() - queue.extend(to_ops) - while queue: - op = queue.popleft() - # We are interested in this op. - if reached_ops[op._id]: - inputs.append(op) - # Clear the boolean so we won't add the inputs again. - reached_ops[op._id] = False - for inp in op.inputs: - queue.append(inp.op) - return inputs - - -def _PendingCount(graph, to_ops, from_ops, colocate_gradients_with_ops): - """Initialize the pending count for ops between two lists of Operations. - - 'pending_count[op._id]' indicates the number of backprop inputs - to this operation. - - Args: - graph: a Graph. - to_ops: list of Operations. - from_ops: list of Operations. - colocate_gradients_with_ops: Python bool. See docstring of gradients(). - - Returns: - A tuple containing: (1) a list of integers indexed by operation id, - indicating the number of backprop inputs to this operation, and (2) - a ControlFlowState object which is not None if the ops between from_ops - and to_ops contain control flow loops. - """ - # Mark reachable ops from from_ops. - reached_ops = [False] * (graph._last_id + 1) - for op in to_ops: - reached_ops[op._id] = True - _MarkReachedOps(from_ops, reached_ops) - - # Mark between ops. - between_ops = [False] * (graph._last_id + 1) - between_op_list = [] - queue = collections.deque() - queue.extend(to_ops) - while queue: - op = queue.popleft() - # We are interested in this op. - if reached_ops[op._id]: - between_ops[op._id] = True - between_op_list.append(op) - # Clear the boolean so we won't add the inputs again. - reached_ops[op._id] = False - for inp in op.inputs: - queue.append(inp.op) - - # 'loop_state' is None if there are no while loops. - loop_state = control_flow_ops.MaybeCreateControlFlowState( - between_op_list, between_ops, colocate_gradients_with_ops) - - # Initialize pending count for between ops. - pending_count = [0] * (graph._last_id + 1) - for op in between_op_list: - for x in op.inputs: - if between_ops[x.op._id]: - pending_count[x.op._id] += 1 - - return pending_count, loop_state - - -def _AsList(x): - return x if isinstance(x, (list, tuple)) else [x] - - -def _DefaultGradYs(grad_ys, ys, colocate_gradients_with_ops): - """Fill in default values for grad_ys. - - Args: - grad_ys: List of gradients, can contain None. - ys: List of tensors. - colocate_gradients_with_ops: If True, try colocating gradients with - the corresponding op. - - Returns: - A list of gradients to use, without None. - - Raises: - ValueError: If one of the grad_ys is invalid. - """ - if len(grad_ys) != len(ys): - raise ValueError("Passed %d grad_ys for %d ys" % (len(grad_ys), len(ys))) - grad_ys = ops.convert_n_to_tensor_or_indexed_slices(grad_ys, name="grad_y") - for i in xrange(len(grad_ys)): - grad_y = grad_ys[i] - y = ys[i] - if grad_y is None: - with _maybe_colocate_with(y.op, colocate_gradients_with_ops): - grad_ys[i] = array_ops.fill( - array_ops.shape(y), constant_op.constant( - 1, dtype=y.dtype)) - else: - if grad_y.dtype != y.dtype: - raise ValueError("Y and ys_grad must be of the same type, " - "not y: %s, ys_grad: %s " % - (dtypes.as_dtype(y.dtype).name, - dtypes.as_dtype(grad_y.dtype).name)) - return grad_ys - - -def _IsTrainable(tensor): - dtype = dtypes.as_dtype(tensor.dtype) - return dtype.base_dtype in (dtypes.float16, dtypes.float32, dtypes.float64, - dtypes.complex64, dtypes.complex128) - - -def _VerifyGeneratedGradients(grads, op): - """Verify that gradients are valid in number and type. - - Args: - grads: List of generated gradients. - op: Operation for which the gradients where generated. - - Raises: - ValueError: if the gradients are invalid. - """ - if len(grads) != len(op.inputs): - raise ValueError("Num gradients %d generated for op %s do not match num " - "inputs %d" % (len(grads), op.node_def, len(op.inputs))) - for i in xrange(len(grads)): - grad = grads[i] - inp = op.inputs[i] - if grad is not None: - if not grad.dtype.is_compatible_with(inp.dtype): - raise ValueError("Gradient type %s generated for op %s does " - "not match input type %s" % - (dtypes.as_dtype(grad.dtype).name, op.node_def, - dtypes.as_dtype(inp.dtype).name)) - - -def _StopOps(from_ops, pending_count): - """The set of ops that terminate the gradient computation. - - This computes the frontier of the forward graph *before* which backprop - should stop. Operations in the returned set will not be differentiated. - This set is defined as the subset of `from_ops` containing ops that have - no predecessor in `from_ops`. `pending_count` is the result of - `_PendingCount(g, xs, from_ops)`. An 'op' has predecessors in `from_ops` - iff pending_count[op._id] > 0. - - Args: - from_ops: list of Operations. - pending_count: List of integers, indexed by operation id. - - Returns: - The set of operations. - """ - stop_ops = set() - for op in from_ops: - is_stop_op = True - for inp in op.inputs: - if pending_count[inp.op._id] > 0: - is_stop_op = False - break - if is_stop_op: - stop_ops.add(op._id) - return stop_ops - - -@contextlib.contextmanager -def _maybe_colocate_with(op, colocate_gradients_with_ops): - """Context to colocate with `op` if `colocate_gradients_with_ops`.""" - if colocate_gradients_with_ops: - with ops.colocate_with(op): - yield - else: - yield - - -def _SymGrad(op, out_grads): - """Backprop through a function call node op given its outputs' gradients.""" - f_in = [x for x in op.inputs] + out_grads - f_types = [x.dtype for x in op.inputs] - f = attr_value_pb2.NameAttrList() - f.name = op.type - for k in op.node_def.attr: - f.attr[k].CopyFrom(op.node_def.attr[k]) - # pylint: disable=protected-access - in_grads = functional_ops._symbolic_gradient(input=f_in, Tout=f_types, f=f) - # pylint: enable=protected-access - return in_grads - - -def gradients(ys, - xs, - grad_ys=None, - name="gradients", - colocate_gradients_with_ops=False, - gate_gradients=False, - aggregation_method=None): - """Constructs symbolic partial derivatives of sum of `ys` w.r.t. x in `xs`. - - `ys` and `xs` are each a `Tensor` or a list of tensors. `grad_ys` - is a list of `Tensor`, holding the gradients received by the - `ys`. The list must be the same length as `ys`. - - `gradients()` adds ops to the graph to output the partial - derivatives of `ys` with respect to `xs`. It returns a list of - `Tensor` of length `len(xs)` where each tensor is the `sum(dy/dx)` - for y in `ys`. - - `grad_ys` is a list of tensors of the same length as `ys` that holds - the initial gradients for each y in `ys`. When `grad_ys` is None, - we fill in a tensor of '1's of the shape of y for each y in `ys`. A - user can provide their own initial `grad_ys` to compute the - derivatives using a different initial gradient for each y (e.g., if - one wanted to weight the gradient differently for each value in - each y). - - Args: - ys: A `Tensor` or list of tensors to be differentiated. - xs: A `Tensor` or list of tensors to be used for differentiation. - grad_ys: Optional. A `Tensor` or list of tensors the same size as - `ys` and holding the gradients computed for each y in `ys`. - name: Optional name to use for grouping all the gradient ops together. - defaults to 'gradients'. - colocate_gradients_with_ops: If True, try colocating gradients with - the corresponding op. - gate_gradients: If True, add a tuple around the gradients returned - for an operations. This avoids some race conditions. - aggregation_method: Specifies the method used to combine gradient terms. - Accepted values are constants defined in the class `AggregationMethod`. - - Returns: - A list of `sum(dy/dx)` for each x in `xs`. - - Raises: - LookupError: if one of the operations between `x` and `y` does not - have a registered gradient function. - ValueError: if the arguments are invalid. - - """ - ys = _AsList(ys) - xs = _AsList(xs) - if grad_ys is None: - grad_ys = [None] * len(ys) - else: - grad_ys = _AsList(grad_ys) - - with ops.name_scope(name, "gradients", ys + xs + grad_ys): - ys = ops.convert_n_to_tensor_or_indexed_slices(ys, name="y") - xs = ops.convert_n_to_tensor_or_indexed_slices(xs, name="x") - grad_ys = _DefaultGradYs(grad_ys, ys, colocate_gradients_with_ops) - - # The approach we take here is as follows: Create a list of all ops in the - # subgraph between the ys and xs. Visit these ops in reverse order of ids - # to ensure that when we visit an op the gradients w.r.t its outputs have - # been collected. Then aggregate these gradients if needed, call the op's - # gradient function, and add the generated gradients to the gradients for - # its input. - - # Initialize the pending count for ops in the connected subgraph from ys - # to the xs. - to_ops = [t.op for t in ys] - from_ops = [t.op for t in xs] - pending_count, loop_state = _PendingCount(ops.get_default_graph(), to_ops, - from_ops, - colocate_gradients_with_ops) - - # Iterate over the collected ops. - # - # grads: op => list of gradients received on each output endpoint of the - # op. The gradients for each endpoint are initially collected as a list. - # When it is time to call the op's gradient function, for each endpoint we - # aggregate the list of received gradients into a Add() Operation if there - # is more than one. - grads = {} - - # Add the initial gradients for the ys. - for y, grad_y in zip(ys, grad_ys): - _SetGrad(grads, y, grad_y) - - # Initialize queue with to_ops. - queue = collections.deque() - # Add the ops in 'to_ops' into the queue. - to_ops_set = set() - for op in to_ops: - # 'ready' handles the case where one output gradient relies on - # another output's gradient. - # pylint: disable=protected-access - ready = (pending_count[op._id] == 0) - if ready and op._id not in to_ops_set: - to_ops_set.add(op._id) - queue.append(op) - # pylint: enable=protected-access - - if loop_state: - loop_exits = loop_state.ProcessUnusedLoopExits(pending_count, to_ops_set) - for y in loop_exits: - if _IsTrainable(y): - _SetGrad(grads, y, loop_state.ZerosLikeForExit(y)) - queue.append(y.op) - - # The set of 'from_ops'. - stop_ops = _StopOps(from_ops, pending_count) - while queue: - # generate gradient subgraph for op. - op = queue.popleft() - with _maybe_colocate_with(op, colocate_gradients_with_ops): - if loop_state: - loop_state.EnterGradWhileContext(op, before=True) - out_grads = _AggregatedGrads(grads, op, loop_state, aggregation_method) - if loop_state: - loop_state.ExitGradWhileContext(op, before=True) - - grad_fn = None - # pylint: disable=protected-access - is_func_call = ops.get_default_graph()._is_function(op.type) - has_out_grads = any(isinstance(g, ops.Tensor) or g for g in out_grads) - if has_out_grads and (op._id not in stop_ops): - if is_func_call: - grad_fn = ops.get_default_graph()._get_function( - op.type).python_grad_func - # pylint: enable=protected-access - else: - # A grad_fn must be defined, either as a function or as None - # for ops that do not have gradients. - try: - grad_fn = ops.get_gradient_function(op) - except LookupError: - raise LookupError( - "No gradient defined for operation '%s' (op type: %s)" % - (op.name, op.type)) - if loop_state: - loop_state.EnterGradWhileContext(op, before=False) - if (grad_fn or is_func_call) and has_out_grads: - # NOTE: If _AggregatedGrads didn't compute a value for the i'th - # output, it means that the cost does not depend on output[i], - # therefore dC/doutput[i] is 0. - for i, out_grad in enumerate(out_grads): - if (not isinstance(out_grad, ops.Tensor) and - not out_grad) and _IsTrainable(op.outputs[i]): - # Only floating-point outputs get a zero gradient. Gradient - # functions should ignore the gradient for other outputs. - if loop_state: - out_grads[i] = loop_state.ZerosLike(op, i) - else: - out_grads[i] = control_flow_ops.ZerosLikeOutsideLoop(op, i) - with ops.name_scope(op.name + "_grad"): - # pylint: disable=protected-access - with ops.get_default_graph()._original_op(op): - # pylint: enable=protected-access - if grad_fn: - # If grad_fn was found, do not use SymbolicGradient even for - # functions. - in_grads = grad_fn(op, *out_grads) - else: - # For function call ops, we add a 'SymbolicGradient' - # node to the graph to compute gradients. - in_grads = _SymGrad(op, out_grads) - in_grads = _AsList(in_grads) - _VerifyGeneratedGradients(in_grads, op) - if gate_gradients and len( - [x for x in in_grads if x is not None]) > 1: - in_grads = control_flow_ops.tuple(in_grads) - _LogOpGradients(op, out_grads, in_grads) - else: - # If no grad_fn is defined or none of out_grads is available, - # just propagates a list of None backwards. - in_grads = [None] * len(op.inputs) - for t_in, in_grad in zip(op.inputs, in_grads): - if in_grad is not None: - if isinstance(in_grad, ops.Tensor): - in_grad.set_shape(t_in.get_shape()) - _SetGrad(grads, t_in, in_grad) - if loop_state: - loop_state.ExitGradWhileContext(op, before=False) - - # Update pending count for the inputs of op and enqueue ready ops. - _UpdatePendingAndEnqueueReady(grads, op, queue, pending_count, loop_state) - - if loop_state: - loop_state.PostProcessing() - return [_GetGrad(grads, x) for x in xs] - - -def _HasAnyNotNoneGrads(grads, op): - """Return true iff op has real gradient.""" - out_grads = _GetGrads(grads, op) - for out_grad in out_grads: - if isinstance(out_grad, (ops.Tensor, ops.IndexedSlices)): - return True - if out_grad and isinstance(out_grad, collections.Sequence): - if any([g is not None for g in out_grad]): - return True - return False - - -def _UpdatePendingAndEnqueueReady(grads, op, queue, pending_count, loop_state): - """Update pending count for the inputs of op and enqueue ready ops.""" - for x in op.inputs: - # pylint: disable=protected-access - pending_count[x.op._id] -= 1 - ready = (pending_count[x.op._id] == 0) - if loop_state and not ready: - ready = (pending_count[x.op._id] > 0 and - control_flow_ops.IsLoopSwitch(x.op)) - # pylint: enable=protected-access - if ready: - if control_flow_ops.IsLoopExit(x.op): - # if x is an exit without real gradient, defer processing them. - grad_state = loop_state.GetGradState(x.op, before=False) - grad_state.deferred_exits.append(x) - grad_state.pending_exits_count -= 1 - if grad_state.pending_exits_count == 0: - # We now have all the exits so process them. - has_real_grad = False - for y in grad_state.deferred_exits: - if _HasAnyNotNoneGrads(grads, y.op): - has_real_grad = True - queue.append(y.op) - else: - grad_state.unused_exits.append(y) - if has_real_grad: - # For an unused exit, if it has floating-point outputs, backprop - # a zero gradient. Otherwise, just ignore it. - for y in grad_state.unused_exits: - if _IsTrainable(y): - _SetGrad(grads, y, loop_state.ZerosLikeForExit(y)) - queue.append(y.op) - else: - # All exits are "unused" so use None as gradient. - for y in grad_state.unused_exits: - queue.append(y.op) - else: - queue.append(x.op) - - -def _SetGrad(grads, t, grad): - """Sets gradient "grad" in "grads" for tensor "t".""" - op = t.op - op_grads = grads.get(op) - if not op_grads: - op_grads = [[] for _ in xrange(len(op.outputs))] - grads[op] = op_grads - t_grads = op_grads[t.value_index] - if isinstance(t_grads, list): - t_grads.append(grad) - else: - assert control_flow_ops.IsLoopSwitch(op) - op_grads[t.value_index] = grad - - -def _GetGrad(grads, t): - """Gets gradient for tensor "t".""" - op = t.op - op_grads = grads.get(op) - if not op_grads: - return None - t_grad = op_grads[t.value_index] - assert not isinstance(t_grad, list), ( - "gradients list should have been aggregated by now.") - return t_grad - - -def _GetGrads(grads, op): - """Gets all gradients for op.""" - if op in grads: - return grads[op] - else: - return [[] for _ in xrange(len(op.outputs))] - - -def _HandleNestedIndexedSlices(grad): - assert isinstance(grad, ops.IndexedSlices) - if isinstance(grad.values, ops.Tensor): - return grad - else: - assert isinstance(grad.values, ops.IndexedSlices) - g = _HandleNestedIndexedSlices(grad.values) - return ops.IndexedSlices(g.values, - array_ops.gather(grad.indices, g.indices), - g.dense_shape) - - -def _AccumulatorShape(inputs): - shape = tensor_shape.unknown_shape() - for i in inputs: - if isinstance(i, ops.Tensor): - shape = shape.merge_with(i.get_shape()) - return shape - - -def _LogOpGradients(op, out_grads, in_grads): - """Log the in and out grads of an op.""" - logging.vlog(1, "Gradient for '" + op.name + "'") - - def _FilterGrad(x): - if x is None: - return False - if isinstance(x, (list, tuple)): - return bool(x) - else: - return True - - logging.vlog(1, " in --> %s", - ", ".join([x.name for x in out_grads if _FilterGrad(x)])) - logging.vlog(1, " out --> %s", - ", ".join([x.name for x in in_grads if _FilterGrad(x)])) - - -def _MultiDeviceAddN(tensor_list): - """Adds tensors from potentially multiple devices.""" - # Basic function structure comes from control_flow_ops.group(). - # Sort tensors according to their devices. - tensors_on_device = collections.defaultdict(lambda: []) - for tensor in tensor_list: - tensors_on_device[tensor.device].append(tensor) - - # For each device, add the tensors on that device first. - # Then gather the partial sums from multiple devices. - # TODO(sjhwang): Create hierarchical aggregation tree as pbar's suggestion. - # E.g., aggregate per GPU, then per task, and so on. - summands = [] - - def DeviceKey(dev): - return "" if dev is None else dev - - for dev in sorted(six.iterkeys(tensors_on_device), key=DeviceKey): - tensors = tensors_on_device[dev] - with ops.colocate_with(tensors[0].op, ignore_existing=True): - summands.append(math_ops.add_n(tensors)) - - return math_ops.add_n(summands) - - -class AggregationMethod(object): - """A class listing aggregation methods used to combine gradients. - - Computing partial derivatives can require aggregating gradient - contributions. This class lists the various methods that can - be used to combine gradients in the graph: - - * `ADD_N`: All of the gradient terms are summed as part of one - operation using the "AddN" op. It has the property that all - gradients must be ready before any aggregation is performed. - * `DEFAULT`: The system-chosen default aggregation method. - """ - ADD_N = 0 - DEFAULT = ADD_N - # The following are experimental and may not be supported in future releases. - EXPERIMENTAL_TREE = 1 - EXPERIMENTAL_ACCUMULATE_N = 2 - - -def _AggregatedGrads(grads, op, loop_state, aggregation_method=None): - """Get the aggregated gradients for op. - - Args: - grads: The map of memoized gradients. - op: The op to get gradients for. - loop_state: An object for maintaining the state of the while loops in the - graph. It is of type ControlFlowState. None if the graph - contains no while loops. - aggregation_method: Specifies the method used to combine gradient terms. - Accepted values are constants defined in the class `AggregationMethod`. - - Returns: - A list of gradients, one per each output of `op`. If the gradients - for a particular output is a list, this function aggregates it - before returning. - - Raises: - TypeError: if the incoming grads are not Tensors or IndexedSlices. - ValueError: if the arguments are invalid. - - """ - if aggregation_method is None: - aggregation_method = AggregationMethod.DEFAULT - if aggregation_method not in [ - AggregationMethod.ADD_N, AggregationMethod.EXPERIMENTAL_TREE, - AggregationMethod.EXPERIMENTAL_ACCUMULATE_N - ]: - raise ValueError("Invalid aggregation_method specified %s." % - aggregation_method) - out_grads = _GetGrads(grads, op) - for i, out_grad in enumerate(out_grads): - if loop_state: - if isinstance(out_grad, (ops.Tensor, ops.IndexedSlices)): - assert control_flow_ops.IsLoopSwitch(op) - continue - # Grads have to be Tensors or IndexedSlices - if (isinstance(out_grad, collections.Sequence) and not all([ - isinstance(g, (ops.Tensor, ops.IndexedSlices)) for g in out_grad - if g is not None - ])): - raise TypeError("gradients have to be either all Tensors " - "or all IndexedSlices") - # Aggregate multiple gradients, and convert [] to None. - if out_grad: - if len(out_grad) < 2: - used = "nop" - out_grads[i] = out_grad[0] - elif all([isinstance(g, ops.Tensor) for g in out_grad if g is not None]): - tensor_shape = _AccumulatorShape(out_grad) - if (aggregation_method == AggregationMethod.EXPERIMENTAL_ACCUMULATE_N - and len(out_grad) > 2 and tensor_shape.is_fully_defined()): - # The benefit of using AccumulateN is that its inputs can be combined - # in any order and this can allow the expression to be evaluated with - # a smaller memory footprint. When used with gpu_allocator_retry, - # it is possible to compute a sum of terms which are much larger than - # total GPU memory. - # AccumulateN can currently only be used if we know the shape for - # an accumulator variable. If this is not known, or if we only have - # 2 grads then we fall through to the "tree" case below. - used = "accumulate_n" - out_grads[i] = math_ops.accumulate_n(out_grad) - elif aggregation_method in [ - AggregationMethod.EXPERIMENTAL_TREE, - AggregationMethod.EXPERIMENTAL_ACCUMULATE_N - ]: - # Aggregate all gradients by doing pairwise sums: this may - # reduce performance, but it can improve memory because the - # gradients can be released earlier. - # - # TODO(vrv): Consider replacing this with a version of - # tf.AddN() that eagerly frees its inputs as soon as they are - # ready, so the order of this tree does not become a problem. - used = "tree" - with ops.name_scope(op.name + "_gradient_sum"): - running_sum = out_grad[0] - for grad in out_grad[1:]: - running_sum = math_ops.add_n([running_sum, grad]) - out_grads[i] = running_sum - else: - used = "add_n" - out_grads[i] = _MultiDeviceAddN(out_grad) - logging.vlog(2, " _AggregatedGrads %d x %s using %s", - len(out_grad), tensor_shape, used) - else: - out_grad = math_ops._as_indexed_slices_list( - [g for g in out_grad if g is not None]) - out_grad = [_HandleNestedIndexedSlices(x) for x in out_grad] - # Form IndexedSlices out of the concatenated values and - # indices. - out_grads[i] = ops.IndexedSlices( - array_ops.concat(0, [x.values for x in out_grad]), - array_ops.concat(0, [x.indices for x in out_grad]), - out_grad[0].dense_shape) - else: - out_grads[i] = [] - return out_grads - - -# TODO(vrv): Make this available when we want to make it public. -def _hessian_vector_product(ys, xs, v): - """Multiply the Hessian of `ys` wrt `xs` by `v`. - - This is an efficient construction that uses a backprop-like approach - to compute the product between the Hessian and another vector. The - Hessian is usually too large to be explicitly computed or even - represented, but this method allows us to at least multiply by it - for the same big-O cost as backprop. - - Implicit Hessian-vector products are the main practical, scalable way - of using second derivatives with neural networks. They allow us to - do things like construct Krylov subspaces and approximate conjugate - gradient descent. - - Example: if `y` = 1/2 `x`^T A `x`, then `hessian_vector_product(y, - x, v)` will return an expression that evaluates to the same values - as (A + A.T) `v`. - - Args: - ys: A scalar value, or a tensor or list of tensors to be summed to - yield a scalar. - xs: A list of tensors that we should construct the Hessian over. - v: A list of tensors, with the same shapes as xs, that we want to - multiply by the Hessian. - - Returns: - A list of tensors (or if the list would be length 1, a single tensor) - containing the product between the Hessian and `v`. - - Raises: - ValueError: `xs` and `v` have different length. - - """ - - # Validate the input - length = len(xs) - if len(v) != length: - raise ValueError("xs and v must have the same length.") - - # First backprop - grads = gradients(ys, xs) - - assert len(grads) == length - elemwise_products = [ - math_ops.mul(grad_elem, array_ops.stop_gradient(v_elem)) - for grad_elem, v_elem in zip(grads, v) if grad_elem is not None - ] - - # Second backprop - return gradients(elemwise_products, xs) +# pylint: disable=unused-import +from tensorflow.python.ops.gradients_impl import AggregationMethod +from tensorflow.python.ops.gradients_impl import gradients +# pylint: enable=unused-import +from tensorflow.python.util.all_util import remove_undocumented + +_allowed_symbols = [ + # TODO(drpng): find a good place to reference this. + "AggregationMethod", + "gradients", # tf.gradients.gradients. +] +remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/python/ops/gradients_impl.py b/tensorflow/python/ops/gradients_impl.py new file mode 100644 index 00000000000..ee98bfb1f42 --- /dev/null +++ b/tensorflow/python/ops/gradients_impl.py @@ -0,0 +1,831 @@ +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Implements the graph generation for computation of gradients.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import contextlib +import warnings + +import numpy as np +import six +from six.moves import xrange # pylint: disable=redefined-builtin + +from tensorflow.core.framework import attr_value_pb2 +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import tensor_util +from tensorflow.python.ops import array_grad # pylint: disable=unused-import +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import control_flow_grad # pylint: disable=unused-import +from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import image_grad # pylint: disable=unused-import +from tensorflow.python.ops import logging_ops # pylint: disable=unused-import +from tensorflow.python.ops import linalg_grad # pylint: disable=unused-import +from tensorflow.python.ops import math_grad # pylint: disable=unused-import +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import linalg_ops +from tensorflow.python.ops import functional_ops +from tensorflow.python.platform import tf_logging as logging + + +# Warn the user if we convert a sparse representation to dense with at +# least this number of elements. +_LARGE_SPARSE_NUM_ELEMENTS = 100000000 + + +def _IndexedSlicesToTensor(value, dtype=None, name=None, as_ref=False): + """Converts an IndexedSlices object `value` to a Tensor. + + NOTE(mrry): This function is potentially expensive. + + Args: + value: An ops.IndexedSlices object. + dtype: The dtype of the Tensor to be returned. + name: Optional name to use for the returned Tensor. + as_ref: True if a ref is requested. + + Returns: + A dense Tensor representing the values in the given IndexedSlices. + + Raises: + ValueError: If the IndexedSlices does not have the same dtype. + """ + _ = as_ref + if dtype and not dtype.is_compatible_with(value.dtype): + raise ValueError( + "Tensor conversion requested dtype %s for IndexedSlices with dtype %s" % + (dtype.name, value.dtype.name)) + if value.dense_shape is None: + raise ValueError( + "Tensor conversion requested for IndexedSlices without dense_shape: %s" + % str(value)) + # TODO(mrry): Consider adding static shape information to + # IndexedSlices, to avoid using numpy here. + dense_shape_value = tensor_util.constant_value(value.dense_shape) + if dense_shape_value is not None: + num_elements = np.prod(dense_shape_value) + if num_elements >= _LARGE_SPARSE_NUM_ELEMENTS: + warnings.warn( + "Converting sparse IndexedSlices to a dense Tensor with %d elements. " + "This may consume a large amount of memory." % num_elements) + else: + warnings.warn( + "Converting sparse IndexedSlices to a dense Tensor of unknown shape. " + "This may consume a large amount of memory.") + return math_ops.unsorted_segment_sum( + value.values, value.indices, value.dense_shape[0], name=name) + + +ops.register_tensor_conversion_function(ops.IndexedSlices, + _IndexedSlicesToTensor) + + +def _MarkReachedOps(from_ops, reached_ops): + """Mark all ops reached from "from_ops". + + Args: + from_ops: list of Operations. + reached_ops: list of booleans, indexed by operation id. + """ + queue = collections.deque() + queue.extend(from_ops) + while queue: + op = queue.popleft() + if not reached_ops[op._id]: + reached_ops[op._id] = True + for output in op.outputs: + queue.extend(output.consumers()) + + +def _GatherInputs(to_ops, reached_ops): + """List all inputs of to_ops that are in reached_ops. + + Args: + to_ops: list of Operations. + reached_ops: list of booleans, indexed by operation id. + + Returns: + The list of all inputs of to_ops that are in reached_ops. + That list includes all elements of to_ops. + """ + inputs = [] + queue = collections.deque() + queue.extend(to_ops) + while queue: + op = queue.popleft() + # We are interested in this op. + if reached_ops[op._id]: + inputs.append(op) + # Clear the boolean so we won't add the inputs again. + reached_ops[op._id] = False + for inp in op.inputs: + queue.append(inp.op) + return inputs + + +def _PendingCount(graph, to_ops, from_ops, colocate_gradients_with_ops): + """Initialize the pending count for ops between two lists of Operations. + + 'pending_count[op._id]' indicates the number of backprop inputs + to this operation. + + Args: + graph: a Graph. + to_ops: list of Operations. + from_ops: list of Operations. + colocate_gradients_with_ops: Python bool. See docstring of gradients(). + + Returns: + A tuple containing: (1) a list of integers indexed by operation id, + indicating the number of backprop inputs to this operation, and (2) + a ControlFlowState object which is not None if the ops between from_ops + and to_ops contain control flow loops. + """ + # Mark reachable ops from from_ops. + reached_ops = [False] * (graph._last_id + 1) + for op in to_ops: + reached_ops[op._id] = True + _MarkReachedOps(from_ops, reached_ops) + + # Mark between ops. + between_ops = [False] * (graph._last_id + 1) + between_op_list = [] + queue = collections.deque() + queue.extend(to_ops) + while queue: + op = queue.popleft() + # We are interested in this op. + if reached_ops[op._id]: + between_ops[op._id] = True + between_op_list.append(op) + # Clear the boolean so we won't add the inputs again. + reached_ops[op._id] = False + for inp in op.inputs: + queue.append(inp.op) + + # 'loop_state' is None if there are no while loops. + loop_state = control_flow_ops.MaybeCreateControlFlowState( + between_op_list, between_ops, colocate_gradients_with_ops) + + # Initialize pending count for between ops. + pending_count = [0] * (graph._last_id + 1) + for op in between_op_list: + for x in op.inputs: + if between_ops[x.op._id]: + pending_count[x.op._id] += 1 + + return pending_count, loop_state + + +def _AsList(x): + return x if isinstance(x, (list, tuple)) else [x] + + +def _DefaultGradYs(grad_ys, ys, colocate_gradients_with_ops): + """Fill in default values for grad_ys. + + Args: + grad_ys: List of gradients, can contain None. + ys: List of tensors. + colocate_gradients_with_ops: If True, try colocating gradients with + the corresponding op. + + Returns: + A list of gradients to use, without None. + + Raises: + ValueError: If one of the grad_ys is invalid. + """ + if len(grad_ys) != len(ys): + raise ValueError("Passed %d grad_ys for %d ys" % (len(grad_ys), len(ys))) + grad_ys = ops.convert_n_to_tensor_or_indexed_slices(grad_ys, name="grad_y") + for i in xrange(len(grad_ys)): + grad_y = grad_ys[i] + y = ys[i] + if grad_y is None: + with _maybe_colocate_with(y.op, colocate_gradients_with_ops): + grad_ys[i] = array_ops.fill( + array_ops.shape(y), constant_op.constant( + 1, dtype=y.dtype)) + else: + if grad_y.dtype != y.dtype: + raise ValueError("Y and ys_grad must be of the same type, " + "not y: %s, ys_grad: %s " % + (dtypes.as_dtype(y.dtype).name, + dtypes.as_dtype(grad_y.dtype).name)) + return grad_ys + + +def _IsTrainable(tensor): + dtype = dtypes.as_dtype(tensor.dtype) + return dtype.base_dtype in (dtypes.float16, dtypes.float32, dtypes.float64, + dtypes.complex64, dtypes.complex128) + + +def _VerifyGeneratedGradients(grads, op): + """Verify that gradients are valid in number and type. + + Args: + grads: List of generated gradients. + op: Operation for which the gradients where generated. + + Raises: + ValueError: if the gradients are invalid. + """ + if len(grads) != len(op.inputs): + raise ValueError("Num gradients %d generated for op %s do not match num " + "inputs %d" % (len(grads), op.node_def, len(op.inputs))) + for i in xrange(len(grads)): + grad = grads[i] + inp = op.inputs[i] + if grad is not None: + if not grad.dtype.is_compatible_with(inp.dtype): + raise ValueError("Gradient type %s generated for op %s does " + "not match input type %s" % + (dtypes.as_dtype(grad.dtype).name, op.node_def, + dtypes.as_dtype(inp.dtype).name)) + + +def _StopOps(from_ops, pending_count): + """The set of ops that terminate the gradient computation. + + This computes the frontier of the forward graph *before* which backprop + should stop. Operations in the returned set will not be differentiated. + This set is defined as the subset of `from_ops` containing ops that have + no predecessor in `from_ops`. `pending_count` is the result of + `_PendingCount(g, xs, from_ops)`. An 'op' has predecessors in `from_ops` + iff pending_count[op._id] > 0. + + Args: + from_ops: list of Operations. + pending_count: List of integers, indexed by operation id. + + Returns: + The set of operations. + """ + stop_ops = set() + for op in from_ops: + is_stop_op = True + for inp in op.inputs: + if pending_count[inp.op._id] > 0: + is_stop_op = False + break + if is_stop_op: + stop_ops.add(op._id) + return stop_ops + + +@contextlib.contextmanager +def _maybe_colocate_with(op, colocate_gradients_with_ops): + """Context to colocate with `op` if `colocate_gradients_with_ops`.""" + if colocate_gradients_with_ops: + with ops.colocate_with(op): + yield + else: + yield + + +def _SymGrad(op, out_grads): + """Backprop through a function call node op given its outputs' gradients.""" + f_in = [x for x in op.inputs] + out_grads + f_types = [x.dtype for x in op.inputs] + f = attr_value_pb2.NameAttrList() + f.name = op.type + for k in op.node_def.attr: + f.attr[k].CopyFrom(op.node_def.attr[k]) + # pylint: disable=protected-access + in_grads = functional_ops._symbolic_gradient(input=f_in, Tout=f_types, f=f) + # pylint: enable=protected-access + return in_grads + + +def gradients(ys, + xs, + grad_ys=None, + name="gradients", + colocate_gradients_with_ops=False, + gate_gradients=False, + aggregation_method=None): + """Constructs symbolic partial derivatives of sum of `ys` w.r.t. x in `xs`. + + `ys` and `xs` are each a `Tensor` or a list of tensors. `grad_ys` + is a list of `Tensor`, holding the gradients received by the + `ys`. The list must be the same length as `ys`. + + `gradients()` adds ops to the graph to output the partial + derivatives of `ys` with respect to `xs`. It returns a list of + `Tensor` of length `len(xs)` where each tensor is the `sum(dy/dx)` + for y in `ys`. + + `grad_ys` is a list of tensors of the same length as `ys` that holds + the initial gradients for each y in `ys`. When `grad_ys` is None, + we fill in a tensor of '1's of the shape of y for each y in `ys`. A + user can provide their own initial `grad_ys` to compute the + derivatives using a different initial gradient for each y (e.g., if + one wanted to weight the gradient differently for each value in + each y). + + Args: + ys: A `Tensor` or list of tensors to be differentiated. + xs: A `Tensor` or list of tensors to be used for differentiation. + grad_ys: Optional. A `Tensor` or list of tensors the same size as + `ys` and holding the gradients computed for each y in `ys`. + name: Optional name to use for grouping all the gradient ops together. + defaults to 'gradients'. + colocate_gradients_with_ops: If True, try colocating gradients with + the corresponding op. + gate_gradients: If True, add a tuple around the gradients returned + for an operations. This avoids some race conditions. + aggregation_method: Specifies the method used to combine gradient terms. + Accepted values are constants defined in the class `AggregationMethod`. + + Returns: + A list of `sum(dy/dx)` for each x in `xs`. + + Raises: + LookupError: if one of the operations between `x` and `y` does not + have a registered gradient function. + ValueError: if the arguments are invalid. + + """ + ys = _AsList(ys) + xs = _AsList(xs) + if grad_ys is None: + grad_ys = [None] * len(ys) + else: + grad_ys = _AsList(grad_ys) + + with ops.name_scope(name, "gradients", ys + xs + grad_ys): + ys = ops.convert_n_to_tensor_or_indexed_slices(ys, name="y") + xs = ops.convert_n_to_tensor_or_indexed_slices(xs, name="x") + grad_ys = _DefaultGradYs(grad_ys, ys, colocate_gradients_with_ops) + + # The approach we take here is as follows: Create a list of all ops in the + # subgraph between the ys and xs. Visit these ops in reverse order of ids + # to ensure that when we visit an op the gradients w.r.t its outputs have + # been collected. Then aggregate these gradients if needed, call the op's + # gradient function, and add the generated gradients to the gradients for + # its input. + + # Initialize the pending count for ops in the connected subgraph from ys + # to the xs. + to_ops = [t.op for t in ys] + from_ops = [t.op for t in xs] + pending_count, loop_state = _PendingCount(ops.get_default_graph(), to_ops, + from_ops, + colocate_gradients_with_ops) + + # Iterate over the collected ops. + # + # grads: op => list of gradients received on each output endpoint of the + # op. The gradients for each endpoint are initially collected as a list. + # When it is time to call the op's gradient function, for each endpoint we + # aggregate the list of received gradients into a Add() Operation if there + # is more than one. + grads = {} + + # Add the initial gradients for the ys. + for y, grad_y in zip(ys, grad_ys): + _SetGrad(grads, y, grad_y) + + # Initialize queue with to_ops. + queue = collections.deque() + # Add the ops in 'to_ops' into the queue. + to_ops_set = set() + for op in to_ops: + # 'ready' handles the case where one output gradient relies on + # another output's gradient. + # pylint: disable=protected-access + ready = (pending_count[op._id] == 0) + if ready and op._id not in to_ops_set: + to_ops_set.add(op._id) + queue.append(op) + # pylint: enable=protected-access + + if loop_state: + loop_exits = loop_state.ProcessUnusedLoopExits(pending_count, to_ops_set) + for y in loop_exits: + if _IsTrainable(y): + _SetGrad(grads, y, loop_state.ZerosLikeForExit(y)) + queue.append(y.op) + + # The set of 'from_ops'. + stop_ops = _StopOps(from_ops, pending_count) + while queue: + # generate gradient subgraph for op. + op = queue.popleft() + with _maybe_colocate_with(op, colocate_gradients_with_ops): + if loop_state: + loop_state.EnterGradWhileContext(op, before=True) + out_grads = _AggregatedGrads(grads, op, loop_state, aggregation_method) + if loop_state: + loop_state.ExitGradWhileContext(op, before=True) + + grad_fn = None + # pylint: disable=protected-access + is_func_call = ops.get_default_graph()._is_function(op.type) + has_out_grads = any(isinstance(g, ops.Tensor) or g for g in out_grads) + if has_out_grads and (op._id not in stop_ops): + if is_func_call: + grad_fn = ops.get_default_graph()._get_function( + op.type).python_grad_func + # pylint: enable=protected-access + else: + # A grad_fn must be defined, either as a function or as None + # for ops that do not have gradients. + try: + grad_fn = ops.get_gradient_function(op) + except LookupError: + raise LookupError( + "No gradient defined for operation '%s' (op type: %s)" % + (op.name, op.type)) + if loop_state: + loop_state.EnterGradWhileContext(op, before=False) + if (grad_fn or is_func_call) and has_out_grads: + # NOTE: If _AggregatedGrads didn't compute a value for the i'th + # output, it means that the cost does not depend on output[i], + # therefore dC/doutput[i] is 0. + for i, out_grad in enumerate(out_grads): + if (not isinstance(out_grad, ops.Tensor) and + not out_grad) and _IsTrainable(op.outputs[i]): + # Only floating-point outputs get a zero gradient. Gradient + # functions should ignore the gradient for other outputs. + if loop_state: + out_grads[i] = loop_state.ZerosLike(op, i) + else: + out_grads[i] = control_flow_ops.ZerosLikeOutsideLoop(op, i) + with ops.name_scope(op.name + "_grad"): + # pylint: disable=protected-access + with ops.get_default_graph()._original_op(op): + # pylint: enable=protected-access + if grad_fn: + # If grad_fn was found, do not use SymbolicGradient even for + # functions. + in_grads = grad_fn(op, *out_grads) + else: + # For function call ops, we add a 'SymbolicGradient' + # node to the graph to compute gradients. + in_grads = _SymGrad(op, out_grads) + in_grads = _AsList(in_grads) + _VerifyGeneratedGradients(in_grads, op) + if gate_gradients and len( + [x for x in in_grads if x is not None]) > 1: + in_grads = control_flow_ops.tuple(in_grads) + _LogOpGradients(op, out_grads, in_grads) + else: + # If no grad_fn is defined or none of out_grads is available, + # just propagates a list of None backwards. + in_grads = [None] * len(op.inputs) + for t_in, in_grad in zip(op.inputs, in_grads): + if in_grad is not None: + if isinstance(in_grad, ops.Tensor): + in_grad.set_shape(t_in.get_shape()) + _SetGrad(grads, t_in, in_grad) + if loop_state: + loop_state.ExitGradWhileContext(op, before=False) + + # Update pending count for the inputs of op and enqueue ready ops. + _UpdatePendingAndEnqueueReady(grads, op, queue, pending_count, loop_state) + + if loop_state: + loop_state.PostProcessing() + return [_GetGrad(grads, x) for x in xs] + + +def _HasAnyNotNoneGrads(grads, op): + """Return true iff op has real gradient.""" + out_grads = _GetGrads(grads, op) + for out_grad in out_grads: + if isinstance(out_grad, (ops.Tensor, ops.IndexedSlices)): + return True + if out_grad and isinstance(out_grad, collections.Sequence): + if any([g is not None for g in out_grad]): + return True + return False + + +def _UpdatePendingAndEnqueueReady(grads, op, queue, pending_count, loop_state): + """Update pending count for the inputs of op and enqueue ready ops.""" + for x in op.inputs: + # pylint: disable=protected-access + pending_count[x.op._id] -= 1 + ready = (pending_count[x.op._id] == 0) + if loop_state and not ready: + ready = (pending_count[x.op._id] > 0 and + control_flow_ops.IsLoopSwitch(x.op)) + # pylint: enable=protected-access + if ready: + if control_flow_ops.IsLoopExit(x.op): + # if x is an exit without real gradient, defer processing them. + grad_state = loop_state.GetGradState(x.op, before=False) + grad_state.deferred_exits.append(x) + grad_state.pending_exits_count -= 1 + if grad_state.pending_exits_count == 0: + # We now have all the exits so process them. + has_real_grad = False + for y in grad_state.deferred_exits: + if _HasAnyNotNoneGrads(grads, y.op): + has_real_grad = True + queue.append(y.op) + else: + grad_state.unused_exits.append(y) + if has_real_grad: + # For an unused exit, if it has floating-point outputs, backprop + # a zero gradient. Otherwise, just ignore it. + for y in grad_state.unused_exits: + if _IsTrainable(y): + _SetGrad(grads, y, loop_state.ZerosLikeForExit(y)) + queue.append(y.op) + else: + # All exits are "unused" so use None as gradient. + for y in grad_state.unused_exits: + queue.append(y.op) + else: + queue.append(x.op) + + +def _SetGrad(grads, t, grad): + """Sets gradient "grad" in "grads" for tensor "t".""" + op = t.op + op_grads = grads.get(op) + if not op_grads: + op_grads = [[] for _ in xrange(len(op.outputs))] + grads[op] = op_grads + t_grads = op_grads[t.value_index] + if isinstance(t_grads, list): + t_grads.append(grad) + else: + assert control_flow_ops.IsLoopSwitch(op) + op_grads[t.value_index] = grad + + +def _GetGrad(grads, t): + """Gets gradient for tensor "t".""" + op = t.op + op_grads = grads.get(op) + if not op_grads: + return None + t_grad = op_grads[t.value_index] + assert not isinstance(t_grad, list), ( + "gradients list should have been aggregated by now.") + return t_grad + + +def _GetGrads(grads, op): + """Gets all gradients for op.""" + if op in grads: + return grads[op] + else: + return [[] for _ in xrange(len(op.outputs))] + + +def _HandleNestedIndexedSlices(grad): + assert isinstance(grad, ops.IndexedSlices) + if isinstance(grad.values, ops.Tensor): + return grad + else: + assert isinstance(grad.values, ops.IndexedSlices) + g = _HandleNestedIndexedSlices(grad.values) + return ops.IndexedSlices(g.values, + array_ops.gather(grad.indices, g.indices), + g.dense_shape) + + +def _AccumulatorShape(inputs): + shape = tensor_shape.unknown_shape() + for i in inputs: + if isinstance(i, ops.Tensor): + shape = shape.merge_with(i.get_shape()) + return shape + + +def _LogOpGradients(op, out_grads, in_grads): + """Log the in and out grads of an op.""" + logging.vlog(1, "Gradient for '" + op.name + "'") + + def _FilterGrad(x): + if x is None: + return False + if isinstance(x, (list, tuple)): + return bool(x) + else: + return True + + logging.vlog(1, " in --> %s", + ", ".join([x.name for x in out_grads if _FilterGrad(x)])) + logging.vlog(1, " out --> %s", + ", ".join([x.name for x in in_grads if _FilterGrad(x)])) + + +def _MultiDeviceAddN(tensor_list): + """Adds tensors from potentially multiple devices.""" + # Basic function structure comes from control_flow_ops.group(). + # Sort tensors according to their devices. + tensors_on_device = collections.defaultdict(lambda: []) + for tensor in tensor_list: + tensors_on_device[tensor.device].append(tensor) + + # For each device, add the tensors on that device first. + # Then gather the partial sums from multiple devices. + # TODO(sjhwang): Create hierarchical aggregation tree as pbar's suggestion. + # E.g., aggregate per GPU, then per task, and so on. + summands = [] + + def DeviceKey(dev): + return "" if dev is None else dev + + for dev in sorted(six.iterkeys(tensors_on_device), key=DeviceKey): + tensors = tensors_on_device[dev] + with ops.colocate_with(tensors[0].op, ignore_existing=True): + summands.append(math_ops.add_n(tensors)) + + return math_ops.add_n(summands) + + +class AggregationMethod(object): + """A class listing aggregation methods used to combine gradients. + + Computing partial derivatives can require aggregating gradient + contributions. This class lists the various methods that can + be used to combine gradients in the graph: + + * `ADD_N`: All of the gradient terms are summed as part of one + operation using the "AddN" op. It has the property that all + gradients must be ready before any aggregation is performed. + * `DEFAULT`: The system-chosen default aggregation method. + """ + ADD_N = 0 + DEFAULT = ADD_N + # The following are experimental and may not be supported in future releases. + EXPERIMENTAL_TREE = 1 + EXPERIMENTAL_ACCUMULATE_N = 2 + + +def _AggregatedGrads(grads, op, loop_state, aggregation_method=None): + """Get the aggregated gradients for op. + + Args: + grads: The map of memoized gradients. + op: The op to get gradients for. + loop_state: An object for maintaining the state of the while loops in the + graph. It is of type ControlFlowState. None if the graph + contains no while loops. + aggregation_method: Specifies the method used to combine gradient terms. + Accepted values are constants defined in the class `AggregationMethod`. + + Returns: + A list of gradients, one per each output of `op`. If the gradients + for a particular output is a list, this function aggregates it + before returning. + + Raises: + TypeError: if the incoming grads are not Tensors or IndexedSlices. + ValueError: if the arguments are invalid. + + """ + if aggregation_method is None: + aggregation_method = AggregationMethod.DEFAULT + if aggregation_method not in [ + AggregationMethod.ADD_N, AggregationMethod.EXPERIMENTAL_TREE, + AggregationMethod.EXPERIMENTAL_ACCUMULATE_N + ]: + raise ValueError("Invalid aggregation_method specified %s." % + aggregation_method) + out_grads = _GetGrads(grads, op) + for i, out_grad in enumerate(out_grads): + if loop_state: + if isinstance(out_grad, (ops.Tensor, ops.IndexedSlices)): + assert control_flow_ops.IsLoopSwitch(op) + continue + # Grads have to be Tensors or IndexedSlices + if (isinstance(out_grad, collections.Sequence) and not all([ + isinstance(g, (ops.Tensor, ops.IndexedSlices)) for g in out_grad + if g is not None + ])): + raise TypeError("gradients have to be either all Tensors " + "or all IndexedSlices") + # Aggregate multiple gradients, and convert [] to None. + if out_grad: + if len(out_grad) < 2: + used = "nop" + out_grads[i] = out_grad[0] + elif all([isinstance(g, ops.Tensor) for g in out_grad if g is not None]): + tensor_shape = _AccumulatorShape(out_grad) + if (aggregation_method == AggregationMethod.EXPERIMENTAL_ACCUMULATE_N + and len(out_grad) > 2 and tensor_shape.is_fully_defined()): + # The benefit of using AccumulateN is that its inputs can be combined + # in any order and this can allow the expression to be evaluated with + # a smaller memory footprint. When used with gpu_allocator_retry, + # it is possible to compute a sum of terms which are much larger than + # total GPU memory. + # AccumulateN can currently only be used if we know the shape for + # an accumulator variable. If this is not known, or if we only have + # 2 grads then we fall through to the "tree" case below. + used = "accumulate_n" + out_grads[i] = math_ops.accumulate_n(out_grad) + elif aggregation_method in [ + AggregationMethod.EXPERIMENTAL_TREE, + AggregationMethod.EXPERIMENTAL_ACCUMULATE_N + ]: + # Aggregate all gradients by doing pairwise sums: this may + # reduce performance, but it can improve memory because the + # gradients can be released earlier. + # + # TODO(vrv): Consider replacing this with a version of + # tf.AddN() that eagerly frees its inputs as soon as they are + # ready, so the order of this tree does not become a problem. + used = "tree" + with ops.name_scope(op.name + "_gradient_sum"): + running_sum = out_grad[0] + for grad in out_grad[1:]: + running_sum = math_ops.add_n([running_sum, grad]) + out_grads[i] = running_sum + else: + used = "add_n" + out_grads[i] = _MultiDeviceAddN(out_grad) + logging.vlog(2, " _AggregatedGrads %d x %s using %s", + len(out_grad), tensor_shape, used) + else: + out_grad = math_ops._as_indexed_slices_list( + [g for g in out_grad if g is not None]) + out_grad = [_HandleNestedIndexedSlices(x) for x in out_grad] + # Form IndexedSlices out of the concatenated values and + # indices. + out_grads[i] = ops.IndexedSlices( + array_ops.concat(0, [x.values for x in out_grad]), + array_ops.concat(0, [x.indices for x in out_grad]), + out_grad[0].dense_shape) + else: + out_grads[i] = [] + return out_grads + + +# TODO(vrv): Make this available when we want to make it public. +def _hessian_vector_product(ys, xs, v): + """Multiply the Hessian of `ys` wrt `xs` by `v`. + + This is an efficient construction that uses a backprop-like approach + to compute the product between the Hessian and another vector. The + Hessian is usually too large to be explicitly computed or even + represented, but this method allows us to at least multiply by it + for the same big-O cost as backprop. + + Implicit Hessian-vector products are the main practical, scalable way + of using second derivatives with neural networks. They allow us to + do things like construct Krylov subspaces and approximate conjugate + gradient descent. + + Example: if `y` = 1/2 `x`^T A `x`, then `hessian_vector_product(y, + x, v)` will return an expression that evaluates to the same values + as (A + A.T) `v`. + + Args: + ys: A scalar value, or a tensor or list of tensors to be summed to + yield a scalar. + xs: A list of tensors that we should construct the Hessian over. + v: A list of tensors, with the same shapes as xs, that we want to + multiply by the Hessian. + + Returns: + A list of tensors (or if the list would be length 1, a single tensor) + containing the product between the Hessian and `v`. + + Raises: + ValueError: `xs` and `v` have different length. + + """ + + # Validate the input + length = len(xs) + if len(v) != length: + raise ValueError("xs and v must have the same length.") + + # First backprop + grads = gradients(ys, xs) + + assert len(grads) == length + elemwise_products = [ + math_ops.mul(grad_elem, array_ops.stop_gradient(v_elem)) + for grad_elem, v_elem in zip(grads, v) if grad_elem is not None + ] + + # Second backprop + return gradients(elemwise_products, xs) diff --git a/tensorflow/python/ops/gradients_test.py b/tensorflow/python/ops/gradients_test.py index 1c444de510c..b8627f5f49a 100644 --- a/tensorflow/python/ops/gradients_test.py +++ b/tensorflow/python/ops/gradients_test.py @@ -34,6 +34,7 @@ from tensorflow.python.ops import array_ops from tensorflow.python.ops import data_flow_grad # pylint: disable=unused-import from tensorflow.python.ops import data_flow_ops # pylint: disable=unused-import from tensorflow.python.ops import gradients +from tensorflow.python.ops import gradients_impl from tensorflow.python.ops import math_grad # pylint: disable=unused-import from tensorflow.python.ops import math_ops from tensorflow.python.ops import nn_grad # pylint: disable=unused-import @@ -66,8 +67,8 @@ def _OpsBetween(graph, to_ops, from_ops): # output ops as reached to avoid recursing past them. for op in to_ops: reached_ops[op._id] = True - gradients._MarkReachedOps(from_ops, reached_ops) - between_ops = gradients._GatherInputs(to_ops, reached_ops) + gradients_impl._MarkReachedOps(from_ops, reached_ops) + between_ops = gradients_impl._GatherInputs(to_ops, reached_ops) between_ops.sort(key=lambda x: -x._id) return between_ops @@ -414,7 +415,7 @@ class HessianVectorProductTest(test_util.TensorFlowTestCase): x = constant_op.constant(x_value) mat_x = math_ops.matmul(mat, x, name="Ax") x_mat_x = math_ops.matmul(array_ops.transpose(x), mat_x, name="xAx") - hess_v = gradients._hessian_vector_product(x_mat_x, [x], [v])[0] + hess_v = gradients_impl._hessian_vector_product(x_mat_x, [x], [v])[0] hess_v_actual = hess_v.eval() self.assertAllClose(hess_v_value, hess_v_actual) From 54ab19d23562c26e88a493e190deb1494d8550bf Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 3 Nov 2016 10:56:51 -0800 Subject: [PATCH 52/54] Remove duplicate PredictionKeys. Change: 138099098 --- .../learn/estimators/dnn_linear_combined.py | 41 ++++++++------ .../learn/python/learn/estimators/head.py | 8 --- .../learn/python/learn/estimators/linear.py | 55 +++++++++++-------- .../learn/python/learn/estimators/svm.py | 32 ++++++----- 4 files changed, 75 insertions(+), 61 deletions(-) diff --git a/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined.py b/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined.py index 57a98e419c6..3146a1e7c81 100644 --- a/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined.py +++ b/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined.py @@ -35,6 +35,7 @@ from tensorflow.contrib.learn.python.learn import trainable from tensorflow.contrib.learn.python.learn.estimators import composable_model from tensorflow.contrib.learn.python.learn.estimators import estimator from tensorflow.contrib.learn.python.learn.estimators import head as head_lib +from tensorflow.contrib.learn.python.learn.estimators import prediction_key from tensorflow.contrib.learn.python.learn.utils import export from tensorflow.python.framework import ops from tensorflow.python.ops import control_flow_ops @@ -747,13 +748,16 @@ class DNNLinearCombinedClassifier(evaluable.Evaluable, trainable.Trainable): Numpy array of predicted classes (or an iterable of predicted classes if as_iterable is True). """ - preds = self._estimator.predict(x=x, input_fn=input_fn, - batch_size=batch_size, - outputs=[head_lib.PredictionKey.CLASSES], - as_iterable=as_iterable) + key = prediction_key.PredictionKey.CLASSES + preds = self._estimator.predict( + x=x, + input_fn=input_fn, + batch_size=batch_size, + outputs=[key], + as_iterable=as_iterable) if as_iterable: - return _as_iterable(preds, output=head_lib.PredictionKey.CLASSES) - return preds[head_lib.PredictionKey.CLASSES].reshape(-1) + return _as_iterable(preds, output=key) + return preds[key].reshape(-1) @deprecated_arg_values( estimator.AS_ITERABLE_DATE, estimator.AS_ITERABLE_INSTRUCTIONS, @@ -775,20 +779,22 @@ class DNNLinearCombinedClassifier(evaluable.Evaluable, trainable.Trainable): Numpy array of predicted probabilities (or an iterable of predicted probabilities if as_iterable is True). """ + key = prediction_key.PredictionKey.PROBABILITIES preds = self._estimator.predict( - x=x, input_fn=input_fn, + x=x, + input_fn=input_fn, batch_size=batch_size, - outputs=[head_lib.PredictionKey.PROBABILITIES], + outputs=[key], as_iterable=as_iterable) if as_iterable: - return _as_iterable(preds, output=head_lib.PredictionKey.PROBABILITIES) - return preds[head_lib.PredictionKey.PROBABILITIES] + return _as_iterable(preds, output=key) + return preds[key] def _get_predict_ops(self, features): """See `Estimator` class.""" # pylint: disable=protected-access return self._estimator._get_predict_ops(features)[ - head_lib.PredictionKey.PROBABILITIES] + prediction_key.PredictionKey.PROBABILITIES] def get_variable_names(self): """Returns list of all variable names in this model. @@ -826,9 +832,9 @@ class DNNLinearCombinedClassifier(evaluable.Evaluable, trainable.Trainable): input_fn=input_fn or default_input_fn, input_feature_key=input_feature_key, use_deprecated_input_fn=use_deprecated_input_fn, - signature_fn=( - signature_fn or export.classification_signature_fn_with_prob), - prediction_key=head_lib.PredictionKey.PROBABILITIES, + signature_fn=(signature_fn or + export.classification_signature_fn_with_prob), + prediction_key=prediction_key.PredictionKey.PROBABILITIES, default_batch_size=default_batch_size, exports_to_keep=exports_to_keep) @@ -1041,10 +1047,11 @@ class DNNLinearCombinedRegressor(_DNNLinearCombinedBaseEstimator): head=head, config=config, feature_engineering_fn=feature_engineering_fn, - default_prediction_key=head_lib.PredictionKey.SCORES, + default_prediction_key=prediction_key.PredictionKey.SCORES, enable_centered_bias=enable_centered_bias) def _get_predict_ops(self, features): """See base class.""" - return super(DNNLinearCombinedRegressor, self)._get_predict_ops(features)[ - head_lib.PredictionKey.SCORES] + return super( + DNNLinearCombinedRegressor, + self)._get_predict_ops(features)[prediction_key.PredictionKey.SCORES] diff --git a/tensorflow/contrib/learn/python/learn/estimators/head.py b/tensorflow/contrib/learn/python/learn/estimators/head.py index b38e671dc49..77e3067bbde 100644 --- a/tensorflow/contrib/learn/python/learn/estimators/head.py +++ b/tensorflow/contrib/learn/python/learn/estimators/head.py @@ -867,11 +867,3 @@ def _streaming_at_threshold(streaming_metrics_fn, threshold): return array_ops.squeeze(precision_tensor), update_op return _streaming_metrics - - -class PredictionKey(object): - CLASSES = "classes" - PROBABILITIES = "probabilities" - LOGITS = "logits" - LOGISTIC = "logistic" - SCORES = "scores" diff --git a/tensorflow/contrib/learn/python/learn/estimators/linear.py b/tensorflow/contrib/learn/python/learn/estimators/linear.py index b09373f8e87..80407e56302 100644 --- a/tensorflow/contrib/learn/python/learn/estimators/linear.py +++ b/tensorflow/contrib/learn/python/learn/estimators/linear.py @@ -32,6 +32,7 @@ from tensorflow.contrib.learn.python.learn import evaluable from tensorflow.contrib.learn.python.learn import trainable from tensorflow.contrib.learn.python.learn.estimators import estimator from tensorflow.contrib.learn.python.learn.estimators import head as head_lib +from tensorflow.contrib.learn.python.learn.estimators import prediction_key from tensorflow.contrib.learn.python.learn.utils import export from tensorflow.contrib.linear_optimizer.python import sdca_optimizer from tensorflow.python.framework import dtypes @@ -462,13 +463,16 @@ class LinearClassifier(evaluable.Evaluable, trainable.Trainable): as_iterable=False) def predict(self, x=None, input_fn=None, batch_size=None, as_iterable=True): """Runs inference to determine the predicted class.""" - preds = self._estimator.predict(x=x, input_fn=input_fn, - batch_size=batch_size, - outputs=[head_lib.PredictionKey.CLASSES], - as_iterable=as_iterable) + key = prediction_key.PredictionKey.CLASSES + preds = self._estimator.predict( + x=x, + input_fn=input_fn, + batch_size=batch_size, + outputs=[key], + as_iterable=as_iterable) if as_iterable: - return _as_iterable(preds, output=head_lib.PredictionKey.CLASSES) - return preds[head_lib.PredictionKey.CLASSES] + return _as_iterable(preds, output=key) + return preds[key] @deprecated_arg_values( estimator.AS_ITERABLE_DATE, estimator.AS_ITERABLE_INSTRUCTIONS, @@ -476,14 +480,16 @@ class LinearClassifier(evaluable.Evaluable, trainable.Trainable): def predict_proba(self, x=None, input_fn=None, batch_size=None, outputs=None, as_iterable=True): """Runs inference to determine the class probability predictions.""" - preds = self._estimator.predict(x=x, input_fn=input_fn, - batch_size=batch_size, - outputs=[ - head_lib.PredictionKey.PROBABILITIES], - as_iterable=as_iterable) + key = prediction_key.PredictionKey.PROBABILITIES + preds = self._estimator.predict( + x=x, + input_fn=input_fn, + batch_size=batch_size, + outputs=[key], + as_iterable=as_iterable) if as_iterable: - return _as_iterable(preds, output=head_lib.PredictionKey.PROBABILITIES) - return preds[head_lib.PredictionKey.PROBABILITIES] + return _as_iterable(preds, output=key) + return preds[key] def get_variable_names(self): return self._estimator.get_variable_names() @@ -509,9 +515,9 @@ class LinearClassifier(evaluable.Evaluable, trainable.Trainable): input_fn=input_fn or default_input_fn, input_feature_key=input_feature_key, use_deprecated_input_fn=use_deprecated_input_fn, - signature_fn=( - signature_fn or export.classification_signature_fn_with_prob), - prediction_key=head_lib.PredictionKey.PROBABILITIES, + signature_fn=(signature_fn or + export.classification_signature_fn_with_prob), + prediction_key=prediction_key.PredictionKey.PROBABILITIES, default_batch_size=default_batch_size, exports_to_keep=exports_to_keep) @@ -725,13 +731,16 @@ class LinearRegressor(evaluable.Evaluable, trainable.Trainable): as_iterable=False) def predict(self, x=None, input_fn=None, batch_size=None, as_iterable=True): """Runs inference to determine the predicted class.""" - preds = self._estimator.predict(x=x, input_fn=input_fn, - batch_size=batch_size, - outputs=[head_lib.PredictionKey.SCORES], - as_iterable=as_iterable) + key = prediction_key.PredictionKey.SCORES + preds = self._estimator.predict( + x=x, + input_fn=input_fn, + batch_size=batch_size, + outputs=[key], + as_iterable=as_iterable) if as_iterable: - return _as_iterable(preds, output=head_lib.PredictionKey.SCORES) - return preds[head_lib.PredictionKey.SCORES] + return _as_iterable(preds, output=key) + return preds[key] def get_variable_names(self): return self._estimator.get_variable_names() @@ -758,7 +767,7 @@ class LinearRegressor(evaluable.Evaluable, trainable.Trainable): input_feature_key=input_feature_key, use_deprecated_input_fn=use_deprecated_input_fn, signature_fn=(signature_fn or export.regression_signature_fn), - prediction_key=head_lib.PredictionKey.SCORES, + prediction_key=prediction_key.PredictionKey.SCORES, default_batch_size=default_batch_size, exports_to_keep=exports_to_keep) diff --git a/tensorflow/contrib/learn/python/learn/estimators/svm.py b/tensorflow/contrib/learn/python/learn/estimators/svm.py index 6fd675e1b8b..0af33baeeb5 100644 --- a/tensorflow/contrib/learn/python/learn/estimators/svm.py +++ b/tensorflow/contrib/learn/python/learn/estimators/svm.py @@ -30,6 +30,7 @@ from tensorflow.contrib.learn.python.learn import trainable from tensorflow.contrib.learn.python.learn.estimators import estimator from tensorflow.contrib.learn.python.learn.estimators import head as head_lib from tensorflow.contrib.learn.python.learn.estimators import linear +from tensorflow.contrib.learn.python.learn.estimators import prediction_key from tensorflow.contrib.linear_optimizer.python import sdca_optimizer @@ -188,13 +189,16 @@ class SVM(trainable.Trainable, evaluable.Evaluable): as_iterable=False) def predict(self, x=None, input_fn=None, batch_size=None, as_iterable=True): """Runs inference to determine the predicted class.""" - preds = self._estimator.predict(x=x, input_fn=input_fn, - batch_size=batch_size, - outputs=[head_lib.PredictionKey.CLASSES], - as_iterable=as_iterable) + key = prediction_key.PredictionKey.CLASSES + preds = self._estimator.predict( + x=x, + input_fn=input_fn, + batch_size=batch_size, + outputs=[key], + as_iterable=as_iterable) if as_iterable: - return _as_iterable(preds, output=head_lib.PredictionKey.CLASSES) - return preds[head_lib.PredictionKey.CLASSES] + return _as_iterable(preds, output=key) + return preds[key] @deprecated_arg_values( estimator.AS_ITERABLE_DATE, estimator.AS_ITERABLE_INSTRUCTIONS, @@ -202,14 +206,16 @@ class SVM(trainable.Trainable, evaluable.Evaluable): def predict_proba(self, x=None, input_fn=None, batch_size=None, outputs=None, as_iterable=True): """Runs inference to determine the class probability predictions.""" - preds = self._estimator.predict(x=x, input_fn=input_fn, - batch_size=batch_size, - outputs=[ - head_lib.PredictionKey.PROBABILITIES], - as_iterable=as_iterable) + key = prediction_key.PredictionKey.PROBABILITIES + preds = self._estimator.predict( + x=x, + input_fn=input_fn, + batch_size=batch_size, + outputs=[key], + as_iterable=as_iterable) if as_iterable: - return _as_iterable(preds, output=head_lib.PredictionKey.PROBABILITIES) - return preds[head_lib.PredictionKey.PROBABILITIES] + return _as_iterable(preds, output=key) + return preds[key] # pylint: enable=protected-access def get_variable_names(self): From 88c47c9d5690e42e61e8887db5e05fd9331677ed Mon Sep 17 00:00:00 2001 From: Sukriti Ramesh Date: Thu, 3 Nov 2016 11:06:14 -0800 Subject: [PATCH 53/54] Add readme for SavedModel py. Change: 138100290 --- tensorflow/python/saved_model/README.md | 155 ++++++++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 tensorflow/python/saved_model/README.md diff --git a/tensorflow/python/saved_model/README.md b/tensorflow/python/saved_model/README.md new file mode 100644 index 00000000000..1323c91f86a --- /dev/null +++ b/tensorflow/python/saved_model/README.md @@ -0,0 +1,155 @@ +# TensorFlow SavedModel + +[TOC] + +## Overview +This document describes SavedModel, the universal serialization format for +[TensorFlow](https://www.tensorflow.org/) models. + +SavedModel provides a language-neutral format to save machine-learned models +that is recoverable and hermetic. It enables higher-level systems and tools to +produce, consume and transform TensorFlow models. + +## Features + +The following is a summary of the features in SavedModel: + +* Multiple graphs sharing a single set of variables and assets can be added to a + single SavedModel. Each graph is associated with a specific set of tags to + allow identification during a load or restore operation. +* Support for `SignatureDefs` + * Graphs that are used for inference tasks typically have a set of inputs + and outputs. This is called a `Signature`. + * SavedModel uses [SignatureDefs](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/protobuf/meta_graph.proto) + to allow generic support for signatures that may need to be saved with the graphs. +* Support for `Assets`. + * For cases where ops depend on external files for initialization, such as + vocabularies, SavedModel supports this via `assets`. + * Assets are copied to the SavedModel location and can be read when loading + a specific meta graph def. +* Support to clear devices before generating the SavedModel. + +The following is a summary of features that are NOT supported in SavedModel. +Higher-level frameworks and tools that use SavedModel may provide these. + +* Implicit versioning. +* Garbage collection. +* Atomic writes to the SavedModel location. + +## Background +SavedModel manages and builds upon existing TensorFlow primitives such as +`TensorFlow Saver` and `MetaGraphDef`. Specifically, SavedModel wraps a [TensorFlow Saver](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/python/training/saver.py). +The Saver is primarily used to generate the variable checkpoints. SavedModel +will replace the existing [TensorFlow Inference Model Format](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/session_bundle/README.md) +as the canonical way to export TensorFlow graphs for serving. + +## Components +A SavedModel directory has the following structure: + +``` +assets/ +assets.extra/ +variables/ + variables.data-?????-of-????? + variables.index +saved_model.pb +``` + +* SavedModel protocol buffer + * `saved_model.pb` or `saved_model.pbtxt` + * Includes the graph definitions as `MetaGraphDef` protocol buffers. +* Assets + * Subfolder called `assets`. + * Contains auxiliary files such as vocabularies, etc. +* Extra assets + * Subfolder where higher-level libraries and users can add their own assets + that co-exist with the model, but are not loaded by the graph. + * This subfolder is not managed by the SavedModel libraries. +* Variables + * Subfolder called `variables`. + * Includes output from the [TensorFlow Saver](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/python/training/saver.py). + * `variables.data-?????-of-?????` + * `variables.index` + +## APIs +The APIs for building and loading a SavedModel are described in this section. + +### Builder +The SavedModel [builder](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/builder.py) +is implemented in Python. + +The `SavedModelBuilder` class provides functionality to save multiple meta graph +defs, associated variables and assets. + +To build a SavedModel, the first meta graph must be saved with variables. +Subsequent meta graphs will simply be saved with their graph definitions. If +assets need to be saved and written or copied to disk, they can be provided +when the meta graph def is added. If multiple meta graph defs are associated +with an asset of the same name, only the first version is retained. + +#### Tags +Each meta graph added to the SavedModel must be annotated with user specified +tags. The tags provide a means to identify the specific meta graph to load and +restore, along with the shared set of variables and assets. These tags +typically annotate a MetaGraph with it's functionality (e.g. serving or +training), and possibly hardware specific aspects such as GPU. + +A subset of commonly used tags is specified in [Python](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/tag_constants.py) +and [C++](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/cc/saved_model/tag_constants.h) +for the purpose of easy and consistent usability. + +#### Usage +The typical usage of `builder` is as follows: + +~~~python +export_dir = ... +... +builder = saved_model_builder.SavedModelBuilder(export_dir) +with tf.Session(graph=tf.Graph()) as sess: +... +builder.add_meta_graph_and_variables(sess, + [tag_constants.TRAINING], + signature_def_map=foo_signatures, + assets_collection=foo_assets) +... +with tf.Session(graph=tf.Graph()) as sess: + ... + builder.add_meta_graph(["bar-tag", "baz-tag"]) +... +builder.save() +~~~ + +### Loader +The SavedModel loader is implemented in C++ and Python. + +#### Python +The Python version of the SavedModel [loader](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/builder.py) +provides load and restore capability for a SavedModel. The `load` operation +requires the session in which to restore the graph definition and variables, the +tags used to identify the meta graph def to load and the location of the +SavedModel. Upon a load, the subset of variables and assets supplied as part of +the specific meta graph def, will be restored into the supplied session. + +~~~python +export_dir = ... +... +with tf.Session(graph=tf.Graph()) as sess: + loader.load(sess, [tag_constants.TRAINING], export_dir) + ... +~~~ + +#### C++ +The C++ version of the SavedModel [loader](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/cc/saved_model/loader.h) +provides an API to load a SavedModel from a path, while allowing +`SessionOptions` and `RunOptions`. Similar to the Python version, the C++ +version requires the tags associated with the graph to be loaded, to be +specified. The loaded version of SavedModel is referred to as `SavedModelBundle` +and contains the meta graph def and the session within which it is loaded. + +~~~c++ +const string export_dir = ... +SavedModelBundle bundle; +... +LoadSavedModel(session_options, run_options, export_dir, {kSavedModelTagTrain}, + &bundle); +~~~ From 5904c0e5f2631b198d47a1f4229bf93991daf3f7 Mon Sep 17 00:00:00 2001 From: Jonathan Hseu Date: Thu, 3 Nov 2016 11:13:50 -0800 Subject: [PATCH 54/54] Fix windows build. Change: 138101249 --- tensorflow/core/platform/windows/windows_file_system.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/core/platform/windows/windows_file_system.h b/tensorflow/core/platform/windows/windows_file_system.h index 12b579bc86a..64da239d96d 100644 --- a/tensorflow/core/platform/windows/windows_file_system.h +++ b/tensorflow/core/platform/windows/windows_file_system.h @@ -16,6 +16,7 @@ limitations under the License. #ifndef TENSORFLOW_CORE_PLATFORM_WINDOWS_WINDOWS_FILE_SYSTEM_H_ #define TENSORFLOW_CORE_PLATFORM_WINDOWS_WINDOWS_FILE_SYSTEM_H_ +#include "tensorflow/core/lib/io/path.h" #include "tensorflow/core/platform/file_system.h" #ifdef PLATFORM_WINDOWS @@ -68,7 +69,7 @@ class LocalWinFileSystem : public WindowsFileSystem { public: string TranslateName(const string& name) const override { StringPiece scheme, host, path; - ParseURI(name, &scheme, &host, &path); + io::ParseURI(name, &scheme, &host, &path); return path.ToString(); } };