From 3d13afe4ecd59396d5b438751cda1b2d3d12465b Mon Sep 17 00:00:00 2001 From: Vagif Date: Wed, 6 Feb 2019 15:46:49 +0900 Subject: [PATCH] Fix misleading comment in layer normalization Comment states that moments are calculated across the last dimension, however this is not true for convolutional layers, where the moments are calculated on all dimensions except the one with index 0. I changed the comment from "Calculate the moments on the last axis (layer activations)." to "By default, compute the moments across all the dimensions except the one with index 0." --- tensorflow/contrib/layers/python/layers/layers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/layers/python/layers/layers.py b/tensorflow/contrib/layers/python/layers/layers.py index 403b522ce45..9d9524e4e4b 100644 --- a/tensorflow/contrib/layers/python/layers/layers.py +++ b/tensorflow/contrib/layers/python/layers/layers.py @@ -2308,7 +2308,7 @@ def layer_norm(inputs, initializer=init_ops.ones_initializer(), collections=gamma_collections, trainable=trainable) - # Calculate the moments on the last axis (layer activations). + # By default, compute the moments across all the dimensions except the one with index 0. norm_axes = list(range(begin_norm_axis, inputs_rank)) mean, variance = nn.moments(inputs, norm_axes, keep_dims=True) # Compute layer normalization using the batch_normalization function.