diff --git a/tensorflow/python/training/moving_averages.py b/tensorflow/python/training/moving_averages.py
index a2b3b866cf1..9a2969aaeb1 100644
--- a/tensorflow/python/training/moving_averages.py
+++ b/tensorflow/python/training/moving_averages.py
@@ -444,7 +444,7 @@ class ExponentialMovingAverage(object):
                     "Variable", "VariableV2", "VarHandleOp"
                 ]))
             if self._zero_debias:
-              zero_debias_true.add(avg)
+              zero_debias_true.add(avg.experimental_ref())
         self._averages[var.experimental_ref()] = avg
 
     with ops.name_scope(self.name) as scope: