diff --git a/tensorflow/core/kernels/training_op_helpers.h b/tensorflow/core/kernels/training_op_helpers.h
index 98e2b3c0f2f..715dd8af7da 100644
--- a/tensorflow/core/kernels/training_op_helpers.h
+++ b/tensorflow/core/kernels/training_op_helpers.h
@@ -178,7 +178,7 @@ VariableInputLockHolder MaybeLockVariableInputMutexesInOrder(
     mutex* mu = GetTrainingVariableMutex<Device, T>(ctx, input, sparse, &var);
     core::ScopedUnref scoped_unref(var);
     if (mu != nullptr) {
-      if (do_lock) {
+      if (!sparse || do_lock) {
         locks->emplace_back(*mu);
       } else {
         shared_locks->emplace_back(*mu);