diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py
index c94c74a67e..51b656d74a 100644
--- a/synapse/federation/transaction_queue.py
+++ b/synapse/federation/transaction_queue.py
@@ -462,6 +462,13 @@ class TransactionQueue(object):
                     code = e.code
                     response = e.response
 
+                    if e.code == 429 or 500 <= e.code:
+                        logger.info(
+                            "TX [%s] {%s} got %d response",
+                            destination, txn_id, code
+                        )
+                        raise e
+
                 logger.info(
                     "TX [%s] {%s} got %d response",
                     destination, txn_id, code
diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py
index 46ef5a8ec7..218029e8a8 100644
--- a/synapse/util/retryutils.py
+++ b/synapse/util/retryutils.py
@@ -123,7 +123,7 @@ class RetryDestinationLimiter(object):
     def __exit__(self, exc_type, exc_val, exc_tb):
         valid_err_code = False
         if exc_type is not None and issubclass(exc_type, CodeMessageException):
-            valid_err_code = 0 <= exc_val.code < 500
+            valid_err_code = valid_err_code != 429 and 0 <= exc_val.code < 500
 
         if exc_type is None or valid_err_code:
             # We connected successfully.