diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py
index cb106c6a1b..bb3d9258a6 100644
--- a/synapse/federation/transaction_queue.py
+++ b/synapse/federation/transaction_queue.py
@@ -504,7 +504,7 @@ class TransactionQueue(object):
                     code = e.code
                     response = e.response
 
-                    if e.code == 429 or 500 <= e.code:
+                    if e.code in (401, 404, 429) or 500 <= e.code:
                         logger.info(
                             "TX [%s] {%s} got %d response",
                             destination, txn_id, code
diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py
index b94ae369cf..153ef001ad 100644
--- a/synapse/util/retryutils.py
+++ b/synapse/util/retryutils.py
@@ -129,11 +129,13 @@ class RetryDestinationLimiter(object):
             # APIs may expect to never received e.g. a 404. It's important to
             # handle 404 as some remote servers will return a 404 when the HS
             # has been decommissioned.
+            # If we get a 401, then we should probably back off since they
+            # won't accept our requests for at least a while.
+            # 429 is us being aggresively rate limited, so lets rate limit
+            # ourselves.
             if exc_val.code == 404 and self.backoff_on_404:
                 valid_err_code = False
-            elif exc_val.code == 429:
-                # 429 is us being aggresively rate limited, so lets rate limit
-                # ourselves.
+            elif exc_val.code in (401, 429):
                 valid_err_code = False
             elif exc_val.code < 500:
                 valid_err_code = True