diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py
index 9ab119f29e..888b7ef2e9 100644
--- a/synapse/util/retryutils.py
+++ b/synapse/util/retryutils.py
@@ -89,6 +89,9 @@ class RetryDestinationLimiter(object):
 
         if exc_type is None and exc_val is None and exc_tb is None:
             # We connected successfully.
+            if not self.retry_interval:
+                return
+
             retry_last_ts = 0
             self.retry_interval = 0
         else: