diff --git a/synapse/notifier.py b/synapse/notifier.py
index 1a6a2c780c..76637b176d 100644
--- a/synapse/notifier.py
+++ b/synapse/notifier.py
@@ -350,14 +350,15 @@ class Notifier(object):
             while not result and not timed_out[0]:
                 new_token = yield deferred
 
-                deferred = defer.Deferred()
-                listener = _NotificationListener(deferred, timeout)
-                listeners.append(listener)
-                user_stream.listeners.add(listener)
-
                 result = yield callback(current_token, new_token)
                 current_token = new_token
 
+                if not result:
+                    deferred = defer.Deferred()
+                    listener = _NotificationListener(deferred, timeout)
+                    listeners.append(listener)
+                    user_stream.listeners.add(listener)
+
         if timer[0] is not None:
             try:
                 self.clock.cancel_call_later(timer[0])