diff --git a/changelog.d/4204.misc b/changelog.d/4204.misc
new file mode 100644
index 0000000000..efd1f4abd6
--- /dev/null
+++ b/changelog.d/4204.misc
@@ -0,0 +1 @@
+Fix logcontext leaks in EmailPusher and in tests
\ No newline at end of file
diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py
index ebcb93bfc7..1eb5be0957 100644
--- a/synapse/push/mailer.py
+++ b/synapse/push/mailer.py
@@ -36,6 +36,7 @@ from synapse.push.presentable_names import (
 )
 from synapse.types import UserID
 from synapse.util.async_helpers import concurrently_execute
+from synapse.util.logcontext import make_deferred_yieldable
 from synapse.visibility import filter_events_for_client
 
 logger = logging.getLogger(__name__)
@@ -192,7 +193,7 @@ class Mailer(object):
 
         logger.info("Sending email push notification to %s" % email_address)
 
-        yield self.sendmail(
+        yield make_deferred_yieldable(self.sendmail(
             self.hs.config.email_smtp_host,
             raw_from, raw_to, multipart_msg.as_string().encode('utf8'),
             reactor=self.hs.get_reactor(),
@@ -201,7 +202,7 @@ class Mailer(object):
             username=self.hs.config.email_smtp_user,
             password=self.hs.config.email_smtp_pass,
             requireTransportSecurity=self.hs.config.require_transport_security
-        )
+        ))
 
     @defer.inlineCallbacks
     def get_room_vars(self, room_id, user_id, notifs, notif_events, room_state_ids):
diff --git a/tests/push/test_http.py b/tests/push/test_http.py
index addc01ab7f..6dc45e8506 100644
--- a/tests/push/test_http.py
+++ b/tests/push/test_http.py
@@ -18,6 +18,7 @@ from mock import Mock
 from twisted.internet.defer import Deferred
 
 from synapse.rest.client.v1 import admin, login, room
+from synapse.util.logcontext import make_deferred_yieldable
 
 from tests.unittest import HomeserverTestCase
 
@@ -47,7 +48,7 @@ class HTTPPusherTests(HomeserverTestCase):
         def post_json_get_json(url, body):
             d = Deferred()
             self.push_attempts.append((d, url, body))
-            return d
+            return make_deferred_yieldable(d)
 
         m.post_json_get_json = post_json_get_json
 
diff --git a/tests/rest/media/v1/test_url_preview.py b/tests/rest/media/v1/test_url_preview.py
index 86c813200a..c62f71b44a 100644
--- a/tests/rest/media/v1/test_url_preview.py
+++ b/tests/rest/media/v1/test_url_preview.py
@@ -20,6 +20,7 @@ from mock import Mock
 from twisted.internet.defer import Deferred
 
 from synapse.config.repository import MediaStorageProviderConfig
+from synapse.util.logcontext import make_deferred_yieldable
 from synapse.util.module_loader import load_module
 
 from tests import unittest
@@ -77,7 +78,7 @@ class URLPreviewTests(unittest.HomeserverTestCase):
             d = Deferred()
             d.addCallback(write_to)
             self.fetches.append((d, url))
-            return d
+            return make_deferred_yieldable(d)
 
         client = Mock()
         client.get_file = get_file
diff --git a/tests/test_federation.py b/tests/test_federation.py
index e1a34ccffd..1a5dc32c88 100644
--- a/tests/test_federation.py
+++ b/tests/test_federation.py
@@ -123,8 +123,8 @@ class MessageAcceptTests(unittest.TestCase):
                 "test.serv", lying_event, sent_to_us_directly=True
             )
 
-        # Step the reactor, so the database fetches come back
-        self.reactor.advance(1)
+            # Step the reactor, so the database fetches come back
+            self.reactor.advance(1)
 
         # on_receive_pdu should throw an error
         failure = self.failureResultOf(d)
diff --git a/tests/test_server.py b/tests/test_server.py
index f0e6291b7e..634a8fbca5 100644
--- a/tests/test_server.py
+++ b/tests/test_server.py
@@ -27,6 +27,7 @@ from synapse.api.errors import Codes, SynapseError
 from synapse.http.server import JsonResource
 from synapse.http.site import SynapseSite, logger
 from synapse.util import Clock
+from synapse.util.logcontext import make_deferred_yieldable
 
 from tests import unittest
 from tests.server import FakeTransport, make_request, render, setup_test_homeserver
@@ -95,7 +96,7 @@ class JsonResourceTests(unittest.TestCase):
             d = Deferred()
             d.addCallback(_throw)
             self.reactor.callLater(1, d.callback, True)
-            return d
+            return make_deferred_yieldable(d)
 
         res = JsonResource(self.homeserver)
         res.register_paths("GET", [re.compile("^/_matrix/foo$")], _callback)