diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index e96d747b99..aee9b8a14f 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -397,13 +397,24 @@ class Auth(object):
             (EventTypes.ThirdPartyInvite, token,)
         )
         if not invite_event:
+            logger.info("Failing 3pid invite because no invite found for token %s", token)
             return False
         try:
             public_key = join_third_party_invite["public_key"]
             key_validity_url = join_third_party_invite["key_validity_url"]
             if invite_event.content["public_key"] != public_key:
+                logger.info(
+                    "Failing 3pid invite because public key invite: %s != join: %s",
+                    invite_event.content["public_key"],
+                    public_key
+                )
                 return False
             if invite_event.content["key_validity_url"] != key_validity_url:
+                logger.info(
+                    "Failing 3pid invite because key_validity_url invite: %s != join: %s",
+                    invite_event.content["key_validity_url"],
+                    key_validity_url
+                )
                 return False
             verify_key = nacl.signing.VerifyKey(decode_base64(public_key))
             encoded_signature = join_third_party_invite["signature"]
diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index 9ffa521aad..3f0cde56f0 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -22,11 +22,16 @@ from synapse.types import UserID, RoomAlias, RoomID
 from synapse.api.constants import (
     EventTypes, Membership, JoinRules, RoomCreationPreset,
 )
-from synapse.api.errors import StoreError, SynapseError
+from synapse.api.errors import AuthError, StoreError, SynapseError
 from synapse.util import stringutils, unwrapFirstError
 from synapse.util.async import run_on_reactor
 
+from signedjson.sign import verify_signed_json
+from signedjson.key import decode_verify_key_bytes
+
 from collections import OrderedDict
+from unpaddedbase64 import decode_base64
+
 import logging
 import string
 
@@ -614,12 +619,34 @@ class RoomMemberHandler(BaseHandler):
             )
 
             if "mxid" in data:
-                # TODO: Validate the response signature and such
+                if "signatures" not in data:
+                    raise AuthError(401, "No signatures on 3pid binding")
+                self.verify_any_signature(data, id_server)
                 defer.returnValue(data["mxid"])
+
         except IOError as e:
             logger.warn("Error from identity server lookup: %s" % (e,))
             defer.returnValue(None)
 
+    @defer.inlineCallbacks
+    def verify_any_signature(self, data, server_hostname):
+        if server_hostname not in data["signatures"]:
+            raise AuthError(401, "No signature from server %s" % (server_hostname,))
+        for key_name, signature in data["signatures"][server_hostname].items():
+            key_data = yield self.hs.get_simple_http_client().get_json(
+                "https://%s/_matrix/identity/api/v1/pubkey/%s" %
+                (server_hostname, key_name,),
+            )
+            if "public_key" not in key_data:
+                raise AuthError(401, "No public key named %s from %s" %
+                                (key_name, server_hostname,))
+            verify_signed_json(
+                data,
+                server_hostname,
+                decode_verify_key_bytes(key_name, decode_base64(key_data["public_key"]))
+            )
+            return
+
     @defer.inlineCallbacks
     def _make_and_store_3pid_invite(
             self,
diff --git a/synapse/http/client.py b/synapse/http/client.py
index 9a5869abee..27e5190224 100644
--- a/synapse/http/client.py
+++ b/synapse/http/client.py
@@ -24,7 +24,6 @@ from canonicaljson import encode_canonical_json
 from twisted.internet import defer, reactor, ssl
 from twisted.web.client import (
     Agent, readBody, FileBodyProducer, PartialDownloadError,
-    HTTPConnectionPool,
 )
 from twisted.web.http_headers import Headers
 
@@ -59,11 +58,8 @@ class SimpleHttpClient(object):
         # The default context factory in Twisted 14.0.0 (which we require) is
         # BrowserLikePolicyForHTTPS which will do regular cert validation
         # 'like a browser'
-        pool = HTTPConnectionPool(reactor)
-        pool.maxPersistentPerHost = 10
         self.agent = Agent(
             reactor,
-            pool=pool,
             connectTimeout=15,
             contextFactory=hs.get_http_client_context_factory()
         )
diff --git a/synapse/util/third_party_invites.py b/synapse/util/third_party_invites.py
index 41e597d5b9..335a9755b2 100644
--- a/synapse/util/third_party_invites.py
+++ b/synapse/util/third_party_invites.py
@@ -63,7 +63,7 @@ def check_key_valid(http_client, event):
             event.content["third_party_invite"]["key_validity_url"],
             {"public_key": event.content["third_party_invite"]["public_key"]}
         )
-        if not response["valid"]:
-            raise AuthError(403, "Third party certificate was invalid")
-    except IOError:
+    except Exception:
         raise AuthError(502, "Third party certificate could not be checked")
+    if "valid" not in response or not response["valid"]:
+        raise AuthError(403, "Third party certificate was invalid")