From 9f863d34663a4d58cf59cd15e0b0aa4a8e8581a5 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 14 Aug 2014 09:52:20 +0100
Subject: [PATCH 01/46] Start phasing out HttpServer: we should be using
 Resources instead. Added resource_for_client/federation/web_client to the
 HomeServer and hooked the C-S servlets to operate on resource_for_client.
 Dynamically construct the Resource tree.

---
 synapse/app/homeserver.py   | 77 ++++++++++++++++++++++++++++++++++++-
 synapse/http/server.py      |  5 +++
 synapse/rest/__init__.py    | 24 +++++-------
 synapse/rest/base.py        |  4 +-
 synapse/server.py           |  8 ++--
 tests/rest/test_presence.py |  3 ++
 tests/rest/test_profile.py  |  1 +
 7 files changed, 101 insertions(+), 21 deletions(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 82afb04c7d..1acc87e99c 100644
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -21,8 +21,12 @@ from synapse.server import HomeServer
 from twisted.internet import reactor
 from twisted.enterprise import adbapi
 from twisted.python.log import PythonLoggingObserver
-from synapse.http.server import TwistedHttpServer
+from twisted.web.resource import Resource
+from twisted.web.static import File
+from twisted.web.server import Site
+from synapse.http.server import TwistedHttpServer, JsonResource
 from synapse.http.client import TwistedHttpClient
+from synapse.rest.base import CLIENT_PREFIX
 
 from daemonize import Daemonize
 
@@ -41,6 +45,15 @@ class SynapseHomeServer(HomeServer):
     def build_http_client(self):
         return TwistedHttpClient()
 
+    def build_resource_for_client(self):
+        return JsonResource()
+
+    def build_resource_for_federation(self):
+        return JsonResource()
+
+    def build_resource_for_web_client(self):
+        return File("webclient")
+
     def build_db_pool(self):
         """ Set up all the dbs. Since all the *.sql have IF NOT EXISTS, so we
         don't have to worry about overwriting existing content.
@@ -73,6 +86,65 @@ class SynapseHomeServer(HomeServer):
 
         return pool
 
+    def create_resource_tree(self):
+        """Create the resource tree for this Home Server.
+
+        This in unduly complicated because Twisted does not support putting
+        child resources more than 1 level deep at a time.
+        """
+        desired_tree = (  # list of tuples containing (path_str, Resource)
+            ("/matrix/client", self.get_resource_for_web_client()),
+            (CLIENT_PREFIX, self.get_resource_for_client())
+        )
+
+        self.root_resource = Resource()
+        # ideally we'd just use getChild and putChild but getChild doesn't work
+        # unless you give it a Request object IN ADDITION to the name :/ So
+        # instead, we'll store a copy of this mapping so we can actually add
+        # extra resources to existing nodes. See self._resource_id for the key.
+        resource_mappings = {}
+        for (full_path, resource) in desired_tree:
+            logging.info("Attaching %s to path %s", resource, full_path)
+            last_resource = self.root_resource
+            for path_seg in full_path.split('/')[1:-1]:
+                if not path_seg in last_resource.listNames():
+                    # resource doesn't exist
+                    child_resource = Resource()
+                    last_resource.putChild(path_seg, child_resource)
+                    res_id = self._resource_id(last_resource, path_seg)
+                    resource_mappings[res_id] = child_resource
+                    last_resource = child_resource
+                else:
+                    # we have an existing Resource, pull it out.
+                    res_id = self._resource_id(last_resource, path_seg)
+                    last_resource = resource_mappings[res_id]
+
+            # now attach the actual resource
+            last_path_seg = full_path.split('/')[-1]
+            last_resource.putChild(last_path_seg, resource)
+            res_id = self._resource_id(last_resource, last_path_seg)
+            resource_mappings[res_id] = resource
+
+        return self.root_resource
+
+    def _resource_id(self, resource, path_seg):
+        """Construct an arbitrary resource ID so you can retrieve the mapping
+        later.
+
+        If you want to represent resource A putChild resource B with path C,
+        the mapping should looks like _resource_id(A,C) = B.
+
+        Args:
+            resource (Resource): The *parent* Resource
+            path_seg (str): The name of the child Resource to be attached.
+        Returns:
+            str: A unique string which can be a key to the child Resource.
+        """
+        return "%s-%s" % (resource, path_seg)
+
+    def start_listening(self, port):
+        reactor.listenTCP(port, Site(self.root_resource))
+
 
 def setup_logging(verbosity=0, filename=None, config_path=None):
     """ Sets up logging with verbosity levels.
@@ -150,7 +222,8 @@ def setup():
 
     hs.register_servlets()
 
-    hs.get_http_server().start_listening(args.port)
+    hs.create_resource_tree()
+    hs.start_listening(args.port)
 
     hs.build_db_pool()
 
diff --git a/synapse/http/server.py b/synapse/http/server.py
index d7f4b691bc..32dfd836cd 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -185,3 +185,8 @@ def respond_with_json_bytes(request, code, json_bytes, send_cors=False):
     request.write(json_bytes)
     request.finish()
     return NOT_DONE_YET
+
+
+# FIXME: Temp, just so the new name can be used without breaking the world.
+class JsonResource(TwistedHttpServer):
+    pass
\ No newline at end of file
diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py
index 74a372e2ff..82c09f4c4a 100644
--- a/synapse/rest/__init__.py
+++ b/synapse/rest/__init__.py
@@ -32,19 +32,15 @@ class RestServletFactory(object):
     """
 
     def __init__(self, hs):
-        http_server = hs.get_http_server()
+        client_resource = hs.get_resource_for_client()
 
         # TODO(erikj): There *must* be a better way of doing this.
-        room.register_servlets(hs, http_server)
-        events.register_servlets(hs, http_server)
-        register.register_servlets(hs, http_server)
-        login.register_servlets(hs, http_server)
-        profile.register_servlets(hs, http_server)
-        public.register_servlets(hs, http_server)
-        presence.register_servlets(hs, http_server)
-        im.register_servlets(hs, http_server)
-        directory.register_servlets(hs, http_server)
-
-    def register_web_client(self, hs):
-        http_server = hs.get_http_server()
-        webclient.register_servlets(hs, http_server)
+        room.register_servlets(hs, client_resource)
+        events.register_servlets(hs, client_resource)
+        register.register_servlets(hs, client_resource)
+        login.register_servlets(hs, client_resource)
+        profile.register_servlets(hs, client_resource)
+        public.register_servlets(hs, client_resource)
+        presence.register_servlets(hs, client_resource)
+        im.register_servlets(hs, client_resource)
+        directory.register_servlets(hs, client_resource)
diff --git a/synapse/rest/base.py b/synapse/rest/base.py
index 65d417f757..124b3a2e0c 100644
--- a/synapse/rest/base.py
+++ b/synapse/rest/base.py
@@ -16,6 +16,8 @@
 """ This module contains base REST classes for constructing REST servlets. """
 import re
 
+CLIENT_PREFIX = "/matrix/client/api/v1"
+
 
 def client_path_pattern(path_regex):
     """Creates a regex compiled client path with the correct client path
@@ -27,7 +29,7 @@ def client_path_pattern(path_regex):
     Returns:
         SRE_Pattern
     """
-    return re.compile("^/matrix/client/api/v1" + path_regex)
+    return re.compile("^" + CLIENT_PREFIX + path_regex)
 
 
 class RestServlet(object):
diff --git a/synapse/server.py b/synapse/server.py
index 96830a88b1..537e431375 100644
--- a/synapse/server.py
+++ b/synapse/server.py
@@ -70,6 +70,9 @@ class BaseHomeServer(object):
         'room_lock_manager',
         'notifier',
         'distributor',
+        'resource_for_client',
+        'resource_for_federation',
+        'resource_for_web_client',
     ]
 
     def __init__(self, hostname, **kwargs):
@@ -178,9 +181,6 @@ class HomeServer(BaseHomeServer):
 
     def register_servlets(self):
         """ Register all servlets associated with this HomeServer.
-
-        Args:
-            host_web_client (bool): True to host the web client as well.
         """
         # Simply building the ServletFactory is sufficient to have it register
-        factory = self.get_rest_servlet_factory()
+        self.get_rest_servlet_factory()
diff --git a/tests/rest/test_presence.py b/tests/rest/test_presence.py
index f013abbee4..98b308dcdb 100644
--- a/tests/rest/test_presence.py
+++ b/tests/rest/test_presence.py
@@ -51,6 +51,7 @@ class PresenceStateTestCase(unittest.TestCase):
         hs = HomeServer("test",
             db_pool=None,
             http_client=None,
+            resource_for_client=self.mock_server,
             http_server=self.mock_server,
         )
 
@@ -108,6 +109,7 @@ class PresenceListTestCase(unittest.TestCase):
         hs = HomeServer("test",
             db_pool=None,
             http_client=None,
+            resource_for_client=self.mock_server,
             http_server=self.mock_server,
         )
 
@@ -184,6 +186,7 @@ class PresenceEventStreamTestCase(unittest.TestCase):
             db_pool=None,
             http_client=None,
             http_server=self.mock_server,
+            resource_for_client=self.mock_server,
             datastore=Mock(spec=[
                 "set_presence_state",
                 "get_presence_list",
diff --git a/tests/rest/test_profile.py b/tests/rest/test_profile.py
index 46e6137775..975d32f64d 100644
--- a/tests/rest/test_profile.py
+++ b/tests/rest/test_profile.py
@@ -44,6 +44,7 @@ class ProfileTestCase(unittest.TestCase):
             db_pool=None,
             http_client=None,
             http_server=self.mock_server,
+            resource_for_client=self.mock_server,
             federation=Mock(),
             replication_layer=Mock(),
         )

From 29aa13f0d4424729813bea99b84d34a3b3d4e68c Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 14 Aug 2014 10:05:06 +0100
Subject: [PATCH 02/46] Make federation use resource_for_federation as well.

---
 synapse/app/homeserver.py      |  4 ++-
 synapse/federation/__init__.py |  2 +-
 synapse/rest/__init__.py       |  3 +--
 synapse/rest/webclient.py      | 45 ----------------------------------
 4 files changed, 5 insertions(+), 49 deletions(-)
 delete mode 100644 synapse/rest/webclient.py

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 1acc87e99c..d9494cb054 100644
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -27,6 +27,7 @@ from twisted.web.server import Site
 from synapse.http.server import TwistedHttpServer, JsonResource
 from synapse.http.client import TwistedHttpClient
 from synapse.rest.base import CLIENT_PREFIX
+from synapse.federation.transport import PREFIX
 
 from daemonize import Daemonize
 
@@ -94,7 +95,8 @@ class SynapseHomeServer(HomeServer):
         """
         desired_tree = (  # list of tuples containing (path_str, Resource)
             ("/matrix/client", self.get_resource_for_web_client()),
-            (CLIENT_PREFIX, self.get_resource_for_client())
+            (CLIENT_PREFIX, self.get_resource_for_client()),
+            (PREFIX, self.get_resource_for_federation())
         )
 
         self.root_resource = Resource()
diff --git a/synapse/federation/__init__.py b/synapse/federation/__init__.py
index ac0c10dc33..b15e7cf941 100644
--- a/synapse/federation/__init__.py
+++ b/synapse/federation/__init__.py
@@ -23,7 +23,7 @@ from .transport import TransportLayer
 def initialize_http_replication(homeserver):
     transport = TransportLayer(
         homeserver.hostname,
-        server=homeserver.get_http_server(),
+        server=homeserver.get_resource_for_federation(),
         client=homeserver.get_http_client()
     )
 
diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py
index 82c09f4c4a..da18933b63 100644
--- a/synapse/rest/__init__.py
+++ b/synapse/rest/__init__.py
@@ -15,8 +15,7 @@
 
 
 from . import (
-    room, events, register, login, profile, public, presence, im, directory,
-    webclient
+    room, events, register, login, profile, public, presence, im, directory
 )
 
 
diff --git a/synapse/rest/webclient.py b/synapse/rest/webclient.py
deleted file mode 100644
index 75a425c14c..0000000000
--- a/synapse/rest/webclient.py
+++ /dev/null
@@ -1,45 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from synapse.rest.base import RestServlet
-
-import logging
-import re
-
-logger = logging.getLogger(__name__)
-
-
-class WebClientRestServlet(RestServlet):
-    # No PATTERN; we have custom dispatch rules here
-
-    def register(self, http_server):
-        http_server.register_path("GET",
-                                  re.compile("^/$"),
-                                  self.on_GET_redirect)
-        http_server.register_path("GET",
-                                  re.compile("^/matrix/client$"),
-                                  self.on_GET)
-
-    def on_GET(self, request):
-        return (200, "not implemented")
-
-    def on_GET_redirect(self, request):
-        request.setHeader("Location", request.uri + "matrix/client")
-        return (302, None)
-
-
-def register_servlets(hs, http_server):
-    logger.info("Registering web client.")
-    WebClientRestServlet(hs).register(http_server)
\ No newline at end of file

From 9a1638ed21ea716b999853c8d63c30073e677c46 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 14 Aug 2014 10:18:54 +0100
Subject: [PATCH 03/46] Removed http_server from HomeServer. Updated unit tests
 to use either resource_for_federation or resource_for_client depending on
 what is being tested.

---
 synapse/app/homeserver.py           |  4 +---
 synapse/http/server.py              | 12 +++---------
 synapse/server.py                   |  5 +++--
 tests/federation/test_federation.py |  2 +-
 tests/handlers/test_directory.py    |  2 +-
 tests/handlers/test_federation.py   |  2 +-
 tests/handlers/test_presence.py     |  8 ++++----
 tests/handlers/test_presencelike.py |  2 +-
 tests/handlers/test_profile.py      |  4 ++--
 tests/handlers/test_room.py         |  3 +--
 tests/rest/test_presence.py         |  6 +++---
 tests/rest/test_profile.py          |  1 -
 12 files changed, 21 insertions(+), 30 deletions(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index d9494cb054..ea6f0985ef 100644
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -24,7 +24,7 @@ from twisted.python.log import PythonLoggingObserver
 from twisted.web.resource import Resource
 from twisted.web.static import File
 from twisted.web.server import Site
-from synapse.http.server import TwistedHttpServer, JsonResource
+from synapse.http.server import JsonResource
 from synapse.http.client import TwistedHttpClient
 from synapse.rest.base import CLIENT_PREFIX
 from synapse.federation.transport import PREFIX
@@ -40,8 +40,6 @@ logger = logging.getLogger(__name__)
 
 
 class SynapseHomeServer(HomeServer):
-    def build_http_server(self):
-        return TwistedHttpServer()
 
     def build_http_client(self):
         return TwistedHttpClient()
diff --git a/synapse/http/server.py b/synapse/http/server.py
index 32dfd836cd..87b4fc8a5f 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -52,10 +52,9 @@ class HttpServer(object):
         pass
 
 
-# The actual HTTP server impl, using twisted http server
-class TwistedHttpServer(HttpServer, resource.Resource):
-    """ This wraps the twisted HTTP server, and triggers the correct callbacks
-    on the transport_layer.
+class JsonResource(HttpServer, resource.Resource):
+    """ This implements the HttpServer interface and provides JSON support for
+    Resources.
 
     Register callbacks via register_path()
     """
@@ -185,8 +184,3 @@ def respond_with_json_bytes(request, code, json_bytes, send_cors=False):
     request.write(json_bytes)
     request.finish()
     return NOT_DONE_YET
-
-
-# FIXME: Temp, just so the new name can be used without breaking the world.
-class JsonResource(TwistedHttpServer):
-    pass
\ No newline at end of file
diff --git a/synapse/server.py b/synapse/server.py
index 537e431375..0f7ac352ae 100644
--- a/synapse/server.py
+++ b/synapse/server.py
@@ -55,7 +55,6 @@ class BaseHomeServer(object):
 
     DEPENDENCIES = [
         'clock',
-        'http_server',
         'http_client',
         'db_pool',
         'persistence_service',
@@ -138,7 +137,9 @@ class HomeServer(BaseHomeServer):
     required.
 
     It still requires the following to be specified by the caller:
-        http_server
+        resource_for_client
+        resource_for_web_client
+        resource_for_federation
         http_client
         db_pool
     """
diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py
index ec39c7ee33..478ddd879e 100644
--- a/tests/federation/test_federation.py
+++ b/tests/federation/test_federation.py
@@ -70,7 +70,7 @@ class FederationTestCase(unittest.TestCase):
         )
         self.clock = MockClock()
         hs = HomeServer("test",
-                http_server=self.mock_http_server,
+                resource_for_federation=self.mock_http_server,
                 http_client=self.mock_http_client,
                 db_pool=None,
                 datastore=self.mock_persistence,
diff --git a/tests/handlers/test_directory.py b/tests/handlers/test_directory.py
index 0ace2d0c9a..88ac8933f8 100644
--- a/tests/handlers/test_directory.py
+++ b/tests/handlers/test_directory.py
@@ -51,7 +51,7 @@ class DirectoryTestCase(unittest.TestCase):
                 "get_association_from_room_alias",
             ]),
             http_client=None,
-            http_server=Mock(),
+            resource_for_federation=Mock(),
             replication_layer=self.mock_federation,
         )
         hs.handlers = DirectoryHandlers(hs)
diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py
index bdee7cfad4..ab9c242579 100644
--- a/tests/handlers/test_federation.py
+++ b/tests/handlers/test_federation.py
@@ -42,7 +42,7 @@ class FederationTestCase(unittest.TestCase):
                 "persist_event",
                 "store_room",
             ]),
-            http_server=NonCallableMock(),
+            resource_for_federation=NonCallableMock(),
             http_client=NonCallableMock(spec_set=[]),
             notifier=NonCallableMock(spec_set=["on_new_room_event"]),
             handlers=NonCallableMock(spec_set=[
diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py
index b365741d99..61c2547af4 100644
--- a/tests/handlers/test_presence.py
+++ b/tests/handlers/test_presence.py
@@ -66,7 +66,7 @@ class PresenceStateTestCase(unittest.TestCase):
                     "set_presence_list_accepted",
                 ]),
                 handlers=None,
-                http_server=Mock(),
+                resource_for_federation=Mock(),
                 http_client=None,
             )
         hs.handlers = JustPresenceHandlers(hs)
@@ -188,7 +188,7 @@ class PresenceInvitesTestCase(unittest.TestCase):
                     "del_presence_list",
                 ]),
                 handlers=None,
-                http_server=Mock(),
+                resource_for_client=Mock(),
                 http_client=None,
                 replication_layer=self.replication
             )
@@ -402,7 +402,7 @@ class PresencePushTestCase(unittest.TestCase):
                     "set_presence_state",
                 ]),
                 handlers=None,
-                http_server=Mock(),
+                resource_for_client=Mock(),
                 http_client=None,
                 replication_layer=self.replication,
             )
@@ -727,7 +727,7 @@ class PresencePollingTestCase(unittest.TestCase):
                 db_pool=None,
                 datastore=Mock(spec=[]),
                 handlers=None,
-                http_server=Mock(),
+                resource_for_client=Mock(),
                 http_client=None,
                 replication_layer=self.replication,
             )
diff --git a/tests/handlers/test_presencelike.py b/tests/handlers/test_presencelike.py
index 6eeb1bb522..bba5dd4e53 100644
--- a/tests/handlers/test_presencelike.py
+++ b/tests/handlers/test_presencelike.py
@@ -71,7 +71,7 @@ class PresenceProfilelikeDataTestCase(unittest.TestCase):
                     "set_profile_displayname",
                 ]),
                 handlers=None,
-                http_server=Mock(),
+                resource_for_federation=Mock(),
                 http_client=None,
                 replication_layer=MockReplication(),
             )
diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py
index eb1df2a4cf..87a8139920 100644
--- a/tests/handlers/test_profile.py
+++ b/tests/handlers/test_profile.py
@@ -56,7 +56,7 @@ class ProfileTestCase(unittest.TestCase):
                     "set_profile_avatar_url",
                 ]),
                 handlers=None,
-                http_server=Mock(),
+                resource_for_federation=Mock(),
                 replication_layer=self.mock_federation,
             )
         hs.handlers = ProfileHandlers(hs)
@@ -139,7 +139,7 @@ class ProfileTestCase(unittest.TestCase):
         mocked_set = self.datastore.set_profile_avatar_url
         mocked_set.return_value = defer.succeed(())
 
-        yield self.handler.set_avatar_url(self.frank, self.frank, 
+        yield self.handler.set_avatar_url(self.frank, self.frank,
                 "http://my.server/pic.gif")
 
         mocked_set.assert_called_with("1234ABCD", "http://my.server/pic.gif")
diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py
index 99067da6a5..fd2d66db38 100644
--- a/tests/handlers/test_room.py
+++ b/tests/handlers/test_room.py
@@ -46,7 +46,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase):
                 "get_room",
                 "store_room",
             ]),
-            http_server=NonCallableMock(),
+            resource_for_federation=NonCallableMock(),
             http_client=NonCallableMock(spec_set=[]),
             notifier=NonCallableMock(spec_set=["on_new_room_event"]),
             handlers=NonCallableMock(spec_set=[
@@ -317,7 +317,6 @@ class RoomCreationTest(unittest.TestCase):
             datastore=NonCallableMock(spec_set=[
                 "store_room",
             ]),
-            http_server=NonCallableMock(),
             http_client=NonCallableMock(spec_set=[]),
             notifier=NonCallableMock(spec_set=["on_new_room_event"]),
             handlers=NonCallableMock(spec_set=[
diff --git a/tests/rest/test_presence.py b/tests/rest/test_presence.py
index 98b308dcdb..91d4d1ff6c 100644
--- a/tests/rest/test_presence.py
+++ b/tests/rest/test_presence.py
@@ -52,7 +52,7 @@ class PresenceStateTestCase(unittest.TestCase):
             db_pool=None,
             http_client=None,
             resource_for_client=self.mock_server,
-            http_server=self.mock_server,
+            resource_for_federation=self.mock_server,
         )
 
         def _get_user_by_token(token=None):
@@ -110,7 +110,7 @@ class PresenceListTestCase(unittest.TestCase):
             db_pool=None,
             http_client=None,
             resource_for_client=self.mock_server,
-            http_server=self.mock_server,
+            resource_for_federation=self.mock_server
         )
 
         def _get_user_by_token(token=None):
@@ -185,8 +185,8 @@ class PresenceEventStreamTestCase(unittest.TestCase):
         hs = HomeServer("test",
             db_pool=None,
             http_client=None,
-            http_server=self.mock_server,
             resource_for_client=self.mock_server,
+            resource_for_federation=self.mock_server,
             datastore=Mock(spec=[
                 "set_presence_state",
                 "get_presence_list",
diff --git a/tests/rest/test_profile.py b/tests/rest/test_profile.py
index 975d32f64d..ff1e92805e 100644
--- a/tests/rest/test_profile.py
+++ b/tests/rest/test_profile.py
@@ -43,7 +43,6 @@ class ProfileTestCase(unittest.TestCase):
         hs = HomeServer("test",
             db_pool=None,
             http_client=None,
-            http_server=self.mock_server,
             resource_for_client=self.mock_server,
             federation=Mock(),
             replication_layer=Mock(),

From de65c34fcf996b83febada7f786f9863c67c675d Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 14 Aug 2014 10:24:17 +0100
Subject: [PATCH 04/46] Honour the -w flag to enable the web client at
 /matrix/client

---
 synapse/app/homeserver.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index ea6f0985ef..07d38b5035 100644
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -85,17 +85,20 @@ class SynapseHomeServer(HomeServer):
 
         return pool
 
-    def create_resource_tree(self):
+    def create_resource_tree(self, web_client):
         """Create the resource tree for this Home Server.
 
         This in unduly complicated because Twisted does not support putting
         child resources more than 1 level deep at a time.
         """
-        desired_tree = (  # list of tuples containing (path_str, Resource)
-            ("/matrix/client", self.get_resource_for_web_client()),
+        desired_tree = [  # list containing (path_str, Resource)
             (CLIENT_PREFIX, self.get_resource_for_client()),
             (PREFIX, self.get_resource_for_federation())
-        )
+        ]
+        if web_client:
+            logger.info("Adding the web client.")
+            desired_tree.append(("/matrix/client",  # TODO constant please
+                                self.get_resource_for_web_client()))
 
         self.root_resource = Resource()
         # ideally we'd just use getChild and putChild but getChild doesn't work
@@ -222,7 +225,7 @@ def setup():
 
     hs.register_servlets()
 
-    hs.create_resource_tree()
+    hs.create_resource_tree(web_client=args.webclient)
     hs.start_listening(args.port)
 
     hs.build_db_pool()

From 7dc0a28e17bff5172c303497b6c1ca40af23e7e9 Mon Sep 17 00:00:00 2001
From: Emmanuel ROHEE <erohee@amdocs.com>
Date: Thu, 14 Aug 2014 11:36:11 +0200
Subject: [PATCH 05/46] Created m-file-input. A directive to open a file
 selection dialog on whatever HTML element

---
 .../fileInput/file-input-directive.js         | 43 +++++++++++++++++++
 webclient/index.html                          |  1 +
 2 files changed, 44 insertions(+)
 create mode 100644 webclient/components/fileInput/file-input-directive.js

diff --git a/webclient/components/fileInput/file-input-directive.js b/webclient/components/fileInput/file-input-directive.js
new file mode 100644
index 0000000000..9b73f877e9
--- /dev/null
+++ b/webclient/components/fileInput/file-input-directive.js
@@ -0,0 +1,43 @@
+/*
+ Copyright 2014 matrix.org
+ 
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+ 
+ http://www.apache.org/licenses/LICENSE-2.0
+ 
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+'use strict';
+
+/*
+ * Transform an element into an image file input button.
+ * Watch to the passed variable change. It will contain the selected HTML5 file object.
+ */
+angular.module('mFileInput', [])
+.directive('mFileInput', function() {
+    return {
+        restrict: 'A',
+        transclude: 'true',
+        template: '<div ng-transclude></div><input ng-hide="true" type="file" accept="image/*"/>',
+        scope: {
+            selectedFile: '=mFileInput'
+        },
+        
+        link: function(scope, element, attrs, ctrl) {
+            element.bind("click", function() {
+                element.find("input")[0].click();
+                element.find("input").bind("change", function(e) {
+                    scope.selectedFile = this.files[0];
+                    scope.$apply();
+                });
+            });
+      }
+    };
+});
\ No newline at end of file
diff --git a/webclient/index.html b/webclient/index.html
index ddc9ab5e32..f4b791ecdb 100644
--- a/webclient/index.html
+++ b/webclient/index.html
@@ -14,6 +14,7 @@
     <script src="room/room-controller.js"></script>
     <script src="rooms/rooms-controller.js"></script>
     <script src="components/matrix/matrix-service.js"></script>
+    <script src="components/fileInput/file-input-directive.js"></script>
 </head>
 
 <body>

From 28a49a9eaf79f1b05fc2f793ad264d45c017de4c Mon Sep 17 00:00:00 2001
From: Emmanuel ROHEE <erohee@amdocs.com>
Date: Thu, 14 Aug 2014 11:39:03 +0200
Subject: [PATCH 06/46] Show avatar in profile section and added a button to
 select a file (not yet wired to upload service)

---
 webclient/app.css                   | 14 ++++++++++++++
 webclient/rooms/rooms-controller.js | 12 ++++++++++--
 webclient/rooms/rooms.html          | 29 +++++++++++++++++++++++------
 3 files changed, 47 insertions(+), 8 deletions(-)

diff --git a/webclient/app.css b/webclient/app.css
index 65049c95c9..122f25c9ff 100644
--- a/webclient/app.css
+++ b/webclient/app.css
@@ -219,6 +219,20 @@ h1 {
     background-color: #fff ! important;
 }
 
+/*** Profile ***/
+
+.profile-avatar {
+    width: 160px;
+    height: 160px;
+    display:table-cell;
+    vertical-align: middle;
+}
+
+.profile-avatar img {
+    max-width: 100%;
+    max-height: 100%;
+}
+
 /******************************/
 
 .header {
diff --git a/webclient/rooms/rooms-controller.js b/webclient/rooms/rooms-controller.js
index 293ea8bc8b..b7f19bb2b5 100644
--- a/webclient/rooms/rooms-controller.js
+++ b/webclient/rooms/rooms-controller.js
@@ -16,7 +16,7 @@ limitations under the License.
 
 'use strict';
 
-angular.module('RoomsController', ['matrixService'])
+angular.module('RoomsController', ['matrixService', 'mFileInput'])
 .controller('RoomsController', ['$scope', '$location', 'matrixService',
                                function($scope, $location, matrixService) {
                                    
@@ -40,7 +40,8 @@ angular.module('RoomsController', ['matrixService'])
 
     $scope.newProfileInfo = {
         name: matrixService.config().displayName,
-        avatar: matrixService.config().avatarUrl
+        avatar: matrixService.config().avatarUrl,
+        avatarFile: undefined
     };
 
     $scope.linkedEmails = {
@@ -163,6 +164,13 @@ angular.module('RoomsController', ['matrixService'])
         );
     };
 
+
+    $scope.$watch("newProfileInfo.avatarFile", function(newValue, oldValue) {
+        if ($scope.newProfileInfo.avatarFile) {
+            //@TODO: Upload this HTML5 image file to somewhere
+        }
+    });
+
     $scope.setAvatar = function(newUrl) {
         console.log("Updating avatar to "+newUrl);
         matrixService.setProfilePictureUrl(newUrl).then(
diff --git a/webclient/rooms/rooms.html b/webclient/rooms/rooms.html
index d303e143b9..66b89caf00 100644
--- a/webclient/rooms/rooms.html
+++ b/webclient/rooms/rooms.html
@@ -3,18 +3,35 @@
     <div class="page">
     <div class="wrapper">
             
+    <div>
+        <form>
+            <table>
+                <tr>
+                    <td>
+                        <div class="profile-avatar">
+                            <img  src="{{ newProfileInfo.avatar || 'img/default-profile.jpg'}}" m-file-input="newProfileInfo.avatarFile"/>
+                        </div>
+                    </td>
+                    <td>
+                        <button  m-file-input="newProfileInfo.avatarFile">Upload new Avatar</button> 
+                        or use an existing image URL:
+                        <div>
+                            <input size="40" ng-model="newProfileInfo.avatar" ng-enter="setAvatar(newProfileInfo.avatar)" />
+                            <button ng-disabled="!newProfileInfo.avatar" ng-click="setAvatar(newProfileInfo.avatar)">Update Avatar</button>   
+                        </div>
+                    </td>
+                </tr>
+            </table>
+        </form>
+    </div>
+
     <div>
         <form>
             <input size="40" ng-model="newProfileInfo.name" ng-enter="setDisplayName(newProfileInfo.name)" />
             <button ng-disabled="!newProfileInfo.name" ng-click="setDisplayName(newProfileInfo.name)">Update Name</button>    
         </form>
     </div>
-    <div>
-        <form>
-            <input size="40" ng-model="newProfileInfo.avatar" ng-enter="setAvatar(newProfileInfo.avatar)" />
-            <button ng-disabled="!newProfileInfo.avatar" ng-click="setAvatar(newProfileInfo.avatar)">Update Avatar</button>    
-        </form>
-    </div>
+
     <br/>
 
     <div>

From 60b0fca1036fd0cd9db8bcf518171ca55c4d1af8 Mon Sep 17 00:00:00 2001
From: Emmanuel ROHEE <erohee@amdocs.com>
Date: Thu, 14 Aug 2014 11:51:31 +0200
Subject: [PATCH 07/46] Use ng-src

---
 webclient/rooms/rooms.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/webclient/rooms/rooms.html b/webclient/rooms/rooms.html
index 66b89caf00..5974bd940c 100644
--- a/webclient/rooms/rooms.html
+++ b/webclient/rooms/rooms.html
@@ -9,7 +9,7 @@
                 <tr>
                     <td>
                         <div class="profile-avatar">
-                            <img  src="{{ newProfileInfo.avatar || 'img/default-profile.jpg'}}" m-file-input="newProfileInfo.avatarFile"/>
+                            <img  ng-src="{{ newProfileInfo.avatar || 'img/default-profile.jpg' }}" m-file-input="newProfileInfo.avatarFile"/>
                         </div>
                     </td>
                     <td>

From e543d6a91df64b87ba1c178af4d77792f4909081 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 14 Aug 2014 11:17:58 +0100
Subject: [PATCH 08/46] Fixed dynamic resource mapping to clobber dummy
 Resources with the actual desired Resource in the event of a collision (as is
 the case for '/matrix/client' and '/matrix/client/api/v1')

---
 synapse/app/homeserver.py | 32 +++++++++++++++++++++++++++-----
 1 file changed, 27 insertions(+), 5 deletions(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 07d38b5035..28eb223d9a 100644
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -51,7 +51,7 @@ class SynapseHomeServer(HomeServer):
         return JsonResource()
 
     def build_resource_for_web_client(self):
-        return File("webclient")
+        return File("webclient")  # TODO configurable?
 
     def build_db_pool(self):
         """ Set up all the dbs. Since all the *.sql have IF NOT EXISTS, so we
@@ -90,8 +90,13 @@ class SynapseHomeServer(HomeServer):
 
         This in unduly complicated because Twisted does not support putting
         child resources more than 1 level deep at a time.
+
+        Args:
+            web_client (bool): True to enable the web client.
         """
-        desired_tree = [  # list containing (path_str, Resource)
+        # list containing (path_str, Resource) e.g:
+        # [ ("/aaa/bbb/cc", Resource1), ("/aaa/dummy", Resource2) ]
+        desired_tree = [
             (CLIENT_PREFIX, self.get_resource_for_client()),
             (PREFIX, self.get_resource_for_federation())
         ]
@@ -111,19 +116,36 @@ class SynapseHomeServer(HomeServer):
             last_resource = self.root_resource
             for path_seg in full_path.split('/')[1:-1]:
                 if not path_seg in last_resource.listNames():
-                    # resource doesn't exist
+                    # resource doesn't exist, so make a "dummy resource"
                     child_resource = Resource()
                     last_resource.putChild(path_seg, child_resource)
                     res_id = self._resource_id(last_resource, path_seg)
                     resource_mappings[res_id] = child_resource
                     last_resource = child_resource
                 else:
-                    # we have an existing Resource, pull it out.
+                    # we have an existing Resource, use that instead.
                     res_id = self._resource_id(last_resource, path_seg)
                     last_resource = resource_mappings[res_id]
 
-            # now attach the actual resource
+            # ===========================
+            # now attach the actual desired resource
             last_path_seg = full_path.split('/')[-1]
+
+            # if there is already a resource here, thieve its children and
+            # replace it
+            res_id = self._resource_id(last_resource, last_path_seg)
+            if res_id in resource_mappings:
+                # there is a dummy resource at this path already, which needs
+                # to be replaced with the desired resource.
+                existing_dummy_resource = resource_mappings[res_id]
+                for child_name in existing_dummy_resource.listNames():
+                    child_res_id = self._resource_id(existing_dummy_resource,
+                                                     child_name)
+                    child_resource = resource_mappings[child_res_id]
+                    # steal the children
+                    resource.putChild(child_name, child_resource)
+
+            # finally, insert the desired resource in the right place
             last_resource.putChild(last_path_seg, resource)
             res_id = self._resource_id(last_resource, last_path_seg)
             resource_mappings[res_id] = resource

From 9fd445eb92ed9def98764968a3f3fceb0ae706c9 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 14 Aug 2014 11:37:13 +0100
Subject: [PATCH 09/46] If the web client is enabled, automatically redirect
 root '/' to the web client path.

---
 synapse/app/homeserver.py | 17 +++++++++++++----
 synapse/http/server.py    | 17 +++++++++++++++++
 2 files changed, 30 insertions(+), 4 deletions(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 28eb223d9a..e2489abc86 100644
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -24,7 +24,7 @@ from twisted.python.log import PythonLoggingObserver
 from twisted.web.resource import Resource
 from twisted.web.static import File
 from twisted.web.server import Site
-from synapse.http.server import JsonResource
+from synapse.http.server import JsonResource, RootRedirect
 from synapse.http.client import TwistedHttpClient
 from synapse.rest.base import CLIENT_PREFIX
 from synapse.federation.transport import PREFIX
@@ -85,7 +85,7 @@ class SynapseHomeServer(HomeServer):
 
         return pool
 
-    def create_resource_tree(self, web_client):
+    def create_resource_tree(self, web_client, redirect_root_to_web_client):
         """Create the resource tree for this Home Server.
 
         This in unduly complicated because Twisted does not support putting
@@ -93,6 +93,9 @@ class SynapseHomeServer(HomeServer):
 
         Args:
             web_client (bool): True to enable the web client.
+            redirect_root_to_web_client (bool): True to redirect '/' to the
+            location of the web client. This does nothing if web_client is not
+            True.
         """
         # list containing (path_str, Resource) e.g:
         # [ ("/aaa/bbb/cc", Resource1), ("/aaa/dummy", Resource2) ]
@@ -105,7 +108,11 @@ class SynapseHomeServer(HomeServer):
             desired_tree.append(("/matrix/client",  # TODO constant please
                                 self.get_resource_for_web_client()))
 
-        self.root_resource = Resource()
+        if web_client and redirect_root_to_web_client:
+            self.root_resource = RootRedirect("/matrix/client")
+        else:
+            self.root_resource = Resource()
+
         # ideally we'd just use getChild and putChild but getChild doesn't work
         # unless you give it a Request object IN ADDITION to the name :/ So
         # instead, we'll store a copy of this mapping so we can actually add
@@ -247,7 +254,9 @@ def setup():
 
     hs.register_servlets()
 
-    hs.create_resource_tree(web_client=args.webclient)
+    hs.create_resource_tree(
+        web_client=args.webclient,
+        redirect_root_to_web_client=True)
     hs.start_listening(args.port)
 
     hs.build_db_pool()
diff --git a/synapse/http/server.py b/synapse/http/server.py
index 87b4fc8a5f..bad2738bde 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -22,6 +22,7 @@ from synapse.api.errors import cs_exception, CodeMessageException
 from twisted.internet import defer, reactor
 from twisted.web import server, resource
 from twisted.web.server import NOT_DONE_YET
+from twisted.web.util import redirectTo
 
 import collections
 import logging
@@ -159,6 +160,22 @@ class JsonResource(HttpServer, resource.Resource):
         return False
 
 
+class RootRedirect(resource.Resource):
+    """Redirects the root '/' path to another path."""
+
+    def __init__(self, path):
+        resource.Resource.__init__(self)
+        self.url = path
+
+    def render_GET(self, request):
+        return redirectTo(self.url, request)
+
+    def getChild(self, name, request):
+        if len(name) == 0:
+            return self  # select ourselves as the child to render
+        return resource.Resource.getChild(self, name, request)
+
+
 def respond_with_json_bytes(request, code, json_bytes, send_cors=False):
     """Sends encoded JSON in response to the given request.
 

From c75add6ec87cb548b5cc74b7e6e0c56d78d5eae9 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 14 Aug 2014 11:52:56 +0100
Subject: [PATCH 10/46] Added a urls module for keeping client and federation
 prefixes.

---
 synapse/api/urls.py             | 19 +++++++++++++++++++
 synapse/app/homeserver.py       |  5 ++---
 synapse/federation/transport.py |  4 +---
 synapse/handlers/directory.py   |  6 ------
 synapse/rest/base.py            |  3 +--
 5 files changed, 23 insertions(+), 14 deletions(-)
 create mode 100644 synapse/api/urls.py

diff --git a/synapse/api/urls.py b/synapse/api/urls.py
new file mode 100644
index 0000000000..7a6fff7d18
--- /dev/null
+++ b/synapse/api/urls.py
@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+# Copyright 2014 matrix.org
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Contains the URL paths to prefix various aspects of the server with. """
+
+CLIENT_PREFIX = "/matrix/client/api/v1"
+FEDERATION_PREFIX = "/matrix/federation/v1"
\ No newline at end of file
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index e2489abc86..7c356e7855 100644
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -26,8 +26,7 @@ from twisted.web.static import File
 from twisted.web.server import Site
 from synapse.http.server import JsonResource, RootRedirect
 from synapse.http.client import TwistedHttpClient
-from synapse.rest.base import CLIENT_PREFIX
-from synapse.federation.transport import PREFIX
+from synapse.api.urls import CLIENT_PREFIX, FEDERATION_PREFIX
 
 from daemonize import Daemonize
 
@@ -101,7 +100,7 @@ class SynapseHomeServer(HomeServer):
         # [ ("/aaa/bbb/cc", Resource1), ("/aaa/dummy", Resource2) ]
         desired_tree = [
             (CLIENT_PREFIX, self.get_resource_for_client()),
-            (PREFIX, self.get_resource_for_federation())
+            (FEDERATION_PREFIX, self.get_resource_for_federation())
         ]
         if web_client:
             logger.info("Adding the web client.")
diff --git a/synapse/federation/transport.py b/synapse/federation/transport.py
index e09dfc2670..50c3df4a5d 100644
--- a/synapse/federation/transport.py
+++ b/synapse/federation/transport.py
@@ -23,6 +23,7 @@ over a different (albeit still reliable) protocol.
 
 from twisted.internet import defer
 
+from synapse.api.urls import FEDERATION_PREFIX as PREFIX
 from synapse.util.logutils import log_function
 
 import logging
@@ -33,9 +34,6 @@ import re
 logger = logging.getLogger(__name__)
 
 
-PREFIX = "/matrix/federation/v1"
-
-
 class TransportLayer(object):
     """This is a basic implementation of the transport layer that translates
     transactions and other requests to/from HTTP.
diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py
index df98e39f69..7c89150d99 100644
--- a/synapse/handlers/directory.py
+++ b/synapse/handlers/directory.py
@@ -20,17 +20,11 @@ from ._base import BaseHandler
 from synapse.api.errors import SynapseError
 
 import logging
-import json
-import urllib
 
 
 logger = logging.getLogger(__name__)
 
 
-# TODO(erikj): This needs to be factored out somewere
-PREFIX = "/matrix/client/api/v1"
-
-
 class DirectoryHandler(BaseHandler):
 
     def __init__(self, hs):
diff --git a/synapse/rest/base.py b/synapse/rest/base.py
index 124b3a2e0c..6a88cbe866 100644
--- a/synapse/rest/base.py
+++ b/synapse/rest/base.py
@@ -14,10 +14,9 @@
 # limitations under the License.
 
 """ This module contains base REST classes for constructing REST servlets. """
+from synapse.api.urls import CLIENT_PREFIX
 import re
 
-CLIENT_PREFIX = "/matrix/client/api/v1"
-
 
 def client_path_pattern(path_regex):
     """Creates a regex compiled client path with the correct client path

From d253a3553967948e277929549a4ae6367584342f Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 14 Aug 2014 11:54:37 +0100
Subject: [PATCH 11/46] Added web client prefix

---
 synapse/api/urls.py       | 3 ++-
 synapse/app/homeserver.py | 6 +++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/synapse/api/urls.py b/synapse/api/urls.py
index 7a6fff7d18..04970adb71 100644
--- a/synapse/api/urls.py
+++ b/synapse/api/urls.py
@@ -16,4 +16,5 @@
 """Contains the URL paths to prefix various aspects of the server with. """
 
 CLIENT_PREFIX = "/matrix/client/api/v1"
-FEDERATION_PREFIX = "/matrix/federation/v1"
\ No newline at end of file
+FEDERATION_PREFIX = "/matrix/federation/v1"
+WEB_CLIENT_PREFIX = "/matrix/client"
\ No newline at end of file
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 7c356e7855..fc12e0dba5 100644
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -26,7 +26,7 @@ from twisted.web.static import File
 from twisted.web.server import Site
 from synapse.http.server import JsonResource, RootRedirect
 from synapse.http.client import TwistedHttpClient
-from synapse.api.urls import CLIENT_PREFIX, FEDERATION_PREFIX
+from synapse.api.urls import CLIENT_PREFIX, FEDERATION_PREFIX, WEB_CLIENT_PREFIX
 
 from daemonize import Daemonize
 
@@ -104,11 +104,11 @@ class SynapseHomeServer(HomeServer):
         ]
         if web_client:
             logger.info("Adding the web client.")
-            desired_tree.append(("/matrix/client",  # TODO constant please
+            desired_tree.append((WEB_CLIENT_PREFIX,
                                 self.get_resource_for_web_client()))
 
         if web_client and redirect_root_to_web_client:
-            self.root_resource = RootRedirect("/matrix/client")
+            self.root_resource = RootRedirect(WEB_CLIENT_PREFIX)
         else:
             self.root_resource = Resource()
 

From 2a793a6c422e24a064b034f7dcdab816a667af73 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 14 Aug 2014 11:57:25 +0100
Subject: [PATCH 12/46] Default error code BAD_PAGINATION for EventStreamErrors

---
 synapse/api/errors.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/synapse/api/errors.py b/synapse/api/errors.py
index 8b9766fab7..b5970c959b 100644
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -74,7 +74,10 @@ class AuthError(SynapseError):
 
 class EventStreamError(SynapseError):
     """An error raised when there a problem with the event stream."""
-    pass
+    def __init__(self, *args, **kwargs):
+        if "errcode" not in kwargs:
+            kwargs["errcode"] = Codes.BAD_PAGINATION
+        super(EventStreamError, self).__init__(*args, **kwargs)
 
 
 class LoginError(SynapseError):

From d5033849a5e42e401ddee3faa731a55ad11d11ca Mon Sep 17 00:00:00 2001
From: Emmanuel ROHEE <erohee@amdocs.com>
Date: Thu, 14 Aug 2014 13:51:35 +0200
Subject: [PATCH 13/46] BF: Use ng-src

---
 webclient/room/room.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/webclient/room/room.html b/webclient/room/room.html
index 8fc7d5d360..91e900c678 100644
--- a/webclient/room/room.html
+++ b/webclient/room/room.html
@@ -35,7 +35,7 @@
                     <div class="bubble">
                         {{ msg.content.msgtype === "m.emote" ? ("* " + (members[msg.user_id].displayname || msg.user_id) + " " + msg.content.body) : "" }}
                         {{ msg.content.msgtype === "m.text" ? msg.content.body : "" }}
-                        <img class="image" ng-hide='msg.content.msgtype !== "m.image"' src="{{ msg.content.url }}" alt="{{ msg.content.body }}"/>
+                        <img class="image" ng-hide='msg.content.msgtype !== "m.image"' ng-src="{{ msg.content.url }}" alt="{{ msg.content.body }}"/>
                     </div>
                 </td>
                 <td class="rightBlock">

From 61933f8e5278fda03e34eb25b729dd6a3987b6ab Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 14 Aug 2014 13:47:39 +0100
Subject: [PATCH 14/46] Added M_UNKNOWN_TOKEN error code and send it when there
 is an unrecognised access_token

---
 docs/client-server/specification.rst | 5 ++++-
 synapse/api/auth.py                  | 5 +++--
 synapse/api/errors.py                | 1 +
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/docs/client-server/specification.rst b/docs/client-server/specification.rst
index 7df2bb14c5..97c8587a6d 100644
--- a/docs/client-server/specification.rst
+++ b/docs/client-server/specification.rst
@@ -262,7 +262,10 @@ the error, but the keys 'error' and 'errcode' will always be present.
 Some standard error codes are below:
 
 M_FORBIDDEN:
-Forbidden access, e.g. bad access token, failed login.
+Forbidden access, e.g. joining a room without permission, failed login.
+
+M_UNKNOWN_TOKEN:
+The access token specified was not recognised.
 
 M_BAD_JSON:
 Request contained valid JSON, but it was malformed in some way, e.g. missing
diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index 8d2ba242e1..31852b29a5 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -18,7 +18,7 @@
 from twisted.internet import defer
 
 from synapse.api.constants import Membership
-from synapse.api.errors import AuthError, StoreError
+from synapse.api.errors import AuthError, StoreError, Codes
 from synapse.api.events.room import (RoomTopicEvent, RoomMemberEvent,
                                      MessageEvent, FeedbackEvent)
 
@@ -163,4 +163,5 @@ class Auth(object):
             user_id = yield self.store.get_user_by_token(token=token)
             defer.returnValue(self.hs.parse_userid(user_id))
         except StoreError:
-            raise AuthError(403, "Unrecognised access token.")
+            raise AuthError(403, "Unrecognised access token.",
+                            errcode=Codes.UNKNOWN_TOKEN)
diff --git a/synapse/api/errors.py b/synapse/api/errors.py
index b5970c959b..21ededc5ae 100644
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -27,6 +27,7 @@ class Codes(object):
     BAD_PAGINATION = "M_BAD_PAGINATION"
     UNKNOWN = "M_UNKNOWN"
     NOT_FOUND = "M_NOT_FOUND"
+    UNKNOWN_TOKEN = "M_UNKNOWN_TOKEN"
 
 
 class CodeMessageException(Exception):

From 613e468b89cac37e4537b1798aee98d824b55cb3 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 14 Aug 2014 13:57:55 +0100
Subject: [PATCH 15/46] Guess the home server URL on the login screen by
 inspecting the URL of the web client.

---
 webclient/login/login-controller.js | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/webclient/login/login-controller.js b/webclient/login/login-controller.js
index 26590da686..fa91bf4253 100644
--- a/webclient/login/login-controller.js
+++ b/webclient/login/login-controller.js
@@ -3,8 +3,16 @@ angular.module('LoginController', ['matrixService'])
                                     function($scope, $location, matrixService) {
     'use strict';
     
+    
+    // Assume that this is hosted on the home server, in which case the URL
+    // contains the home server.
+    var hs_url = $location.protocol() + "://" + $location.host();
+    if ($location.port()) {
+        hs_url += ":" + $location.port();
+    }
+    
     $scope.account = {
-        homeserver: "http://localhost:8080",
+        homeserver: hs_url,
         desired_user_name: "",
         user_id: "",
         password: "",

From 7143f358f1487d4044cc5ad64056f621a5aa2139 Mon Sep 17 00:00:00 2001
From: Emmanuel ROHEE <erohee@amdocs.com>
Date: Thu, 14 Aug 2014 14:59:33 +0200
Subject: [PATCH 16/46] Detect when the user access token is no more valid and
 log the user out in this case

---
 webclient/app-controller.js                   | 10 ++++++++--
 webclient/components/matrix/matrix-service.js | 14 ++++++++++++--
 2 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/webclient/app-controller.js b/webclient/app-controller.js
index 41055bdcd2..086fa3d946 100644
--- a/webclient/app-controller.js
+++ b/webclient/app-controller.js
@@ -55,8 +55,14 @@ angular.module('MatrixWebClientController', ['matrixService'])
         
         // And go to the login page
         $location.path("login");
-    };    
-                          
+    };
+
+    // Listen to the event indicating that the access token is no more valid.
+    // In this case, the user needs to log in again.
+    $scope.$on("M_UNKNOWN_TOKEN", function() {
+        console.log("Invalid access token -> log user out");
+        $scope.logout();
+    });
 }]);
 
    
\ No newline at end of file
diff --git a/webclient/components/matrix/matrix-service.js b/webclient/components/matrix/matrix-service.js
index f054bf301e..81ccdc2cc0 100644
--- a/webclient/components/matrix/matrix-service.js
+++ b/webclient/components/matrix/matrix-service.js
@@ -17,7 +17,7 @@ limitations under the License.
 'use strict';
 
 angular.module('matrixService', [])
-.factory('matrixService', ['$http', '$q', function($http, $q) {
+.factory('matrixService', ['$http', '$q', '$rootScope', function($http, $q, $rootScope) {
         
    /* 
     * Permanent storage of user information
@@ -60,7 +60,6 @@ angular.module('matrixService', [])
             headers: headers
         })
         .success(function(data, status, headers, config) {
-            // @TODO: We could detect a bad access token here and make an automatic logout
             deferred.resolve(data, status, headers, config);
         })
         .error(function(data, status, headers, config) {
@@ -70,6 +69,11 @@ angular.module('matrixService', [])
                 reason = JSON.stringify(data);
             }
             deferred.reject(reason, data, status, headers, config);
+
+            if (403 === status && "M_UNKNOWN_TOKEN" === data.errcode) {
+                // The access token is no more valid, broadcast the issue
+                $rootScope.$broadcast("M_UNKNOWN_TOKEN");
+            }
         });
 
         return deferred.promise;
@@ -301,6 +305,12 @@ angular.module('matrixService', [])
             return doBaseRequest(config.identityServer, "POST", path, {}, data, headers); 
         },
         
+        
+        // 
+        testLogin: function() {
+            
+        },
+        
         /****** Permanent storage of user information ******/
         
         // Returns the current config

From e37de2aef38a6de5987694f2dc5d7a74c7f623b2 Mon Sep 17 00:00:00 2001
From: "Paul \"LeoNerd\" Evans" <paul@matrix.org>
Date: Thu, 14 Aug 2014 14:05:05 +0100
Subject: [PATCH 17/46] chmod +x homeserver.py

---
 synapse/app/homeserver.py | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 synapse/app/homeserver.py

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
old mode 100644
new mode 100755

From e4061383b8bb5b8c3b250a81ea7f0d5b6dd04a0e Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 14 Aug 2014 14:07:14 +0100
Subject: [PATCH 18/46] Change relative db paths to absolute paths in case we
 daemonize.

---
 synapse/app/homeserver.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index fc12e0dba5..3429a29a6b 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -34,6 +34,7 @@ import argparse
 import logging
 import logging.config
 import sqlite3
+import os
 
 logger = logging.getLogger(__name__)
 
@@ -234,9 +235,15 @@ def setup():
 
     verbosity = int(args.verbose) if args.verbose else None
 
+    # Because if/when we daemonize we change to root dir.
+    db_name = os.path.abspath(args.db)
+    log_file = args.log_file
+    if log_file:
+        log_file = os.path.abspath(log_file)
+
     setup_logging(
         verbosity=verbosity,
-        filename=args.log_file,
+        filename=log_file,
         config_path=args.log_config,
     )
 
@@ -244,7 +251,7 @@ def setup():
 
     hs = SynapseHomeServer(
         args.host,
-        db_name=args.db
+        db_name=db_name
     )
 
     # This object doesn't need to be saved because it's set as the handler for

From 0fa05ea3314779e3e01e87c0240331825b8115a3 Mon Sep 17 00:00:00 2001
From: "Paul \"LeoNerd\" Evans" <paul@matrix.org>
Date: Thu, 14 Aug 2014 14:15:54 +0100
Subject: [PATCH 19/46] Round Presence mtime and mtime_age to nearest msec;
 avoids floats for msec values over the wire

---
 synapse/handlers/presence.py | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py
index 8bdb0fe5c7..351ff305dc 100644
--- a/synapse/handlers/presence.py
+++ b/synapse/handlers/presence.py
@@ -177,7 +177,9 @@ class PresenceHandler(BaseHandler):
             state = self._get_or_offline_usercache(target_user).get_state()
 
         if "mtime" in state:
-            state["mtime_age"] = self.clock.time_msec() - state.pop("mtime")
+            state["mtime_age"] = int(
+                self.clock.time_msec() - state.pop("mtime")
+            )
         defer.returnValue(state)
 
     @defer.inlineCallbacks
@@ -367,7 +369,9 @@ class PresenceHandler(BaseHandler):
             p["observed_user"] = observed_user
             p.update(self._get_or_offline_usercache(observed_user).get_state())
             if "mtime" in p:
-                p["mtime_age"] = self.clock.time_msec() - p.pop("mtime")
+                p["mtime_age"] = int(
+                    self.clock.time_msec() - p.pop("mtime")
+                )
 
         defer.returnValue(presence)
 
@@ -560,7 +564,9 @@ class PresenceHandler(BaseHandler):
 
         if "mtime" in state:
             state = dict(state)
-            state["mtime_age"] = self.clock.time_msec() - state.pop("mtime")
+            state["mtime_age"] = int(
+                self.clock.time_msec() - state.pop("mtime")
+            )
 
         yield self.federation.send_edu(
             destination=destination,
@@ -598,7 +604,9 @@ class PresenceHandler(BaseHandler):
             del state["user_id"]
 
             if "mtime_age" in state:
-                state["mtime"] = self.clock.time_msec() - state.pop("mtime_age")
+                state["mtime"] = int(
+                    self.clock.time_msec() - state.pop("mtime_age")
+                )
 
             statuscache = self._get_or_make_usercache(user)
 
@@ -720,6 +728,8 @@ class UserPresenceCache(object):
         content["user_id"] = user.to_string()
 
         if "mtime" in content:
-            content["mtime_age"] = clock.time_msec() - content.pop("mtime")
+            content["mtime_age"] = int(
+                clock.time_msec() - content.pop("mtime")
+            )
 
         return {"type": "m.presence", "content": content}

From 5a5f37ca17fdee8149ec0f6ce78f83259ed9d530 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 14 Aug 2014 14:29:01 +0100
Subject: [PATCH 20/46] Send forbidden codes when doing login attempts.

---
 synapse/handlers/login.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/synapse/handlers/login.py b/synapse/handlers/login.py
index ca69829d77..0220fa0604 100644
--- a/synapse/handlers/login.py
+++ b/synapse/handlers/login.py
@@ -16,7 +16,7 @@
 from twisted.internet import defer
 
 from ._base import BaseHandler
-from synapse.api.errors import LoginError
+from synapse.api.errors import LoginError, Codes
 
 import bcrypt
 import logging
@@ -51,7 +51,7 @@ class LoginHandler(BaseHandler):
         user_info = yield self.store.get_user_by_id(user_id=user)
         if not user_info:
             logger.warn("Attempted to login as %s but they do not exist.", user)
-            raise LoginError(403, "")
+            raise LoginError(403, "", errcode=Codes.FORBIDDEN)
 
         stored_hash = user_info[0]["password_hash"]
         if bcrypt.checkpw(password, stored_hash):
@@ -62,4 +62,4 @@ class LoginHandler(BaseHandler):
             defer.returnValue(token)
         else:
             logger.warn("Failed password login for user %s", user)
-            raise LoginError(403, "")
\ No newline at end of file
+            raise LoginError(403, "", errcode=Codes.FORBIDDEN)
\ No newline at end of file

From 76005c44f7ec4ea28ba0d5eecccfa64f4df6d664 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 14 Aug 2014 15:21:39 +0100
Subject: [PATCH 21/46] Added an access token interceptor to check unknown
 tokens.

---
 webclient/app.js | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/webclient/app.js b/webclient/app.js
index 651aeeaa77..f869309449 100644
--- a/webclient/app.js
+++ b/webclient/app.js
@@ -23,8 +23,8 @@ var matrixWebClient = angular.module('matrixWebClient', [
     'matrixService'
 ]);
 
-matrixWebClient.config(['$routeProvider',
-    function($routeProvider) {
+matrixWebClient.config(['$routeProvider', '$provide', '$httpProvider',
+    function($routeProvider, $provide, $httpProvider) {
         $routeProvider.
             when('/login', {
                 templateUrl: 'login/login.html',
@@ -41,6 +41,22 @@ matrixWebClient.config(['$routeProvider',
             otherwise({
                 redirectTo: '/rooms'
             });
+            
+        $provide.factory('AccessTokenInterceptor', function ($q) {
+            return {
+                responseError: function(rejection) {
+                    console.log("Rejection: " + JSON.stringify(rejection));
+                    if (rejection.status === 403 && "data" in rejection && 
+                            "errcode" in rejection.data && 
+                            rejection.data.errcode === "M_UNKNOWN_TOKEN") {
+                        console.log("TODO: Got a 403 with an unknown token. Logging out.")
+                        // TODO logout
+                    }
+                    return $q.reject(rejection);
+                }
+            };
+        });
+        $httpProvider.interceptors.push('AccessTokenInterceptor');
     }]);
 
 matrixWebClient.run(['$location', 'matrixService' , function($location, matrixService) {
@@ -75,4 +91,4 @@ matrixWebClient
         return function(text) {
             return $sce.trustAsHtml(text);
         };
-    }]);
\ No newline at end of file
+    }]);

From db3e1d73c6a81bda3b2624596ea9b3f113242d38 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 14 Aug 2014 15:36:40 +0100
Subject: [PATCH 22/46] Move the unknown token broadcast to the interceptor.
 Return the $http promise and not a wrapped one via $q. Everything now needs a
 level deeper nesting. Fixed registration and login.

---
 webclient/app.js                              | 10 ++++-----
 webclient/components/matrix/matrix-service.js | 21 +------------------
 webclient/login/login-controller.js           | 14 ++++++++-----
 webclient/login/login.html                    |  1 +
 4 files changed, 16 insertions(+), 30 deletions(-)

diff --git a/webclient/app.js b/webclient/app.js
index f869309449..0b613fa206 100644
--- a/webclient/app.js
+++ b/webclient/app.js
@@ -42,20 +42,20 @@ matrixWebClient.config(['$routeProvider', '$provide', '$httpProvider',
                 redirectTo: '/rooms'
             });
             
-        $provide.factory('AccessTokenInterceptor', function ($q) {
+        $provide.factory('AccessTokenInterceptor', ['$q', '$rootScope', 
+            function ($q, $rootScope) {
             return {
                 responseError: function(rejection) {
-                    console.log("Rejection: " + JSON.stringify(rejection));
                     if (rejection.status === 403 && "data" in rejection && 
                             "errcode" in rejection.data && 
                             rejection.data.errcode === "M_UNKNOWN_TOKEN") {
-                        console.log("TODO: Got a 403 with an unknown token. Logging out.")
-                        // TODO logout
+                        console.log("Got a 403 with an unknown token. Logging out.")
+                        $rootScope.$broadcast("M_UNKNOWN_TOKEN");
                     }
                     return $q.reject(rejection);
                 }
             };
-        });
+        }]);
         $httpProvider.interceptors.push('AccessTokenInterceptor');
     }]);
 
diff --git a/webclient/components/matrix/matrix-service.js b/webclient/components/matrix/matrix-service.js
index 81ccdc2cc0..132c996f7a 100644
--- a/webclient/components/matrix/matrix-service.js
+++ b/webclient/components/matrix/matrix-service.js
@@ -49,32 +49,13 @@ angular.module('matrixService', [])
         if (path.indexOf(prefixPath) !== 0) {
             path = prefixPath + path;
         }
-        // Do not directly return the $http instance but return a promise
-        // with enriched or cleaned information
-        var deferred = $q.defer();
-        $http({
+        return $http({
             method: method,
             url: baseUrl + path,
             params: params,
             data: data,
             headers: headers
         })
-        .success(function(data, status, headers, config) {
-            deferred.resolve(data, status, headers, config);
-        })
-        .error(function(data, status, headers, config) {
-            // Enrich the error callback with an human readable error reason
-            var reason = data.error;
-            if (!data.error) {
-                reason = JSON.stringify(data);
-            }
-            deferred.reject(reason, data, status, headers, config);
-
-            if (403 === status && "M_UNKNOWN_TOKEN" === data.errcode) {
-                // The access token is no more valid, broadcast the issue
-                $rootScope.$broadcast("M_UNKNOWN_TOKEN");
-            }
-        });
 
         return deferred.promise;
     };
diff --git a/webclient/login/login-controller.js b/webclient/login/login-controller.js
index fa91bf4253..c519f7698c 100644
--- a/webclient/login/login-controller.js
+++ b/webclient/login/login-controller.js
@@ -39,14 +39,13 @@ angular.module('LoginController', ['matrixService'])
         }
 
         matrixService.register($scope.account.desired_user_name, $scope.account.pwd1).then(
-            function(data) {
+            function(response) {
                 $scope.feedback = "Success";
-
                 // Update the current config 
                 var config = matrixService.config();
                 angular.extend(config, {
-                    access_token: data.access_token,
-                    user_id: data.user_id
+                    access_token: response.data.access_token,
+                    user_id: response.data.user_id
                 });
                 matrixService.setConfig(config);
 
@@ -74,7 +73,7 @@ angular.module('LoginController', ['matrixService'])
                     matrixService.setConfig({
                         homeserver: $scope.account.homeserver,
                         user_id: $scope.account.user_id,
-                        access_token: response.access_token
+                        access_token: response.data.access_token
                     });
                     matrixService.saveConfig();
                     $location.path("rooms");
@@ -82,6 +81,11 @@ angular.module('LoginController', ['matrixService'])
                 else {
                     $scope.feedback = "Failed to login: " + JSON.stringify(response);
                 }
+            },
+            function(error) {
+                if (error.data.errcode === "M_FORBIDDEN") {
+                    $scope.login_error_msg = "Incorrect username or password.";
+                }
             }
         );
     };
diff --git a/webclient/login/login.html b/webclient/login/login.html
index 508ff5e4bf..f02dde89a6 100644
--- a/webclient/login/login.html
+++ b/webclient/login/login.html
@@ -22,6 +22,7 @@
     <h3>Got an account?</h3>
     <form novalidate>
         <!-- Login with an registered user -->
+        <div>{{ login_error_msg }} </div>
         <div>
             <input id="user_id" size="70" type="text" auto-focus ng-model="account.user_id" placeholder="User ID (ex:@bob:localhost)"/>
             <br />

From 24bd133d9d4c9a30c4609cf6d55f02ab6f05c142 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 14 Aug 2014 15:43:16 +0100
Subject: [PATCH 23/46] Added extra nesting .data and rename callback to be
 response not data

---
 webclient/login/login-controller.js |  4 +-
 webclient/rooms/rooms-controller.js | 58 ++++++++++++++---------------
 2 files changed, 31 insertions(+), 31 deletions(-)

diff --git a/webclient/login/login-controller.js b/webclient/login/login-controller.js
index c519f7698c..015868b0b9 100644
--- a/webclient/login/login-controller.js
+++ b/webclient/login/login-controller.js
@@ -68,7 +68,7 @@ angular.module('LoginController', ['matrixService'])
         // try to login
         matrixService.login($scope.account.user_id, $scope.account.password).then(
             function(response) {
-                if ("access_token" in response) {
+                if ("access_token" in response.data) {
                     $scope.feedback = "Login successful.";
                     matrixService.setConfig({
                         homeserver: $scope.account.homeserver,
@@ -79,7 +79,7 @@ angular.module('LoginController', ['matrixService'])
                     $location.path("rooms");
                 }
                 else {
-                    $scope.feedback = "Failed to login: " + JSON.stringify(response);
+                    $scope.feedback = "Failed to login: " + JSON.stringify(response.data);
                 }
             },
             function(error) {
diff --git a/webclient/rooms/rooms-controller.js b/webclient/rooms/rooms-controller.js
index b7f19bb2b5..d0924f5887 100644
--- a/webclient/rooms/rooms-controller.js
+++ b/webclient/rooms/rooms-controller.js
@@ -75,18 +75,18 @@ angular.module('RoomsController', ['matrixService', 'mFileInput'])
         // List all rooms joined or been invited to
         $scope.rooms = matrixService.rooms();
         matrixService.rooms().then(
-            function(data) {
-                data = assignRoomAliases(data);
+            function(response) {
+                var data = assignRoomAliases(response.data);
                 $scope.feedback = "Success";
                 $scope.rooms = data;
             },
-            function(reason) {
-                $scope.feedback = "Failure: " + reason;
+            function(error) {
+                $scope.feedback = "Failure: " + error.data;
             });
         
         matrixService.publicRooms().then(
-            function(data) {
-                $scope.public_rooms = assignRoomAliases(data.chunk);
+            function(response) {
+                $scope.public_rooms = assignRoomAliases(response.data.chunk);
             }
         );
     };
@@ -101,14 +101,14 @@ angular.module('RoomsController', ['matrixService', 'mFileInput'])
         matrixService.create(room_id, visibility).then(
             function(response) { 
                 // This room has been created. Refresh the rooms list
-                console.log("Created room " + response.room_alias + " with id: "+
-                response.room_id);
+                console.log("Created room " + response.data.room_alias + " with id: "+
+                response.data.room_id);
                 matrixService.createRoomIdToAliasMapping(
-                    response.room_id, response.room_alias);
+                    response.data.room_id, response.data.room_alias);
                 $scope.refresh();
             },
-            function(reason) {
-                $scope.feedback = "Failure: " + reason;
+            function(error) {
+                $scope.feedback = "Failure: " + error.data;
             });
     };
     
@@ -118,17 +118,17 @@ angular.module('RoomsController', ['matrixService', 'mFileInput'])
         //$location.path("room/" + room_id);
         matrixService.join(room_id).then(
             function(response) {
-                if (response.hasOwnProperty("room_id")) {
-                    if (response.room_id != room_id) {
-                        $location.path("room/" + response.room_id);
+                if (response.data.hasOwnProperty("room_id")) {
+                    if (response.data.room_id != room_id) {
+                        $location.path("room/" + response.data.room_id);
                         return;
                      }
                 }
 
                 $location.path("room/" + room_id);
             },
-            function(reason) {
-                $scope.feedback = "Can't join room: " + reason;
+            function(error) {
+                $scope.feedback = "Can't join room: " + error.data;
             }
         );
     };
@@ -136,15 +136,15 @@ angular.module('RoomsController', ['matrixService', 'mFileInput'])
     $scope.joinAlias = function(room_alias) {
         matrixService.joinAlias(room_alias).then(
             function(response) {
-                if (response.hasOwnProperty("room_id")) {
-                    $location.path("room/" + response.room_id);
+                if (response.data.hasOwnProperty("room_id")) {
+                    $location.path("room/" + response.data.room_id);
                     return;
                 } else {
                     // TODO (erikj): Do something here?
                 }
             },
-            function(reason) {
-                $scope.feedback = "Can't join room: " + reason;
+            function(error) {
+                $scope.feedback = "Can't join room: " + error.data;
             }
         );
     };
@@ -158,8 +158,8 @@ angular.module('RoomsController', ['matrixService', 'mFileInput'])
                 matrixService.setConfig(config);
                 matrixService.saveConfig();
             },
-            function(reason) {
-                $scope.feedback = "Can't update display name: " + reason;
+            function(error) {
+                $scope.feedback = "Can't update display name: " + error.data;
             }
         );
     };
@@ -182,8 +182,8 @@ angular.module('RoomsController', ['matrixService', 'mFileInput'])
                 matrixService.setConfig(config);
                 matrixService.saveConfig();
             },
-            function(reason) {
-                $scope.feedback = "Can't update avatar: " + reason;
+            function(error) {
+                $scope.feedback = "Can't update avatar: " + error.data;
             }
         );
     };
@@ -191,8 +191,8 @@ angular.module('RoomsController', ['matrixService', 'mFileInput'])
     $scope.linkEmail = function(email) {
         matrixService.linkEmail(email).then(
             function(response) {
-                if (response.success === true) {
-                    $scope.linkedEmails.authTokenId = response.tokenId;
+                if (response.data.success === true) {
+                    $scope.linkedEmails.authTokenId = response.data.tokenId;
                     $scope.emailFeedback = "You have been sent an email.";
                     $scope.linkedEmails.emailBeingAuthed = email;
                 }
@@ -200,8 +200,8 @@ angular.module('RoomsController', ['matrixService', 'mFileInput'])
                     $scope.emailFeedback = "Failed to send email.";
                 }
             },
-            function(reason) {
-                $scope.emailFeedback = "Can't send email: " + reason;
+            function(error) {
+                $scope.emailFeedback = "Can't send email: " + error.data;
             }
         );
     };
@@ -214,7 +214,7 @@ angular.module('RoomsController', ['matrixService', 'mFileInput'])
         }
         matrixService.authEmail(matrixService.config().user_id, tokenId, code).then(
             function(response) {
-                if ("success" in response && response.success === false) {
+                if ("success" in response.data && response.data.success === false) {
                     $scope.emailFeedback = "Failed to authenticate email.";
                     return;
                 }

From 40c998336d3562018162bfc14b4dade1139f414c Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 14 Aug 2014 15:47:38 +0100
Subject: [PATCH 24/46] Finish up room controller too. May have missed one or
 two, but testing didn't pick anything up.

---
 webclient/room/room-controller.js | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/webclient/room/room-controller.js b/webclient/room/room-controller.js
index 470f41521a..cec19f7994 100644
--- a/webclient/room/room-controller.js
+++ b/webclient/room/room-controller.js
@@ -99,8 +99,8 @@ angular.module('RoomController', [])
                     function(response) {
                         var member = $scope.members[chunk.target_user_id];
                         if (member !== undefined) {
-                            console.log("Updated displayname "+chunk.target_user_id+" to " + response.displayname);
-                            member.displayname = response.displayname;
+                            console.log("Updated displayname "+chunk.target_user_id+" to " + response.data.displayname);
+                            member.displayname = response.data.displayname;
                         }
                     }
                 ); 
@@ -108,8 +108,8 @@ angular.module('RoomController', [])
                     function(response) {
                          var member = $scope.members[chunk.target_user_id];
                          if (member !== undefined) {
-                            console.log("Updated image for "+chunk.target_user_id+" to " + response.avatar_url);
-                            member.avatar_url = response.avatar_url;
+                            console.log("Updated image for "+chunk.target_user_id+" to " + response.data.avatar_url);
+                            member.avatar_url = response.data.avatar_url;
                          }
                     }
                 );
@@ -171,8 +171,8 @@ angular.module('RoomController', [])
                 console.log("Sent message");
                 $scope.textInput = "";
             },
-            function(reason) {
-                $scope.feedback = "Failed to send: " + reason;
+            function(error) {
+                $scope.feedback = "Failed to send: " + error.data.error;
             });               
     };
 
@@ -190,13 +190,13 @@ angular.module('RoomController', [])
                 // Get the current member list
                 matrixService.getMemberList($scope.room_id).then(
                     function(response) {
-                        for (var i = 0; i < response.chunk.length; i++) {
-                            var chunk = response.chunk[i];
+                        for (var i = 0; i < response.data.chunk.length; i++) {
+                            var chunk = response.data.chunk[i];
                             updateMemberList(chunk);
                         }
                     },
-                    function(reason) {
-                        $scope.feedback = "Failed get member list: " + reason;
+                    function(error) {
+                        $scope.feedback = "Failed get member list: " + error.data.error;
                     }
                 );
             },
@@ -224,8 +224,8 @@ angular.module('RoomController', [])
                 console.log("Left room ");
                 $location.path("rooms");
             },
-            function(reason) {
-                $scope.feedback = "Failed to leave room: " + reason;
+            function(error) {
+                $scope.feedback = "Failed to leave room: " + error.data.error;
             });
     };
 
@@ -234,8 +234,8 @@ angular.module('RoomController', [])
             function() {
                 console.log("Image sent");
             },
-            function(reason) {
-                $scope.feedback = "Failed to send image: " + reason;
+            function(error) {
+                $scope.feedback = "Failed to send image: " + error.data.error;
             });
     };
 

From fb93e14e530d6dfddf22d77eb42be5758f2d50f5 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 14 Aug 2014 16:03:04 +0100
Subject: [PATCH 25/46] Be more helpful when failing to register/login, stating
 why (communication error, user in user, wrong credentials, etc). Make the HS
 send M_USER_IN_USE.

---
 synapse/storage/registration.py     |  4 ++--
 webclient/login/login-controller.js | 20 ++++++++++++++++----
 2 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py
index 68cdfbb4ca..b1e4196435 100644
--- a/synapse/storage/registration.py
+++ b/synapse/storage/registration.py
@@ -17,7 +17,7 @@ from twisted.internet import defer
 
 from sqlite3 import IntegrityError
 
-from synapse.api.errors import StoreError
+from synapse.api.errors import StoreError, Codes
 
 from ._base import SQLBaseStore
 
@@ -73,7 +73,7 @@ class RegistrationStore(SQLBaseStore):
                         "VALUES (?,?,?)",
                         [user_id, password_hash, now])
         except IntegrityError:
-            raise StoreError(400, "User ID already taken.")
+            raise StoreError(400, "User ID already taken.", errcode=Codes.USER_IN_USE)
 
         # it's possible for this to get a conflict, but only for a single user
         # since tokens are namespaced based on their user ID
diff --git a/webclient/login/login-controller.js b/webclient/login/login-controller.js
index 015868b0b9..53756be9ea 100644
--- a/webclient/login/login-controller.js
+++ b/webclient/login/login-controller.js
@@ -55,8 +55,15 @@ angular.module('LoginController', ['matrixService'])
                  // Go to the user's rooms list page
                 $location.path("rooms");
             },
-            function(reason) {
-                $scope.feedback = "Failure: " + reason;
+            function(error) {
+                if (error.data) {
+                    if (error.data.errcode === "M_USER_IN_USE") {
+                        $scope.feedback = "Username already taken.";
+                    }
+                }
+                else if (error.status === 0) {
+                    $scope.feedback = "Unable to talk to the server.";
+                }
             });
     };
 
@@ -83,8 +90,13 @@ angular.module('LoginController', ['matrixService'])
                 }
             },
             function(error) {
-                if (error.data.errcode === "M_FORBIDDEN") {
-                    $scope.login_error_msg = "Incorrect username or password.";
+                if (error.data) {
+                    if (error.data.errcode === "M_FORBIDDEN") {
+                        $scope.login_error_msg = "Incorrect username or password.";
+                    }
+                }
+                else if (error.status === 0) {
+                    $scope.login_error_msg = "Unable to talk to the server.";
                 }
             }
         );

From 657ab9ba9d0b1afda959267d05479983ce8bfaa4 Mon Sep 17 00:00:00 2001
From: "Paul \"LeoNerd\" Evans" <paul@matrix.org>
Date: Thu, 14 Aug 2014 16:06:02 +0100
Subject: [PATCH 26/46] Put some DEBUG logging in lockutils.py so we can debug
 roomlocks

---
 synapse/util/lockutils.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/synapse/util/lockutils.py b/synapse/util/lockutils.py
index 758be0b901..d0bb50d035 100644
--- a/synapse/util/lockutils.py
+++ b/synapse/util/lockutils.py
@@ -24,9 +24,10 @@ logger = logging.getLogger(__name__)
 
 class Lock(object):
 
-    def __init__(self, deferred):
+    def __init__(self, deferred, key):
         self._deferred = deferred
         self.released = False
+        self.key = key
 
     def release(self):
         self.released = True
@@ -38,9 +39,10 @@ class Lock(object):
             self.release()
 
     def __enter__(self):
-            return self
+        return self
 
     def __exit__(self, type, value, traceback):
+        logger.debug("Releasing lock for key=%r", self.key)
         self.release()
 
 
@@ -63,6 +65,10 @@ class LockManager(object):
         self._lock_deferreds[key] = new_deferred
 
         if old_deferred:
+            logger.debug("Queueing on lock for key=%r", key)
             yield old_deferred
+            logger.debug("Obtained lock for key=%r", key)
+        else:
+            logger.debug("Entering uncontended lock for key=%r", key)
 
-        defer.returnValue(Lock(new_deferred))
+        defer.returnValue(Lock(new_deferred, key))

From 6f925f61ff69392d1fbc478150de99ecad7ca6f5 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 14 Aug 2014 16:08:14 +0100
Subject: [PATCH 27/46] Auto-correct the username when logging in if there
 isn't an @

---
 webclient/login/login-controller.js | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/webclient/login/login-controller.js b/webclient/login/login-controller.js
index 53756be9ea..826a533873 100644
--- a/webclient/login/login-controller.js
+++ b/webclient/login/login-controller.js
@@ -68,6 +68,12 @@ angular.module('LoginController', ['matrixService'])
     };
 
     $scope.login = function() {
+        if ($scope.account.user_id.indexOf("@") !== 0) {
+            // technically should be the host of account.homeserver
+            $scope.account.user_id = "@" + $scope.account.user_id + ":" + 
+                                     $location.host()
+        }
+    
         matrixService.setConfig({
             homeserver: $scope.account.homeserver,
             user_id: $scope.account.user_id

From 93a8be7befa291c766dc2a1c2274c2a5f9de9c0a Mon Sep 17 00:00:00 2001
From: "Paul \"LeoNerd\" Evans" <paul@matrix.org>
Date: Thu, 14 Aug 2014 16:15:53 +0100
Subject: [PATCH 28/46] We really don't need debug logging of all the SQL
 statements we execute; we're quite happy these all work now

---
 synapse/storage/stream.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py
index 1dedffac49..47a1f2c45a 100644
--- a/synapse/storage/stream.py
+++ b/synapse/storage/stream.py
@@ -72,7 +72,6 @@ class StreamStore(SQLBaseStore):
             "messages", query, query_args, from_pkey, to_pkey, limit=limit
         )
 
-        logger.debug("[SQL] %s : %s", query, query_args)
         cursor = txn.execute(query, query_args)
         return self._as_events(cursor, MessagesTable, from_pkey)
 
@@ -110,7 +109,6 @@ class StreamStore(SQLBaseStore):
             limit=limit, group_by=" GROUP BY messages.id "
         )
 
-        logger.debug("[SQL] %s : %s", query, query_args)
         cursor = txn.execute(query, query_args)
 
         # convert the result set into events
@@ -195,7 +193,6 @@ class StreamStore(SQLBaseStore):
             "feedback", query, query_args, from_pkey, to_pkey, limit=limit
         )
 
-        logger.debug("[SQL] %s : %s", query, query_args)
         cursor = txn.execute(query, query_args)
         return self._as_events(cursor, FeedbackTable, from_pkey)
 
@@ -227,7 +224,6 @@ class StreamStore(SQLBaseStore):
             "room_data", query, query_args, from_pkey, to_pkey, limit=limit
         )
 
-        logger.debug("[SQL] %s : %s", query, query_args)
         cursor = txn.execute(query, query_args)
         return self._as_events(cursor, RoomDataTable, from_pkey)
 

From 53147e5ae4c02f1d9bc4a4b5242a8ac1f476b1b8 Mon Sep 17 00:00:00 2001
From: "Paul \"LeoNerd\" Evans" <paul@matrix.org>
Date: Thu, 14 Aug 2014 16:22:08 +0100
Subject: [PATCH 29/46] Reflect user's messages up to themselves before pushing
 it to federatoin; also release roomlock before touching federation so we
 don't halt progress on the world

---
 synapse/handlers/room.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index eae40765b3..5d0379254b 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -94,10 +94,10 @@ class MessageHandler(BaseHandler):
                 event.room_id
             )
 
-            yield self.hs.get_federation().handle_new_event(event)
-
             self.notifier.on_new_room_event(event, store_id)
 
+        yield self.hs.get_federation().handle_new_event(event)
+
     @defer.inlineCallbacks
     def get_messages(self, user_id=None, room_id=None, pagin_config=None,
                      feedback=False):

From ca3747fb2f9a1e2e6d5d9e55cd0760e373b57a90 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 14 Aug 2014 16:29:24 +0100
Subject: [PATCH 30/46] hs: Make /login accept full user IDs or just local
 parts. webclient: Only enable Register button when both password fields
 match.

---
 synapse/handlers/login.py           | 6 +++++-
 webclient/login/login-controller.js | 6 ------
 webclient/login/login.html          | 2 +-
 3 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/synapse/handlers/login.py b/synapse/handlers/login.py
index 0220fa0604..5c7d503a24 100644
--- a/synapse/handlers/login.py
+++ b/synapse/handlers/login.py
@@ -16,6 +16,7 @@
 from twisted.internet import defer
 
 from ._base import BaseHandler
+from synapse.types import UserID
 from synapse.api.errors import LoginError, Codes
 
 import bcrypt
@@ -35,7 +36,7 @@ class LoginHandler(BaseHandler):
         """Login as the specified user with the specified password.
 
         Args:
-            user (str): The user ID.
+            user (str): The user ID or username.
             password (str): The password.
         Returns:
             The newly allocated access token.
@@ -47,6 +48,9 @@ class LoginHandler(BaseHandler):
         if not hasattr(self, "reg_handler"):
             self.reg_handler = self.hs.get_handlers().registration_handler
 
+        if not user.startswith('@'):
+            user = UserID.create_local(user, self.hs).to_string()
+
         # pull out the hash for this user if they exist
         user_info = yield self.store.get_user_by_id(user_id=user)
         if not user_info:
diff --git a/webclient/login/login-controller.js b/webclient/login/login-controller.js
index 826a533873..53756be9ea 100644
--- a/webclient/login/login-controller.js
+++ b/webclient/login/login-controller.js
@@ -68,12 +68,6 @@ angular.module('LoginController', ['matrixService'])
     };
 
     $scope.login = function() {
-        if ($scope.account.user_id.indexOf("@") !== 0) {
-            // technically should be the host of account.homeserver
-            $scope.account.user_id = "@" + $scope.account.user_id + ":" + 
-                                     $location.host()
-        }
-    
         matrixService.setConfig({
             homeserver: $scope.account.homeserver,
             user_id: $scope.account.user_id
diff --git a/webclient/login/login.html b/webclient/login/login.html
index f02dde89a6..0fbeeabed7 100644
--- a/webclient/login/login.html
+++ b/webclient/login/login.html
@@ -15,7 +15,7 @@
         <!-- New user registration -->
         <div>
             <br/>
-            <button ng-click="register()" ng-disabled="!account.desired_user_name || !account.homeserver || !account.identityServer || !account.pwd1 || !account.pwd2">Register</button>
+            <button ng-click="register()" ng-disabled="!account.desired_user_name || !account.homeserver || !account.identityServer || !account.pwd1 || !account.pwd2 || account.pwd1 !== account.pwd2">Register</button>
         </div>
     </form>
 

From fef3183461a60715cb5fb0638abcde335c61db82 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 14 Aug 2014 16:40:15 +0100
Subject: [PATCH 31/46] Pass back the user_id in the response to /login in case
 it has changed. Store and use that on the webclient rather than the input
 field.

---
 synapse/handlers/login.py           | 6 +-----
 synapse/rest/login.py               | 6 ++++++
 webclient/login/login-controller.js | 2 +-
 webclient/login/login.html          | 2 +-
 4 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/synapse/handlers/login.py b/synapse/handlers/login.py
index 5c7d503a24..0220fa0604 100644
--- a/synapse/handlers/login.py
+++ b/synapse/handlers/login.py
@@ -16,7 +16,6 @@
 from twisted.internet import defer
 
 from ._base import BaseHandler
-from synapse.types import UserID
 from synapse.api.errors import LoginError, Codes
 
 import bcrypt
@@ -36,7 +35,7 @@ class LoginHandler(BaseHandler):
         """Login as the specified user with the specified password.
 
         Args:
-            user (str): The user ID or username.
+            user (str): The user ID.
             password (str): The password.
         Returns:
             The newly allocated access token.
@@ -48,9 +47,6 @@ class LoginHandler(BaseHandler):
         if not hasattr(self, "reg_handler"):
             self.reg_handler = self.hs.get_handlers().registration_handler
 
-        if not user.startswith('@'):
-            user = UserID.create_local(user, self.hs).to_string()
-
         # pull out the hash for this user if they exist
         user_info = yield self.store.get_user_by_id(user_id=user)
         if not user_info:
diff --git a/synapse/rest/login.py b/synapse/rest/login.py
index 88a3218332..bcf63fd2ab 100644
--- a/synapse/rest/login.py
+++ b/synapse/rest/login.py
@@ -16,6 +16,7 @@
 from twisted.internet import defer
 
 from synapse.api.errors import SynapseError
+from synapse.types import UserID
 from base import RestServlet, client_path_pattern
 
 import json
@@ -45,12 +46,17 @@ class LoginRestServlet(RestServlet):
 
     @defer.inlineCallbacks
     def do_password_login(self, login_submission):
+        if not login_submission["user"].startswith('@'):
+            login_submission["user"] = UserID.create_local(
+                login_submission["user"], self.hs).to_string()
+
         handler = self.handlers.login_handler
         token = yield handler.login(
             user=login_submission["user"],
             password=login_submission["password"])
 
         result = {
+            "user_id": login_submission["user"],  # may have changed
             "access_token": token,
             "home_server": self.hs.hostname,
         }
diff --git a/webclient/login/login-controller.js b/webclient/login/login-controller.js
index 53756be9ea..8bd6a4e84f 100644
--- a/webclient/login/login-controller.js
+++ b/webclient/login/login-controller.js
@@ -79,7 +79,7 @@ angular.module('LoginController', ['matrixService'])
                     $scope.feedback = "Login successful.";
                     matrixService.setConfig({
                         homeserver: $scope.account.homeserver,
-                        user_id: $scope.account.user_id,
+                        user_id: response.data.user_id,
                         access_token: response.data.access_token
                     });
                     matrixService.saveConfig();
diff --git a/webclient/login/login.html b/webclient/login/login.html
index 0fbeeabed7..a8b2b1f12d 100644
--- a/webclient/login/login.html
+++ b/webclient/login/login.html
@@ -24,7 +24,7 @@
         <!-- Login with an registered user -->
         <div>{{ login_error_msg }} </div>
         <div>
-            <input id="user_id" size="70" type="text" auto-focus ng-model="account.user_id" placeholder="User ID (ex:@bob:localhost)"/>
+            <input id="user_id" size="70" type="text" auto-focus ng-model="account.user_id" placeholder="User ID (ex:@bob:localhost or bob)"/>
             <br />
             <input id="password" size="70" type="password" ng-model="account.password" placeholder="Password"/><br />
             <br/>

From 30da8c81c761a1f58c9643f41450240bfe1d6cc5 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 14 Aug 2014 17:23:47 +0100
Subject: [PATCH 32/46] webclient: You can now paginate in rooms. Defaults to
 10 messages, with a button to get more (needs to be hooked into
 infini-scrolling).

---
 docs/client-server/specification.rst          |  3 +
 webclient/components/matrix/matrix-service.js | 11 +++
 webclient/room/room-controller.js             | 80 ++++++++++++++-----
 webclient/room/room.html                      |  1 +
 4 files changed, 74 insertions(+), 21 deletions(-)

diff --git a/docs/client-server/specification.rst b/docs/client-server/specification.rst
index 97c8587a6d..b82093f2d3 100644
--- a/docs/client-server/specification.rst
+++ b/docs/client-server/specification.rst
@@ -414,6 +414,9 @@ The server checks this, finds it is valid, and returns:
 {
   "access_token": "abcdef0123456789"
 }
+The server may optionally return "user_id" to confirm or change the user's ID.
+This is particularly useful if the home server wishes to support localpart entry
+of usernames (e.g. "bob" rather than "@bob:matrix.org").
 
 OAuth2-based
 ------------
diff --git a/webclient/components/matrix/matrix-service.js b/webclient/components/matrix/matrix-service.js
index 132c996f7a..6d66111469 100644
--- a/webclient/components/matrix/matrix-service.js
+++ b/webclient/components/matrix/matrix-service.js
@@ -212,6 +212,17 @@ angular.module('matrixService', [])
             path = path.replace("$room_id", room_id);
             return doRequest("GET", path);
         },
+        
+        paginateBackMessages: function(room_id, from_token, limit) {
+            var path = "/rooms/$room_id/messages/list";
+            path = path.replace("$room_id", room_id);
+            var params = {
+                from: from_token,
+                to: "START",
+                limit: limit
+            };
+            return doRequest("GET", path, params);
+        },
 
         // get a list of public rooms on your home server
         publicRooms: function() {
diff --git a/webclient/room/room-controller.js b/webclient/room/room-controller.js
index cec19f7994..8003105654 100644
--- a/webclient/room/room-controller.js
+++ b/webclient/room/room-controller.js
@@ -18,11 +18,14 @@ angular.module('RoomController', [])
 .controller('RoomController', ['$scope', '$http', '$timeout', '$routeParams', '$location', 'matrixService',
                                function($scope, $http, $timeout, $routeParams, $location, matrixService) {
    'use strict';
+    var MESSAGES_PER_PAGINATION = 10;
     $scope.room_id = $routeParams.room_id;
     $scope.room_alias = matrixService.getRoomIdToAliasMapping($scope.room_id);
     $scope.state = {
         user_id: matrixService.config().user_id,
-        events_from: "START"
+        events_from: "END", // when to start the event stream from.
+        earliest_token: "END", // stores how far back we've paginated.
+        can_paginate: true
     };
     $scope.messages = [];
     $scope.members = {};
@@ -30,6 +33,53 @@ angular.module('RoomController', [])
 
     $scope.imageURLToSend = "";
     $scope.userIDToInvite = "";
+    
+    var scrollToBottom = function() {
+        $timeout(function() {
+            var objDiv = document.getElementsByClassName("messageTableWrapper")[0];
+            objDiv.scrollTop = objDiv.scrollHeight;
+        },0);
+    };
+    
+    var parseChunk = function(chunks, appendToStart) {
+        for (var i = 0; i < chunks.length; i++) {
+            var chunk = chunks[i];
+            if (chunk.room_id == $scope.room_id && chunk.type == "m.room.message") {
+                if ("membership_target" in chunk.content) {
+                    chunk.user_id = chunk.content.membership_target;
+                }
+                if (appendToStart) {
+                    $scope.messages.unshift(chunk);
+                }
+                else {
+                    $scope.messages.push(chunk);
+                    scrollToBottom();
+                }
+            }
+            else if (chunk.room_id == $scope.room_id && chunk.type == "m.room.member") {
+                updateMemberList(chunk);
+            }
+            else if (chunk.type === "m.presence") {
+                updatePresence(chunk);
+            }
+        }
+    };
+    
+    var paginate = function(numItems) {
+        matrixService.paginateBackMessages($scope.room_id, $scope.state.earliest_token, numItems).then(
+            function(response) {
+                parseChunk(response.data.chunk, true);
+                $scope.state.earliest_token = response.data.end;
+                if (response.data.chunk.length < MESSAGES_PER_PAGINATION) {
+                    // no more messages to paginate :(
+                    $scope.state.can_paginate = false;
+                }
+            },
+            function(error) {
+                console.log("paginateBackMessages Ruh roh: " + JSON.stringify(error));
+            }
+        )
+    };
 
     var shortPoll = function() {
         $http.get(matrixService.config().homeserver + matrixService.prefix + "/events", {
@@ -41,28 +91,10 @@ angular.module('RoomController', [])
             .then(function(response) {
                 console.log("Got response from "+$scope.state.events_from+" to "+response.data.end);
                 $scope.state.events_from = response.data.end;
-
                 $scope.feedback = "";
 
-                for (var i = 0; i < response.data.chunk.length; i++) {
-                    var chunk = response.data.chunk[i];
-                    if (chunk.room_id == $scope.room_id && chunk.type == "m.room.message") {
-                        if ("membership_target" in chunk.content) {
-                            chunk.user_id = chunk.content.membership_target;
-                        }
-                        $scope.messages.push(chunk);
-                        $timeout(function() {
-                            var objDiv = document.getElementsByClassName("messageTableWrapper")[0];
-                            objDiv.scrollTop = objDiv.scrollHeight;
-                        },0);
-                    }
-                    else if (chunk.room_id == $scope.room_id && chunk.type == "m.room.member") {
-                        updateMemberList(chunk);
-                    }
-                    else if (chunk.type === "m.presence") {
-                        updatePresence(chunk);
-                    }
-                }
+                parseChunk(response.data.chunk, false);
+                
                 if ($scope.stopPoll) {
                     console.log("Stopping polling.");
                 }
@@ -199,6 +231,8 @@ angular.module('RoomController', [])
                         $scope.feedback = "Failed get member list: " + error.data.error;
                     }
                 );
+                
+                paginate(MESSAGES_PER_PAGINATION);
             },
             function(reason) {
                 $scope.feedback = "Can't join room: " + reason;
@@ -238,6 +272,10 @@ angular.module('RoomController', [])
                 $scope.feedback = "Failed to send image: " + error.data.error;
             });
     };
+    
+    $scope.loadMoreHistory = function() {
+        paginate(MESSAGES_PER_PAGINATION);
+    };
 
     $scope.$on('$destroy', function(e) {
         console.log("onDestroyed: Stopping poll.");
diff --git a/webclient/room/room.html b/webclient/room/room.html
index 91e900c678..0f86a158ec 100644
--- a/webclient/room/room.html
+++ b/webclient/room/room.html
@@ -86,6 +86,7 @@
                     <button ng-click="inviteUser(userIDToInvite)">Invite</button>
             </span>
             <button ng-click="leaveRoom()">Leave</button>
+            <button ng-click="loadMoreHistory()" ng-disabled="!state.can_paginate">MOAR HISTORY</button>
         </div>
     </div>
 

From f5973d8ddb1d972221370022459e9c750c079ad8 Mon Sep 17 00:00:00 2001
From: Emmanuel ROHEE <erohee@amdocs.com>
Date: Thu, 14 Aug 2014 18:38:42 +0200
Subject: [PATCH 33/46] Create a temporary upload service server side (by
 hacking demos/webserver.py) and client side with an angularjs service
 component.

---
 demo/webserver.py                             | 25 +++++++++-
 .../fileUpload/file-upload-service.js         | 47 +++++++++++++++++++
 2 files changed, 71 insertions(+), 1 deletion(-)
 create mode 100644 webclient/components/fileUpload/file-upload-service.js

diff --git a/demo/webserver.py b/demo/webserver.py
index 78f3213540..875095c877 100644
--- a/demo/webserver.py
+++ b/demo/webserver.py
@@ -2,9 +2,32 @@ import argparse
 import BaseHTTPServer
 import os
 import SimpleHTTPServer
+import cgi, logging
 
 from daemonize import Daemonize
 
+class SimpleHTTPRequestHandlerWithPOST(SimpleHTTPServer.SimpleHTTPRequestHandler):
+    UPLOAD_PATH = "upload"
+
+    """
+    Accept all post request as file upload
+    """
+    def do_POST(self):
+
+        path = os.path.join(self.UPLOAD_PATH, os.path.basename(self.path))
+        length = self.headers['content-length']
+        data = self.rfile.read(int(length))
+
+        with open(path, 'wb') as fh:
+            fh.write(data)
+
+        self.send_response(200)
+        self.send_header('Content-Type', 'application/json')
+        self.end_headers()
+
+        # Return the absolute path of the uploaded file
+        self.wfile.write('{"url":"/%s"}' % path)
+
 
 def setup():
     parser = argparse.ArgumentParser()
@@ -19,7 +42,7 @@ def setup():
 
     httpd = BaseHTTPServer.HTTPServer(
         ('', args.port),
-        SimpleHTTPServer.SimpleHTTPRequestHandler
+        SimpleHTTPRequestHandlerWithPOST
     )
 
     def run():
diff --git a/webclient/components/fileUpload/file-upload-service.js b/webclient/components/fileUpload/file-upload-service.js
new file mode 100644
index 0000000000..5729d5da48
--- /dev/null
+++ b/webclient/components/fileUpload/file-upload-service.js
@@ -0,0 +1,47 @@
+/*
+ Copyright 2014 matrix.org
+ 
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+ 
+ http://www.apache.org/licenses/LICENSE-2.0
+ 
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+'use strict';
+
+/*
+ * Upload an HTML5 file to a server
+ */
+angular.module('mFileUpload', [])
+.service('mFileUpload', ['$http', '$q', function ($http, $q) {
+        
+    /*
+     * Upload an HTML5 file to a server and returned a promise
+     * that will provide the URL of the uploaded file.
+     */
+    this.uploadFile = function(file) {
+        var deferred = $q.defer();
+        
+        // @TODO: This service runs with the do_POST hacky implementation of /synapse/demos/webserver.py.
+        // This is temporary until we have a true file upload service
+        console.log("Uploading " + file.name + "...");
+        $http.post(file.name, file)
+        .success(function(data, status, headers, config) {
+            deferred.resolve(location.origin + data.url);
+            console.log("   -> Successfully uploaded! Available at " + location.origin + data.url);
+        }).
+        error(function(data, status, headers, config) {
+            console.log("   -> Failed to upload"  + file.name);
+            deferred.reject();
+        });
+        
+        return deferred.promise;
+    };
+}]);
\ No newline at end of file

From deae7f4f5d5cbb5e396465166bcb282c2a25a294 Mon Sep 17 00:00:00 2001
From: Emmanuel ROHEE <erohee@amdocs.com>
Date: Thu, 14 Aug 2014 18:39:23 +0200
Subject: [PATCH 34/46] Create a temporary upload service server side (by
 hacking demos/webserver.py) and client side with an angularjs service
 component.

---
 webclient/index.html | 1 +
 1 file changed, 1 insertion(+)

diff --git a/webclient/index.html b/webclient/index.html
index f4b791ecdb..e62ec39669 100644
--- a/webclient/index.html
+++ b/webclient/index.html
@@ -15,6 +15,7 @@
     <script src="rooms/rooms-controller.js"></script>
     <script src="components/matrix/matrix-service.js"></script>
     <script src="components/fileInput/file-input-directive.js"></script>
+    <script src="components/fileUpload/file-upload-service.js"></script>
 </head>
 
 <body>

From e6c62d5d7f45234ee574595eeb5ab8b7df41a1ed Mon Sep 17 00:00:00 2001
From: Emmanuel ROHEE <erohee@amdocs.com>
Date: Thu, 14 Aug 2014 18:40:20 +0200
Subject: [PATCH 35/46] We can now upload avatar image somewhere

---
 webclient/rooms/rooms-controller.js | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/webclient/rooms/rooms-controller.js b/webclient/rooms/rooms-controller.js
index d0924f5887..2ce14e1d49 100644
--- a/webclient/rooms/rooms-controller.js
+++ b/webclient/rooms/rooms-controller.js
@@ -16,9 +16,9 @@ limitations under the License.
 
 'use strict';
 
-angular.module('RoomsController', ['matrixService', 'mFileInput'])
-.controller('RoomsController', ['$scope', '$location', 'matrixService',
-                               function($scope, $location, matrixService) {
+angular.module('RoomsController', ['matrixService', 'mFileInput', 'mFileUpload'])
+.controller('RoomsController', ['$scope', '$location', 'matrixService', 'mFileUpload',
+                               function($scope, $location, matrixService, mFileUpload) {
                                    
     $scope.rooms = [];
     $scope.public_rooms = [];
@@ -167,7 +167,16 @@ angular.module('RoomsController', ['matrixService', 'mFileInput'])
 
     $scope.$watch("newProfileInfo.avatarFile", function(newValue, oldValue) {
         if ($scope.newProfileInfo.avatarFile) {
-            //@TODO: Upload this HTML5 image file to somewhere
+            console.log("Uploading new avatar file...");
+            mFileUpload.uploadFile($scope.newProfileInfo.avatarFile).then(
+                function(url) {
+                    $scope.newProfileInfo.avatar = url;
+                    $scope.setAvatar($scope.newProfileInfo.avatar);
+                },
+                function(error) {
+                    $scope.feedback = "Can't upload image";
+                } 
+            );
         }
     });
 

From 5de086b736218d43bd51c3b83ca26118806488a2 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 14 Aug 2014 17:40:27 +0100
Subject: [PATCH 36/46] More helpful display when the event stream fails,
 wiping it when the connection is regained.

---
 webclient/room/room-controller.js | 10 ++++++----
 webclient/room/room.html          |  5 ++++-
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/webclient/room/room-controller.js b/webclient/room/room-controller.js
index 8003105654..fb6e2025fc 100644
--- a/webclient/room/room-controller.js
+++ b/webclient/room/room-controller.js
@@ -25,7 +25,8 @@ angular.module('RoomController', [])
         user_id: matrixService.config().user_id,
         events_from: "END", // when to start the event stream from.
         earliest_token: "END", // stores how far back we've paginated.
-        can_paginate: true
+        can_paginate: true, // this is toggled off when we run out of items
+        stream_failure: undefined // the response when the stream fails
     };
     $scope.messages = [];
     $scope.members = {};
@@ -76,7 +77,7 @@ angular.module('RoomController', [])
                 }
             },
             function(error) {
-                console.log("paginateBackMessages Ruh roh: " + JSON.stringify(error));
+                console.log("Failed to paginateBackMessages: " + JSON.stringify(error));
             }
         )
     };
@@ -89,6 +90,7 @@ angular.module('RoomController', [])
                 "timeout": 5000
             }})
             .then(function(response) {
+                $scope.state.stream_failure = undefined;
                 console.log("Got response from "+$scope.state.events_from+" to "+response.data.end);
                 $scope.state.events_from = response.data.end;
                 $scope.feedback = "";
@@ -102,7 +104,7 @@ angular.module('RoomController', [])
                     $timeout(shortPoll, 0);
                 }
             }, function(response) {
-                $scope.feedback = "Can't stream: " + response.data;
+                $scope.state.stream_failure = response;
 
                 if (response.status == 403) {
                     $scope.stopPoll = true;
@@ -215,7 +217,7 @@ angular.module('RoomController', [])
         // Join the room
         matrixService.join($scope.room_id).then(
             function() {
-                console.log("Joined room");
+                console.log("Joined room "+$scope.room_id);
                 // Now start reading from the stream
                 $timeout(shortPoll, 0);
 
diff --git a/webclient/room/room.html b/webclient/room/room.html
index 0f86a158ec..3b9ba713de 100644
--- a/webclient/room/room.html
+++ b/webclient/room/room.html
@@ -86,7 +86,10 @@
                     <button ng-click="inviteUser(userIDToInvite)">Invite</button>
             </span>
             <button ng-click="leaveRoom()">Leave</button>
-            <button ng-click="loadMoreHistory()" ng-disabled="!state.can_paginate">MOAR HISTORY</button>
+            <button ng-click="loadMoreHistory()" ng-disabled="!state.can_paginate">Load more history</button>
+            <div ng-hide="!state.stream_failure">
+                {{ state.stream_failure.data.error || "Connection failure" }}
+            </div>
         </div>
     </div>
 

From 856f29c03c2a43212a6eb1ddf6edcbe03f710949 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Thu, 14 Aug 2014 17:44:21 +0100
Subject: [PATCH 37/46] fix linewrap

---
 README.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.rst b/README.rst
index 8131172d8e..cb47fd4812 100644
--- a/README.rst
+++ b/README.rst
@@ -189,7 +189,8 @@ Running a Demo Federation of Homeservers
 
 If you want to get up and running quickly with a trio of homeservers in a
 private federation (``localhost:8080``, ``localhost:8081`` and
-``localhost:8082``) which you can then access through the webclient running at http://localhost:8080.  Simply run::
+``localhost:8082``) which you can then access through the webclient running at
+http://localhost:8080. Simply run::
 
     $ demo/start.sh
 

From 94eb2560f47295ebc6ced75161430a27419d89e9 Mon Sep 17 00:00:00 2001
From: "Paul \"LeoNerd\" Evans" <paul@matrix.org>
Date: Thu, 14 Aug 2014 17:50:43 +0100
Subject: [PATCH 38/46] Add documentation about Federation Queries and EDUs

---
 docs/server-server/specification.rst | 68 ++++++++++++++++++++++++----
 1 file changed, 59 insertions(+), 9 deletions(-)

diff --git a/docs/server-server/specification.rst b/docs/server-server/specification.rst
index f3c571aa86..5a1f28745c 100644
--- a/docs/server-server/specification.rst
+++ b/docs/server-server/specification.rst
@@ -29,15 +29,40 @@ can also be performed.
  |                  |<--------( HTTP )-----------|                  |
  +------------------+                            +------------------+
 
+There are three main kinds of communication that occur between home servers:
 
-Transactions and PDUs
-=====================
+ * Queries
+   These are single request/response interactions between a given pair of
+   servers, initiated by one side sending an HTTP request to obtain some
+   information, and responded by the other. They are not persisted and contain
+   no long-term significant history. They simply request a snapshot state at the
+   instant the query is made.
 
-The communication between home servers is performed by a bidirectional exchange
-of messages. These messages are called Transactions, and are encoded as JSON
-objects with a dict as the top-level element, passed over HTTP. A Transaction is
-meaningful only to the pair of home servers that exchanged it; they are not
-globally-meaningful.
+ * EDUs - Ephemeral Data Units
+   These are notifications of events that are pushed from one home server to
+   another. They are not persisted and contain no long-term significant history,
+   nor does the receiving home server have to reply to them.
+
+ * PDUs - Persisted Data Units
+   These are notifications of events that are broadcast from one home server to
+   any others that are interested in the same "context" (namely, a Room ID).
+   They are persisted to long-term storage and form the record of history for
+   that context.
+
+Where Queries are presented directly across the HTTP connection as GET requests
+to specific URLs, EDUs and PDUs are further wrapped in an envelope called a
+Transaction, which is transferred from the origin to the destination home server
+using a PUT request.
+
+
+Transactions and EDUs/PDUs
+==========================
+
+The transfer of EDUs and PDUs between home servers is performed by an exchange
+of Transaction messages, which are encoded as JSON objects with a dict as the
+top-level element, passed over an HTTP PUT request. A Transaction is meaningful
+only to the pair of home servers that exchanged it; they are not globally-
+meaningful.
 
 Each transaction has an opaque ID and timestamp (UNIX epoch time in miliseconds)
 generated by its origin server, an origin and destination server name, a list of
@@ -49,7 +74,8 @@ Transaction carries.
   "origin":"red",
   "destination":"blue",
   "prev_ids":["e1da392e61898be4d2009b9fecce5325"],
-  "pdus":[...]}
+  "pdus":[...],
+  "edus":[...]}
 
 The "previous IDs" field will contain a list of previous transaction IDs that
 the origin server has sent to this destination. Its purpose is to act as a
@@ -58,7 +84,9 @@ successfully received that Transaction, or ask for a retransmission if not.
 
 The "pdus" field of a transaction is a list, containing zero or more PDUs.[*]
 Each PDU is itself a dict containing a number of keys, the exact details of
-which will vary depending on the type of PDU.
+which will vary depending on the type of PDU. Similarly, the "edus" field is
+another list containing the EDUs. This key may be entirely absent if there are
+no EDUs to transfer.
 
 (* Normally the PDU list will be non-empty, but the server should cope with
 receiving an "empty" transaction, as this is useful for informing peers of other
@@ -112,6 +140,15 @@ so on. This part needs refining. And writing in its own document as the details
 relate to the server/system as a whole, not specifically to server-server
 federation.]]
 
+EDUs, by comparison to PDUs, do not have an ID, a context, or a list of
+"previous" IDs. The only mandatory fields for these are the type, origin and
+destination home server names, and the actual nested content.
+
+ {"edu_type":"m.presence",
+  "origin":"blue",
+  "destination":"orange",
+  "content":...}
+
 
 Protocol URLs
 =============
@@ -179,3 +216,16 @@ To stream events all the events:
   Retrieves all of the transactions later than any version given by the "v"
   arguments. [[TODO(paul): I'm not sure what the "origin" argument does because
   I think at some point in the code it's got swapped around.]]
+
+
+To make a query:
+
+  GET .../query/:query_type
+    Query args: as specified by the individual query types
+
+  Response: JSON encoding of a response object
+
+  Performs a single query request on the receiving home server. The Query Type
+  part of the path specifies the kind of query being made, and its query
+  arguments have a meaning specific to that kind of query. The response is a
+  JSON-encoded object whose meaning also depends on the kind of query.

From 24dfdb4a7d62775594d3b0c395f1818a515ceb71 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 14 Aug 2014 17:51:12 +0100
Subject: [PATCH 39/46] Update README to mention -w and remove SimpleHTTPServer

---
 README.rst | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/README.rst b/README.rst
index cb47fd4812..729df13b12 100644
--- a/README.rst
+++ b/README.rst
@@ -197,15 +197,9 @@ http://localhost:8080. Simply run::
 Running The Demo Web Client
 ===========================
 
-At the present time, the web client is not directly served by the homeserver's
-HTTP server. To serve this in a form the web browser can reach, arrange for the
-'webclient' sub-directory to be made available by any sort of HTTP server that
-can serve static files. For example, python's SimpleHTTPServer will suffice::
-
-    $ cd webclient
-    $ python -m SimpleHTTPServer
-
-You can now point your browser at  http://localhost:8000/  to find the client.
+You can run the web client when you run the homeserver by adding ``-w`` to the
+command to run ``homeserver.py``. The web client can be accessed via 
+http://localhost/matrix/client
 
 If this is the first time you have used the client from that browser (it uses
 HTML5 local storage to remember its config), you will need to log in to your

From 3ddfc949dc38eaeb2174e548138b446d77c6070c Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Thu, 14 Aug 2014 17:54:40 +0100
Subject: [PATCH 40/46] manual syutil

---
 README.rst | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/README.rst b/README.rst
index 729df13b12..5ff6fbaab2 100644
--- a/README.rst
+++ b/README.rst
@@ -120,6 +120,10 @@ may need to also run:
     $ sudo apt-get install python-pip
     $ sudo pip install --upgrade setuptools
     
+If you don't have access to github, then you may need to install ``syutil``
+manually by checking it out and running ``python setup.py develop --user`` on it
+too.
+    
 If you get errors about ``sodium.h`` being missing, you may also need to
 manually install a newer PyNaCl via pip as setuptools installs an old one. Or
 you can check PyNaCl out of git directly (https://github.com/pyca/pynacl) and

From 7a025d63687e43c96532f69860c68ee6c19543ba Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Thu, 14 Aug 2014 18:58:01 +0100
Subject: [PATCH 41/46] It's called Matrix :)

---
 docs/server-server/specification.rst | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/docs/server-server/specification.rst b/docs/server-server/specification.rst
index 5a1f28745c..a9ab9bff66 100644
--- a/docs/server-server/specification.rst
+++ b/docs/server-server/specification.rst
@@ -1,8 +1,8 @@
-============================
-Synapse Server-to-Server API
-============================
+===========================
+Matrix Server-to-Server API
+===========================
 
-A description of the protocol used to communicate between Synapse home servers;
+A description of the protocol used to communicate between Matrix home servers;
 also known as Federation.
 
 
@@ -10,7 +10,7 @@ Overview
 ========
 
 The server-server API is a mechanism by which two home servers can exchange
-Synapse event messages, both as a real-time push of current events, and as a
+Matrix event messages, both as a real-time push of current events, and as a
 historic fetching mechanism to synchronise past history for clients to view. It
 uses HTTP connections between each pair of servers involved as the underlying
 transport. Messages are exchanged between servers in real-time by active pushing
@@ -19,7 +19,7 @@ historic data for the purpose of back-filling scrollback buffers and the like
 can also be performed.
 
 
- { Synapse entities }                            { Synapse entities }
+  { Matrix clients }                              { Matrix clients }
      ^          |                                    ^          |
      |  events  |                                    |  events  |
      |          V                                    |          V
@@ -64,10 +64,10 @@ top-level element, passed over an HTTP PUT request. A Transaction is meaningful
 only to the pair of home servers that exchanged it; they are not globally-
 meaningful.
 
-Each transaction has an opaque ID and timestamp (UNIX epoch time in miliseconds)
-generated by its origin server, an origin and destination server name, a list of
-"previous IDs", and a list of PDUs - the actual message payload that the
-Transaction carries.
+Each transaction has an opaque ID and timestamp (UNIX epoch time in
+milliseconds) generated by its origin server, an origin and destination server
+name, a list of "previous IDs", and a list of PDUs - the actual message payload
+that the Transaction carries.
 
  {"transaction_id":"916d630ea616342b42e98a3be0b74113",
   "ts":1404835423000,
@@ -114,7 +114,7 @@ field of a PDU refers to PDUs that any origin server has sent, rather than
 previous IDs that this origin has sent. This list may refer to other PDUs sent
 by the same origin as the current one, or other origins.
 
-Because of the distributed nature of participants in a Synapse conversation, it
+Because of the distributed nature of participants in a Matrix conversation, it
 is impossible to establish a globally-consistent total ordering on the events.
 However, by annotating each outbound PDU at its origin with IDs of other PDUs it
 has received, a partial ordering can be constructed allowing causallity

From 0b179db36de8b4665e0ac33102a6998a7329d263 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Thu, 14 Aug 2014 18:58:40 +0100
Subject: [PATCH 42/46] s/Synapse/Matrix/

---
 docs/client-server/specification.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/client-server/specification.rst b/docs/client-server/specification.rst
index b82093f2d3..3367884ad4 100644
--- a/docs/client-server/specification.rst
+++ b/docs/client-server/specification.rst
@@ -1,6 +1,6 @@
-=========================
-Synapse Client-Server API
-=========================
+========================
+Matrix Client-Server API
+========================
 
 The following specification outlines how a client can send and receive data from 
 a home server.

From 286e90e58f077f955b2b516737a6b80dddbf73aa Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Fri, 15 Aug 2014 09:29:39 +0100
Subject: [PATCH 43/46] Updated README about -w in all the places.

---
 README.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.rst b/README.rst
index 5ff6fbaab2..378b460d0b 100644
--- a/README.rst
+++ b/README.rst
@@ -26,8 +26,8 @@ To get up and running:
       with ``python setup.py develop --user`` and then run one with
       ``python synapse/app/homeserver.py``
       
-    - To run your own webclient:
-      ``cd webclient; python -m SimpleHTTPServer`` and hit http://localhost:8000
+    - To run your own webclient, add ``-w``:
+      ``python synapse/app/homeserver.py -w`` and hit http://localhost:8080/matrix/client
       in your web browser (a recent Chrome, Safari or Firefox for now,
       please...)
              
@@ -203,7 +203,7 @@ Running The Demo Web Client
 
 You can run the web client when you run the homeserver by adding ``-w`` to the
 command to run ``homeserver.py``. The web client can be accessed via 
-http://localhost/matrix/client
+http://localhost:8080/matrix/client
 
 If this is the first time you have used the client from that browser (it uses
 HTML5 local storage to remember its config), you will need to log in to your

From 33d62c2c6674cf19f5261817bff81cd5209408bd Mon Sep 17 00:00:00 2001
From: "Paul \"LeoNerd\" Evans" <paul@matrix.org>
Date: Fri, 15 Aug 2014 11:40:58 +0100
Subject: [PATCH 44/46] Remember to reflect membership LEAVE events to the
 leaving member so they know it happened

---
 synapse/api/notifier.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/synapse/api/notifier.py b/synapse/api/notifier.py
index 105a11401b..65b5a4ebb3 100644
--- a/synapse/api/notifier.py
+++ b/synapse/api/notifier.py
@@ -56,6 +56,10 @@ class Notifier(object):
         if (event.type == RoomMemberEvent.TYPE and
                 event.content["membership"] == Membership.INVITE):
             member_list.append(event.target_user_id)
+        # similarly, LEAVEs must be sent to the person leaving
+        if (event.type == RoomMemberEvent.TYPE and
+                event.content["membership"] == Membership.LEAVE):
+            member_list.append(event.target_user_id)
 
         for user_id in member_list:
             if user_id in self.stored_event_listeners:

From 1a26905cc9b0c957c9619f55705b88f7d08c3071 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 15 Aug 2014 11:41:11 +0100
Subject: [PATCH 45/46] Fix pontenial bug in state resolution handler that
 compared dicts rather than their id's

---
 synapse/state.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/synapse/state.py b/synapse/state.py
index b081de8f4f..d8977b61ea 100644
--- a/synapse/state.py
+++ b/synapse/state.py
@@ -157,7 +157,10 @@ class StateHandler(object):
             defer.returnValue(True)
             return
 
-        if new_branch[-1] == current_branch[-1]:
+        n = new_branch[-1]
+        c = current_branch[-1]
+
+        if n.pdu_id == c.pdu_id and n.origin == c.origin:
             # We have all the PDUs we need, so we can just do the conflict
             # resolution.
 

From c5f2da587532c80cda066acc715344b74a9d19d5 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 15 Aug 2014 11:47:01 +0100
Subject: [PATCH 46/46] Add a check to make sure that during state conflict res
 we only request a PDU we don't have.

---
 synapse/state.py    | 12 ++++++++++--
 tests/test_state.py |  3 +++
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/synapse/state.py b/synapse/state.py
index d8977b61ea..4f8b4d9760 100644
--- a/synapse/state.py
+++ b/synapse/state.py
@@ -191,10 +191,18 @@ class StateHandler(object):
                 key=lambda x: x.depth
             )
 
+            pdu_id = missing_prev.prev_state_id
+            origin = missing_prev.prev_state_origin
+
+            is_missing = yield self.store.get_pdu(pdu_id, origin) is None
+
+            if not is_missing:
+                raise Exception("Conflict resolution failed.")
+
             yield self._replication.get_pdu(
                 destination=missing_prev.origin,
-                pdu_origin=missing_prev.prev_state_origin,
-                pdu_id=missing_prev.prev_state_id,
+                pdu_origin=origin,
+                pdu_id=pdu_id,
                 outlier=True
             )
 
diff --git a/tests/test_state.py b/tests/test_state.py
index a2908a2eac..aaf873a856 100644
--- a/tests/test_state.py
+++ b/tests/test_state.py
@@ -37,6 +37,7 @@ class StateTestCase(unittest.TestCase):
             "update_current_state",
             "get_latest_pdus_in_context",
             "get_current_state",
+            "get_pdu",
         ])
         self.replication = Mock(spec=["get_pdu"])
 
@@ -220,6 +221,8 @@ class StateTestCase(unittest.TestCase):
 
         self.replication.get_pdu.side_effect = set_return_tree
 
+        self.persistence.get_pdu.return_value = None
+
         is_new = yield self.state.handle_new_state(new_pdu)
 
         self.assertTrue(is_new)