From 613748804a302ced5de124c86606aff3a36acf42 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 9 Dec 2015 17:35:55 +0000 Subject: [PATCH 01/36] Changelog for v0.12.0 --- CHANGES.rst | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 5c38c1915f..d151badc98 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,37 @@ +Changes in synapse v0.12.0 (2015-12-09) +======================================= + +* Host the client APIs released as r0 by + https://matrix.org/docs/spec/r0.0.0/client_server.html + on paths prefixed by /_matrix/client/r0. (PR #430, PR #415, PR #400) +* Updates the client APIs to match r0 of the matrix specification. + + * All APIs return events in the new event format, old APIs also include + the fields needed to parse the event using the old format for + compatibility. (PR #402) + * Search results are now given as a JSON array rather than + a JSON object (PR #405) + * Miscellaneous changes to search (PR #403, PR #406, PR #412) + * Filter JSON objects may now be passed as query parameters to /sync + (PR #431) + * Fix implementation of /admin/whois (PR #418) + * Only include the rooms that user has left in /sync if the client requests + them in the filter (PR #423) + * Don't push for m.room.message by default (PR #411) + * Add API for setting per account user data (PR #392) + * Allow users to forget rooms (PR #385) + +* Performance improvements and monitoring: + + * Add per-request counters for CPU time spent on the main python thread. + (PR #421, PR #420) + * Add per-request counters for time spent in the database (PR #429) + * Make state updates in the C+S API idempotent (PR #416) + * Only fire user_joined_room if the user has actually joined. (PR #410) + * Reuse a single http client, rather than creating new ones (PR #413) + +* Fixed a bug upgrading from older versions of synapse on postgresql (PR #417) + Changes in synapse v0.11.1 (2015-11-20) ======================================= From 5bdb93c2a6b4efcd25bb9a5974f9c4eebb040b23 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 9 Dec 2015 17:45:35 +0000 Subject: [PATCH 02/36] Add to changelog --- CHANGES.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index d151badc98..03dc975762 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -3,7 +3,7 @@ Changes in synapse v0.12.0 (2015-12-09) * Host the client APIs released as r0 by https://matrix.org/docs/spec/r0.0.0/client_server.html - on paths prefixed by /_matrix/client/r0. (PR #430, PR #415, PR #400) + on paths prefixed by ``/_matrix/client/r0``. (PR #430, PR #415, PR #400) * Updates the client APIs to match r0 of the matrix specification. * All APIs return events in the new event format, old APIs also include @@ -12,12 +12,12 @@ Changes in synapse v0.12.0 (2015-12-09) * Search results are now given as a JSON array rather than a JSON object (PR #405) * Miscellaneous changes to search (PR #403, PR #406, PR #412) - * Filter JSON objects may now be passed as query parameters to /sync + * Filter JSON objects may now be passed as query parameters to ``/sync`` (PR #431) - * Fix implementation of /admin/whois (PR #418) - * Only include the rooms that user has left in /sync if the client requests - them in the filter (PR #423) - * Don't push for m.room.message by default (PR #411) + * Fix implementation of ``/admin/whois`` (PR #418) + * Only include the rooms that user has left in ``/sync`` if the client + requests them in the filter (PR #423) + * Don't push for ``m.room.message`` by default (PR #411) * Add API for setting per account user data (PR #392) * Allow users to forget rooms (PR #385) @@ -27,7 +27,7 @@ Changes in synapse v0.12.0 (2015-12-09) (PR #421, PR #420) * Add per-request counters for time spent in the database (PR #429) * Make state updates in the C+S API idempotent (PR #416) - * Only fire user_joined_room if the user has actually joined. (PR #410) + * Only fire ``user_joined_room`` if the user has actually joined. (PR #410) * Reuse a single http client, rather than creating new ones (PR #413) * Fixed a bug upgrading from older versions of synapse on postgresql (PR #417) From 05f6cb42db1cc1a9720fa7214a14a26613a8b699 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 9 Dec 2015 17:48:02 +0000 Subject: [PATCH 03/36] Bump synapse version to v0.12.0 --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index 3e7e26bf60..5db4eae354 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.11.1" +__version__ = "0.12.0" From dd9430e758ed103af8883392e0bc4cc0ac600a4c Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 10 Dec 2015 11:26:58 +0000 Subject: [PATCH 04/36] Update release date --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 03dc975762..6247d1b389 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,4 +1,4 @@ -Changes in synapse v0.12.0 (2015-12-09) +Changes in synapse v0.12.0 (2015-12-10) ======================================= * Host the client APIs released as r0 by From a8589d1ff3ca9f473ffa492e0c96778333928882 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 10 Dec 2015 11:39:00 +0000 Subject: [PATCH 05/36] Mark the version as a -rc1 release candidate --- CHANGES.rst | 4 ++-- synapse/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 6247d1b389..f81a51dc7f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,5 +1,5 @@ -Changes in synapse v0.12.0 (2015-12-10) -======================================= +Changes in synapse v0.12.0-rc1 (2015-12-10) +=========================================== * Host the client APIs released as r0 by https://matrix.org/docs/spec/r0.0.0/client_server.html diff --git a/synapse/__init__.py b/synapse/__init__.py index 5db4eae354..c357f8f9c2 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.12.0" +__version__ = "0.12.0-rc1" From 99afb4b750f9ba5074f8e7dd79144cf678c668f1 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 10 Dec 2015 17:08:21 +0000 Subject: [PATCH 06/36] Ensure that the event that gets persisted is the one that was signed --- synapse/handlers/federation.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 2855f2d7c3..e7ad48c948 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -596,7 +596,7 @@ class FederationHandler(BaseHandler): handled_events = set() try: - new_event = self._sign_event(event) + event = self._sign_event(event) # Try the host we successfully got a response to /make_join/ # request first. try: @@ -604,7 +604,7 @@ class FederationHandler(BaseHandler): target_hosts.insert(0, origin) except ValueError: pass - ret = yield self.replication_layer.send_join(target_hosts, new_event) + ret = yield self.replication_layer.send_join(target_hosts, event) origin = ret["origin"] state = ret["state"] @@ -613,12 +613,12 @@ class FederationHandler(BaseHandler): handled_events.update([s.event_id for s in state]) handled_events.update([a.event_id for a in auth_chain]) - handled_events.add(new_event.event_id) + handled_events.add(event.event_id) logger.debug("do_invite_join auth_chain: %s", auth_chain) logger.debug("do_invite_join state: %s", state) - logger.debug("do_invite_join event: %s", new_event) + logger.debug("do_invite_join event: %s", event) try: yield self.store.store_room( @@ -636,14 +636,14 @@ class FederationHandler(BaseHandler): with PreserveLoggingContext(): d = self.notifier.on_new_room_event( - new_event, event_stream_id, max_stream_id, + event, event_stream_id, max_stream_id, extra_users=[joinee] ) def log_failure(f): logger.warn( "Failed to notify about %s: %s", - new_event.event_id, f.value + event.event_id, f.value ) d.addErrback(log_failure) From 7d6b3133125aef802dad36d120ad23d5e33948bf Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 10 Dec 2015 17:49:34 +0000 Subject: [PATCH 07/36] Add caches for whether a room has been forgotten by a user --- synapse/handlers/room.py | 2 +- synapse/storage/roommember.py | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 116a998c42..a72c3fda9f 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -755,7 +755,7 @@ class RoomMemberHandler(BaseHandler): defer.returnValue((token, public_key, key_validity_url, display_name)) def forget(self, user, room_id): - self.store.forget(user.to_string(), room_id) + return self.store.forget(user.to_string(), room_id) class RoomListHandler(BaseHandler): diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 69398b7c8e..e1777d7afa 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -18,7 +18,7 @@ from twisted.internet import defer from collections import namedtuple from ._base import SQLBaseStore -from synapse.util.caches.descriptors import cached +from synapse.util.caches.descriptors import cached, cachedInlineCallbacks from synapse.api.constants import Membership from synapse.types import UserID @@ -270,6 +270,7 @@ class RoomMemberStore(SQLBaseStore): defer.returnValue(ret) + @defer.inlineCallbacks def forget(self, user_id, room_id): """Indicate that user_id wishes to discard history for room_id.""" def f(txn): @@ -284,9 +285,11 @@ class RoomMemberStore(SQLBaseStore): " room_id = ?" ) txn.execute(sql, (user_id, room_id)) - self.runInteraction("forget_membership", f) + yield self.runInteraction("forget_membership", f) + self.was_forgotten_at.invalidate_all() + self.did_forget.invalidate((user_id, room_id)) - @defer.inlineCallbacks + @cachedInlineCallbacks(num_args=2) def did_forget(self, user_id, room_id): """Returns whether user_id has elected to discard history for room_id. @@ -310,7 +313,7 @@ class RoomMemberStore(SQLBaseStore): count = yield self.runInteraction("did_forget_membership", f) defer.returnValue(count == 0) - @defer.inlineCallbacks + @cachedInlineCallbacks(num_args=3) def was_forgotten_at(self, user_id, room_id, event_id): """Returns whether user_id has elected to discard history for room_id at event_id. From 515548a47ae0418203224a4315b88531cf28a9de Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 10 Dec 2015 17:54:23 +0000 Subject: [PATCH 08/36] Missing yield --- synapse/rest/client/v1/room.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 53cc29becb..6fe53f70e5 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -490,7 +490,7 @@ class RoomMembershipRestServlet(ClientV1RestServlet): ) if membership_action == "forget": - self.handlers.room_member_handler.forget(user, room_id) + yield self.handlers.room_member_handler.forget(user, room_id) defer.returnValue((200, {})) From 5577a6109052e6c953a1532ecb3b473db709905e Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 10 Dec 2015 19:03:06 +0000 Subject: [PATCH 09/36] throwaway 1-liner for generating password hashes --- scripts/gen_password | 1 + 1 file changed, 1 insertion(+) create mode 100644 scripts/gen_password diff --git a/scripts/gen_password b/scripts/gen_password new file mode 100644 index 0000000000..7afd3a5dfd --- /dev/null +++ b/scripts/gen_password @@ -0,0 +1 @@ +perl -MCrypt::Random -MCrypt::Eksblowfish::Bcrypt -e 'print Crypt::Eksblowfish::Bcrypt::bcrypt("secret", "\$2\$12\$" . Crypt::Eksblowfish::Bcrypt::en_base64(Crypt::Random::makerandom_octet(Length=>16)))."\n"' From 51fb590c0e787c385bea1d595fa8bceea23c26e5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 11 Dec 2015 11:12:57 +0000 Subject: [PATCH 10/36] Use more efficient query form --- synapse/storage/search.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/synapse/storage/search.py b/synapse/storage/search.py index 39f600f53c..c39d54a7ca 100644 --- a/synapse/storage/search.py +++ b/synapse/storage/search.py @@ -143,7 +143,7 @@ class SearchStore(BackgroundUpdateStore): search_query = search_query = _parse_query(self.database_engine, search_term) - args = [search_query] + args = [] # Make sure we don't explode because the person is in too many rooms. # We filter the results below regardless. @@ -164,16 +164,19 @@ class SearchStore(BackgroundUpdateStore): if isinstance(self.database_engine, PostgresEngine): sql = ( - "SELECT ts_rank_cd(vector, query) AS rank, room_id, event_id" - " FROM to_tsquery('english', ?) as query, event_search" - " WHERE vector @@ query" + "SELECT ts_rank_cd(vector, to_tsquery('english', ?)) AS rank," + " room_id, event_id" + " FROM event_search" + " WHERE vector @@ to_tsquery('english', ?)" ) + args = [search_query, search_query] + args elif isinstance(self.database_engine, Sqlite3Engine): sql = ( "SELECT rank(matchinfo(event_search)) as rank, room_id, event_id" " FROM event_search" " WHERE value MATCH ?" ) + args = [search_query] + args else: # This should be unreachable. raise Exception("Unrecognized database engine") @@ -232,7 +235,7 @@ class SearchStore(BackgroundUpdateStore): search_query = search_query = _parse_query(self.database_engine, search_term) - args = [search_query] + args = [] # Make sure we don't explode because the person is in too many rooms. # We filter the results below regardless. @@ -267,12 +270,13 @@ class SearchStore(BackgroundUpdateStore): if isinstance(self.database_engine, PostgresEngine): sql = ( - "SELECT ts_rank_cd(vector, query) as rank," + "SELECT ts_rank_cd(vector, to_tsquery('english', ?)) as rank," " origin_server_ts, stream_ordering, room_id, event_id" - " FROM to_tsquery('english', ?) as query, event_search" + " FROM event_search" " NATURAL JOIN events" - " WHERE vector @@ query AND " + " WHERE vector @@ to_tsquery('english', ?) AND " ) + args = [search_term, search_term] + args elif isinstance(self.database_engine, Sqlite3Engine): # We use CROSS JOIN here to ensure we use the right indexes. # https://sqlite.org/optoverview.html#crossjoin @@ -292,6 +296,7 @@ class SearchStore(BackgroundUpdateStore): " CROSS JOIN events USING (event_id)" " WHERE " ) + args = [search_term] + args else: # This should be unreachable. raise Exception("Unrecognized database engine") From d9a5c56930c22b02268f5deca4df84eba345ec2c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 11 Dec 2015 11:40:23 +0000 Subject: [PATCH 11/36] Include approximate count of search results --- synapse/handlers/search.py | 8 +++++- synapse/storage/search.py | 56 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py index bc79564287..99ef56871c 100644 --- a/synapse/handlers/search.py +++ b/synapse/handlers/search.py @@ -152,11 +152,15 @@ class SearchHandler(BaseHandler): highlights = set() + count = None + if order_by == "rank": search_result = yield self.store.search_msgs( room_ids, search_term, keys ) + count = search_result["count"] + if search_result["highlights"]: highlights.update(search_result["highlights"]) @@ -207,6 +211,8 @@ class SearchHandler(BaseHandler): if search_result["highlights"]: highlights.update(search_result["highlights"]) + count = search_result["count"] + results = search_result["results"] results_map = {r["event"].event_id: r for r in results} @@ -359,7 +365,7 @@ class SearchHandler(BaseHandler): rooms_cat_res = { "results": results, - "count": len(results), + "count": count, "highlights": list(highlights), } diff --git a/synapse/storage/search.py b/synapse/storage/search.py index c39d54a7ca..efd87d99bb 100644 --- a/synapse/storage/search.py +++ b/synapse/storage/search.py @@ -162,6 +162,9 @@ class SearchStore(BackgroundUpdateStore): "(%s)" % (" OR ".join(local_clauses),) ) + count_args = args + count_clauses = clauses + if isinstance(self.database_engine, PostgresEngine): sql = ( "SELECT ts_rank_cd(vector, to_tsquery('english', ?)) AS rank," @@ -170,6 +173,12 @@ class SearchStore(BackgroundUpdateStore): " WHERE vector @@ to_tsquery('english', ?)" ) args = [search_query, search_query] + args + + count_sql = ( + "SELECT room_id, count(*) as count FROM event_search" + " WHERE vector @@ to_tsquery('english', ?)" + ) + count_args = [search_query] + count_args elif isinstance(self.database_engine, Sqlite3Engine): sql = ( "SELECT rank(matchinfo(event_search)) as rank, room_id, event_id" @@ -177,6 +186,12 @@ class SearchStore(BackgroundUpdateStore): " WHERE value MATCH ?" ) args = [search_query] + args + + count_sql = ( + "SELECT room_id, count(*) as count FROM event_search" + " WHERE value MATCH ? AND " + ) + count_args = [search_term] + count_args else: # This should be unreachable. raise Exception("Unrecognized database engine") @@ -184,6 +199,9 @@ class SearchStore(BackgroundUpdateStore): for clause in clauses: sql += " AND " + clause + for clause in count_clauses: + count_sql += " AND " + clause + # We add an arbitrary limit here to ensure we don't try to pull the # entire table from the database. sql += " ORDER BY rank DESC LIMIT 500" @@ -205,6 +223,14 @@ class SearchStore(BackgroundUpdateStore): if isinstance(self.database_engine, PostgresEngine): highlights = yield self._find_highlights_in_postgres(search_query, events) + count_sql += " GROUP BY room_id" + + count_results = yield self._execute( + "search_rooms_count", self.cursor_to_dict, count_sql, *count_args + ) + + count = sum(row["count"] for row in count_results if row["room_id"] in room_ids) + defer.returnValue({ "results": [ { @@ -215,6 +241,7 @@ class SearchStore(BackgroundUpdateStore): if r["event_id"] in event_map ], "highlights": highlights, + "count": count, }) @defer.inlineCallbacks @@ -254,6 +281,9 @@ class SearchStore(BackgroundUpdateStore): "(%s)" % (" OR ".join(local_clauses),) ) + count_args = args + count_clauses = clauses + if pagination_token: try: origin_server_ts, stream = pagination_token.split(",") @@ -276,7 +306,13 @@ class SearchStore(BackgroundUpdateStore): " NATURAL JOIN events" " WHERE vector @@ to_tsquery('english', ?) AND " ) - args = [search_term, search_term] + args + args = [search_query, search_query] + args + + count_sql = ( + "SELECT room_id, count(*) as count FROM event_search" + " WHERE vector @@ to_tsquery('english', ?) AND " + ) + count_args = [search_query] + count_args elif isinstance(self.database_engine, Sqlite3Engine): # We use CROSS JOIN here to ensure we use the right indexes. # https://sqlite.org/optoverview.html#crossjoin @@ -296,12 +332,19 @@ class SearchStore(BackgroundUpdateStore): " CROSS JOIN events USING (event_id)" " WHERE " ) - args = [search_term] + args + args = [search_query] + args + + count_sql = ( + "SELECT room_id, count(*) as count FROM event_search" + " WHERE value MATCH ? AND " + ) + count_args = [search_term] + count_args else: # This should be unreachable. raise Exception("Unrecognized database engine") sql += " AND ".join(clauses) + count_sql += " AND ".join(count_clauses) # We add an arbitrary limit here to ensure we don't try to pull the # entire table from the database. @@ -326,6 +369,14 @@ class SearchStore(BackgroundUpdateStore): if isinstance(self.database_engine, PostgresEngine): highlights = yield self._find_highlights_in_postgres(search_query, events) + count_sql += " GROUP BY room_id" + + count_results = yield self._execute( + "search_rooms_count", self.cursor_to_dict, count_sql, *count_args + ) + + count = sum(row["count"] for row in count_results if row["room_id"] in room_ids) + defer.returnValue({ "results": [ { @@ -339,6 +390,7 @@ class SearchStore(BackgroundUpdateStore): if r["event_id"] in event_map ], "highlights": highlights, + "count": count, }) def _find_highlights_in_postgres(self, search_query, events): From 5a3e4e43d893d73f3a6b3eab985a6482c8e33e78 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 11 Dec 2015 11:38:03 +0000 Subject: [PATCH 12/36] SYN-90: We don't need --proccess-dependency-links When installing synapse since all its dependencies are on PyPI --- README.rst | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index 80e1b26e60..06f06fd353 100644 --- a/README.rst +++ b/README.rst @@ -130,7 +130,7 @@ To install the synapse homeserver run:: virtualenv -p python2.7 ~/.synapse source ~/.synapse/bin/activate pip install --upgrade setuptools - pip install --process-dependency-links https://github.com/matrix-org/synapse/tarball/master + pip install https://github.com/matrix-org/synapse/tarball/master This installs synapse, along with the libraries it uses, into a virtual environment under ``~/.synapse``. Feel free to pick a different directory @@ -235,8 +235,7 @@ pip may be outdated (6.0.7-1 and needs to be upgraded to 6.0.8-1 ):: You also may need to explicitly specify python 2.7 again during the install request:: - pip2.7 install --process-dependency-links \ - https://github.com/matrix-org/synapse/tarball/master + pip2.7 install https://github.com/matrix-org/synapse/tarball/master If you encounter an error with lib bcrypt causing an Wrong ELF Class: ELFCLASS32 (x64 Systems), you may need to reinstall py-bcrypt to correctly @@ -295,8 +294,7 @@ Troubleshooting Troubleshooting Installation ---------------------------- -Synapse requires pip 1.7 or later, so if your OS provides too old a version and -you get errors about ``error: no such option: --process-dependency-links`` you +Synapse requires pip 1.7 or later, so if your OS provides too old a version you may need to manually upgrade it:: sudo pip install --upgrade pip From 1ee7280c4c7a6ad99236a10a861fde3cd013892b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 11 Dec 2015 16:48:20 +0000 Subject: [PATCH 13/36] Do the /sync in parallel accross the rooms like /initialSync does --- synapse/handlers/sync.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 24c2b2fad6..7088c20cb4 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -17,6 +17,7 @@ from ._base import BaseHandler from synapse.streams.config import PaginationConfig from synapse.api.constants import Membership, EventTypes +from synapse.util import unwrapFirstError from twisted.internet import defer @@ -209,9 +210,10 @@ class SyncHandler(BaseHandler): joined = [] invited = [] archived = [] + deferreds = [] for event in room_list: if event.membership == Membership.JOIN: - room_sync = yield self.full_state_sync_for_joined_room( + room_sync_deferred = self.full_state_sync_for_joined_room( room_id=event.room_id, sync_config=sync_config, now_token=now_token, @@ -220,7 +222,8 @@ class SyncHandler(BaseHandler): tags_by_room=tags_by_room, account_data_by_room=account_data_by_room, ) - joined.append(room_sync) + room_sync_deferred.addCallback(joined.append) + deferreds.append(room_sync_deferred) elif event.membership == Membership.INVITE: invite = yield self.store.get_event(event.event_id) invited.append(InvitedSyncResult( @@ -231,7 +234,7 @@ class SyncHandler(BaseHandler): leave_token = now_token.copy_and_replace( "room_key", "s%d" % (event.stream_ordering,) ) - room_sync = yield self.full_state_sync_for_archived_room( + room_sync_deferred = self.full_state_sync_for_archived_room( sync_config=sync_config, room_id=event.room_id, leave_event_id=event.event_id, @@ -240,7 +243,12 @@ class SyncHandler(BaseHandler): tags_by_room=tags_by_room, account_data_by_room=account_data_by_room, ) - archived.append(room_sync) + room_sync_deferred.addCallback(archived.append) + deferreds.append(room_sync_deferred) + + yield defer.gatherResults( + deferreds, consumeErrors=True + ).addErrback(unwrapFirstError) defer.returnValue(SyncResult( presence=presence, From bfc52a2342999a7887dcc5ba653b67454c0fc2c8 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 14 Dec 2015 11:38:11 +0000 Subject: [PATCH 14/36] Fix typo in sql for full text search on sqlite3 --- synapse/storage/search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/search.py b/synapse/storage/search.py index efd87d99bb..00f89ff02f 100644 --- a/synapse/storage/search.py +++ b/synapse/storage/search.py @@ -189,7 +189,7 @@ class SearchStore(BackgroundUpdateStore): count_sql = ( "SELECT room_id, count(*) as count FROM event_search" - " WHERE value MATCH ? AND " + " WHERE value MATCH ?" ) count_args = [search_term] + count_args else: From 76e69cc8de186c42be5763be0492d074319060cc Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 14 Dec 2015 12:38:55 +0000 Subject: [PATCH 15/36] Fix typo --- synapse/storage/roommember.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index e1777d7afa..4e0e9ab59a 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -121,7 +121,7 @@ class RoomMemberStore(SQLBaseStore): return self.get_rooms_for_user_where_membership_is( user_id, [Membership.INVITE] ).addCallback(lambda invites: self._get_events([ - invites.event_id for invite in invites + invite.event_id for invite in invites ])) def get_leave_and_ban_events_for_user(self, user_id): From 338c0a8a69096c188f4739c235f74a072a62e92f Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 14 Dec 2015 13:50:50 +0000 Subject: [PATCH 16/36] Include errcode on Internal Server Error --- synapse/http/server.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/synapse/http/server.py b/synapse/http/server.py index c44bdfc888..1b936b6892 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -15,7 +15,7 @@ from synapse.api.errors import ( - cs_exception, SynapseError, CodeMessageException, UnrecognizedRequestError + cs_exception, SynapseError, CodeMessageException, UnrecognizedRequestError, Codes ) from synapse.util.logcontext import LoggingContext, PreserveLoggingContext import synapse.metrics @@ -127,7 +127,10 @@ def request_handler(request_handler): respond_with_json( request, 500, - {"error": "Internal server error"}, + { + "error": "Internal server error", + "errcode": Codes.M_UNKNOWN, + }, send_cors=True ) return wrapped_request_handler From 98dfa7d24f91ff083b36f1379ce2426c8e6cdb75 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 14 Dec 2015 13:55:46 +0000 Subject: [PATCH 17/36] Skip events that where the body, name or topic isn't a string when back populating the FTS index --- synapse/storage/search.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/synapse/storage/search.py b/synapse/storage/search.py index 39f600f53c..04246101df 100644 --- a/synapse/storage/search.py +++ b/synapse/storage/search.py @@ -85,6 +85,11 @@ class SearchStore(BackgroundUpdateStore): # skip over it. continue + if not isinstance(value, basestring): + # If the event body, name or topic isn't a string + # then skip over it + continue + event_search_rows.append((event_id, room_id, key, value)) if isinstance(self.database_engine, PostgresEngine): From 834924248f4034a209271828d7cca47eee01f328 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 14 Dec 2015 14:09:21 +0000 Subject: [PATCH 18/36] Check whether prev_content or prev_sender is set before trying to rollback state --- synapse/rest/client/v2_alpha/sync.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index f0a637a6da..7cba981c04 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -357,14 +357,19 @@ class SyncRestServlet(RestServlet): if prev_event_id is None: del result[event_key] else: - result[event_key] = FrozenEvent({ - "type": timeline_event.type, - "state_key": timeline_event.state_key, - "content": timeline_event.unsigned['prev_content'], - "sender": timeline_event.unsigned['prev_sender'], - "event_id": prev_event_id, - "room_id": timeline_event.room_id, - }) + prev_content = timeline_event.unsigned.get('prev_content') + prev_sender = timeline_event.unsigned.get('prev_sender') + if prev_content and prev_sender: + result[event_key] = FrozenEvent({ + "type": timeline_event.type, + "state_key": timeline_event.state_key, + "content": prev_content, + "sender": prev_sender, + "event_id": prev_event_id, + "room_id": timeline_event.room_id, + }) + else: + del result[event_key] logger.debug("New value: %r", result.get(event_key)) return result From 070e28e203e52fd8968564bec8e73c96c1ab290b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 14 Dec 2015 14:34:04 +0000 Subject: [PATCH 19/36] Combine the prev content tests --- synapse/rest/client/v2_alpha/sync.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 7cba981c04..3f8ce701dc 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -354,22 +354,20 @@ class SyncRestServlet(RestServlet): logger.debug("Replacing %s with %s in state dict", timeline_event.event_id, prev_event_id) - if prev_event_id is None: + prev_content = timeline_event.unsigned.get('prev_content') + prev_sender = timeline_event.unsigned.get('prev_sender') + if prev_event_id is None or not prev_content or not prev_sender: del result[event_key] else: - prev_content = timeline_event.unsigned.get('prev_content') - prev_sender = timeline_event.unsigned.get('prev_sender') - if prev_content and prev_sender: - result[event_key] = FrozenEvent({ - "type": timeline_event.type, - "state_key": timeline_event.state_key, - "content": prev_content, - "sender": prev_sender, - "event_id": prev_event_id, - "room_id": timeline_event.room_id, - }) - else: - del result[event_key] + result[event_key] = FrozenEvent({ + "type": timeline_event.type, + "state_key": timeline_event.state_key, + "content": prev_content, + "sender": prev_sender, + "event_id": prev_event_id, + "room_id": timeline_event.room_id, + }) + logger.debug("New value: %r", result.get(event_key)) return result From 28c5181dfebbce99a4981584a5761285522ed29b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 14 Dec 2015 14:50:51 +0000 Subject: [PATCH 20/36] Add commentary for fix in PR#442 --- synapse/rest/client/v2_alpha/sync.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 3f8ce701dc..adf77e13bf 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -356,6 +356,12 @@ class SyncRestServlet(RestServlet): prev_content = timeline_event.unsigned.get('prev_content') prev_sender = timeline_event.unsigned.get('prev_sender') + # Empircally it seems possible for the event to have a + # "replaces_state" key but not a prev_content or prev_sender + # markjh conjectures that it could be due to the server not + # having a copy of that event. + # If this is the case the we ignore the previous event. This will + # cause the displayname calculations on the client to be incorrect if prev_event_id is None or not prev_content or not prev_sender: del result[event_key] else: From dbe7892e03e2e0e6a50c54109c30b22fe4194894 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 14 Dec 2015 15:09:41 +0000 Subject: [PATCH 21/36] Fix a race between started/stopped stream --- synapse/handlers/events.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index fe300433e6..576d77e0e7 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -69,7 +69,12 @@ class EventStreamHandler(BaseHandler): A deferred that completes once their presence has been updated. """ if user not in self._streams_per_user: - self._streams_per_user[user] = 0 + # Make sure we set the streams per user to 1 here rather than + # setting it to zero and incrementing the value below. + # Otherwise this may race with stopped_stream causing the + # user to be erased from the map before we have a chance + # to increment it. + self._streams_per_user[user] = 1 if user in self._stop_timer_per_user: try: self.clock.cancel_call_later( @@ -79,8 +84,8 @@ class EventStreamHandler(BaseHandler): logger.exception("Failed to cancel event timer") else: yield started_user_eventstream(self.distributor, user) - - self._streams_per_user[user] += 1 + else: + self._streams_per_user[user] += 1 def stopped_stream(self, user): """If there are no streams for a user this starts a timer that will From 2acae8300fa272caeb774f24d19b80632eca7ae3 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 14 Dec 2015 15:19:37 +0000 Subject: [PATCH 22/36] Fix logging to lie less --- synapse/rest/client/v2_alpha/sync.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index adf77e13bf..b16831246d 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -351,8 +351,6 @@ class SyncRestServlet(RestServlet): continue prev_event_id = timeline_event.unsigned.get("replaces_state", None) - logger.debug("Replacing %s with %s in state dict", - timeline_event.event_id, prev_event_id) prev_content = timeline_event.unsigned.get('prev_content') prev_sender = timeline_event.unsigned.get('prev_sender') @@ -363,8 +361,17 @@ class SyncRestServlet(RestServlet): # If this is the case the we ignore the previous event. This will # cause the displayname calculations on the client to be incorrect if prev_event_id is None or not prev_content or not prev_sender: + logger.debug( + "Removing %r from the state dict, as it is missing " + " prev_content (prev_event_id=%r)", + timeline_event.event_id, prev_event_id + ) del result[event_key] else: + logger.debug( + "Replacing %r with %r in state dict", + timeline_event.event_id, prev_event_id + ) result[event_key] = FrozenEvent({ "type": timeline_event.type, "state_key": timeline_event.state_key, From 3ddf0b97223fe1f5818251256332c8bad6909020 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 14 Dec 2015 15:20:59 +0000 Subject: [PATCH 23/36] Fix spacing --- synapse/rest/client/v2_alpha/sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index b16831246d..73b44e92eb 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -362,7 +362,7 @@ class SyncRestServlet(RestServlet): # cause the displayname calculations on the client to be incorrect if prev_event_id is None or not prev_content or not prev_sender: logger.debug( - "Removing %r from the state dict, as it is missing " + "Removing %r from the state dict, as it is missing" " prev_content (prev_event_id=%r)", timeline_event.event_id, prev_event_id ) From 63fdd9fe0bc5ed3aabe27af7e082d697e4863c83 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 14 Dec 2015 16:26:59 +0000 Subject: [PATCH 24/36] Changelog and version bump for v0.12.0-rc2 --- CHANGES.rst | 11 +++++++++++ synapse/__init__.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index f81a51dc7f..6f427f677b 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,14 @@ +Changes in synapse v0.12.0-rc2 (2015-12-14) +=========================================== + +* Add caches for whether rooms have been forgotten by a user (PR #434) +* Remove instructions to use ``--process-dependency-link`` since all of the + dependencies of synapse are on PyPI (PR #436) +* Parallelise the processing of ``/sync`` requests (PR #437) +* Fix race updating presence in ``/events`` (PR #444) +* Fix bug back-populating search results (PR #441) +* Fix bug calculating state in ``/sync`` requests (PR #442) + Changes in synapse v0.12.0-rc1 (2015-12-10) =========================================== diff --git a/synapse/__init__.py b/synapse/__init__.py index c357f8f9c2..e07c26ccd0 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.12.0-rc1" +__version__ = "0.12.0-rc2" From dcfc70e8ed263256b2a3cf59e7d21e54f39fc287 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 15 Dec 2015 17:02:21 +0000 Subject: [PATCH 25/36] Allow users to change which account a 3pid is bound to --- synapse/storage/registration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 2e5eddd259..09a05b08ef 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -258,10 +258,10 @@ class RegistrationStore(SQLBaseStore): @defer.inlineCallbacks def user_add_threepid(self, user_id, medium, address, validated_at, added_at): yield self._simple_upsert("user_threepids", { - "user_id": user_id, "medium": medium, "address": address, }, { + "user_id": user_id, "validated_at": validated_at, "added_at": added_at, }) From 0311612ce9c70d2748cdf2badbd87c854ef5ba8d Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 16 Dec 2015 13:05:32 +0000 Subject: [PATCH 26/36] Give the IS a bunch more 3pid invite context This allows it to form richer emails --- synapse/handlers/room.py | 67 +++++++++++++++++++++++++++++++++++----- 1 file changed, 60 insertions(+), 7 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index a72c3fda9f..6a482dacc9 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -704,13 +704,48 @@ class RoomMemberHandler(BaseHandler): token_id, txn_id ): + room_state = yield self.hs.get_state_handler().get_current_state(room_id) + + inviter_display_name = "" + inviter_avatar_url = "" + member_event = room_state.get((EventTypes.Member, user.to_string())) + if member_event: + inviter_display_name = member_event.content.get("displayname", "") + inviter_avatar_url = member_event.content.get("avatar_url", "") + + canonical_room_alias = "" + canonical_alias_event = room_state.get((EventTypes.CanonicalAlias, "")) + if canonical_alias_event: + canonical_room_alias = canonical_alias_event.content.get("alias", "") + + room_name = "" + room_name_event = room_state.get((EventTypes.Name, "")) + if room_name_event: + room_name = room_name_event.content.get("name", "") + + room_join_rules = "" + join_rules_event = room_state.get((EventTypes.JoinRules, "")) + if join_rules_event: + room_join_rules = join_rules_event.content.get("join_rule", "") + + room_avatar_url = "" + room_avatar_event = room_state.get((EventTypes.RoomAvatar, "")) + if room_avatar_event: + room_avatar_url = room_avatar_event.content.get("url", "") + token, public_key, key_validity_url, display_name = ( yield self._ask_id_server_for_third_party_invite( - id_server, - medium, - address, - room_id, - user.to_string() + id_server=id_server, + medium=medium, + address=address, + room_id=room_id, + inviter_user_id=user.to_string(), + room_alias=canonical_room_alias, + room_avatar_url=room_avatar_url, + room_join_rules=room_join_rules, + room_name=room_name, + inviter_display_name=inviter_display_name, + inviter_avatar_url=inviter_avatar_url ) ) msg_handler = self.hs.get_handlers().message_handler @@ -732,7 +767,19 @@ class RoomMemberHandler(BaseHandler): @defer.inlineCallbacks def _ask_id_server_for_third_party_invite( - self, id_server, medium, address, room_id, sender): + self, + id_server, + medium, + address, + room_id, + inviter_user_id, + room_alias, + room_avatar_url, + room_join_rules, + room_name, + inviter_display_name, + inviter_avatar_url + ): is_url = "%s%s/_matrix/identity/api/v1/store-invite" % ( id_server_scheme, id_server, ) @@ -742,7 +789,13 @@ class RoomMemberHandler(BaseHandler): "medium": medium, "address": address, "room_id": room_id, - "sender": sender, + "room_alias": room_alias, + "room_avatar_url": room_avatar_url, + "room_join_rules": room_join_rules, + "room_name": room_name, + "sender": inviter_user_id, + "sender_display_name": inviter_display_name, + "sender_avatar_url": inviter_avatar_url, } ) # TODO: Check for success From 2b0f8a948286424212e79b9ac5d22d5be6707f1f Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 16 Dec 2015 17:59:44 +0100 Subject: [PATCH 27/36] Fix typo --- synapse/http/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/http/server.py b/synapse/http/server.py index 1b936b6892..682b6b379b 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -129,7 +129,7 @@ def request_handler(request_handler): 500, { "error": "Internal server error", - "errcode": Codes.M_UNKNOWN, + "errcode": Codes.UNKNOWN, }, send_cors=True ) From a64f9bbfe0fc592043a3da8979b7f2545187dbb6 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 17 Dec 2015 12:47:26 +0000 Subject: [PATCH 28/36] Fix 500 error when back-paginating search results We were mistakenly adding pagination clauses to the count query, which then failed because the count query doesn't join to the events table. --- synapse/storage/search.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/synapse/storage/search.py b/synapse/storage/search.py index 57c9cc1c5f..6cb5e73b6e 100644 --- a/synapse/storage/search.py +++ b/synapse/storage/search.py @@ -286,8 +286,10 @@ class SearchStore(BackgroundUpdateStore): "(%s)" % (" OR ".join(local_clauses),) ) - count_args = args - count_clauses = clauses + # take copies of the current args and clauses lists, before adding + # pagination clauses to main query. + count_args = list(args) + count_clauses = list(clauses) if pagination_token: try: From 8c5f252edbb0c62663116c6a541ce8691414996a Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 17 Dec 2015 18:09:51 +0100 Subject: [PATCH 29/36] Strip address and such out of 3pid invites We're not meant to leak that into the graph --- synapse/api/auth.py | 2 +- synapse/handlers/federation.py | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index b9c3e6d2c4..adb7d64482 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -778,7 +778,7 @@ class Auth(object): if "third_party_invite" in event.content: key = ( EventTypes.ThirdPartyInvite, - event.content["third_party_invite"]["token"] + event.content["third_party_invite"]["signed"]["token"] ) third_party_invite = current_state.get(key) if third_party_invite: diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index e7ad48c948..1255241461 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1650,11 +1650,22 @@ class FederationHandler(BaseHandler): sender = invite["sender"] room_id = invite["room_id"] + if "signed" not in invite: + logger.info( + "Discarding received notification of third party invite " + "without signed: %s" % (invite,) + ) + return + + third_party_invite = { + "signed": invite["signed"], + } + event_dict = { "type": EventTypes.Member, "content": { "membership": Membership.INVITE, - "third_party_invite": invite, + "third_party_invite": third_party_invite, }, "room_id": room_id, "sender": sender, From bdacee476d2642753cfa54f5092e56ecb148ff56 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 17 Dec 2015 18:31:20 +0100 Subject: [PATCH 30/36] Add display_name to 3pid invite in m.room.member invites --- synapse/handlers/federation.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 1255241461..28f2ff68d6 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1650,7 +1650,7 @@ class FederationHandler(BaseHandler): sender = invite["sender"] room_id = invite["room_id"] - if "signed" not in invite: + if "signed" not in invite or "token" not in invite["signed"]: logger.info( "Discarding received notification of third party invite " "without signed: %s" % (invite,) @@ -1676,6 +1676,11 @@ class FederationHandler(BaseHandler): builder = self.event_builder_factory.new(event_dict) EventValidator().validate_new(builder) event, context = yield self._create_new_client_event(builder=builder) + + event, context = yield self.add_display_name_to_third_party_invite( + event_dict, event, context + ) + self.auth.check(event, context.current_state) yield self._validate_keyserver(event, auth_events=context.current_state) member_handler = self.hs.get_handlers().room_member_handler @@ -1697,6 +1702,10 @@ class FederationHandler(BaseHandler): builder=builder, ) + event, context = yield self.add_display_name_to_third_party_invite( + event_dict, event, context + ) + self.auth.check(event, auth_events=context.current_state) yield self._validate_keyserver(event, auth_events=context.current_state) @@ -1706,6 +1715,27 @@ class FederationHandler(BaseHandler): member_handler = self.hs.get_handlers().room_member_handler yield member_handler.change_membership(event, context) + @defer.inlineCallbacks + def add_display_name_to_third_party_invite(self, event_dict, event, context): + key = ( + EventTypes.ThirdPartyInvite, + event.content["third_party_invite"]["signed"]["token"] + ) + original_invite = context.current_state.get(key) + if not original_invite: + logger.info( + "Could not find invite event for third_party_invite - " + "discarding: %s" % (event_dict,) + ) + return + + display_name = original_invite.content["display_name"] + event_dict["content"]["third_party_invite"]["display_name"] = display_name + builder = self.event_builder_factory.new(event_dict) + EventValidator().validate_new(builder) + event, context = yield self._create_new_client_event(builder=builder) + defer.returnValue((event, context)) + @defer.inlineCallbacks def _validate_keyserver(self, event, auth_events): token = event.content["third_party_invite"]["signed"]["token"] From 772ad4f71503866842eb9033b220b757ef20e711 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 17 Dec 2015 23:04:20 +0000 Subject: [PATCH 31/36] stop generating default identicons. reverts most of 582019f870adbc4a8a8a9ef97b527e0fead77761 and solves vector-web/vector-im#346 --- synapse/handlers/register.py | 22 +++------------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index a037da0f70..8a365c20f9 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -132,25 +132,9 @@ class RegistrationHandler(BaseHandler): raise RegistrationError( 500, "Cannot generate user ID.") - # create a default avatar for the user - # XXX: ideally clients would explicitly specify one, but given they don't - # and we want consistent and pretty identicons for random users, we'll - # do it here. - try: - auth_user = UserID.from_string(user_id) - media_repository = self.hs.get_resource_for_media_repository() - identicon_resource = media_repository.getChildWithDefault("identicon", None) - upload_resource = media_repository.getChildWithDefault("upload", None) - identicon_bytes = identicon_resource.generate_identicon(user_id, 320, 320) - content_uri = yield upload_resource.create_content( - "image/png", None, identicon_bytes, len(identicon_bytes), auth_user - ) - profile_handler = self.hs.get_handlers().profile_handler - profile_handler.set_avatar_url( - auth_user, auth_user, ("%s#auto" % (content_uri,)) - ) - except NotImplementedError: - pass # make tests pass without messing around creating default avatars + # We used to generate default identicons here, but nowadays + # we want clients to generate their own as part of their branding + # rather than there being consistent matrix-wide ones, so we don't. defer.returnValue((user_id, token)) From 64374bda5b47e043a5ff3c0af23bd29461596059 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 17 Dec 2015 23:04:53 +0000 Subject: [PATCH 32/36] fix indentation level --- synapse/handlers/register.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 8a365c20f9..698e7d4479 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -132,9 +132,9 @@ class RegistrationHandler(BaseHandler): raise RegistrationError( 500, "Cannot generate user ID.") - # We used to generate default identicons here, but nowadays - # we want clients to generate their own as part of their branding - # rather than there being consistent matrix-wide ones, so we don't. + # We used to generate default identicons here, but nowadays + # we want clients to generate their own as part of their branding + # rather than there being consistent matrix-wide ones, so we don't. defer.returnValue((user_id, token)) From ce4999268a06ccc716d1340b0f4c3e88103d7084 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 18 Dec 2015 10:06:56 +0000 Subject: [PATCH 33/36] Fix typo that broke registration on the mobile clients --- synapse/handlers/register.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index a037da0f70..19df5aa852 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -42,7 +42,7 @@ class RegistrationHandler(BaseHandler): self.distributor = hs.get_distributor() self.distributor.declare("registered_user") - self.captch_client = CaptchaServerHttpClient(hs) + self.captcha_client = CaptchaServerHttpClient(hs) @defer.inlineCallbacks def check_username(self, localpart): From 2f871ad143988199a5c3ceac918f721752968e71 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 18 Dec 2015 20:44:47 +0000 Subject: [PATCH 34/36] Generate code coverage report when running jenkins.sh --- jenkins.sh | 21 ++++++++++++++++----- tox.ini | 3 ++- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/jenkins.sh b/jenkins.sh index 0018ca610a..7075b1a51a 100755 --- a/jenkins.sh +++ b/jenkins.sh @@ -5,9 +5,10 @@ export PYTHONDONTWRITEBYTECODE=yep # Output test results as junit xml export TRIAL_FLAGS="--reporter=subunit" export TOXSUFFIX="| subunit-1to2 | subunit2junitxml --no-passthrough --output-to=results.xml" - -# Output coverage to coverage.xml -export DUMP_COVERAGE_COMMAND="coverage xml -o coverage.xml" +# Write coverage reports to a separate file for each process +# Include branch coverage +export COVERAGE_OPTS="-p" +export DUMP_COVERAGE_COMMAND="coverage help" # Output flake8 violations to violations.flake8.log # Don't exit with non-0 status code on Jenkins, @@ -15,6 +16,8 @@ export DUMP_COVERAGE_COMMAND="coverage xml -o coverage.xml" # UNSTABLE or FAILURE this build. export PEP8SUFFIX="--output-file=violations.flake8.log || echo flake8 finished with status code \$?" +rm .coverage.* || echo "No files to remove" + tox : ${GIT_BRANCH:="origin/$(git rev-parse --abbrev-ref HEAD)"} @@ -45,7 +48,7 @@ export PERL5LIB PERL_MB_OPT PERL_MM_OPT : ${PORT_BASE:=8000} echo >&2 "Running sytest with SQLite3"; -./run-tests.pl -O tap --synapse-directory .. --all --port-base $PORT_BASE > results-sqlite3.tap +./run-tests.pl --coverage -O tap --synapse-directory .. --all --port-base $PORT_BASE > results-sqlite3.tap RUN_POSTGRES="" @@ -64,7 +67,15 @@ done if test $RUN_POSTGRES = ":$(($PORT_BASE + 1)):$(($PORT_BASE + 2))"; then echo >&2 "Running sytest with PostgreSQL"; pip install psycopg2 - ./run-tests.pl -O tap --synapse-directory .. --all --port-base $PORT_BASE > results-postgresql.tap + ./run-tests.pl --coverage -O tap --synapse-directory .. --all --port-base $PORT_BASE > results-postgresql.tap else echo >&2 "Skipping running sytest with PostgreSQL, $RUN_POSTGRES" fi + +cd .. +cp sytest/.coverage.* . + +# Combine the coverage reports +python -m coverage combine +# Output coverage to coverage.xml +coverage xml -o coverage.xml diff --git a/tox.ini b/tox.ini index 95424765c3..bd313a4f36 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,8 @@ deps = setenv = PYTHONDONTWRITEBYTECODE = no_byte_code commands = - /bin/bash -c "coverage run --source=synapse {envbindir}/trial {env:TRIAL_FLAGS:} {posargs:tests} {env:TOXSUFFIX:}" + /bin/bash -c "coverage run {env:COVERAGE_OPTS:} --source={toxinidir}/synapse \ + {envbindir}/trial {env:TRIAL_FLAGS:} {posargs:tests} {env:TOXSUFFIX:}" {env:DUMP_COVERAGE_COMMAND:coverage report -m} [testenv:packaging] From a6ba41e0785f8f597713bd023e1f6dc3a3d966ea Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Fri, 18 Dec 2015 21:36:42 +0000 Subject: [PATCH 35/36] Actually look up required remote server key IDs set.union() is a side-effect-free function that returns the union of two sets. This clearly wanted .update(), which is the side-effecting mutator version. --- synapse/crypto/keyring.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index bc5bb5cdb1..1fea568eed 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -230,7 +230,9 @@ class Keyring(object): missing_keys = {} for group in group_id_to_group.values(): - missing_keys.setdefault(group.server_name, set()).union(group.key_ids) + missing_keys.setdefault(group.server_name, set()).update( + group.key_ids + ) for fn in key_fetch_fns: results = yield fn(missing_keys.items()) From 64b660682492cd9addaa4a681b0b7780fc23d9d9 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 21 Dec 2015 15:22:03 +0000 Subject: [PATCH 36/36] Remove accidentally committed debug logging --- synapse/rest/client/v2_alpha/sync.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 73b44e92eb..697df03dda 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -104,7 +104,6 @@ class SyncRestServlet(RestServlet): ) if filter_id and filter_id.startswith('{'): - logging.error("MJH %r", filter_id) try: filter_object = json.loads(filter_id) except: