From 74c38797601f6d7d1a02d21fc54ceb1a54629c64 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 19 Nov 2014 18:20:59 +0000 Subject: [PATCH 001/180] Start creating a module to do generic notifications (just prints them to stdout currently!) --- synapse/api/errors.py | 1 + synapse/app/homeserver.py | 2 + synapse/push/__init__.py | 76 ++++++++++++++++++++++ synapse/push/httppusher.py | 40 ++++++++++++ synapse/push/pusherpool.py | 94 +++++++++++++++++++++++++++ synapse/rest/__init__.py | 3 +- synapse/rest/pusher.py | 71 +++++++++++++++++++++ synapse/server.py | 5 ++ synapse/storage/__init__.py | 6 +- synapse/storage/pusher.py | 98 +++++++++++++++++++++++++++++ synapse/storage/schema/delta/v7.sql | 28 +++++++++ synapse/storage/schema/pusher.sql | 28 +++++++++ 12 files changed, 449 insertions(+), 3 deletions(-) create mode 100644 synapse/push/__init__.py create mode 100644 synapse/push/httppusher.py create mode 100644 synapse/push/pusherpool.py create mode 100644 synapse/rest/pusher.py create mode 100644 synapse/storage/pusher.py create mode 100644 synapse/storage/schema/delta/v7.sql create mode 100644 synapse/storage/schema/pusher.sql diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 33d15072af..97750ca2b0 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -32,6 +32,7 @@ class Codes(object): LIMIT_EXCEEDED = "M_LIMIT_EXCEEDED" CAPTCHA_NEEDED = "M_CAPTCHA_NEEDED" CAPTCHA_INVALID = "M_CAPTCHA_INVALID" + MISSING_PARAM = "M_MISSING_PARAM" class CodeMessageException(Exception): diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 85284a4919..de16d8a2e1 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -242,6 +242,8 @@ def setup(): bind_port = None hs.start_listening(bind_port, config.unsecure_port) + hs.get_pusherpool().start() + if config.daemonize: print config.pid_file daemon = Daemonize( diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py new file mode 100644 index 0000000000..df0b91a8e9 --- /dev/null +++ b/synapse/push/__init__.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from synapse.streams.config import PaginationConfig +from synapse.types import StreamToken + +import synapse.util.async + +import logging + +logger = logging.getLogger(__name__) + +class Pusher(object): + INITIAL_BACKOFF = 1000 + MAX_BACKOFF = 10 * 60 * 1000 + + def __init__(self, _hs, user_name, app, app_display_name, device_display_name, pushkey, data, last_token): + self.hs = _hs + self.evStreamHandler = self.hs.get_handlers().event_stream_handler + self.store = self.hs.get_datastore() + self.user_name = user_name + self.app = app + self.app_display_name = app_display_name + self.device_display_name = device_display_name + self.pushkey = pushkey + self.data = data + self.last_token = last_token + self.backoff_delay = Pusher.INITIAL_BACKOFF + + @defer.inlineCallbacks + def start(self): + if not self.last_token: + # First-time setup: get a token to start from (we can't just start from no token, ie. 'now' + # because we need the result to be reproduceable in case we fail to dispatch the push) + config = PaginationConfig(from_token=None, limit='1') + chunk = yield self.evStreamHandler.get_stream(self.user_name, config, timeout=0) + self.last_token = chunk['end'] + self.store.update_pusher_last_token(self.user_name, self.pushkey, self.last_token) + logger.info("Pusher %s for user %s starting from token %s", + self.pushkey, self.user_name, self.last_token) + + while True: + from_tok = StreamToken.from_string(self.last_token) + config = PaginationConfig(from_token=from_tok, limit='1') + chunk = yield self.evStreamHandler.get_stream(self.user_name, config, timeout=100*365*24*60*60*1000) + + if (self.dispatchPush(chunk['chunk'][0])): + self.backoff_delay = Pusher.INITIAL_BACKOFF + self.last_token = chunk['end'] + self.store.update_pusher_last_token(self.user_name, self.pushkey, self.last_token) + else: + logger.warn("Failed to dispatch push for user %s. Trying again in %dms", + self.user_name, self.backoff_delay) + yield synapse.util.async.sleep(self.backoff_delay / 1000.0) + self.backoff_delay *=2 + if self.backoff_delay > Pusher.MAX_BACKOFF: + self.backoff_delay = Pusher.MAX_BACKOFF + + +class PusherConfigException(Exception): + def __init__(self, msg): + super(PusherConfigException, self).__init__(msg) \ No newline at end of file diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py new file mode 100644 index 0000000000..988c4e32f5 --- /dev/null +++ b/synapse/push/httppusher.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from synapse.push import Pusher, PusherConfigException +from synapse.http.client import + +import logging + +logger = logging.getLogger(__name__) + +class HttpPusher(Pusher): + def __init__(self, _hs, user_name, app, app_display_name, device_display_name, pushkey, data, last_token): + super(HttpPusher, self).__init__(_hs, + user_name, + app, + app_display_name, + device_display_name, + pushkey, + data, + last_token) + if 'url' not in data: + raise PusherConfigException("'url' required in data for HTTP pusher") + self.url = data['url'] + + def dispatchPush(self, event): + print event + return True + diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py new file mode 100644 index 0000000000..436040f123 --- /dev/null +++ b/synapse/push/pusherpool.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from httppusher import HttpPusher +from synapse.push import PusherConfigException + +import logging +import json + +logger = logging.getLogger(__name__) + +class PusherPool: + def __init__(self, _hs): + self.hs = _hs + self.store = self.hs.get_datastore() + self.pushers = [] + self.last_pusher_started = -1 + + def start(self): + self._pushers_added() + + def add_pusher(self, user_name, kind, app, app_display_name, device_display_name, pushkey, data): + # we try to create the pusher just to validate the config: it will then get pulled out of the database, + # recreated, added and started: this means we have only one code path adding pushers. + self._create_pusher({ + "user_name": user_name, + "kind": kind, + "app": app, + "app_display_name": app_display_name, + "device_display_name": device_display_name, + "pushkey": pushkey, + "data": data, + "last_token": None + }) + self._add_pusher_to_store(user_name, kind, app, app_display_name, device_display_name, pushkey, data) + + @defer.inlineCallbacks + def _add_pusher_to_store(self, user_name, kind, app, app_display_name, device_display_name, pushkey, data): + yield self.store.add_pusher(user_name=user_name, + kind=kind, + app=app, + app_display_name=app_display_name, + device_display_name=device_display_name, + pushkey=pushkey, + data=json.dumps(data)) + self._pushers_added() + + def _create_pusher(self, pusherdict): + if pusherdict['kind'] == 'http': + return HttpPusher(self.hs, + user_name=pusherdict['user_name'], + app=pusherdict['app'], + app_display_name=pusherdict['app_display_name'], + device_display_name=pusherdict['device_display_name'], + pushkey=pusherdict['pushkey'], + data=pusherdict['data'], + last_token=pusherdict['last_token'] + ) + else: + raise PusherConfigException("Unknown pusher type '%s' for user %s" % + (pusherdict['kind'], pusherdict['user_name'])) + + @defer.inlineCallbacks + def _pushers_added(self): + pushers = yield self.store.get_all_pushers_after_id(self.last_pusher_started) + for p in pushers: + p['data'] = json.loads(p['data']) + if (len(pushers)): + self.last_pusher_started = pushers[-1]['id'] + + self._start_pushers(pushers) + + def _start_pushers(self, pushers): + logger.info("Starting %d pushers", (len(pushers))) + for pusherdict in pushers: + p = self._create_pusher(pusherdict) + if p: + self.pushers.append(p) + p.start() \ No newline at end of file diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index e391e5678d..c38cf27690 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -16,7 +16,7 @@ from . import ( room, events, register, login, profile, presence, initial_sync, directory, - voip, admin, + voip, admin, pusher, ) @@ -45,3 +45,4 @@ class RestServletFactory(object): directory.register_servlets(hs, client_resource) voip.register_servlets(hs, client_resource) admin.register_servlets(hs, client_resource) + pusher.register_servlets(hs, client_resource) diff --git a/synapse/rest/pusher.py b/synapse/rest/pusher.py new file mode 100644 index 0000000000..85d0d1c8cd --- /dev/null +++ b/synapse/rest/pusher.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from synapse.api.errors import SynapseError, Codes +from synapse.push import PusherConfigException +from base import RestServlet, client_path_pattern + +import json + + +class PusherRestServlet(RestServlet): + PATTERN = client_path_pattern("/pushers/(?P[\w]*)$") + + @defer.inlineCallbacks + def on_PUT(self, request, pushkey): + user = yield self.auth.get_user_by_req(request) + + content = _parse_json(request) + + reqd = ['kind', 'app', 'app_display_name', 'device_display_name', 'data'] + missing = [] + for i in reqd: + if i not in content: + missing.append(i) + if len(missing): + raise SynapseError(400, "Missing parameters: "+','.join(missing), errcode=Codes.MISSING_PARAM) + + pusher_pool = self.hs.get_pusherpool() + try: + pusher_pool.add_pusher(user_name=user.to_string(), + kind=content['kind'], + app=content['app'], + app_display_name=content['app_display_name'], + device_display_name=content['device_display_name'], + pushkey=pushkey, + data=content['data']) + except PusherConfigException as pce: + raise SynapseError(400, "Config Error: "+pce.message, errcode=Codes.MISSING_PARAM) + + defer.returnValue((200, {})) + + def on_OPTIONS(self, request): + return (200, {}) + +# XXX: C+ped from rest/room.py - surely this should be common? +def _parse_json(request): + try: + content = json.loads(request.content.read()) + if type(content) != dict: + raise SynapseError(400, "Content must be a JSON object.", + errcode=Codes.NOT_JSON) + return content + except ValueError: + raise SynapseError(400, "Content not JSON.", errcode=Codes.NOT_JSON) + +def register_servlets(hs, http_server): + PusherRestServlet(hs).register(http_server) diff --git a/synapse/server.py b/synapse/server.py index da0a44433a..cfbe7d5e38 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -36,6 +36,7 @@ from synapse.util.lockutils import LockManager from synapse.streams.events import EventSources from synapse.api.ratelimiting import Ratelimiter from synapse.crypto.keyring import Keyring +from synapse.push.pusherpool import PusherPool class BaseHomeServer(object): @@ -82,6 +83,7 @@ class BaseHomeServer(object): 'ratelimiter', 'keyring', 'event_validator', + 'pusherpool' ] def __init__(self, hostname, **kwargs): @@ -228,6 +230,9 @@ class HomeServer(BaseHomeServer): def build_event_validator(self): return EventValidator(self) + def build_pusherpool(self): + return PusherPool(self) + def register_servlets(self): """ Register all servlets associated with this HomeServer. """ diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index c36d938d96..5957f938a4 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -33,6 +33,7 @@ from .stream import StreamStore from .transactions import TransactionStore from .keys import KeyStore from .event_federation import EventFederationStore +from .pusher import PusherStore from .state import StateStore from .signatures import SignatureStore @@ -62,12 +63,13 @@ SCHEMAS = [ "state", "event_edges", "event_signatures", + "pusher" ] # Remember to update this number every time an incompatible change is made to # database schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 6 +SCHEMA_VERSION = 7 class _RollbackButIsFineException(Exception): @@ -81,7 +83,7 @@ class DataStore(RoomMemberStore, RoomStore, RegistrationStore, StreamStore, ProfileStore, FeedbackStore, PresenceStore, TransactionStore, DirectoryStore, KeyStore, StateStore, SignatureStore, - EventFederationStore, ): + EventFederationStore, PusherStore, ): def __init__(self, hs): super(DataStore, self).__init__(hs) diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py new file mode 100644 index 0000000000..047a5f42d9 --- /dev/null +++ b/synapse/storage/pusher.py @@ -0,0 +1,98 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections + +from ._base import SQLBaseStore, Table +from twisted.internet import defer + +from sqlite3 import IntegrityError +from synapse.api.errors import StoreError + +import logging + +logger = logging.getLogger(__name__) + +class PusherStore(SQLBaseStore): + @defer.inlineCallbacks + def get_all_pushers_after_id(self, min_id): + sql = ( + "SELECT id, user_name, kind, app, app_display_name, device_display_name, pushkey, data, last_token " + "FROM pushers " + "WHERE id > ?" + ) + + rows = yield self._execute(None, sql, min_id) + + ret = [ + { + "id": r[0], + "user_name": r[1], + "kind": r[2], + "app": r[3], + "app_display_name": r[4], + "device_display_name": r[5], + "pushkey": r[6], + "data": r[7], + "last_token": r[8] + + } + for r in rows + ] + + defer.returnValue(ret) + + @defer.inlineCallbacks + def add_pusher(self, user_name, kind, app, app_display_name, device_display_name, pushkey, data): + try: + yield self._simple_insert(PushersTable.table_name, dict( + user_name=user_name, + kind=kind, + app=app, + app_display_name=app_display_name, + device_display_name=device_display_name, + pushkey=pushkey, + data=data + )) + except IntegrityError: + raise StoreError(409, "Pushkey in use.") + except Exception as e: + logger.error("create_pusher with failed: %s", e) + raise StoreError(500, "Problem creating pusher.") + + @defer.inlineCallbacks + def update_pusher_last_token(self, user_name, pushkey, last_token): + yield self._simple_update_one(PushersTable.table_name, + {'user_name': user_name, 'pushkey': pushkey}, + {'last_token': last_token} + ) + + +class PushersTable(Table): + table_name = "pushers" + + fields = [ + "id", + "user_name", + "kind", + "app" + "app_display_name", + "device_display_name", + "pushkey", + "data", + "last_token" + ] + + EntryType = collections.namedtuple("PusherEntry", fields) \ No newline at end of file diff --git a/synapse/storage/schema/delta/v7.sql b/synapse/storage/schema/delta/v7.sql new file mode 100644 index 0000000000..7f6852485d --- /dev/null +++ b/synapse/storage/schema/delta/v7.sql @@ -0,0 +1,28 @@ +/* Copyright 2014 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +-- Push notification endpoints that users have configured +CREATE TABLE IF NOT EXISTS pushers ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_name TEXT NOT NULL, + kind varchar(8) NOT NULL, + app varchar(64) NOT NULL, + app_display_name varchar(64) NOT NULL, + device_display_name varchar(128) NOT NULL, + pushkey blob NOT NULL, + data text, + last_token TEXT, + FOREIGN KEY(user_name) REFERENCES users(name), + UNIQUE (user_name, pushkey) +); diff --git a/synapse/storage/schema/pusher.sql b/synapse/storage/schema/pusher.sql new file mode 100644 index 0000000000..7f6852485d --- /dev/null +++ b/synapse/storage/schema/pusher.sql @@ -0,0 +1,28 @@ +/* Copyright 2014 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +-- Push notification endpoints that users have configured +CREATE TABLE IF NOT EXISTS pushers ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_name TEXT NOT NULL, + kind varchar(8) NOT NULL, + app varchar(64) NOT NULL, + app_display_name varchar(64) NOT NULL, + device_display_name varchar(128) NOT NULL, + pushkey blob NOT NULL, + data text, + last_token TEXT, + FOREIGN KEY(user_name) REFERENCES users(name), + UNIQUE (user_name, pushkey) +); From 051b18581109022d79d9f270d5f5e565688d89d0 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 19 Nov 2014 18:37:00 +0000 Subject: [PATCH 002/180] remove random half-line --- synapse/push/httppusher.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 988c4e32f5..f3c3ca8191 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -14,7 +14,6 @@ # limitations under the License. from synapse.push import Pusher, PusherConfigException -from synapse.http.client import import logging From eb6aedf92c0fe467fd4724623262907ad78573bb Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 21 Nov 2014 12:21:00 +0000 Subject: [PATCH 003/180] More work on pushers. Attempt to do HTTP pokes. Not sure if the actual HTTP pokes work or not yet but the retry semantics are pretty good. --- synapse/http/client.py | 19 ++++++++++ synapse/push/__init__.py | 58 ++++++++++++++++++++++++----- synapse/push/httppusher.py | 55 +++++++++++++++++++++++++-- synapse/push/pusherpool.py | 8 +++- synapse/storage/pusher.py | 26 +++++++++++-- synapse/storage/schema/delta/v7.sql | 2 + synapse/storage/schema/pusher.sql | 2 + 7 files changed, 150 insertions(+), 20 deletions(-) diff --git a/synapse/http/client.py b/synapse/http/client.py index 048a428905..82e80385ce 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -60,6 +60,25 @@ class SimpleHttpClient(object): defer.returnValue(json.loads(body)) + @defer.inlineCallbacks + def post_json_get_json(self, uri, post_json): + json_str = json.dumps(post_json) + + logger.info("HTTP POST %s -> %s", json_str, uri) + + response = yield self.agent.request( + "POST", + uri.encode("ascii"), + headers=Headers({ + "Content-Type": ["application/json"] + }), + bodyProducer=FileBodyProducer(StringIO(json_str)) + ) + + body = yield readBody(response) + + defer.returnValue(json.loads(body)) + @defer.inlineCallbacks def get_json(self, uri, args={}): """ Get's some json from the given host and path diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index df0b91a8e9..a96f0f0183 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -26,12 +26,15 @@ logger = logging.getLogger(__name__) class Pusher(object): INITIAL_BACKOFF = 1000 - MAX_BACKOFF = 10 * 60 * 1000 + MAX_BACKOFF = 60 * 60 * 1000 + GIVE_UP_AFTER = 24 * 60 * 60 * 1000 - def __init__(self, _hs, user_name, app, app_display_name, device_display_name, pushkey, data, last_token): + def __init__(self, _hs, user_name, app, app_display_name, device_display_name, pushkey, data, + last_token, last_success, failing_since): self.hs = _hs self.evStreamHandler = self.hs.get_handlers().event_stream_handler self.store = self.hs.get_datastore() + self.clock = self.hs.get_clock() self.user_name = user_name self.app = app self.app_display_name = app_display_name @@ -40,6 +43,7 @@ class Pusher(object): self.data = data self.last_token = last_token self.backoff_delay = Pusher.INITIAL_BACKOFF + self.failing_since = None @defer.inlineCallbacks def start(self): @@ -58,17 +62,51 @@ class Pusher(object): config = PaginationConfig(from_token=from_tok, limit='1') chunk = yield self.evStreamHandler.get_stream(self.user_name, config, timeout=100*365*24*60*60*1000) - if (self.dispatchPush(chunk['chunk'][0])): + # limiting to 1 may get 1 event plus 1 presence event, so pick out the actual event + singleEvent = None + for c in chunk['chunk']: + if 'event_id' in c: # Hmmm... + singleEvent = c + break + if not singleEvent: + continue + + ret = yield self.dispatchPush(singleEvent) + if (ret): self.backoff_delay = Pusher.INITIAL_BACKOFF self.last_token = chunk['end'] - self.store.update_pusher_last_token(self.user_name, self.pushkey, self.last_token) + self.store.update_pusher_last_token_and_success(self.user_name, self.pushkey, + self.last_token, self.clock.time_msec()) + if self.failing_since: + self.failing_since = None + self.store.update_pusher_failing_since(self.user_name, self.pushkey, self.failing_since) else: - logger.warn("Failed to dispatch push for user %s. Trying again in %dms", - self.user_name, self.backoff_delay) - yield synapse.util.async.sleep(self.backoff_delay / 1000.0) - self.backoff_delay *=2 - if self.backoff_delay > Pusher.MAX_BACKOFF: - self.backoff_delay = Pusher.MAX_BACKOFF + if not self.failing_since: + self.failing_since = self.clock.time_msec() + self.store.update_pusher_failing_since(self.user_name, self.pushkey, self.failing_since) + + if self.failing_since and self.failing_since < self.clock.time_msec() - Pusher.GIVE_UP_AFTER: + # we really only give up so that if the URL gets fixed, we don't suddenly deliver a load + # of old notifications. + logger.warn("Giving up on a notification to user %s, pushkey %s", + self.user_name, self.pushkey) + self.backoff_delay = Pusher.INITIAL_BACKOFF + self.last_token = chunk['end'] + self.store.update_pusher_last_token(self.user_name, self.pushkey, self.last_token) + + self.failing_since = None + self.store.update_pusher_failing_since(self.user_name, self.pushkey, self.failing_since) + else: + logger.warn("Failed to dispatch push for user %s (failing for %dms)." + "Trying again in %dms", + self.user_name, + self.clock.time_msec() - self.failing_since, + self.backoff_delay + ) + yield synapse.util.async.sleep(self.backoff_delay / 1000.0) + self.backoff_delay *=2 + if self.backoff_delay > Pusher.MAX_BACKOFF: + self.backoff_delay = Pusher.MAX_BACKOFF class PusherConfigException(Exception): diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index f3c3ca8191..33d735b974 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -14,13 +14,17 @@ # limitations under the License. from synapse.push import Pusher, PusherConfigException +from synapse.http.client import SimpleHttpClient + +from twisted.internet import defer import logging logger = logging.getLogger(__name__) class HttpPusher(Pusher): - def __init__(self, _hs, user_name, app, app_display_name, device_display_name, pushkey, data, last_token): + def __init__(self, _hs, user_name, app, app_display_name, device_display_name, pushkey, data, + last_token, last_success, failing_since): super(HttpPusher, self).__init__(_hs, user_name, app, @@ -28,12 +32,55 @@ class HttpPusher(Pusher): device_display_name, pushkey, data, - last_token) + last_token, + last_success, + failing_since) if 'url' not in data: raise PusherConfigException("'url' required in data for HTTP pusher") self.url = data['url'] + self.httpCli = SimpleHttpClient(self.hs) + self.data_minus_url = {} + self.data_minus_url.update(self.data) + del self.data_minus_url['url'] + def _build_notification_dict(self, event): + # we probably do not want to push for every presence update + # (we may want to be able to set up notifications when specific + # people sign in, but we'd want to only deliver the pertinent ones) + # Actually, presence events will not get this far now because we + # need to filter them out in the main Pusher code. + if 'event_id' not in event: + return None + + return { + 'notification': { + 'transition' : 'new', # everything is new for now: we don't have read receipts + 'id': event['event_id'], + 'type': event['type'], + 'from': event['user_id'], + # we may have to fetch this over federation and we can't trust it anyway: is it worth it? + #'fromDisplayName': 'Steve Stevington' + }, + #'counts': { -- we don't mark messages as read yet so we have no way of knowing + # 'unread': 1, + # 'missedCalls': 2 + # }, + 'devices': { + self.pushkey: { + 'data' : self.data_minus_url + } + } + } + + @defer.inlineCallbacks def dispatchPush(self, event): - print event - return True + notificationDict = self._build_notification_dict(event) + if not notificationDict: + defer.returnValue(True) + try: + yield self.httpCli.post_json_get_json(self.url, notificationDict) + except: + logger.exception("Failed to push %s ", self.url) + defer.returnValue(False) + defer.returnValue(True) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 436040f123..3fa5a4c4ff 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -45,7 +45,9 @@ class PusherPool: "device_display_name": device_display_name, "pushkey": pushkey, "data": data, - "last_token": None + "last_token": None, + "last_success": None, + "failing_since": None }) self._add_pusher_to_store(user_name, kind, app, app_display_name, device_display_name, pushkey, data) @@ -69,7 +71,9 @@ class PusherPool: device_display_name=pusherdict['device_display_name'], pushkey=pusherdict['pushkey'], data=pusherdict['data'], - last_token=pusherdict['last_token'] + last_token=pusherdict['last_token'], + last_success=pusherdict['last_success'], + failing_since=pusherdict['failing_since'] ) else: raise PusherConfigException("Unknown pusher type '%s' for user %s" % diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index 047a5f42d9..ce158c4b18 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -29,7 +29,8 @@ class PusherStore(SQLBaseStore): @defer.inlineCallbacks def get_all_pushers_after_id(self, min_id): sql = ( - "SELECT id, user_name, kind, app, app_display_name, device_display_name, pushkey, data, last_token " + "SELECT id, user_name, kind, app, app_display_name, device_display_name, pushkey, data, " + "last_token, last_success, failing_since " "FROM pushers " "WHERE id > ?" ) @@ -46,8 +47,9 @@ class PusherStore(SQLBaseStore): "device_display_name": r[5], "pushkey": r[6], "data": r[7], - "last_token": r[8] - + "last_token": r[8], + "last_success": r[9], + "failing_since": r[10] } for r in rows ] @@ -79,6 +81,20 @@ class PusherStore(SQLBaseStore): {'last_token': last_token} ) + @defer.inlineCallbacks + def update_pusher_last_token_and_success(self, user_name, pushkey, last_token, last_success): + yield self._simple_update_one(PushersTable.table_name, + {'user_name': user_name, 'pushkey': pushkey}, + {'last_token': last_token, 'last_success': last_success} + ) + + @defer.inlineCallbacks + def update_pusher_failing_since(self, user_name, pushkey, failing_since): + yield self._simple_update_one(PushersTable.table_name, + {'user_name': user_name, 'pushkey': pushkey}, + {'failing_since': failing_since} + ) + class PushersTable(Table): table_name = "pushers" @@ -92,7 +108,9 @@ class PushersTable(Table): "device_display_name", "pushkey", "data", - "last_token" + "last_token", + "last_success", + "failing_since" ] EntryType = collections.namedtuple("PusherEntry", fields) \ No newline at end of file diff --git a/synapse/storage/schema/delta/v7.sql b/synapse/storage/schema/delta/v7.sql index 7f6852485d..e83f7e7436 100644 --- a/synapse/storage/schema/delta/v7.sql +++ b/synapse/storage/schema/delta/v7.sql @@ -23,6 +23,8 @@ CREATE TABLE IF NOT EXISTS pushers ( pushkey blob NOT NULL, data text, last_token TEXT, + last_success BIGINT, + failing_since BIGINT, FOREIGN KEY(user_name) REFERENCES users(name), UNIQUE (user_name, pushkey) ); diff --git a/synapse/storage/schema/pusher.sql b/synapse/storage/schema/pusher.sql index 7f6852485d..e83f7e7436 100644 --- a/synapse/storage/schema/pusher.sql +++ b/synapse/storage/schema/pusher.sql @@ -23,6 +23,8 @@ CREATE TABLE IF NOT EXISTS pushers ( pushkey blob NOT NULL, data text, last_token TEXT, + last_success BIGINT, + failing_since BIGINT, FOREIGN KEY(user_name) REFERENCES users(name), UNIQUE (user_name, pushkey) ); From bdc21e72820e148941bbecb36200d51ca340748d Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 2 Dec 2014 14:10:24 +0000 Subject: [PATCH 004/180] convert to spaces before I start a holy war --- contrib/jitsimeetbridge/jitsimeetbridge.py | 372 ++++++++++----------- 1 file changed, 186 insertions(+), 186 deletions(-) diff --git a/contrib/jitsimeetbridge/jitsimeetbridge.py b/contrib/jitsimeetbridge/jitsimeetbridge.py index dbc6f6ffa5..15f8e1c48b 100644 --- a/contrib/jitsimeetbridge/jitsimeetbridge.py +++ b/contrib/jitsimeetbridge/jitsimeetbridge.py @@ -39,43 +39,43 @@ ROOMDOMAIN="meet.jit.si" #ROOMDOMAIN="conference.jitsi.vuc.me" class TrivialMatrixClient: - def __init__(self, access_token): - self.token = None - self.access_token = access_token + def __init__(self, access_token): + self.token = None + self.access_token = access_token - def getEvent(self): - while True: - url = MATRIXBASE+'events?access_token='+self.access_token+"&timeout=60000" - if self.token: - url += "&from="+self.token - req = grequests.get(url) - resps = grequests.map([req]) - obj = json.loads(resps[0].content) - print "incoming from matrix",obj - if 'end' not in obj: - continue - self.token = obj['end'] - if len(obj['chunk']): - return obj['chunk'][0] + def getEvent(self): + while True: + url = MATRIXBASE+'events?access_token='+self.access_token+"&timeout=60000" + if self.token: + url += "&from="+self.token + req = grequests.get(url) + resps = grequests.map([req]) + obj = json.loads(resps[0].content) + print "incoming from matrix",obj + if 'end' not in obj: + continue + self.token = obj['end'] + if len(obj['chunk']): + return obj['chunk'][0] - def joinRoom(self, roomId): - url = MATRIXBASE+'rooms/'+roomId+'/join?access_token='+self.access_token - print url - headers={ 'Content-Type': 'application/json' } - req = grequests.post(url, headers=headers, data='{}') - resps = grequests.map([req]) - obj = json.loads(resps[0].content) - print "response: ",obj + def joinRoom(self, roomId): + url = MATRIXBASE+'rooms/'+roomId+'/join?access_token='+self.access_token + print url + headers={ 'Content-Type': 'application/json' } + req = grequests.post(url, headers=headers, data='{}') + resps = grequests.map([req]) + obj = json.loads(resps[0].content) + print "response: ",obj - def sendEvent(self, roomId, evType, event): - url = MATRIXBASE+'rooms/'+roomId+'/send/'+evType+'?access_token='+self.access_token - print url - print json.dumps(event) - headers={ 'Content-Type': 'application/json' } - req = grequests.post(url, headers=headers, data=json.dumps(event)) - resps = grequests.map([req]) - obj = json.loads(resps[0].content) - print "response: ",obj + def sendEvent(self, roomId, evType, event): + url = MATRIXBASE+'rooms/'+roomId+'/send/'+evType+'?access_token='+self.access_token + print url + print json.dumps(event) + headers={ 'Content-Type': 'application/json' } + req = grequests.post(url, headers=headers, data=json.dumps(event)) + resps = grequests.map([req]) + obj = json.loads(resps[0].content) + print "response: ",obj @@ -83,178 +83,178 @@ xmppClients = {} def matrixLoop(): - while True: - ev = matrixCli.getEvent() - print ev - if ev['type'] == 'm.room.member': - print 'membership event' - if ev['membership'] == 'invite' and ev['state_key'] == MYUSERNAME: - roomId = ev['room_id'] - print "joining room %s" % (roomId) - matrixCli.joinRoom(roomId) - elif ev['type'] == 'm.room.message': - if ev['room_id'] in xmppClients: - print "already have a bridge for that user, ignoring" - continue - print "got message, connecting" - xmppClients[ev['room_id']] = TrivialXmppClient(ev['room_id'], ev['user_id']) - gevent.spawn(xmppClients[ev['room_id']].xmppLoop) - elif ev['type'] == 'm.call.invite': - print "Incoming call" - #sdp = ev['content']['offer']['sdp'] - #print "sdp: %s" % (sdp) - #xmppClients[ev['room_id']] = TrivialXmppClient(ev['room_id'], ev['user_id']) - #gevent.spawn(xmppClients[ev['room_id']].xmppLoop) - elif ev['type'] == 'm.call.answer': - print "Call answered" - sdp = ev['content']['answer']['sdp'] - if ev['room_id'] not in xmppClients: - print "We didn't have a call for that room" - continue - # should probably check call ID too - xmppCli = xmppClients[ev['room_id']] - xmppCli.sendAnswer(sdp) - elif ev['type'] == 'm.call.hangup': - if ev['room_id'] in xmppClients: - xmppClients[ev['room_id']].stop() - del xmppClients[ev['room_id']] - + while True: + ev = matrixCli.getEvent() + print ev + if ev['type'] == 'm.room.member': + print 'membership event' + if ev['membership'] == 'invite' and ev['state_key'] == MYUSERNAME: + roomId = ev['room_id'] + print "joining room %s" % (roomId) + matrixCli.joinRoom(roomId) + elif ev['type'] == 'm.room.message': + if ev['room_id'] in xmppClients: + print "already have a bridge for that user, ignoring" + continue + print "got message, connecting" + xmppClients[ev['room_id']] = TrivialXmppClient(ev['room_id'], ev['user_id']) + gevent.spawn(xmppClients[ev['room_id']].xmppLoop) + elif ev['type'] == 'm.call.invite': + print "Incoming call" + #sdp = ev['content']['offer']['sdp'] + #print "sdp: %s" % (sdp) + #xmppClients[ev['room_id']] = TrivialXmppClient(ev['room_id'], ev['user_id']) + #gevent.spawn(xmppClients[ev['room_id']].xmppLoop) + elif ev['type'] == 'm.call.answer': + print "Call answered" + sdp = ev['content']['answer']['sdp'] + if ev['room_id'] not in xmppClients: + print "We didn't have a call for that room" + continue + # should probably check call ID too + xmppCli = xmppClients[ev['room_id']] + xmppCli.sendAnswer(sdp) + elif ev['type'] == 'm.call.hangup': + if ev['room_id'] in xmppClients: + xmppClients[ev['room_id']].stop() + del xmppClients[ev['room_id']] + class TrivialXmppClient: - def __init__(self, matrixRoom, userId): - self.rid = 0 - self.matrixRoom = matrixRoom - self.userId = userId - self.running = True + def __init__(self, matrixRoom, userId): + self.rid = 0 + self.matrixRoom = matrixRoom + self.userId = userId + self.running = True - def stop(self): - self.running = False + def stop(self): + self.running = False - def nextRid(self): - self.rid += 1 - return '%d' % (self.rid) + def nextRid(self): + self.rid += 1 + return '%d' % (self.rid) - def sendIq(self, xml): - fullXml = "%s" % (self.nextRid(), self.sid, xml) - #print "\t>>>%s" % (fullXml) - return self.xmppPoke(fullXml) - - def xmppPoke(self, xml): - headers = {'Content-Type': 'application/xml'} - req = grequests.post(HTTPBIND, verify=False, headers=headers, data=xml) - resps = grequests.map([req]) - obj = BeautifulSoup(resps[0].content) - return obj + def sendIq(self, xml): + fullXml = "%s" % (self.nextRid(), self.sid, xml) + #print "\t>>>%s" % (fullXml) + return self.xmppPoke(fullXml) - def sendAnswer(self, answer): - print "sdp from matrix client",answer - p = subprocess.Popen(['node', 'unjingle/unjingle.js', '--sdp'], stdin=subprocess.PIPE, stdout=subprocess.PIPE) - jingle, out_err = p.communicate(answer) - jingle = jingle % { - 'tojid': self.callfrom, - 'action': 'session-accept', - 'initiator': self.callfrom, - 'responder': self.jid, - 'sid': self.callsid - } - print "answer jingle from sdp",jingle - res = self.sendIq(jingle) - print "reply from answer: ",res - - self.ssrcs = {} - jingleSoup = BeautifulSoup(jingle) - for cont in jingleSoup.iq.jingle.findAll('content'): - if cont.description: - self.ssrcs[cont['name']] = cont.description['ssrc'] - print "my ssrcs:",self.ssrcs + def xmppPoke(self, xml): + headers = {'Content-Type': 'application/xml'} + req = grequests.post(HTTPBIND, verify=False, headers=headers, data=xml) + resps = grequests.map([req]) + obj = BeautifulSoup(resps[0].content) + return obj - gevent.joinall([ - gevent.spawn(self.advertiseSsrcs) - ]) - - def advertiseSsrcs(self): + def sendAnswer(self, answer): + print "sdp from matrix client",answer + p = subprocess.Popen(['node', 'unjingle/unjingle.js', '--sdp'], stdin=subprocess.PIPE, stdout=subprocess.PIPE) + jingle, out_err = p.communicate(answer) + jingle = jingle % { + 'tojid': self.callfrom, + 'action': 'session-accept', + 'initiator': self.callfrom, + 'responder': self.jid, + 'sid': self.callsid + } + print "answer jingle from sdp",jingle + res = self.sendIq(jingle) + print "reply from answer: ",res + + self.ssrcs = {} + jingleSoup = BeautifulSoup(jingle) + for cont in jingleSoup.iq.jingle.findAll('content'): + if cont.description: + self.ssrcs[cont['name']] = cont.description['ssrc'] + print "my ssrcs:",self.ssrcs + + gevent.joinall([ + gevent.spawn(self.advertiseSsrcs) + ]) + + def advertiseSsrcs(self): time.sleep(7) - print "SSRC spammer started" - while self.running: - ssrcMsg = "%(nick)s" % { 'tojid': "%s@%s/%s" % (ROOMNAME, ROOMDOMAIN, self.shortJid), 'nick': self.userId, 'assrc': self.ssrcs['audio'], 'vssrc': self.ssrcs['video'] } - res = self.sendIq(ssrcMsg) - print "reply from ssrc announce: ",res - time.sleep(10) - - + print "SSRC spammer started" + while self.running: + ssrcMsg = "%(nick)s" % { 'tojid': "%s@%s/%s" % (ROOMNAME, ROOMDOMAIN, self.shortJid), 'nick': self.userId, 'assrc': self.ssrcs['audio'], 'vssrc': self.ssrcs['video'] } + res = self.sendIq(ssrcMsg) + print "reply from ssrc announce: ",res + time.sleep(10) - def xmppLoop(self): - self.matrixCallId = time.time() - res = self.xmppPoke("" % (self.nextRid(), HOST)) - print res - self.sid = res.body['sid'] - print "sid %s" % (self.sid) - res = self.sendIq("") + def xmppLoop(self): + self.matrixCallId = time.time() + res = self.xmppPoke("" % (self.nextRid(), HOST)) - res = self.xmppPoke("" % (self.nextRid(), self.sid, HOST)) - - res = self.sendIq("") - print res + print res + self.sid = res.body['sid'] + print "sid %s" % (self.sid) - self.jid = res.body.iq.bind.jid.string - print "jid: %s" % (self.jid) - self.shortJid = self.jid.split('-')[0] + res = self.sendIq("") - res = self.sendIq("") + res = self.xmppPoke("" % (self.nextRid(), self.sid, HOST)) - #randomthing = res.body.iq['to'] - #whatsitpart = randomthing.split('-')[0] + res = self.sendIq("") + print res - #print "other random bind thing: %s" % (randomthing) + self.jid = res.body.iq.bind.jid.string + print "jid: %s" % (self.jid) + self.shortJid = self.jid.split('-')[0] - # advertise preence to the jitsi room, with our nick - res = self.sendIq("%s" % (HOST, TURNSERVER, ROOMNAME, ROOMDOMAIN, self.userId)) - self.muc = {'users': []} - for p in res.body.findAll('presence'): - u = {} - u['shortJid'] = p['from'].split('/')[1] - if p.c and p.c.nick: - u['nick'] = p.c.nick.string - self.muc['users'].append(u) - print "muc: ",self.muc + res = self.sendIq("") - # wait for stuff - while True: - print "waiting..." - res = self.sendIq("") - print "got from stream: ",res - if res.body.iq: - jingles = res.body.iq.findAll('jingle') - if len(jingles): - self.callfrom = res.body.iq['from'] - self.handleInvite(jingles[0]) - elif 'type' in res.body and res.body['type'] == 'terminate': - self.running = False - del xmppClients[self.matrixRoom] - return + #randomthing = res.body.iq['to'] + #whatsitpart = randomthing.split('-')[0] + + #print "other random bind thing: %s" % (randomthing) + + # advertise preence to the jitsi room, with our nick + res = self.sendIq("%s" % (HOST, TURNSERVER, ROOMNAME, ROOMDOMAIN, self.userId)) + self.muc = {'users': []} + for p in res.body.findAll('presence'): + u = {} + u['shortJid'] = p['from'].split('/')[1] + if p.c and p.c.nick: + u['nick'] = p.c.nick.string + self.muc['users'].append(u) + print "muc: ",self.muc + + # wait for stuff + while True: + print "waiting..." + res = self.sendIq("") + print "got from stream: ",res + if res.body.iq: + jingles = res.body.iq.findAll('jingle') + if len(jingles): + self.callfrom = res.body.iq['from'] + self.handleInvite(jingles[0]) + elif 'type' in res.body and res.body['type'] == 'terminate': + self.running = False + del xmppClients[self.matrixRoom] + return + + def handleInvite(self, jingle): + self.initiator = jingle['initiator'] + self.callsid = jingle['sid'] + p = subprocess.Popen(['node', 'unjingle/unjingle.js', '--jingle'], stdin=subprocess.PIPE, stdout=subprocess.PIPE) + print "raw jingle invite",str(jingle) + sdp, out_err = p.communicate(str(jingle)) + print "transformed remote offer sdp",sdp + inviteEvent = { + 'offer': { + 'type': 'offer', + 'sdp': sdp + }, + 'call_id': self.matrixCallId, + 'version': 0, + 'lifetime': 30000 + } + matrixCli.sendEvent(self.matrixRoom, 'm.call.invite', inviteEvent) - def handleInvite(self, jingle): - self.initiator = jingle['initiator'] - self.callsid = jingle['sid'] - p = subprocess.Popen(['node', 'unjingle/unjingle.js', '--jingle'], stdin=subprocess.PIPE, stdout=subprocess.PIPE) - print "raw jingle invite",str(jingle) - sdp, out_err = p.communicate(str(jingle)) - print "transformed remote offer sdp",sdp - inviteEvent = { - 'offer': { - 'type': 'offer', - 'sdp': sdp - }, - 'call_id': self.matrixCallId, - 'version': 0, - 'lifetime': 30000 - } - matrixCli.sendEvent(self.matrixRoom, 'm.call.invite', inviteEvent) - matrixCli = TrivialMatrixClient(ACCESS_TOKEN) gevent.joinall([ - gevent.spawn(matrixLoop) + gevent.spawn(matrixLoop) ]) From 88af58d41d561f1d9f6bbbfb2a1e8bd00dbbe638 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 3 Dec 2014 13:37:02 +0000 Subject: [PATCH 005/180] Update to app_id / app_instance_id (partially) and mangle to be PEP8 compliant. --- synapse/push/__init__.py | 99 ++++++++++++++++++++--------- synapse/push/httppusher.py | 75 ++++++++++++---------- synapse/push/pusherpool.py | 75 +++++++++++++--------- synapse/rest/pusher.py | 32 ++++++---- synapse/storage/pusher.py | 54 +++++++++------- synapse/storage/schema/delta/v7.sql | 5 +- synapse/storage/schema/pusher.sql | 5 +- 7 files changed, 214 insertions(+), 131 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index a96f0f0183..5fca3bd772 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -24,90 +24,127 @@ import logging logger = logging.getLogger(__name__) + class Pusher(object): INITIAL_BACKOFF = 1000 MAX_BACKOFF = 60 * 60 * 1000 GIVE_UP_AFTER = 24 * 60 * 60 * 1000 - def __init__(self, _hs, user_name, app, app_display_name, device_display_name, pushkey, data, + def __init__(self, _hs, user_name, app_id, app_instance_id, + app_display_name, device_display_name, pushkey, data, last_token, last_success, failing_since): self.hs = _hs self.evStreamHandler = self.hs.get_handlers().event_stream_handler self.store = self.hs.get_datastore() self.clock = self.hs.get_clock() self.user_name = user_name - self.app = app + self.app_id = app_id + self.app_instance_id = app_instance_id self.app_display_name = app_display_name self.device_display_name = device_display_name self.pushkey = pushkey self.data = data self.last_token = last_token + self.last_success = last_success # not actually used self.backoff_delay = Pusher.INITIAL_BACKOFF - self.failing_since = None + self.failing_since = failing_since @defer.inlineCallbacks def start(self): if not self.last_token: - # First-time setup: get a token to start from (we can't just start from no token, ie. 'now' - # because we need the result to be reproduceable in case we fail to dispatch the push) + # First-time setup: get a token to start from (we can't + # just start from no token, ie. 'now' + # because we need the result to be reproduceable in case + # we fail to dispatch the push) config = PaginationConfig(from_token=None, limit='1') - chunk = yield self.evStreamHandler.get_stream(self.user_name, config, timeout=0) + chunk = yield self.evStreamHandler.get_stream( + self.user_name, config, timeout=0) self.last_token = chunk['end'] - self.store.update_pusher_last_token(self.user_name, self.pushkey, self.last_token) + self.store.update_pusher_last_token( + self.user_name, self.pushkey, self.last_token) logger.info("Pusher %s for user %s starting from token %s", self.pushkey, self.user_name, self.last_token) while True: from_tok = StreamToken.from_string(self.last_token) config = PaginationConfig(from_token=from_tok, limit='1') - chunk = yield self.evStreamHandler.get_stream(self.user_name, config, timeout=100*365*24*60*60*1000) + chunk = yield self.evStreamHandler.get_stream( + self.user_name, config, timeout=100*365*24*60*60*1000) - # limiting to 1 may get 1 event plus 1 presence event, so pick out the actual event - singleEvent = None + # limiting to 1 may get 1 event plus 1 presence event, so + # pick out the actual event + single_event = None for c in chunk['chunk']: - if 'event_id' in c: # Hmmm... - singleEvent = c + if 'event_id' in c: # Hmmm... + single_event = c break - if not singleEvent: + if not single_event: continue - ret = yield self.dispatchPush(singleEvent) - if (ret): + ret = yield self.dispatch_push(single_event) + if ret: self.backoff_delay = Pusher.INITIAL_BACKOFF self.last_token = chunk['end'] - self.store.update_pusher_last_token_and_success(self.user_name, self.pushkey, - self.last_token, self.clock.time_msec()) + self.store.update_pusher_last_token_and_success( + self.user_name, + self.pushkey, + self.last_token, + self.clock.time_msec() + ) if self.failing_since: self.failing_since = None - self.store.update_pusher_failing_since(self.user_name, self.pushkey, self.failing_since) + self.store.update_pusher_failing_since( + self.user_name, + self.pushkey, + self.failing_since) else: if not self.failing_since: self.failing_since = self.clock.time_msec() - self.store.update_pusher_failing_since(self.user_name, self.pushkey, self.failing_since) + self.store.update_pusher_failing_since( + self.user_name, + self.pushkey, + self.failing_since + ) - if self.failing_since and self.failing_since < self.clock.time_msec() - Pusher.GIVE_UP_AFTER: - # we really only give up so that if the URL gets fixed, we don't suddenly deliver a load + if self.failing_since and \ + self.failing_since < \ + self.clock.time_msec() - Pusher.GIVE_UP_AFTER: + # we really only give up so that if the URL gets + # fixed, we don't suddenly deliver a load # of old notifications. - logger.warn("Giving up on a notification to user %s, pushkey %s", + logger.warn("Giving up on a notification to user %s, " + "pushkey %s", self.user_name, self.pushkey) self.backoff_delay = Pusher.INITIAL_BACKOFF self.last_token = chunk['end'] - self.store.update_pusher_last_token(self.user_name, self.pushkey, self.last_token) + self.store.update_pusher_last_token( + self.user_name, + self.pushkey, + self.last_token + ) self.failing_since = None - self.store.update_pusher_failing_since(self.user_name, self.pushkey, self.failing_since) - else: - logger.warn("Failed to dispatch push for user %s (failing for %dms)." - "Trying again in %dms", - self.user_name, - self.clock.time_msec() - self.failing_since, - self.backoff_delay + self.store.update_pusher_failing_since( + self.user_name, + self.pushkey, + self.failing_since ) + else: + logger.warn("Failed to dispatch push for user %s " + "(failing for %dms)." + "Trying again in %dms", + self.user_name, + self.clock.time_msec() - self.failing_since, + self.backoff_delay + ) yield synapse.util.async.sleep(self.backoff_delay / 1000.0) - self.backoff_delay *=2 + self.backoff_delay *= 2 if self.backoff_delay > Pusher.MAX_BACKOFF: self.backoff_delay = Pusher.MAX_BACKOFF + def dispatch_push(self, p): + pass + class PusherConfigException(Exception): def __init__(self, msg): diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 33d735b974..fd7fe4e39c 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -22,21 +22,28 @@ import logging logger = logging.getLogger(__name__) + class HttpPusher(Pusher): - def __init__(self, _hs, user_name, app, app_display_name, device_display_name, pushkey, data, + def __init__(self, _hs, user_name, app_id, app_instance_id, + app_display_name, device_display_name, pushkey, data, last_token, last_success, failing_since): - super(HttpPusher, self).__init__(_hs, - user_name, - app, - app_display_name, - device_display_name, - pushkey, - data, - last_token, - last_success, - failing_since) + super(HttpPusher, self).__init__( + _hs, + user_name, + app_id, + app_instance_id, + app_display_name, + device_display_name, + pushkey, + data, + last_token, + last_success, + failing_since + ) if 'url' not in data: - raise PusherConfigException("'url' required in data for HTTP pusher") + raise PusherConfigException( + "'url' required in data for HTTP pusher" + ) self.url = data['url'] self.httpCli = SimpleHttpClient(self.hs) self.data_minus_url = {} @@ -53,34 +60,36 @@ class HttpPusher(Pusher): return None return { - 'notification': { - 'transition' : 'new', # everything is new for now: we don't have read receipts - 'id': event['event_id'], - 'type': event['type'], - 'from': event['user_id'], - # we may have to fetch this over federation and we can't trust it anyway: is it worth it? - #'fromDisplayName': 'Steve Stevington' - }, - #'counts': { -- we don't mark messages as read yet so we have no way of knowing - # 'unread': 1, - # 'missedCalls': 2 - # }, - 'devices': { - self.pushkey: { - 'data' : self.data_minus_url + 'notification': { + 'transition': 'new', + # everything is new for now: we don't have read receipts + 'id': event['event_id'], + 'type': event['type'], + 'from': event['user_id'], + # we may have to fetch this over federation and we + # can't trust it anyway: is it worth it? + #'fromDisplayName': 'Steve Stevington' + }, + #'counts': { -- we don't mark messages as read yet so + # we have no way of knowing + # 'unread': 1, + # 'missedCalls': 2 + # }, + 'devices': { + self.pushkey: { + 'data': self.data_minus_url } - } + } } @defer.inlineCallbacks - def dispatchPush(self, event): - notificationDict = self._build_notification_dict(event) - if not notificationDict: + def dispatch_push(self, event): + notification_dict = self._build_notification_dict(event) + if not notification_dict: defer.returnValue(True) try: - yield self.httpCli.post_json_get_json(self.url, notificationDict) + yield self.httpCli.post_json_get_json(self.url, notification_dict) except: logger.exception("Failed to push %s ", self.url) defer.returnValue(False) defer.returnValue(True) - diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 3fa5a4c4ff..045c36f3b7 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -34,13 +34,17 @@ class PusherPool: def start(self): self._pushers_added() - def add_pusher(self, user_name, kind, app, app_display_name, device_display_name, pushkey, data): - # we try to create the pusher just to validate the config: it will then get pulled out of the database, - # recreated, added and started: this means we have only one code path adding pushers. + def add_pusher(self, user_name, kind, app_id, app_instance_id, + app_display_name, device_display_name, pushkey, data): + # we try to create the pusher just to validate the config: it + # will then get pulled out of the database, + # recreated, added and started: this means we have only one + # code path adding pushers. self._create_pusher({ "user_name": user_name, "kind": kind, - "app": app, + "app_id": app_id, + "app_instance_id": app_instance_id, "app_display_name": app_display_name, "device_display_name": device_display_name, "pushkey": pushkey, @@ -49,42 +53,55 @@ class PusherPool: "last_success": None, "failing_since": None }) - self._add_pusher_to_store(user_name, kind, app, app_display_name, device_display_name, pushkey, data) + self._add_pusher_to_store(user_name, kind, app_id, app_instance_id, + app_display_name, device_display_name, + pushkey, data) @defer.inlineCallbacks - def _add_pusher_to_store(self, user_name, kind, app, app_display_name, device_display_name, pushkey, data): - yield self.store.add_pusher(user_name=user_name, - kind=kind, - app=app, - app_display_name=app_display_name, - device_display_name=device_display_name, - pushkey=pushkey, - data=json.dumps(data)) + def _add_pusher_to_store(self, user_name, kind, app_id, app_instance_id, + app_display_name, device_display_name, + pushkey, data): + yield self.store.add_pusher( + user_name=user_name, + kind=kind, + app_id=app_id, + app_instance_id=app_instance_id, + app_display_name=app_display_name, + device_display_name=device_display_name, + pushkey=pushkey, + data=json.dumps(data) + ) self._pushers_added() def _create_pusher(self, pusherdict): if pusherdict['kind'] == 'http': - return HttpPusher(self.hs, - user_name=pusherdict['user_name'], - app=pusherdict['app'], - app_display_name=pusherdict['app_display_name'], - device_display_name=pusherdict['device_display_name'], - pushkey=pusherdict['pushkey'], - data=pusherdict['data'], - last_token=pusherdict['last_token'], - last_success=pusherdict['last_success'], - failing_since=pusherdict['failing_since'] - ) + return HttpPusher( + self.hs, + user_name=pusherdict['user_name'], + app_id=pusherdict['app_id'], + app_instance_id=pusherdict['app_instance_id'], + app_display_name=pusherdict['app_display_name'], + device_display_name=pusherdict['device_display_name'], + pushkey=pusherdict['pushkey'], + data=pusherdict['data'], + last_token=pusherdict['last_token'], + last_success=pusherdict['last_success'], + failing_since=pusherdict['failing_since'] + ) else: - raise PusherConfigException("Unknown pusher type '%s' for user %s" % - (pusherdict['kind'], pusherdict['user_name'])) + raise PusherConfigException( + "Unknown pusher type '%s' for user %s" % + (pusherdict['kind'], pusherdict['user_name']) + ) @defer.inlineCallbacks def _pushers_added(self): - pushers = yield self.store.get_all_pushers_after_id(self.last_pusher_started) + pushers = yield self.store.get_all_pushers_after_id( + self.last_pusher_started + ) for p in pushers: p['data'] = json.loads(p['data']) - if (len(pushers)): + if len(pushers): self.last_pusher_started = pushers[-1]['id'] self._start_pushers(pushers) @@ -95,4 +112,4 @@ class PusherPool: p = self._create_pusher(pusherdict) if p: self.pushers.append(p) - p.start() \ No newline at end of file + p.start() diff --git a/synapse/rest/pusher.py b/synapse/rest/pusher.py index 85d0d1c8cd..a39341cd8b 100644 --- a/synapse/rest/pusher.py +++ b/synapse/rest/pusher.py @@ -31,30 +31,37 @@ class PusherRestServlet(RestServlet): content = _parse_json(request) - reqd = ['kind', 'app', 'app_display_name', 'device_display_name', 'data'] + reqd = ['kind', 'app_id', 'app_instance_id', 'app_display_name', + 'device_display_name', 'data'] missing = [] for i in reqd: if i not in content: missing.append(i) if len(missing): - raise SynapseError(400, "Missing parameters: "+','.join(missing), errcode=Codes.MISSING_PARAM) + raise SynapseError(400, "Missing parameters: "+','.join(missing), + errcode=Codes.MISSING_PARAM) pusher_pool = self.hs.get_pusherpool() try: - pusher_pool.add_pusher(user_name=user.to_string(), - kind=content['kind'], - app=content['app'], - app_display_name=content['app_display_name'], - device_display_name=content['device_display_name'], - pushkey=pushkey, - data=content['data']) + pusher_pool.add_pusher( + user_name=user.to_string(), + kind=content['kind'], + app_id=content['app_id'], + app_instance_id=content['app_instance_id'], + app_display_name=content['app_display_name'], + device_display_name=content['device_display_name'], + pushkey=pushkey, + data=content['data'] + ) except PusherConfigException as pce: - raise SynapseError(400, "Config Error: "+pce.message, errcode=Codes.MISSING_PARAM) + raise SynapseError(400, "Config Error: "+pce.message, + errcode=Codes.MISSING_PARAM) defer.returnValue((200, {})) - def on_OPTIONS(self, request): - return (200, {}) + def on_OPTIONS(self, _): + return 200, {} + # XXX: C+ped from rest/room.py - surely this should be common? def _parse_json(request): @@ -67,5 +74,6 @@ def _parse_json(request): except ValueError: raise SynapseError(400, "Content not JSON.", errcode=Codes.NOT_JSON) + def register_servlets(hs, http_server): PusherRestServlet(hs).register(http_server) diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index ce158c4b18..a858e46f3b 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -25,11 +25,13 @@ import logging logger = logging.getLogger(__name__) + class PusherStore(SQLBaseStore): @defer.inlineCallbacks def get_all_pushers_after_id(self, min_id): sql = ( - "SELECT id, user_name, kind, app, app_display_name, device_display_name, pushkey, data, " + "SELECT id, user_name, kind, app_id, app_instance_id," + "app_display_name, device_display_name, pushkey, data, " "last_token, last_success, failing_since " "FROM pushers " "WHERE id > ?" @@ -42,14 +44,15 @@ class PusherStore(SQLBaseStore): "id": r[0], "user_name": r[1], "kind": r[2], - "app": r[3], - "app_display_name": r[4], - "device_display_name": r[5], - "pushkey": r[6], - "data": r[7], - "last_token": r[8], - "last_success": r[9], - "failing_since": r[10] + "app_id": r[3], + "app_instance_id": r[4], + "app_display_name": r[5], + "device_display_name": r[6], + "pushkey": r[7], + "data": r[8], + "last_token": r[9], + "last_success": r[10], + "failing_since": r[11] } for r in rows ] @@ -57,12 +60,14 @@ class PusherStore(SQLBaseStore): defer.returnValue(ret) @defer.inlineCallbacks - def add_pusher(self, user_name, kind, app, app_display_name, device_display_name, pushkey, data): + def add_pusher(self, user_name, kind, app_id, app_instance_id, + app_display_name, device_display_name, pushkey, data): try: yield self._simple_insert(PushersTable.table_name, dict( user_name=user_name, kind=kind, - app=app, + app_id=app_id, + app_instance_id=app_instance_id, app_display_name=app_display_name, device_display_name=device_display_name, pushkey=pushkey, @@ -76,23 +81,27 @@ class PusherStore(SQLBaseStore): @defer.inlineCallbacks def update_pusher_last_token(self, user_name, pushkey, last_token): - yield self._simple_update_one(PushersTable.table_name, - {'user_name': user_name, 'pushkey': pushkey}, - {'last_token': last_token} + yield self._simple_update_one( + PushersTable.table_name, + {'user_name': user_name, 'pushkey': pushkey}, + {'last_token': last_token} ) @defer.inlineCallbacks - def update_pusher_last_token_and_success(self, user_name, pushkey, last_token, last_success): - yield self._simple_update_one(PushersTable.table_name, - {'user_name': user_name, 'pushkey': pushkey}, - {'last_token': last_token, 'last_success': last_success} + def update_pusher_last_token_and_success(self, user_name, pushkey, + last_token, last_success): + yield self._simple_update_one( + PushersTable.table_name, + {'user_name': user_name, 'pushkey': pushkey}, + {'last_token': last_token, 'last_success': last_success} ) @defer.inlineCallbacks def update_pusher_failing_since(self, user_name, pushkey, failing_since): - yield self._simple_update_one(PushersTable.table_name, - {'user_name': user_name, 'pushkey': pushkey}, - {'failing_since': failing_since} + yield self._simple_update_one( + PushersTable.table_name, + {'user_name': user_name, 'pushkey': pushkey}, + {'failing_since': failing_since} ) @@ -103,7 +112,8 @@ class PushersTable(Table): "id", "user_name", "kind", - "app" + "app_id", + "app_instance_id", "app_display_name", "device_display_name", "pushkey", diff --git a/synapse/storage/schema/delta/v7.sql b/synapse/storage/schema/delta/v7.sql index e83f7e7436..b60aeda756 100644 --- a/synapse/storage/schema/delta/v7.sql +++ b/synapse/storage/schema/delta/v7.sql @@ -17,11 +17,12 @@ CREATE TABLE IF NOT EXISTS pushers ( id INTEGER PRIMARY KEY AUTOINCREMENT, user_name TEXT NOT NULL, kind varchar(8) NOT NULL, - app varchar(64) NOT NULL, + app_id varchar(64) NOT NULL, + app_instance_id varchar(64) NOT NULL, app_display_name varchar(64) NOT NULL, device_display_name varchar(128) NOT NULL, pushkey blob NOT NULL, - data text, + data blob, last_token TEXT, last_success BIGINT, failing_since BIGINT, diff --git a/synapse/storage/schema/pusher.sql b/synapse/storage/schema/pusher.sql index e83f7e7436..b60aeda756 100644 --- a/synapse/storage/schema/pusher.sql +++ b/synapse/storage/schema/pusher.sql @@ -17,11 +17,12 @@ CREATE TABLE IF NOT EXISTS pushers ( id INTEGER PRIMARY KEY AUTOINCREMENT, user_name TEXT NOT NULL, kind varchar(8) NOT NULL, - app varchar(64) NOT NULL, + app_id varchar(64) NOT NULL, + app_instance_id varchar(64) NOT NULL, app_display_name varchar(64) NOT NULL, device_display_name varchar(128) NOT NULL, pushkey blob NOT NULL, - data text, + data blob, last_token TEXT, last_success BIGINT, failing_since BIGINT, From 9728c305a34a1f9546d2ce0ef4c54352dc55a16d Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 18 Dec 2014 14:49:22 +0000 Subject: [PATCH 006/180] after a few rethinks, a working implementation of pushers. --- synapse/push/__init__.py | 12 +++-- synapse/push/httppusher.py | 25 ++++----- synapse/push/pusherpool.py | 47 ++++++++-------- synapse/rest/pusher.py | 13 +++-- synapse/storage/_base.py | 45 ++++++++++++++++ synapse/storage/pusher.py | 83 ++++++++++++++++++++--------- synapse/storage/schema/delta/v7.sql | 3 +- synapse/storage/schema/pusher.sql | 3 +- 8 files changed, 158 insertions(+), 73 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 5fca3bd772..5fe8719fe7 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -30,7 +30,7 @@ class Pusher(object): MAX_BACKOFF = 60 * 60 * 1000 GIVE_UP_AFTER = 24 * 60 * 60 * 1000 - def __init__(self, _hs, user_name, app_id, app_instance_id, + def __init__(self, _hs, user_name, app_id, app_display_name, device_display_name, pushkey, data, last_token, last_success, failing_since): self.hs = _hs @@ -39,7 +39,6 @@ class Pusher(object): self.clock = self.hs.get_clock() self.user_name = user_name self.app_id = app_id - self.app_instance_id = app_instance_id self.app_display_name = app_display_name self.device_display_name = device_display_name self.pushkey = pushkey @@ -48,6 +47,7 @@ class Pusher(object): self.last_success = last_success # not actually used self.backoff_delay = Pusher.INITIAL_BACKOFF self.failing_since = failing_since + self.alive = True @defer.inlineCallbacks def start(self): @@ -65,7 +65,7 @@ class Pusher(object): logger.info("Pusher %s for user %s starting from token %s", self.pushkey, self.user_name, self.last_token) - while True: + while self.alive: from_tok = StreamToken.from_string(self.last_token) config = PaginationConfig(from_token=from_tok, limit='1') chunk = yield self.evStreamHandler.get_stream( @@ -81,6 +81,9 @@ class Pusher(object): if not single_event: continue + if not self.alive: + continue + ret = yield self.dispatch_push(single_event) if ret: self.backoff_delay = Pusher.INITIAL_BACKOFF @@ -142,6 +145,9 @@ class Pusher(object): if self.backoff_delay > Pusher.MAX_BACKOFF: self.backoff_delay = Pusher.MAX_BACKOFF + def stop(self): + self.alive = False + def dispatch_push(self, p): pass diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index fd7fe4e39c..f94f673391 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -24,14 +24,13 @@ logger = logging.getLogger(__name__) class HttpPusher(Pusher): - def __init__(self, _hs, user_name, app_id, app_instance_id, + def __init__(self, _hs, user_name, app_id, app_display_name, device_display_name, pushkey, data, last_token, last_success, failing_since): super(HttpPusher, self).__init__( _hs, user_name, app_id, - app_instance_id, app_display_name, device_display_name, pushkey, @@ -69,16 +68,18 @@ class HttpPusher(Pusher): # we may have to fetch this over federation and we # can't trust it anyway: is it worth it? #'fromDisplayName': 'Steve Stevington' - }, - #'counts': { -- we don't mark messages as read yet so - # we have no way of knowing - # 'unread': 1, - # 'missedCalls': 2 - # }, - 'devices': { - self.pushkey: { - 'data': self.data_minus_url - } + #'counts': { -- we don't mark messages as read yet so + # we have no way of knowing + # 'unread': 1, + # 'missedCalls': 2 + # }, + 'devices': [ + { + 'app_id': self.app_id, + 'pushkey': self.pushkey, + 'data': self.data_minus_url + } + ] } } diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 045c36f3b7..d34ef3f6cf 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -24,17 +24,23 @@ import json logger = logging.getLogger(__name__) + class PusherPool: def __init__(self, _hs): self.hs = _hs self.store = self.hs.get_datastore() - self.pushers = [] + self.pushers = {} self.last_pusher_started = -1 + @defer.inlineCallbacks def start(self): - self._pushers_added() + pushers = yield self.store.get_all_pushers() + for p in pushers: + p['data'] = json.loads(p['data']) + self._start_pushers(pushers) - def add_pusher(self, user_name, kind, app_id, app_instance_id, + @defer.inlineCallbacks + def add_pusher(self, user_name, kind, app_id, app_display_name, device_display_name, pushkey, data): # we try to create the pusher just to validate the config: it # will then get pulled out of the database, @@ -44,7 +50,6 @@ class PusherPool: "user_name": user_name, "kind": kind, "app_id": app_id, - "app_instance_id": app_instance_id, "app_display_name": app_display_name, "device_display_name": device_display_name, "pushkey": pushkey, @@ -53,25 +58,26 @@ class PusherPool: "last_success": None, "failing_since": None }) - self._add_pusher_to_store(user_name, kind, app_id, app_instance_id, - app_display_name, device_display_name, - pushkey, data) + yield self._add_pusher_to_store( + user_name, kind, app_id, + app_display_name, device_display_name, + pushkey, data + ) @defer.inlineCallbacks - def _add_pusher_to_store(self, user_name, kind, app_id, app_instance_id, + def _add_pusher_to_store(self, user_name, kind, app_id, app_display_name, device_display_name, pushkey, data): yield self.store.add_pusher( user_name=user_name, kind=kind, app_id=app_id, - app_instance_id=app_instance_id, app_display_name=app_display_name, device_display_name=device_display_name, pushkey=pushkey, data=json.dumps(data) ) - self._pushers_added() + self._refresh_pusher((app_id, pushkey)) def _create_pusher(self, pusherdict): if pusherdict['kind'] == 'http': @@ -79,7 +85,6 @@ class PusherPool: self.hs, user_name=pusherdict['user_name'], app_id=pusherdict['app_id'], - app_instance_id=pusherdict['app_instance_id'], app_display_name=pusherdict['app_display_name'], device_display_name=pusherdict['device_display_name'], pushkey=pusherdict['pushkey'], @@ -95,21 +100,21 @@ class PusherPool: ) @defer.inlineCallbacks - def _pushers_added(self): - pushers = yield self.store.get_all_pushers_after_id( - self.last_pusher_started + def _refresh_pusher(self, app_id_pushkey): + p = yield self.store.get_pushers_by_app_id_and_pushkey( + app_id_pushkey ) - for p in pushers: - p['data'] = json.loads(p['data']) - if len(pushers): - self.last_pusher_started = pushers[-1]['id'] + p['data'] = json.loads(p['data']) - self._start_pushers(pushers) + self._start_pushers([p]) def _start_pushers(self, pushers): - logger.info("Starting %d pushers", (len(pushers))) + logger.info("Starting %d pushers", len(pushers)) for pusherdict in pushers: p = self._create_pusher(pusherdict) if p: - self.pushers.append(p) + fullid = "%s:%s" % (pusherdict['app_id'], pusherdict['pushkey']) + if fullid in self.pushers: + self.pushers[fullid].stop() + self.pushers[fullid] = p p.start() diff --git a/synapse/rest/pusher.py b/synapse/rest/pusher.py index a39341cd8b..5b371318d0 100644 --- a/synapse/rest/pusher.py +++ b/synapse/rest/pusher.py @@ -23,16 +23,16 @@ import json class PusherRestServlet(RestServlet): - PATTERN = client_path_pattern("/pushers/(?P[\w]*)$") + PATTERN = client_path_pattern("/pushers/set$") @defer.inlineCallbacks - def on_PUT(self, request, pushkey): + def on_POST(self, request): user = yield self.auth.get_user_by_req(request) content = _parse_json(request) - reqd = ['kind', 'app_id', 'app_instance_id', 'app_display_name', - 'device_display_name', 'data'] + reqd = ['kind', 'app_id', 'app_display_name', + 'device_display_name', 'pushkey', 'data'] missing = [] for i in reqd: if i not in content: @@ -43,14 +43,13 @@ class PusherRestServlet(RestServlet): pusher_pool = self.hs.get_pusherpool() try: - pusher_pool.add_pusher( + yield pusher_pool.add_pusher( user_name=user.to_string(), kind=content['kind'], app_id=content['app_id'], - app_instance_id=content['app_instance_id'], app_display_name=content['app_display_name'], device_display_name=content['device_display_name'], - pushkey=pushkey, + pushkey=content['pushkey'], data=content['data'] ) except PusherConfigException as pce: diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 4881f03368..eb8cc4a9f3 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -195,6 +195,51 @@ class SQLBaseStore(object): txn.execute(sql, values.values()) return txn.lastrowid + def _simple_upsert(self, table, keyvalues, values): + """ + :param table: The table to upsert into + :param keyvalues: Dict of the unique key tables and their new values + :param values: Dict of all the nonunique columns and their new values + :return: A deferred + """ + return self.runInteraction( + "_simple_upsert", + self._simple_upsert_txn, table, keyvalues, values + ) + + def _simple_upsert_txn(self, txn, table, keyvalues, values): + # Try to update + sql = "UPDATE %s SET %s WHERE %s" % ( + table, + ", ".join("%s = ?" % (k) for k in values), + " AND ".join("%s = ?" % (k) for k in keyvalues) + ) + sqlargs = values.values() + keyvalues.values() + logger.debug( + "[SQL] %s Args=%s", + sql, sqlargs, + ) + + txn.execute(sql, sqlargs) + if txn.rowcount == 0: + # We didn't update and rows so insert a new one + allvalues = {} + allvalues.update(keyvalues) + allvalues.update(values) + + sql = "INSERT INTO %s (%s) VALUES (%s)" % ( + table, + ", ".join(k for k in allvalues), + ", ".join("?" for _ in allvalues) + ) + logger.debug( + "[SQL] %s Args=%s", + sql, keyvalues.values(), + ) + txn.execute(sql, allvalues.values()) + + + def _simple_select_one(self, table, keyvalues, retcols, allow_none=False): """Executes a SELECT query on the named table, which is expected to diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index a858e46f3b..deabd9cd2e 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -28,16 +28,18 @@ logger = logging.getLogger(__name__) class PusherStore(SQLBaseStore): @defer.inlineCallbacks - def get_all_pushers_after_id(self, min_id): + def get_pushers_by_app_id_and_pushkey(self, app_id_and_pushkey): sql = ( - "SELECT id, user_name, kind, app_id, app_instance_id," + "SELECT id, user_name, kind, app_id," "app_display_name, device_display_name, pushkey, data, " "last_token, last_success, failing_since " "FROM pushers " - "WHERE id > ?" + "WHERE app_id = ? AND pushkey = ?" ) - rows = yield self._execute(None, sql, min_id) + rows = yield self._execute( + None, sql, app_id_and_pushkey[0], app_id_and_pushkey[1] + ) ret = [ { @@ -45,14 +47,43 @@ class PusherStore(SQLBaseStore): "user_name": r[1], "kind": r[2], "app_id": r[3], - "app_instance_id": r[4], - "app_display_name": r[5], - "device_display_name": r[6], - "pushkey": r[7], - "data": r[8], - "last_token": r[9], - "last_success": r[10], - "failing_since": r[11] + "app_display_name": r[4], + "device_display_name": r[5], + "pushkey": r[6], + "data": r[7], + "last_token": r[8], + "last_success": r[9], + "failing_since": r[10] + } + for r in rows + ] + + defer.returnValue(ret[0]) + + @defer.inlineCallbacks + def get_all_pushers(self): + sql = ( + "SELECT id, user_name, kind, app_id," + "app_display_name, device_display_name, pushkey, data, " + "last_token, last_success, failing_since " + "FROM pushers" + ) + + rows = yield self._execute(None, sql) + + ret = [ + { + "id": r[0], + "user_name": r[1], + "kind": r[2], + "app_id": r[3], + "app_display_name": r[4], + "device_display_name": r[5], + "pushkey": r[6], + "data": r[7], + "last_token": r[8], + "last_success": r[9], + "failing_since": r[10] } for r in rows ] @@ -60,21 +91,22 @@ class PusherStore(SQLBaseStore): defer.returnValue(ret) @defer.inlineCallbacks - def add_pusher(self, user_name, kind, app_id, app_instance_id, + def add_pusher(self, user_name, kind, app_id, app_display_name, device_display_name, pushkey, data): try: - yield self._simple_insert(PushersTable.table_name, dict( - user_name=user_name, - kind=kind, - app_id=app_id, - app_instance_id=app_instance_id, - app_display_name=app_display_name, - device_display_name=device_display_name, - pushkey=pushkey, - data=data - )) - except IntegrityError: - raise StoreError(409, "Pushkey in use.") + yield self._simple_upsert( + PushersTable.table_name, + dict( + app_id=app_id, + pushkey=pushkey, + ), + dict( + user_name=user_name, + kind=kind, + app_display_name=app_display_name, + device_display_name=device_display_name, + data=data + )) except Exception as e: logger.error("create_pusher with failed: %s", e) raise StoreError(500, "Problem creating pusher.") @@ -113,7 +145,6 @@ class PushersTable(Table): "user_name", "kind", "app_id", - "app_instance_id", "app_display_name", "device_display_name", "pushkey", diff --git a/synapse/storage/schema/delta/v7.sql b/synapse/storage/schema/delta/v7.sql index b60aeda756..799e48d780 100644 --- a/synapse/storage/schema/delta/v7.sql +++ b/synapse/storage/schema/delta/v7.sql @@ -18,7 +18,6 @@ CREATE TABLE IF NOT EXISTS pushers ( user_name TEXT NOT NULL, kind varchar(8) NOT NULL, app_id varchar(64) NOT NULL, - app_instance_id varchar(64) NOT NULL, app_display_name varchar(64) NOT NULL, device_display_name varchar(128) NOT NULL, pushkey blob NOT NULL, @@ -27,5 +26,5 @@ CREATE TABLE IF NOT EXISTS pushers ( last_success BIGINT, failing_since BIGINT, FOREIGN KEY(user_name) REFERENCES users(name), - UNIQUE (user_name, pushkey) + UNIQUE (app_id, pushkey) ); diff --git a/synapse/storage/schema/pusher.sql b/synapse/storage/schema/pusher.sql index b60aeda756..799e48d780 100644 --- a/synapse/storage/schema/pusher.sql +++ b/synapse/storage/schema/pusher.sql @@ -18,7 +18,6 @@ CREATE TABLE IF NOT EXISTS pushers ( user_name TEXT NOT NULL, kind varchar(8) NOT NULL, app_id varchar(64) NOT NULL, - app_instance_id varchar(64) NOT NULL, app_display_name varchar(64) NOT NULL, device_display_name varchar(128) NOT NULL, pushkey blob NOT NULL, @@ -27,5 +26,5 @@ CREATE TABLE IF NOT EXISTS pushers ( last_success BIGINT, failing_since BIGINT, FOREIGN KEY(user_name) REFERENCES users(name), - UNIQUE (user_name, pushkey) + UNIQUE (app_id, pushkey) ); From fc7c5e9cd7e0b1e29984233249311abe5cf23735 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 18 Dec 2014 14:51:29 +0000 Subject: [PATCH 007/180] Rename the pusher SQL delta to v9 which the next free one --- synapse/storage/schema/delta/{v7.sql => v9.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename synapse/storage/schema/delta/{v7.sql => v9.sql} (100%) diff --git a/synapse/storage/schema/delta/v7.sql b/synapse/storage/schema/delta/v9.sql similarity index 100% rename from synapse/storage/schema/delta/v7.sql rename to synapse/storage/schema/delta/v9.sql From 173264b656b480a2f3634f49e78fd6093633af56 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 18 Dec 2014 14:53:10 +0000 Subject: [PATCH 008/180] ...and bump SCHEMA_VERSION --- synapse/storage/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 642e5e289e..348c3b259c 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -69,7 +69,7 @@ SCHEMAS = [ # Remember to update this number every time an incompatible change is made to # database schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 8 +SCHEMA_VERSION = 9 class _RollbackButIsFineException(Exception): From 4c7ad50f6e50b95dfa9e0961a504e2f0d5b6921a Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 18 Dec 2014 14:55:04 +0000 Subject: [PATCH 009/180] Thank you, pyflakes --- synapse/storage/pusher.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index deabd9cd2e..9b5170a5f7 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -18,7 +18,6 @@ import collections from ._base import SQLBaseStore, Table from twisted.internet import defer -from sqlite3 import IntegrityError from synapse.api.errors import StoreError import logging From afa953a29301dcae40606171ed4cdac90eefab63 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 18 Dec 2014 15:11:06 +0000 Subject: [PATCH 010/180] schema version is now 10 --- synapse/storage/__init__.py | 2 +- synapse/storage/schema/delta/{v9.sql => v10.sql} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename synapse/storage/schema/delta/{v9.sql => v10.sql} (100%) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 348c3b259c..ad1765e04d 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -69,7 +69,7 @@ SCHEMAS = [ # Remember to update this number every time an incompatible change is made to # database schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 9 +SCHEMA_VERSION = 10 class _RollbackButIsFineException(Exception): diff --git a/synapse/storage/schema/delta/v9.sql b/synapse/storage/schema/delta/v10.sql similarity index 100% rename from synapse/storage/schema/delta/v9.sql rename to synapse/storage/schema/delta/v10.sql From fead431c181918558207f5115bd678a8984ce8d5 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 18 Dec 2014 18:44:33 +0000 Subject: [PATCH 011/180] If we didn't get any events, advance the token or we'll just keep not getting the same events again. --- synapse/push/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 5fe8719fe7..c5586be7b9 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -79,6 +79,7 @@ class Pusher(object): single_event = c break if not single_event: + self.last_token = chunk['end'] continue if not self.alive: From 70d0a453f353bf350e02d5306a98126f6b318c88 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 13 Jan 2015 13:14:41 +0000 Subject: [PATCH 012/180] Split out function to decide whether to notify or a given event --- synapse/push/__init__.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index c5586be7b9..f4795d559c 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -49,6 +49,17 @@ class Pusher(object): self.failing_since = failing_since self.alive = True + def _should_notify_for_event(self, ev): + """ + This should take into account notification settings that the user + has configured both globally and per-room when we have the ability + to do such things. + """ + if ev['user_id'] == self.user_name: + # let's assume you probably know about messages you sent yourself + return False + return True + @defer.inlineCallbacks def start(self): if not self.last_token: @@ -85,8 +96,12 @@ class Pusher(object): if not self.alive: continue - ret = yield self.dispatch_push(single_event) - if ret: + processed = False + if self._should_notify_for_event(single_event): + processed = yield self.dispatch_push(single_event) + else: + processed = True + if processed: self.backoff_delay = Pusher.INITIAL_BACKOFF self.last_token = chunk['end'] self.store.update_pusher_last_token_and_success( From 2cb30767fa5e428f82c6c3ebced15d568d671c3c Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 13 Jan 2015 19:48:37 +0000 Subject: [PATCH 013/180] Honour the 'rejected' return from push gateways Add a timestamp to push tokens so we know the last time they we got them from the device. Send it to the push gateways so it can determine whether its failure is more recent than the token. Stop and remove pushers that have been rejected. --- synapse/push/__init__.py | 37 +++++++++++++++++++++++++--- synapse/push/httppusher.py | 15 +++++++---- synapse/push/pusherpool.py | 12 +++++++++ synapse/storage/pusher.py | 34 ++++++++++++++++--------- synapse/storage/schema/delta/v10.sql | 1 + synapse/storage/schema/pusher.sql | 1 + 6 files changed, 81 insertions(+), 19 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index f4795d559c..839f666390 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -31,8 +31,8 @@ class Pusher(object): GIVE_UP_AFTER = 24 * 60 * 60 * 1000 def __init__(self, _hs, user_name, app_id, - app_display_name, device_display_name, pushkey, data, - last_token, last_success, failing_since): + app_display_name, device_display_name, pushkey, pushkey_ts, + data, last_token, last_success, failing_since): self.hs = _hs self.evStreamHandler = self.hs.get_handlers().event_stream_handler self.store = self.hs.get_datastore() @@ -42,6 +42,7 @@ class Pusher(object): self.app_display_name = app_display_name self.device_display_name = device_display_name self.pushkey = pushkey + self.pushkey_ts = pushkey_ts self.data = data self.last_token = last_token self.last_success = last_success # not actually used @@ -98,9 +99,31 @@ class Pusher(object): processed = False if self._should_notify_for_event(single_event): - processed = yield self.dispatch_push(single_event) + rejected = yield self.dispatch_push(single_event) + if not rejected == False: + processed = True + for pk in rejected: + if pk != self.pushkey: + # for sanity, we only remove the pushkey if it + # was the one we actually sent... + logger.warn( + ("Ignoring rejected pushkey %s because we" + + "didn't send it"), (pk,) + ) + else: + logger.info( + "Pushkey %s was rejected: removing", + pk + ) + yield self.hs.get_pusherpool().remove_pusher( + self.app_id, pk + ) else: processed = True + + if not self.alive: + continue + if processed: self.backoff_delay = Pusher.INITIAL_BACKOFF self.last_token = chunk['end'] @@ -165,6 +188,14 @@ class Pusher(object): self.alive = False def dispatch_push(self, p): + """ + Overridden by implementing classes to actually deliver the notification + :param p: The event to notify for as a single event from the event stream + :return: If the notification was delivered, an array containing any + pushkeys that were rejected by the push gateway. + False if the notification could not be delivered (ie. + should be retried). + """ pass diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index f94f673391..bcfa06e2ab 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -25,8 +25,8 @@ logger = logging.getLogger(__name__) class HttpPusher(Pusher): def __init__(self, _hs, user_name, app_id, - app_display_name, device_display_name, pushkey, data, - last_token, last_success, failing_since): + app_display_name, device_display_name, pushkey, pushkey_ts, + data, last_token, last_success, failing_since): super(HttpPusher, self).__init__( _hs, user_name, @@ -34,6 +34,7 @@ class HttpPusher(Pusher): app_display_name, device_display_name, pushkey, + pushkey_ts, data, last_token, last_success, @@ -77,6 +78,7 @@ class HttpPusher(Pusher): { 'app_id': self.app_id, 'pushkey': self.pushkey, + 'pushkeyTs': long(self.pushkey_ts / 1000), 'data': self.data_minus_url } ] @@ -87,10 +89,13 @@ class HttpPusher(Pusher): def dispatch_push(self, event): notification_dict = self._build_notification_dict(event) if not notification_dict: - defer.returnValue(True) + defer.returnValue([]) try: - yield self.httpCli.post_json_get_json(self.url, notification_dict) + resp = yield self.httpCli.post_json_get_json(self.url, notification_dict) except: logger.exception("Failed to push %s ", self.url) defer.returnValue(False) - defer.returnValue(True) + rejected = [] + if 'rejected' in resp: + rejected = resp['rejected'] + defer.returnValue(rejected) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index d34ef3f6cf..edddc3003e 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -53,6 +53,7 @@ class PusherPool: "app_display_name": app_display_name, "device_display_name": device_display_name, "pushkey": pushkey, + "pushkey_ts": self.hs.get_clock().time_msec(), "data": data, "last_token": None, "last_success": None, @@ -75,6 +76,7 @@ class PusherPool: app_display_name=app_display_name, device_display_name=device_display_name, pushkey=pushkey, + pushkey_ts=self.hs.get_clock().time_msec(), data=json.dumps(data) ) self._refresh_pusher((app_id, pushkey)) @@ -88,6 +90,7 @@ class PusherPool: app_display_name=pusherdict['app_display_name'], device_display_name=pusherdict['device_display_name'], pushkey=pusherdict['pushkey'], + pushkey_ts=pusherdict['pushkey_ts'], data=pusherdict['data'], last_token=pusherdict['last_token'], last_success=pusherdict['last_success'], @@ -118,3 +121,12 @@ class PusherPool: self.pushers[fullid].stop() self.pushers[fullid] = p p.start() + + @defer.inlineCallbacks + def remove_pusher(self, app_id, pushkey): + fullid = "%s:%s" % (app_id, pushkey) + if fullid in self.pushers: + logger.info("Stopping pusher %s", fullid) + self.pushers[fullid].stop() + del self.pushers[fullid] + yield self.store.delete_pusher_by_app_id_pushkey(app_id, pushkey) \ No newline at end of file diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index 9b5170a5f7..bfc4980256 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -30,7 +30,7 @@ class PusherStore(SQLBaseStore): def get_pushers_by_app_id_and_pushkey(self, app_id_and_pushkey): sql = ( "SELECT id, user_name, kind, app_id," - "app_display_name, device_display_name, pushkey, data, " + "app_display_name, device_display_name, pushkey, ts, data, " "last_token, last_success, failing_since " "FROM pushers " "WHERE app_id = ? AND pushkey = ?" @@ -49,10 +49,11 @@ class PusherStore(SQLBaseStore): "app_display_name": r[4], "device_display_name": r[5], "pushkey": r[6], - "data": r[7], - "last_token": r[8], - "last_success": r[9], - "failing_since": r[10] + "pushkey_ts": r[7], + "data": r[8], + "last_token": r[9], + "last_success": r[10], + "failing_since": r[11] } for r in rows ] @@ -63,7 +64,7 @@ class PusherStore(SQLBaseStore): def get_all_pushers(self): sql = ( "SELECT id, user_name, kind, app_id," - "app_display_name, device_display_name, pushkey, data, " + "app_display_name, device_display_name, pushkey, ts, data, " "last_token, last_success, failing_since " "FROM pushers" ) @@ -79,10 +80,11 @@ class PusherStore(SQLBaseStore): "app_display_name": r[4], "device_display_name": r[5], "pushkey": r[6], - "data": r[7], - "last_token": r[8], - "last_success": r[9], - "failing_since": r[10] + "pushkey_ts": r[7], + "data": r[8], + "last_token": r[9], + "last_success": r[10], + "failing_since": r[11] } for r in rows ] @@ -91,7 +93,8 @@ class PusherStore(SQLBaseStore): @defer.inlineCallbacks def add_pusher(self, user_name, kind, app_id, - app_display_name, device_display_name, pushkey, data): + app_display_name, device_display_name, + pushkey, pushkey_ts, data): try: yield self._simple_upsert( PushersTable.table_name, @@ -104,12 +107,20 @@ class PusherStore(SQLBaseStore): kind=kind, app_display_name=app_display_name, device_display_name=device_display_name, + ts=pushkey_ts, data=data )) except Exception as e: logger.error("create_pusher with failed: %s", e) raise StoreError(500, "Problem creating pusher.") + @defer.inlineCallbacks + def delete_pusher_by_app_id_pushkey(self, app_id, pushkey): + yield self._simple_delete_one( + PushersTable.table_name, + dict(app_id=app_id, pushkey=pushkey) + ) + @defer.inlineCallbacks def update_pusher_last_token(self, user_name, pushkey, last_token): yield self._simple_update_one( @@ -147,6 +158,7 @@ class PushersTable(Table): "app_display_name", "device_display_name", "pushkey", + "pushkey_ts", "data", "last_token", "last_success", diff --git a/synapse/storage/schema/delta/v10.sql b/synapse/storage/schema/delta/v10.sql index 799e48d780..a991e4eb11 100644 --- a/synapse/storage/schema/delta/v10.sql +++ b/synapse/storage/schema/delta/v10.sql @@ -21,6 +21,7 @@ CREATE TABLE IF NOT EXISTS pushers ( app_display_name varchar(64) NOT NULL, device_display_name varchar(128) NOT NULL, pushkey blob NOT NULL, + ts BIGINT NOT NULL, data blob, last_token TEXT, last_success BIGINT, diff --git a/synapse/storage/schema/pusher.sql b/synapse/storage/schema/pusher.sql index 799e48d780..a991e4eb11 100644 --- a/synapse/storage/schema/pusher.sql +++ b/synapse/storage/schema/pusher.sql @@ -21,6 +21,7 @@ CREATE TABLE IF NOT EXISTS pushers ( app_display_name varchar(64) NOT NULL, device_display_name varchar(128) NOT NULL, pushkey blob NOT NULL, + ts BIGINT NOT NULL, data blob, last_token TEXT, last_success BIGINT, From e3e2fc3255665ac888f3e0c0c6e2be39d5fda5f3 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 15 Jan 2015 16:17:21 +0000 Subject: [PATCH 014/180] Don't make the pushers' event streams cause people to appear online --- synapse/handlers/events.py | 45 +++++++++++++++++++------------------- synapse/push/__init__.py | 4 +++- 2 files changed, 26 insertions(+), 23 deletions(-) diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index c9ade253dd..54ab27004f 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -47,11 +47,11 @@ class EventStreamHandler(BaseHandler): @defer.inlineCallbacks @log_function def get_stream(self, auth_user_id, pagin_config, timeout=0, - as_client_event=True): + as_client_event=True, affect_presence=True): auth_user = self.hs.parse_userid(auth_user_id) try: - if auth_user not in self._streams_per_user: + if affect_presence and auth_user not in self._streams_per_user: self._streams_per_user[auth_user] = 0 if auth_user in self._stop_timer_per_user: try: @@ -64,7 +64,7 @@ class EventStreamHandler(BaseHandler): yield self.distributor.fire( "started_user_eventstream", auth_user ) - self._streams_per_user[auth_user] += 1 + self._streams_per_user[auth_user] += 1 if pagin_config.from_token is None: pagin_config.from_token = None @@ -92,28 +92,29 @@ class EventStreamHandler(BaseHandler): defer.returnValue(chunk) finally: - self._streams_per_user[auth_user] -= 1 - if not self._streams_per_user[auth_user]: - del self._streams_per_user[auth_user] + if affect_presence: + self._streams_per_user[auth_user] -= 1 + if not self._streams_per_user[auth_user]: + del self._streams_per_user[auth_user] - # 10 seconds of grace to allow the client to reconnect again - # before we think they're gone - def _later(): - logger.debug( - "_later stopped_user_eventstream %s", auth_user + # 10 seconds of grace to allow the client to reconnect again + # before we think they're gone + def _later(): + logger.debug( + "_later stopped_user_eventstream %s", auth_user + ) + + self._stop_timer_per_user.pop(auth_user, None) + + yield self.distributor.fire( + "stopped_user_eventstream", auth_user + ) + + logger.debug("Scheduling _later: for %s", auth_user) + self._stop_timer_per_user[auth_user] = ( + self.clock.call_later(30, _later) ) - self._stop_timer_per_user.pop(auth_user, None) - - yield self.distributor.fire( - "stopped_user_eventstream", auth_user - ) - - logger.debug("Scheduling _later: for %s", auth_user) - self._stop_timer_per_user[auth_user] = ( - self.clock.call_later(30, _later) - ) - class EventHandler(BaseHandler): diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 839f666390..9cf996fb80 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -81,7 +81,9 @@ class Pusher(object): from_tok = StreamToken.from_string(self.last_token) config = PaginationConfig(from_token=from_tok, limit='1') chunk = yield self.evStreamHandler.get_stream( - self.user_name, config, timeout=100*365*24*60*60*1000) + self.user_name, config, + timeout=100*365*24*60*60*1000, affect_presence=False + ) # limiting to 1 may get 1 event plus 1 presence event, so # pick out the actual event From 2ca2dbc82183f7dbe8c01694bf1c32a8c4c4b9de Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 15 Jan 2015 16:56:18 +0000 Subject: [PATCH 015/180] Send room name and first alias in notification poke. --- synapse/push/__init__.py | 13 +++++++++++++ synapse/push/httppusher.py | 16 +++++++++++++--- synapse/storage/__init__.py | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 3 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 9cf996fb80..5f4e833add 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -61,6 +61,19 @@ class Pusher(object): return False return True + @defer.inlineCallbacks + def get_context_for_event(self, ev): + name_aliases = yield self.store.get_room_name_and_aliases( + ev['room_id'] + ) + + ctx = {'aliases': name_aliases[1]} + if name_aliases[0] is not None: + ctx['name'] = name_aliases[0] + + defer.returnValue(ctx) + + @defer.inlineCallbacks def start(self): if not self.last_token: diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index bcfa06e2ab..7631a741fa 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -50,6 +50,7 @@ class HttpPusher(Pusher): self.data_minus_url.update(self.data) del self.data_minus_url['url'] + @defer.inlineCallbacks def _build_notification_dict(self, event): # we probably do not want to push for every presence update # (we may want to be able to set up notifications when specific @@ -57,9 +58,11 @@ class HttpPusher(Pusher): # Actually, presence events will not get this far now because we # need to filter them out in the main Pusher code. if 'event_id' not in event: - return None + defer.returnValue(None) - return { + ctx = yield self.get_context_for_event(event) + + d = { 'notification': { 'transition': 'new', # everything is new for now: we don't have read receipts @@ -85,9 +88,16 @@ class HttpPusher(Pusher): } } + if len(ctx['aliases']): + d['notification']['roomAlias'] = ctx['aliases'][0] + if 'name' in ctx: + d['notification']['roomName'] = ctx['name'] + + defer.returnValue(d) + @defer.inlineCallbacks def dispatch_push(self, event): - notification_dict = self._build_notification_dict(event) + notification_dict = yield self._build_notification_dict(event) if not notification_dict: defer.returnValue([]) try: diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index fa7ad0eea8..191fe462a5 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -384,6 +384,41 @@ class DataStore(RoomMemberStore, RoomStore, events = yield self._parse_events(results) defer.returnValue(events) + @defer.inlineCallbacks + def get_room_name_and_aliases(self, room_id): + del_sql = ( + "SELECT event_id FROM redactions WHERE redacts = e.event_id " + "LIMIT 1" + ) + + sql = ( + "SELECT e.*, (%(redacted)s) AS redacted FROM events as e " + "INNER JOIN current_state_events as c ON e.event_id = c.event_id " + "INNER JOIN state_events as s ON e.event_id = s.event_id " + "WHERE c.room_id = ? " + ) % { + "redacted": del_sql, + } + + sql += " AND (s.type = 'm.room.name' AND s.state_key = '')" + sql += " OR s.type = 'm.room.aliases'" + args = (room_id,) + + results = yield self._execute_and_decode(sql, *args) + + events = yield self._parse_events(results) + + name = None + aliases = [] + + for e in events: + if e.type == 'm.room.name': + name = e.content['name'] + elif e.type == 'm.room.aliases': + aliases.extend(e.content['aliases']) + + defer.returnValue((name, aliases)) + @defer.inlineCallbacks def _get_min_token(self): row = yield self._execute( From 2d2953cf5fce26625e56fc1abc230735d007ea1e Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 16 Jan 2015 11:24:10 +0000 Subject: [PATCH 016/180] Require device language when adding a pusher. Because this seems like it might be useful to do sooner rather than later. --- synapse/push/pusherpool.py | 8 +++++--- synapse/rest/pusher.py | 3 ++- synapse/storage/pusher.py | 3 ++- synapse/storage/schema/delta/v10.sql | 1 + synapse/storage/schema/pusher.sql | 1 + 5 files changed, 11 insertions(+), 5 deletions(-) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index edddc3003e..8c77f4b668 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -41,7 +41,7 @@ class PusherPool: @defer.inlineCallbacks def add_pusher(self, user_name, kind, app_id, - app_display_name, device_display_name, pushkey, data): + app_display_name, device_display_name, pushkey, lang, data): # we try to create the pusher just to validate the config: it # will then get pulled out of the database, # recreated, added and started: this means we have only one @@ -54,6 +54,7 @@ class PusherPool: "device_display_name": device_display_name, "pushkey": pushkey, "pushkey_ts": self.hs.get_clock().time_msec(), + "lang": lang, "data": data, "last_token": None, "last_success": None, @@ -62,13 +63,13 @@ class PusherPool: yield self._add_pusher_to_store( user_name, kind, app_id, app_display_name, device_display_name, - pushkey, data + pushkey, lang, data ) @defer.inlineCallbacks def _add_pusher_to_store(self, user_name, kind, app_id, app_display_name, device_display_name, - pushkey, data): + pushkey, lang, data): yield self.store.add_pusher( user_name=user_name, kind=kind, @@ -77,6 +78,7 @@ class PusherPool: device_display_name=device_display_name, pushkey=pushkey, pushkey_ts=self.hs.get_clock().time_msec(), + lang=lang, data=json.dumps(data) ) self._refresh_pusher((app_id, pushkey)) diff --git a/synapse/rest/pusher.py b/synapse/rest/pusher.py index 5b371318d0..6b9a59adb6 100644 --- a/synapse/rest/pusher.py +++ b/synapse/rest/pusher.py @@ -32,7 +32,7 @@ class PusherRestServlet(RestServlet): content = _parse_json(request) reqd = ['kind', 'app_id', 'app_display_name', - 'device_display_name', 'pushkey', 'data'] + 'device_display_name', 'pushkey', 'lang', 'data'] missing = [] for i in reqd: if i not in content: @@ -50,6 +50,7 @@ class PusherRestServlet(RestServlet): app_display_name=content['app_display_name'], device_display_name=content['device_display_name'], pushkey=content['pushkey'], + lang=content['lang'], data=content['data'] ) except PusherConfigException as pce: diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index bfc4980256..4eb30c7bdf 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -94,7 +94,7 @@ class PusherStore(SQLBaseStore): @defer.inlineCallbacks def add_pusher(self, user_name, kind, app_id, app_display_name, device_display_name, - pushkey, pushkey_ts, data): + pushkey, pushkey_ts, lang, data): try: yield self._simple_upsert( PushersTable.table_name, @@ -108,6 +108,7 @@ class PusherStore(SQLBaseStore): app_display_name=app_display_name, device_display_name=device_display_name, ts=pushkey_ts, + lang=lang, data=data )) except Exception as e: diff --git a/synapse/storage/schema/delta/v10.sql b/synapse/storage/schema/delta/v10.sql index a991e4eb11..689d2dff8b 100644 --- a/synapse/storage/schema/delta/v10.sql +++ b/synapse/storage/schema/delta/v10.sql @@ -22,6 +22,7 @@ CREATE TABLE IF NOT EXISTS pushers ( device_display_name varchar(128) NOT NULL, pushkey blob NOT NULL, ts BIGINT NOT NULL, + lang varchar(8), data blob, last_token TEXT, last_success BIGINT, diff --git a/synapse/storage/schema/pusher.sql b/synapse/storage/schema/pusher.sql index a991e4eb11..689d2dff8b 100644 --- a/synapse/storage/schema/pusher.sql +++ b/synapse/storage/schema/pusher.sql @@ -22,6 +22,7 @@ CREATE TABLE IF NOT EXISTS pushers ( device_display_name varchar(128) NOT NULL, pushkey blob NOT NULL, ts BIGINT NOT NULL, + lang varchar(8), data blob, last_token TEXT, last_success BIGINT, From afb714f7bebf88ac27eac018cffa2078e2723310 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 20 Jan 2015 11:49:48 +0000 Subject: [PATCH 017/180] add instance_handles to pushers so we have a way to refer to them even if the push token changes. --- synapse/push/__init__.py | 3 +- synapse/push/httppusher.py | 3 +- synapse/push/pusherpool.py | 9 ++++-- synapse/rest/pusher.py | 3 +- synapse/storage/pusher.py | 46 +++++++++++++++------------- synapse/storage/schema/delta/v10.sql | 1 + synapse/storage/schema/pusher.sql | 1 + 7 files changed, 39 insertions(+), 27 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 5f4e833add..3ee652f3bc 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -30,13 +30,14 @@ class Pusher(object): MAX_BACKOFF = 60 * 60 * 1000 GIVE_UP_AFTER = 24 * 60 * 60 * 1000 - def __init__(self, _hs, user_name, app_id, + def __init__(self, _hs, instance_handle, user_name, app_id, app_display_name, device_display_name, pushkey, pushkey_ts, data, last_token, last_success, failing_since): self.hs = _hs self.evStreamHandler = self.hs.get_handlers().event_stream_handler self.store = self.hs.get_datastore() self.clock = self.hs.get_clock() + self.instance_handle = instance_handle, self.user_name = user_name self.app_id = app_id self.app_display_name = app_display_name diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 7631a741fa..9a3e0be15e 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -24,11 +24,12 @@ logger = logging.getLogger(__name__) class HttpPusher(Pusher): - def __init__(self, _hs, user_name, app_id, + def __init__(self, _hs, instance_handle, user_name, app_id, app_display_name, device_display_name, pushkey, pushkey_ts, data, last_token, last_success, failing_since): super(HttpPusher, self).__init__( _hs, + instance_handle, user_name, app_id, app_display_name, diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 8c77f4b668..2dfecf178b 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -40,7 +40,7 @@ class PusherPool: self._start_pushers(pushers) @defer.inlineCallbacks - def add_pusher(self, user_name, kind, app_id, + def add_pusher(self, user_name, instance_handle, kind, app_id, app_display_name, device_display_name, pushkey, lang, data): # we try to create the pusher just to validate the config: it # will then get pulled out of the database, @@ -49,6 +49,7 @@ class PusherPool: self._create_pusher({ "user_name": user_name, "kind": kind, + "instance_handle": instance_handle, "app_id": app_id, "app_display_name": app_display_name, "device_display_name": device_display_name, @@ -61,17 +62,18 @@ class PusherPool: "failing_since": None }) yield self._add_pusher_to_store( - user_name, kind, app_id, + user_name, instance_handle, kind, app_id, app_display_name, device_display_name, pushkey, lang, data ) @defer.inlineCallbacks - def _add_pusher_to_store(self, user_name, kind, app_id, + def _add_pusher_to_store(self, user_name, instance_handle, kind, app_id, app_display_name, device_display_name, pushkey, lang, data): yield self.store.add_pusher( user_name=user_name, + instance_handle=instance_handle, kind=kind, app_id=app_id, app_display_name=app_display_name, @@ -87,6 +89,7 @@ class PusherPool: if pusherdict['kind'] == 'http': return HttpPusher( self.hs, + instance_handle=pusherdict['instance_handle'], user_name=pusherdict['user_name'], app_id=pusherdict['app_id'], app_display_name=pusherdict['app_display_name'], diff --git a/synapse/rest/pusher.py b/synapse/rest/pusher.py index 6b9a59adb6..4659c9b1d9 100644 --- a/synapse/rest/pusher.py +++ b/synapse/rest/pusher.py @@ -31,7 +31,7 @@ class PusherRestServlet(RestServlet): content = _parse_json(request) - reqd = ['kind', 'app_id', 'app_display_name', + reqd = ['instance_handle', 'kind', 'app_id', 'app_display_name', 'device_display_name', 'pushkey', 'lang', 'data'] missing = [] for i in reqd: @@ -45,6 +45,7 @@ class PusherRestServlet(RestServlet): try: yield pusher_pool.add_pusher( user_name=user.to_string(), + instance_handle=content['instance_handle'], kind=content['kind'], app_id=content['app_id'], app_display_name=content['app_display_name'], diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index 4eb30c7bdf..113cdc8a8e 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -29,7 +29,7 @@ class PusherStore(SQLBaseStore): @defer.inlineCallbacks def get_pushers_by_app_id_and_pushkey(self, app_id_and_pushkey): sql = ( - "SELECT id, user_name, kind, app_id," + "SELECT id, user_name, kind, instance_handle, app_id," "app_display_name, device_display_name, pushkey, ts, data, " "last_token, last_success, failing_since " "FROM pushers " @@ -45,15 +45,16 @@ class PusherStore(SQLBaseStore): "id": r[0], "user_name": r[1], "kind": r[2], - "app_id": r[3], - "app_display_name": r[4], - "device_display_name": r[5], - "pushkey": r[6], - "pushkey_ts": r[7], - "data": r[8], - "last_token": r[9], - "last_success": r[10], - "failing_since": r[11] + "instance_handle": r[3], + "app_id": r[4], + "app_display_name": r[5], + "device_display_name": r[6], + "pushkey": r[7], + "pushkey_ts": r[8], + "data": r[9], + "last_token": r[10], + "last_success": r[11], + "failing_since": r[12] } for r in rows ] @@ -63,7 +64,7 @@ class PusherStore(SQLBaseStore): @defer.inlineCallbacks def get_all_pushers(self): sql = ( - "SELECT id, user_name, kind, app_id," + "SELECT id, user_name, kind, instance_handle, app_id," "app_display_name, device_display_name, pushkey, ts, data, " "last_token, last_success, failing_since " "FROM pushers" @@ -76,15 +77,16 @@ class PusherStore(SQLBaseStore): "id": r[0], "user_name": r[1], "kind": r[2], - "app_id": r[3], - "app_display_name": r[4], - "device_display_name": r[5], - "pushkey": r[6], - "pushkey_ts": r[7], - "data": r[8], - "last_token": r[9], - "last_success": r[10], - "failing_since": r[11] + "instance_handle": r[3], + "app_id": r[4], + "app_display_name": r[5], + "device_display_name": r[6], + "pushkey": r[7], + "pushkey_ts": r[8], + "data": r[9], + "last_token": r[10], + "last_success": r[11], + "failing_since": r[12] } for r in rows ] @@ -92,7 +94,7 @@ class PusherStore(SQLBaseStore): defer.returnValue(ret) @defer.inlineCallbacks - def add_pusher(self, user_name, kind, app_id, + def add_pusher(self, user_name, instance_handle, kind, app_id, app_display_name, device_display_name, pushkey, pushkey_ts, lang, data): try: @@ -105,6 +107,7 @@ class PusherStore(SQLBaseStore): dict( user_name=user_name, kind=kind, + instance_handle=instance_handle, app_display_name=app_display_name, device_display_name=device_display_name, ts=pushkey_ts, @@ -155,6 +158,7 @@ class PushersTable(Table): "id", "user_name", "kind", + "instance_handle", "app_id", "app_display_name", "device_display_name", diff --git a/synapse/storage/schema/delta/v10.sql b/synapse/storage/schema/delta/v10.sql index 689d2dff8b..b84ce20ef3 100644 --- a/synapse/storage/schema/delta/v10.sql +++ b/synapse/storage/schema/delta/v10.sql @@ -16,6 +16,7 @@ CREATE TABLE IF NOT EXISTS pushers ( id INTEGER PRIMARY KEY AUTOINCREMENT, user_name TEXT NOT NULL, + instance_handle varchar(32) NOT NULL, kind varchar(8) NOT NULL, app_id varchar(64) NOT NULL, app_display_name varchar(64) NOT NULL, diff --git a/synapse/storage/schema/pusher.sql b/synapse/storage/schema/pusher.sql index 689d2dff8b..b84ce20ef3 100644 --- a/synapse/storage/schema/pusher.sql +++ b/synapse/storage/schema/pusher.sql @@ -16,6 +16,7 @@ CREATE TABLE IF NOT EXISTS pushers ( id INTEGER PRIMARY KEY AUTOINCREMENT, user_name TEXT NOT NULL, + instance_handle varchar(32) NOT NULL, kind varchar(8) NOT NULL, app_id varchar(64) NOT NULL, app_display_name varchar(64) NOT NULL, From 5d5932d493dc769e95472835b438769567ed550e Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 20 Jan 2015 11:52:08 +0000 Subject: [PATCH 018/180] use underscores everywhere, not camelcase. --- synapse/push/httppusher.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 9a3e0be15e..46433ad4a9 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -72,17 +72,17 @@ class HttpPusher(Pusher): 'from': event['user_id'], # we may have to fetch this over federation and we # can't trust it anyway: is it worth it? - #'fromDisplayName': 'Steve Stevington' + #'from_display_name': 'Steve Stevington' #'counts': { -- we don't mark messages as read yet so # we have no way of knowing # 'unread': 1, - # 'missedCalls': 2 + # 'missed_calls': 2 # }, 'devices': [ { 'app_id': self.app_id, 'pushkey': self.pushkey, - 'pushkeyTs': long(self.pushkey_ts / 1000), + 'pushkey_ts': long(self.pushkey_ts / 1000), 'data': self.data_minus_url } ] From 6dcade97be7f1331063fd12ac85e61c6f2cf7dac Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 21 Jan 2015 16:27:04 +0000 Subject: [PATCH 019/180] Implement new state resolution algorithm --- synapse/state.py | 101 +++++++---- tests/test_state.py | 434 ++++++++++++++++++++++++++++++++++++-------- 2 files changed, 430 insertions(+), 105 deletions(-) diff --git a/synapse/state.py b/synapse/state.py index 8144fa02b4..7d58a76ede 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -19,6 +19,7 @@ from twisted.internet import defer from synapse.util.logutils import log_function from synapse.util.async import run_on_reactor from synapse.api.constants import EventTypes +from synapse.api.errors import AuthError from synapse.events.snapshot import EventContext from collections import namedtuple @@ -42,6 +43,8 @@ class StateHandler(object): def __init__(self, hs): self.store = hs.get_datastore() + # self.auth = hs.get_auth() + self.hs = hs @defer.inlineCallbacks def get_current_state(self, room_id, event_type=None, state_key=""): @@ -210,15 +213,22 @@ class StateHandler(object): else: prev_states = [] + auth_events = { + k: e for k, e in unconflicted_state.items() + if k[0] in (EventTypes.Create, EventTypes.Member, EventTypes.PowerLevels,) + } + try: - new_state = {} - new_state.update(unconflicted_state) - for key, events in conflicted_state.items(): - new_state[key] = self._resolve_state_events(events) + resolved_state = self._resolve_state_events( + conflicted_state, auth_events + ) except: logger.exception("Failed to resolve state") raise + new_state = unconflicted_state + new_state.update(resolved_state) + defer.returnValue((None, new_state, prev_states)) def _get_power_level_from_event_state(self, event, user_id): @@ -238,36 +248,65 @@ class StateHandler(object): return 0 @log_function - def _resolve_state_events(self, events): - curr_events = events + def _resolve_state_events(self, conflicted_state, auth_events): + resolved_state = {} + power_key = (EventTypes.PowerLevels, "") + if power_key in conflicted_state.items(): + power_levels = conflicted_state[power_key] + resolved_state[power_key] = self._resolve_auth_events(power_levels) - new_powers = [ - self._get_power_level_from_event_state(e, e.user_id) - for e in curr_events - ] + auth_events.update(resolved_state) - new_powers = [ - int(p) if p else 0 for p in new_powers - ] + for key, events in conflicted_state.items(): + if key[0] == EventTypes.Member: + resolved_state[key] = self._resolve_auth_events( + events, + auth_events + ) - max_power = max(new_powers) + auth_events.update(resolved_state) - curr_events = [ - z[0] for z in zip(curr_events, new_powers) - if z[1] == max_power - ] + for key, events in conflicted_state.items(): + if key not in resolved_state: + resolved_state[key] = self._resolve_normal_events( + events, auth_events + ) - if not curr_events: - raise RuntimeError("Max didn't get a max?") - elif len(curr_events) == 1: - return curr_events[0] + return resolved_state - # TODO: For now, just choose the one with the largest event_id. - return ( - sorted( - curr_events, - key=lambda e: hashlib.sha1( - e.event_id + e.user_id + e.room_id + e.type - ).hexdigest() - )[0] - ) + def _resolve_auth_events(self, events, auth_events): + reverse = [i for i in reversed(self._ordered_events(events))] + + auth_events = dict(auth_events) + + prev_event = reverse[0] + for event in reverse[1:]: + auth_events[(prev_event.type, prev_event.state_key)] = prev_event + try: + # FIXME: hs.get_auth() is bad style, but we need to do it to + # get around circular deps. + self.hs.get_auth().check(event, auth_events) + prev_event = event + except AuthError: + return prev_event + + return event + + def _resolve_normal_events(self, events, auth_events): + for event in self._ordered_events(events): + try: + # FIXME: hs.get_auth() is bad style, but we need to do it to + # get around circular deps. + self.hs.get_auth().check(event, auth_events) + return event + except AuthError as e: + pass + + # Oh dear. + return event + + def _ordered_events(self, events): + def key_func(e): + return -int(e.depth), hashlib.sha1(e.event_id).hexdigest() + + return sorted(events, key=key_func) \ No newline at end of file diff --git a/tests/test_state.py b/tests/test_state.py index 98ad9e54cd..019e794aa2 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -16,11 +16,120 @@ from tests import unittest from twisted.internet import defer +from synapse.events import FrozenEvent +from synapse.api.auth import Auth +from synapse.api.constants import EventTypes, Membership from synapse.state import StateHandler from mock import Mock +_next_event_id = 1000 + + +def create_event(name=None, type=None, state_key=None, depth=2, event_id=None, + prev_events=[], **kwargs): + global _next_event_id + + if not event_id: + _next_event_id += 1 + event_id = str(_next_event_id) + + if not name: + if state_key is not None: + name = "<%s-%s, %s>" % (type, state_key, event_id,) + else: + name = "<%s, %s>" % (type, event_id,) + + d = { + "event_id": event_id, + "type": type, + "sender": "@user_id:example.com", + "room_id": "!room_id:example.com", + "depth": depth, + "prev_events": prev_events, + } + + if state_key is not None: + d["state_key"] = state_key + + d.update(kwargs) + + event = FrozenEvent(d) + + return event + + +class StateGroupStore(object): + def __init__(self): + self._event_to_state_group = {} + self._group_to_state = {} + + self._next_group = 1 + + def get_state_groups(self, event_ids): + groups = {} + for event_id in event_ids: + group = self._event_to_state_group.get(event_id) + if group: + groups[group] = self._group_to_state[group] + + return defer.succeed(groups) + + def store_state_groups(self, event, context): + if context.current_state is None: + return + + state_events = context.current_state + + if event.is_state(): + state_events[(event.type, event.state_key)] = event + + state_group = context.state_group + if not state_group: + state_group = self._next_group + self._next_group += 1 + + self._group_to_state[state_group] = state_events.values() + + self._event_to_state_group[event.event_id] = state_group + + +class DictObj(dict): + def __init__(self, **kwargs): + super(DictObj, self).__init__(kwargs) + self.__dict__ = self + + +class Graph(object): + def __init__(self, nodes, edges): + events = {} + clobbered = set(events.keys()) + + for event_id, fields in nodes.items(): + refs = edges.get(event_id) + if refs: + clobbered.difference_update(refs) + prev_events = [(r, {}) for r in refs] + else: + prev_events = [] + + events[event_id] = create_event( + event_id=event_id, + prev_events=prev_events, + **fields + ) + + self._leaves = clobbered + self._events = sorted(events.values(), key=lambda e: e.depth) + + def walk(self): + return iter(self._events) + + def get_leaves(self): + return (self._events[i] for i in self._leaves) + + class StateTestCase(unittest.TestCase): def setUp(self): self.store = Mock( @@ -29,20 +138,188 @@ class StateTestCase(unittest.TestCase): "add_event_hashes", ] ) - hs = Mock(spec=["get_datastore"]) + hs = Mock(spec=["get_datastore", "get_auth", "get_state_handler"]) hs.get_datastore.return_value = self.store + hs.get_state_handler.return_value = None + hs.get_auth.return_value = Auth(hs) self.state = StateHandler(hs) self.event_id = 0 + @defer.inlineCallbacks + def test_branch_no_conflict(self): + graph = Graph( + nodes={ + "START": DictObj( + type=EventTypes.Create, + state_key="", + depth=1, + ), + "A": DictObj( + type=EventTypes.Message, + depth=2, + ), + "B": DictObj( + type=EventTypes.Message, + depth=3, + ), + "C": DictObj( + type=EventTypes.Name, + state_key="", + depth=3, + ), + "D": DictObj( + type=EventTypes.Message, + depth=4, + ), + }, + edges={ + "A": ["START"], + "B": ["A"], + "C": ["A"], + "D": ["B", "C"] + } + ) + + store = StateGroupStore() + self.store.get_state_groups.side_effect = store.get_state_groups + + context_store = {} + + for event in graph.walk(): + context = yield self.state.compute_event_context(event) + store.store_state_groups(event, context) + context_store[event.event_id] = context + + self.assertEqual(2, len(context_store["D"].current_state)) + + @defer.inlineCallbacks + def test_branch_basic_conflict(self): + graph = Graph( + nodes={ + "START": DictObj( + type=EventTypes.Create, + state_key="creator", + content={"membership": "@user_id:example.com"}, + depth=1, + ), + "A": DictObj( + type=EventTypes.Member, + state_key="@user_id:example.com", + content={"membership": Membership.JOIN}, + membership=Membership.JOIN, + depth=2, + ), + "B": DictObj( + type=EventTypes.Name, + state_key="", + depth=3, + ), + "C": DictObj( + type=EventTypes.Name, + state_key="", + depth=4, + ), + "D": DictObj( + type=EventTypes.Message, + depth=5, + ), + }, + edges={ + "A": ["START"], + "B": ["A"], + "C": ["A"], + "D": ["B", "C"] + } + ) + + store = StateGroupStore() + self.store.get_state_groups.side_effect = store.get_state_groups + + context_store = {} + + for event in graph.walk(): + context = yield self.state.compute_event_context(event) + store.store_state_groups(event, context) + context_store[event.event_id] = context + + self.assertSetEqual( + {"START", "A", "C"}, + {e.event_id for e in context_store["D"].current_state.values()} + ) + + @defer.inlineCallbacks + def test_branch_have_banned_conflict(self): + graph = Graph( + nodes={ + "START": DictObj( + type=EventTypes.Create, + state_key="creator", + content={"membership": "@user_id:example.com"}, + depth=1, + ), + "A": DictObj( + type=EventTypes.Member, + state_key="@user_id:example.com", + content={"membership": Membership.JOIN}, + membership=Membership.JOIN, + depth=2, + ), + "B": DictObj( + type=EventTypes.Name, + state_key="", + depth=3, + ), + "C": DictObj( + type=EventTypes.Member, + state_key="@user_id_2:example.com", + content={"membership": Membership.BAN}, + membership=Membership.BAN, + depth=4, + ), + "D": DictObj( + type=EventTypes.Name, + state_key="", + depth=4, + sender="@user_id_2:example.com", + ), + "E": DictObj( + type=EventTypes.Message, + depth=5, + ), + }, + edges={ + "A": ["START"], + "B": ["A"], + "C": ["B"], + "D": ["B"], + "E": ["C", "D"] + } + ) + + store = StateGroupStore() + self.store.get_state_groups.side_effect = store.get_state_groups + + context_store = {} + + for event in graph.walk(): + context = yield self.state.compute_event_context(event) + store.store_state_groups(event, context) + context_store[event.event_id] = context + + self.assertSetEqual( + {"START", "A", "B", "C"}, + {e.event_id for e in context_store["E"].current_state.values()} + ) + @defer.inlineCallbacks def test_annotate_with_old_message(self): - event = self.create_event(type="test_message", name="event") + event = create_event(type="test_message", name="event") old_state = [ - self.create_event(type="test1", state_key="1"), - self.create_event(type="test1", state_key="2"), - self.create_event(type="test2", state_key=""), + create_event(type="test1", state_key="1"), + create_event(type="test1", state_key="2"), + create_event(type="test2", state_key=""), ] context = yield self.state.compute_event_context( @@ -62,12 +339,12 @@ class StateTestCase(unittest.TestCase): @defer.inlineCallbacks def test_annotate_with_old_state(self): - event = self.create_event(type="state", state_key="", name="event") + event = create_event(type="state", state_key="", name="event") old_state = [ - self.create_event(type="test1", state_key="1"), - self.create_event(type="test1", state_key="2"), - self.create_event(type="test2", state_key=""), + create_event(type="test1", state_key="1"), + create_event(type="test1", state_key="2"), + create_event(type="test2", state_key=""), ] context = yield self.state.compute_event_context( @@ -88,13 +365,12 @@ class StateTestCase(unittest.TestCase): @defer.inlineCallbacks def test_trivial_annotate_message(self): - event = self.create_event(type="test_message", name="event") - event.prev_events = [] + event = create_event(type="test_message", name="event") old_state = [ - self.create_event(type="test1", state_key="1"), - self.create_event(type="test1", state_key="2"), - self.create_event(type="test2", state_key=""), + create_event(type="test1", state_key="1"), + create_event(type="test1", state_key="2"), + create_event(type="test2", state_key=""), ] group_name = "group_name_1" @@ -119,13 +395,12 @@ class StateTestCase(unittest.TestCase): @defer.inlineCallbacks def test_trivial_annotate_state(self): - event = self.create_event(type="state", state_key="", name="event") - event.prev_events = [] + event = create_event(type="state", state_key="", name="event") old_state = [ - self.create_event(type="test1", state_key="1"), - self.create_event(type="test1", state_key="2"), - self.create_event(type="test2", state_key=""), + create_event(type="test1", state_key="1"), + create_event(type="test1", state_key="2"), + create_event(type="test2", state_key=""), ] group_name = "group_name_1" @@ -150,30 +425,21 @@ class StateTestCase(unittest.TestCase): @defer.inlineCallbacks def test_resolve_message_conflict(self): - event = self.create_event(type="test_message", name="event") - event.prev_events = [] + event = create_event(type="test_message", name="event") old_state_1 = [ - self.create_event(type="test1", state_key="1"), - self.create_event(type="test1", state_key="2"), - self.create_event(type="test2", state_key=""), + create_event(type="test1", state_key="1"), + create_event(type="test1", state_key="2"), + create_event(type="test2", state_key=""), ] old_state_2 = [ - self.create_event(type="test1", state_key="1"), - self.create_event(type="test3", state_key="2"), - self.create_event(type="test4", state_key=""), + create_event(type="test1", state_key="1"), + create_event(type="test3", state_key="2"), + create_event(type="test4", state_key=""), ] - group_name_1 = "group_name_1" - group_name_2 = "group_name_2" - - self.store.get_state_groups.return_value = { - group_name_1: old_state_1, - group_name_2: old_state_2, - } - - context = yield self.state.compute_event_context(event) + context = yield self._get_context(event, old_state_1, old_state_2) self.assertEqual(len(context.current_state), 5) @@ -181,21 +447,70 @@ class StateTestCase(unittest.TestCase): @defer.inlineCallbacks def test_resolve_state_conflict(self): - event = self.create_event(type="test4", state_key="", name="event") - event.prev_events = [] + event = create_event(type="test4", state_key="", name="event") old_state_1 = [ - self.create_event(type="test1", state_key="1"), - self.create_event(type="test1", state_key="2"), - self.create_event(type="test2", state_key=""), + create_event(type="test1", state_key="1"), + create_event(type="test1", state_key="2"), + create_event(type="test2", state_key=""), ] old_state_2 = [ - self.create_event(type="test1", state_key="1"), - self.create_event(type="test3", state_key="2"), - self.create_event(type="test4", state_key=""), + create_event(type="test1", state_key="1"), + create_event(type="test3", state_key="2"), + create_event(type="test4", state_key=""), ] + context = yield self._get_context(event, old_state_1, old_state_2) + + self.assertEqual(len(context.current_state), 5) + + self.assertIsNone(context.state_group) + + @defer.inlineCallbacks + def test_standard_depth_conflict(self): + event = create_event(type="test4", name="event") + + member_event = create_event( + type=EventTypes.Member, + state_key="@user_id:example.com", + content={ + "membership": Membership.JOIN, + } + ) + + old_state_1 = [ + member_event, + create_event(type="test1", state_key="1", depth=1), + ] + + old_state_2 = [ + member_event, + create_event(type="test1", state_key="1", depth=2), + ] + + context = yield self._get_context(event, old_state_1, old_state_2) + + self.assertEqual(old_state_2[1], context.current_state[("test1", "1")]) + + # Reverse the depth to make sure we are actually using the depths + # during state resolution. + + old_state_1 = [ + member_event, + create_event(type="test1", state_key="1", depth=2), + ] + + old_state_2 = [ + member_event, + create_event(type="test1", state_key="1", depth=1), + ] + + context = yield self._get_context(event, old_state_1, old_state_2) + + self.assertEqual(old_state_1[1], context.current_state[("test1", "1")]) + + def _get_context(self, event, old_state_1, old_state_2): group_name_1 = "group_name_1" group_name_2 = "group_name_2" @@ -204,33 +519,4 @@ class StateTestCase(unittest.TestCase): group_name_2: old_state_2, } - context = yield self.state.compute_event_context(event) - - self.assertEqual(len(context.current_state), 5) - - self.assertIsNone(context.state_group) - - def create_event(self, name=None, type=None, state_key=None): - self.event_id += 1 - event_id = str(self.event_id) - - if not name: - if state_key is not None: - name = "<%s-%s>" % (type, state_key) - else: - name = "<%s>" % (type, ) - - event = Mock(name=name, spec=[]) - event.type = type - - if state_key is not None: - event.state_key = state_key - event.event_id = event_id - - event.is_state = lambda: (state_key is not None) - event.unsigned = {} - - event.user_id = "@user_id:example.com" - event.room_id = "!room_id:example.com" - - return event + return self.state.compute_event_context(event) From b390bf39f280c64497da089c647402bc3287f4b0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 21 Jan 2015 16:44:04 +0000 Subject: [PATCH 020/180] Remove unused function. Add comment. --- synapse/state.py | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/synapse/state.py b/synapse/state.py index 7d58a76ede..5b622ad3b1 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -231,24 +231,20 @@ class StateHandler(object): defer.returnValue((None, new_state, prev_states)) - def _get_power_level_from_event_state(self, event, user_id): - if hasattr(event, "old_state_events") and event.old_state_events: - key = (EventTypes.PowerLevels, "", ) - power_level_event = event.old_state_events.get(key) - level = None - if power_level_event: - level = power_level_event.content.get("users", {}).get( - user_id - ) - if not level: - level = power_level_event.content.get("users_default", 0) - - return level - else: - return 0 - @log_function def _resolve_state_events(self, conflicted_state, auth_events): + """ This is where we actually decide which of the conflicted state to + use. + + We resolve conflicts in the following order: + 1. power levels + 2. memberships + 3. other events. + + :param conflicted_state: + :param auth_events: + :return: + """ resolved_state = {} power_key = (EventTypes.PowerLevels, "") if power_key in conflicted_state.items(): From b1b85753d759f7127fbb1c4a95005fffd3da7f4d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 22 Jan 2015 15:50:17 +0000 Subject: [PATCH 021/180] Add support for storing rejected events in EventContext and data stores --- synapse/events/snapshot.py | 1 + synapse/storage/__init__.py | 11 +++++++--- synapse/storage/_base.py | 21 +++++++++++------- synapse/storage/rejections.py | 33 ++++++++++++++++++++++++++++ synapse/storage/schema/delta/v12.sql | 21 ++++++++++++++++++ synapse/storage/schema/im.sql | 7 ++++++ 6 files changed, 83 insertions(+), 11 deletions(-) create mode 100644 synapse/storage/rejections.py create mode 100644 synapse/storage/schema/delta/v12.sql diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py index 6bbba8d6ba..7e98bdef28 100644 --- a/synapse/events/snapshot.py +++ b/synapse/events/snapshot.py @@ -20,3 +20,4 @@ class EventContext(object): self.current_state = current_state self.auth_events = auth_events self.state_group = None + self.rejected = False diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 4beb951b9f..015fcc8775 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -30,6 +30,7 @@ from .transactions import TransactionStore from .keys import KeyStore from .event_federation import EventFederationStore from .media_repository import MediaRepositoryStore +from .rejections import RejectionsStore from .state import StateStore from .signatures import SignatureStore @@ -66,7 +67,7 @@ SCHEMAS = [ # Remember to update this number every time an incompatible change is made to # database schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 11 +SCHEMA_VERSION = 12 class _RollbackButIsFineException(Exception): @@ -82,6 +83,7 @@ class DataStore(RoomMemberStore, RoomStore, DirectoryStore, KeyStore, StateStore, SignatureStore, EventFederationStore, MediaRepositoryStore, + RejectionsStore, ): def __init__(self, hs): @@ -224,6 +226,9 @@ class DataStore(RoomMemberStore, RoomStore, if not outlier: self._store_state_groups_txn(txn, event, context) + if context.rejected: + self._store_rejections_txn(txn, event.event_id, context.rejected) + if current_state: txn.execute( "DELETE FROM current_state_events WHERE room_id = ?", @@ -262,7 +267,7 @@ class DataStore(RoomMemberStore, RoomStore, or_replace=True, ) - if is_new_state: + if is_new_state and not context.rejected: self._simple_insert_txn( txn, "current_state_events", @@ -288,7 +293,7 @@ class DataStore(RoomMemberStore, RoomStore, or_ignore=True, ) - if not backfilled: + if not backfilled and not context.rejected: self._simple_insert_txn( txn, table="state_forward_extremities", diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index f660fc6eaf..2075a018b2 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -458,10 +458,12 @@ class SQLBaseStore(object): return [e for e in events if e] def _get_event_txn(self, txn, event_id, check_redacted=True, - get_prev_content=False): + get_prev_content=False, allow_rejected=False): sql = ( - "SELECT internal_metadata, json, r.event_id FROM event_json as e " + "SELECT internal_metadata, json, r.event_id, reason " + "FROM event_json as e " "LEFT JOIN redactions as r ON e.event_id = r.redacts " + "LEFT JOIN rejections as rej on rej.event_id = e.event_id " "WHERE e.event_id = ? " "LIMIT 1 " ) @@ -473,13 +475,16 @@ class SQLBaseStore(object): if not res: return None - internal_metadata, js, redacted = res + internal_metadata, js, redacted, rejected_reason = res - return self._get_event_from_row_txn( - txn, internal_metadata, js, redacted, - check_redacted=check_redacted, - get_prev_content=get_prev_content, - ) + if allow_rejected or not rejected_reason: + return self._get_event_from_row_txn( + txn, internal_metadata, js, redacted, + check_redacted=check_redacted, + get_prev_content=get_prev_content, + ) + else: + return None def _get_event_from_row_txn(self, txn, internal_metadata, js, redacted, check_redacted=True, get_prev_content=False): diff --git a/synapse/storage/rejections.py b/synapse/storage/rejections.py new file mode 100644 index 0000000000..7d38b31f44 --- /dev/null +++ b/synapse/storage/rejections.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ._base import SQLBaseStore + +import logging + +logger = logging.getLogger(__name__) + + +class RejectionsStore(SQLBaseStore): + def _store_rejections_txn(self, txn, event_id, reason): + self._simple_insert_txn( + txn, + table="rejections", + values={ + "event_id": event_id, + "reason": reason, + "last_failure": self._clock.time_msec(), + } + ) diff --git a/synapse/storage/schema/delta/v12.sql b/synapse/storage/schema/delta/v12.sql new file mode 100644 index 0000000000..bd2a8b1bb5 --- /dev/null +++ b/synapse/storage/schema/delta/v12.sql @@ -0,0 +1,21 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS rejections( + event_id TEXT NOT NULL, + reason TEXT NOT NULL, + last_check TEXT NOT NULL, + CONSTRAINT ev_id UNIQUE (event_id) ON CONFLICT REPLACE +); diff --git a/synapse/storage/schema/im.sql b/synapse/storage/schema/im.sql index dd00c1cd2f..bc7c6b6ed5 100644 --- a/synapse/storage/schema/im.sql +++ b/synapse/storage/schema/im.sql @@ -123,3 +123,10 @@ CREATE TABLE IF NOT EXISTS room_hosts( ); CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id); + +CREATE TABLE IF NOT EXISTS rejections( + event_id TEXT NOT NULL, + reason TEXT NOT NULL, + last_check TEXT NOT NULL, + CONSTRAINT ev_id UNIQUE (event_id) ON CONFLICT REPLACE +); From 73dd81ca62885110f7fe0e51e7f4e0183b495d17 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 22 Jan 2015 15:57:08 +0000 Subject: [PATCH 022/180] fix pyflakes --- synapse/state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/state.py b/synapse/state.py index 5b622ad3b1..d9fdfb34be 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -295,7 +295,7 @@ class StateHandler(object): # get around circular deps. self.hs.get_auth().check(event, auth_events) return event - except AuthError as e: + except AuthError: pass # Oh dear. From ca65a9d03e4e6e8bc9b42a3ff5a83c1e5294fc7b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 22 Jan 2015 16:37:08 +0000 Subject: [PATCH 023/180] Split out TransactionQueue from replication layer --- synapse/federation/replication.py | 291 +--------------------- synapse/federation/transaction_queue.py | 314 ++++++++++++++++++++++++ 2 files changed, 316 insertions(+), 289 deletions(-) create mode 100644 synapse/federation/transaction_queue.py diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 6620532a60..accf95e406 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -20,8 +20,8 @@ a given transport. from twisted.internet import defer from .units import Transaction, Edu - from .persistence import TransactionActions +from .transaction_queue import TransactionQueue from synapse.util.logutils import log_function from synapse.util.logcontext import PreserveLoggingContext @@ -62,7 +62,7 @@ class ReplicationLayer(object): # self.pdu_actions = PduActions(self.store) self.transaction_actions = TransactionActions(self.store) - self._transaction_queue = _TransactionQueue( + self._transaction_queue = TransactionQueue( hs, self.transaction_actions, transport_layer ) @@ -662,290 +662,3 @@ class ReplicationLayer(object): event.internal_metadata.outlier = outlier return event - - -class _TransactionQueue(object): - """This class makes sure we only have one transaction in flight at - a time for a given destination. - - It batches pending PDUs into single transactions. - """ - - def __init__(self, hs, transaction_actions, transport_layer): - self.server_name = hs.hostname - self.transaction_actions = transaction_actions - self.transport_layer = transport_layer - - self._clock = hs.get_clock() - self.store = hs.get_datastore() - - # Is a mapping from destinations -> deferreds. Used to keep track - # of which destinations have transactions in flight and when they are - # done - self.pending_transactions = {} - - # Is a mapping from destination -> list of - # tuple(pending pdus, deferred, order) - self.pending_pdus_by_dest = {} - # destination -> list of tuple(edu, deferred) - self.pending_edus_by_dest = {} - - # destination -> list of tuple(failure, deferred) - self.pending_failures_by_dest = {} - - # HACK to get unique tx id - self._next_txn_id = int(self._clock.time_msec()) - - @defer.inlineCallbacks - @log_function - def enqueue_pdu(self, pdu, destinations, order): - # We loop through all destinations to see whether we already have - # a transaction in progress. If we do, stick it in the pending_pdus - # table and we'll get back to it later. - - destinations = set(destinations) - destinations.discard(self.server_name) - destinations.discard("localhost") - - logger.debug("Sending to: %s", str(destinations)) - - if not destinations: - return - - deferreds = [] - - for destination in destinations: - deferred = defer.Deferred() - self.pending_pdus_by_dest.setdefault(destination, []).append( - (pdu, deferred, order) - ) - - def eb(failure): - if not deferred.called: - deferred.errback(failure) - else: - logger.warn("Failed to send pdu", failure) - - with PreserveLoggingContext(): - self._attempt_new_transaction(destination).addErrback(eb) - - deferreds.append(deferred) - - yield defer.DeferredList(deferreds) - - # NO inlineCallbacks - def enqueue_edu(self, edu): - destination = edu.destination - - if destination == self.server_name: - return - - deferred = defer.Deferred() - self.pending_edus_by_dest.setdefault(destination, []).append( - (edu, deferred) - ) - - def eb(failure): - if not deferred.called: - deferred.errback(failure) - else: - logger.warn("Failed to send edu", failure) - - with PreserveLoggingContext(): - self._attempt_new_transaction(destination).addErrback(eb) - - return deferred - - @defer.inlineCallbacks - def enqueue_failure(self, failure, destination): - deferred = defer.Deferred() - - self.pending_failures_by_dest.setdefault( - destination, [] - ).append( - (failure, deferred) - ) - - yield deferred - - @defer.inlineCallbacks - @log_function - def _attempt_new_transaction(self, destination): - - (retry_last_ts, retry_interval) = (0, 0) - retry_timings = yield self.store.get_destination_retry_timings( - destination - ) - if retry_timings: - (retry_last_ts, retry_interval) = ( - retry_timings.retry_last_ts, retry_timings.retry_interval - ) - if retry_last_ts + retry_interval > int(self._clock.time_msec()): - logger.info( - "TX [%s] not ready for retry yet - " - "dropping transaction for now", - destination, - ) - return - else: - logger.info("TX [%s] is ready for retry", destination) - - logger.info("TX [%s] _attempt_new_transaction", destination) - - if destination in self.pending_transactions: - # XXX: pending_transactions can get stuck on by a never-ending - # request at which point pending_pdus_by_dest just keeps growing. - # we need application-layer timeouts of some flavour of these - # requests - return - - # list of (pending_pdu, deferred, order) - pending_pdus = self.pending_pdus_by_dest.pop(destination, []) - pending_edus = self.pending_edus_by_dest.pop(destination, []) - pending_failures = self.pending_failures_by_dest.pop(destination, []) - - if pending_pdus: - logger.info("TX [%s] len(pending_pdus_by_dest[dest]) = %d", - destination, len(pending_pdus)) - - if not pending_pdus and not pending_edus and not pending_failures: - return - - logger.debug( - "TX [%s] Attempting new transaction" - " (pdus: %d, edus: %d, failures: %d)", - destination, - len(pending_pdus), - len(pending_edus), - len(pending_failures) - ) - - # Sort based on the order field - pending_pdus.sort(key=lambda t: t[2]) - - pdus = [x[0] for x in pending_pdus] - edus = [x[0] for x in pending_edus] - failures = [x[0].get_dict() for x in pending_failures] - deferreds = [ - x[1] - for x in pending_pdus + pending_edus + pending_failures - ] - - try: - self.pending_transactions[destination] = 1 - - logger.debug("TX [%s] Persisting transaction...", destination) - - transaction = Transaction.create_new( - origin_server_ts=int(self._clock.time_msec()), - transaction_id=str(self._next_txn_id), - origin=self.server_name, - destination=destination, - pdus=pdus, - edus=edus, - pdu_failures=failures, - ) - - self._next_txn_id += 1 - - yield self.transaction_actions.prepare_to_send(transaction) - - logger.debug("TX [%s] Persisted transaction", destination) - logger.info( - "TX [%s] Sending transaction [%s]", - destination, - transaction.transaction_id, - ) - - # Actually send the transaction - - # FIXME (erikj): This is a bit of a hack to make the Pdu age - # keys work - def json_data_cb(): - data = transaction.get_dict() - now = int(self._clock.time_msec()) - if "pdus" in data: - for p in data["pdus"]: - if "age_ts" in p: - unsigned = p.setdefault("unsigned", {}) - unsigned["age"] = now - int(p["age_ts"]) - del p["age_ts"] - return data - - code, response = yield self.transport_layer.send_transaction( - transaction, json_data_cb - ) - - logger.info("TX [%s] got %d response", destination, code) - - logger.debug("TX [%s] Sent transaction", destination) - logger.debug("TX [%s] Marking as delivered...", destination) - - yield self.transaction_actions.delivered( - transaction, code, response - ) - - logger.debug("TX [%s] Marked as delivered", destination) - logger.debug("TX [%s] Yielding to callbacks...", destination) - - for deferred in deferreds: - if code == 200: - if retry_last_ts: - # this host is alive! reset retry schedule - yield self.store.set_destination_retry_timings( - destination, 0, 0 - ) - deferred.callback(None) - else: - self.set_retrying(destination, retry_interval) - deferred.errback(RuntimeError("Got status %d" % code)) - - # Ensures we don't continue until all callbacks on that - # deferred have fired - try: - yield deferred - except: - pass - - logger.debug("TX [%s] Yielded to callbacks", destination) - - except Exception as e: - # We capture this here as there as nothing actually listens - # for this finishing functions deferred. - logger.warn( - "TX [%s] Problem in _attempt_transaction: %s", - destination, - e, - ) - - self.set_retrying(destination, retry_interval) - - for deferred in deferreds: - if not deferred.called: - deferred.errback(e) - - finally: - # We want to be *very* sure we delete this after we stop processing - self.pending_transactions.pop(destination, None) - - # Check to see if there is anything else to send. - self._attempt_new_transaction(destination) - - @defer.inlineCallbacks - def set_retrying(self, destination, retry_interval): - # track that this destination is having problems and we should - # give it a chance to recover before trying it again - - if retry_interval: - retry_interval *= 2 - # plateau at hourly retries for now - if retry_interval >= 60 * 60 * 1000: - retry_interval = 60 * 60 * 1000 - else: - retry_interval = 2000 # try again at first after 2 seconds - - yield self.store.set_destination_retry_timings( - destination, - int(self._clock.time_msec()), - retry_interval - ) diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py new file mode 100644 index 0000000000..c2cb4a1c49 --- /dev/null +++ b/synapse/federation/transaction_queue.py @@ -0,0 +1,314 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from twisted.internet import defer + +from .units import Transaction + +from synapse.util.logutils import log_function +from synapse.util.logcontext import PreserveLoggingContext + +import logging + + +logger = logging.getLogger(__name__) + + +class TransactionQueue(object): + """This class makes sure we only have one transaction in flight at + a time for a given destination. + + It batches pending PDUs into single transactions. + """ + + def __init__(self, hs, transaction_actions, transport_layer): + self.server_name = hs.hostname + self.transaction_actions = transaction_actions + self.transport_layer = transport_layer + + self._clock = hs.get_clock() + self.store = hs.get_datastore() + + # Is a mapping from destinations -> deferreds. Used to keep track + # of which destinations have transactions in flight and when they are + # done + self.pending_transactions = {} + + # Is a mapping from destination -> list of + # tuple(pending pdus, deferred, order) + self.pending_pdus_by_dest = {} + # destination -> list of tuple(edu, deferred) + self.pending_edus_by_dest = {} + + # destination -> list of tuple(failure, deferred) + self.pending_failures_by_dest = {} + + # HACK to get unique tx id + self._next_txn_id = int(self._clock.time_msec()) + + @defer.inlineCallbacks + @log_function + def enqueue_pdu(self, pdu, destinations, order): + # We loop through all destinations to see whether we already have + # a transaction in progress. If we do, stick it in the pending_pdus + # table and we'll get back to it later. + + destinations = set(destinations) + destinations.discard(self.server_name) + destinations.discard("localhost") + + logger.debug("Sending to: %s", str(destinations)) + + if not destinations: + return + + deferreds = [] + + for destination in destinations: + deferred = defer.Deferred() + self.pending_pdus_by_dest.setdefault(destination, []).append( + (pdu, deferred, order) + ) + + def eb(failure): + if not deferred.called: + deferred.errback(failure) + else: + logger.warn("Failed to send pdu", failure) + + with PreserveLoggingContext(): + self._attempt_new_transaction(destination).addErrback(eb) + + deferreds.append(deferred) + + yield defer.DeferredList(deferreds) + + # NO inlineCallbacks + def enqueue_edu(self, edu): + destination = edu.destination + + if destination == self.server_name: + return + + deferred = defer.Deferred() + self.pending_edus_by_dest.setdefault(destination, []).append( + (edu, deferred) + ) + + def eb(failure): + if not deferred.called: + deferred.errback(failure) + else: + logger.warn("Failed to send edu", failure) + + with PreserveLoggingContext(): + self._attempt_new_transaction(destination).addErrback(eb) + + return deferred + + @defer.inlineCallbacks + def enqueue_failure(self, failure, destination): + deferred = defer.Deferred() + + self.pending_failures_by_dest.setdefault( + destination, [] + ).append( + (failure, deferred) + ) + + yield deferred + + @defer.inlineCallbacks + @log_function + def _attempt_new_transaction(self, destination): + + (retry_last_ts, retry_interval) = (0, 0) + retry_timings = yield self.store.get_destination_retry_timings( + destination + ) + if retry_timings: + (retry_last_ts, retry_interval) = ( + retry_timings.retry_last_ts, retry_timings.retry_interval + ) + if retry_last_ts + retry_interval > int(self._clock.time_msec()): + logger.info( + "TX [%s] not ready for retry yet - " + "dropping transaction for now", + destination, + ) + return + else: + logger.info("TX [%s] is ready for retry", destination) + + logger.info("TX [%s] _attempt_new_transaction", destination) + + if destination in self.pending_transactions: + # XXX: pending_transactions can get stuck on by a never-ending + # request at which point pending_pdus_by_dest just keeps growing. + # we need application-layer timeouts of some flavour of these + # requests + return + + # list of (pending_pdu, deferred, order) + pending_pdus = self.pending_pdus_by_dest.pop(destination, []) + pending_edus = self.pending_edus_by_dest.pop(destination, []) + pending_failures = self.pending_failures_by_dest.pop(destination, []) + + if pending_pdus: + logger.info("TX [%s] len(pending_pdus_by_dest[dest]) = %d", + destination, len(pending_pdus)) + + if not pending_pdus and not pending_edus and not pending_failures: + return + + logger.debug( + "TX [%s] Attempting new transaction" + " (pdus: %d, edus: %d, failures: %d)", + destination, + len(pending_pdus), + len(pending_edus), + len(pending_failures) + ) + + # Sort based on the order field + pending_pdus.sort(key=lambda t: t[2]) + + pdus = [x[0] for x in pending_pdus] + edus = [x[0] for x in pending_edus] + failures = [x[0].get_dict() for x in pending_failures] + deferreds = [ + x[1] + for x in pending_pdus + pending_edus + pending_failures + ] + + try: + self.pending_transactions[destination] = 1 + + logger.debug("TX [%s] Persisting transaction...", destination) + + transaction = Transaction.create_new( + origin_server_ts=int(self._clock.time_msec()), + transaction_id=str(self._next_txn_id), + origin=self.server_name, + destination=destination, + pdus=pdus, + edus=edus, + pdu_failures=failures, + ) + + self._next_txn_id += 1 + + yield self.transaction_actions.prepare_to_send(transaction) + + logger.debug("TX [%s] Persisted transaction", destination) + logger.info( + "TX [%s] Sending transaction [%s]", + destination, + transaction.transaction_id, + ) + + # Actually send the transaction + + # FIXME (erikj): This is a bit of a hack to make the Pdu age + # keys work + def json_data_cb(): + data = transaction.get_dict() + now = int(self._clock.time_msec()) + if "pdus" in data: + for p in data["pdus"]: + if "age_ts" in p: + unsigned = p.setdefault("unsigned", {}) + unsigned["age"] = now - int(p["age_ts"]) + del p["age_ts"] + return data + + code, response = yield self.transport_layer.send_transaction( + transaction, json_data_cb + ) + + logger.info("TX [%s] got %d response", destination, code) + + logger.debug("TX [%s] Sent transaction", destination) + logger.debug("TX [%s] Marking as delivered...", destination) + + yield self.transaction_actions.delivered( + transaction, code, response + ) + + logger.debug("TX [%s] Marked as delivered", destination) + logger.debug("TX [%s] Yielding to callbacks...", destination) + + for deferred in deferreds: + if code == 200: + if retry_last_ts: + # this host is alive! reset retry schedule + yield self.store.set_destination_retry_timings( + destination, 0, 0 + ) + deferred.callback(None) + else: + self.set_retrying(destination, retry_interval) + deferred.errback(RuntimeError("Got status %d" % code)) + + # Ensures we don't continue until all callbacks on that + # deferred have fired + try: + yield deferred + except: + pass + + logger.debug("TX [%s] Yielded to callbacks", destination) + + except Exception as e: + # We capture this here as there as nothing actually listens + # for this finishing functions deferred. + logger.warn( + "TX [%s] Problem in _attempt_transaction: %s", + destination, + e, + ) + + self.set_retrying(destination, retry_interval) + + for deferred in deferreds: + if not deferred.called: + deferred.errback(e) + + finally: + # We want to be *very* sure we delete this after we stop processing + self.pending_transactions.pop(destination, None) + + # Check to see if there is anything else to send. + self._attempt_new_transaction(destination) + + @defer.inlineCallbacks + def set_retrying(self, destination, retry_interval): + # track that this destination is having problems and we should + # give it a chance to recover before trying it again + + if retry_interval: + retry_interval *= 2 + # plateau at hourly retries for now + if retry_interval >= 60 * 60 * 1000: + retry_interval = 60 * 60 * 1000 + else: + retry_interval = 2000 # try again at first after 2 seconds + + yield self.store.set_destination_retry_timings( + destination, + int(self._clock.time_msec()), + retry_interval + ) From dc93860619d56e88844e91f38f66341a32e4c704 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 22 Jan 2015 17:37:12 +0000 Subject: [PATCH 024/180] Add rest API & store for creating push rules Also make unrecognised request error look more like synapse errors because it makes it easier to throw them from within rest classes. --- synapse/rest/push_rule.py | 195 ++++++++++++++++++++++++++++++++++ synapse/storage/push_rule.py | 196 +++++++++++++++++++++++++++++++++++ 2 files changed, 391 insertions(+) create mode 100644 synapse/rest/push_rule.py create mode 100644 synapse/storage/push_rule.py diff --git a/synapse/rest/push_rule.py b/synapse/rest/push_rule.py new file mode 100644 index 0000000000..b5e74479cf --- /dev/null +++ b/synapse/rest/push_rule.py @@ -0,0 +1,195 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from synapse.api.errors import SynapseError, Codes, UnrecognizedRequestError +from base import RestServlet, client_path_pattern +from synapse.storage.push_rule import InconsistentRuleException, RuleNotFoundException + +import json + + +class PushRuleRestServlet(RestServlet): + PATTERN = client_path_pattern("/pushrules/.*$") + + def rule_spec_from_path(self, path): + if len(path) < 2: + raise UnrecognizedRequestError() + if path[0] != 'pushrules': + raise UnrecognizedRequestError() + + scope = path[1] + path = path[2:] + if scope not in ['global', 'device']: + raise UnrecognizedRequestError() + + device = None + if scope == 'device': + if len(path) == 0: + raise UnrecognizedRequestError() + device = path[0] + path = path[1:] + + if len(path) == 0: + raise UnrecognizedRequestError() + + template = path[0] + path = path[1:] + + if len(path) == 0: + raise UnrecognizedRequestError() + + rule_id = path[0] + + spec = { + 'scope' : scope, + 'template': template, + 'rule_id': rule_id + } + if device: + spec['device'] = device + return spec + + def rule_tuple_from_request_object(self, rule_template, rule_id, req_obj): + if rule_template in ['override', 'underride']: + if 'conditions' not in req_obj: + raise InvalidRuleException("Missing 'conditions'") + conditions = req_obj['conditions'] + for c in conditions: + if 'kind' not in c: + raise InvalidRuleException("Condition without 'kind'") + elif rule_template == 'room': + conditions = [{ + 'kind': 'event_match', + 'key': 'room_id', + 'pattern': rule_id + }] + elif rule_template == 'sender': + conditions = [{ + 'kind': 'event_match', + 'key': 'user_id', + 'pattern': rule_id + }] + elif rule_template == 'content': + if 'pattern' not in req_obj: + raise InvalidRuleException("Content rule missing 'pattern'") + conditions = [{ + 'kind': 'event_match', + 'key': 'content.body', + 'pattern': req_obj['pattern'] + }] + else: + raise InvalidRuleException("Unknown rule template: %s" % (rule_template)) + + if 'actions' not in req_obj: + raise InvalidRuleException("No actions found") + actions = req_obj['actions'] + + for a in actions: + if a in ['notify', 'dont-notify', 'coalesce']: + pass + elif isinstance(a, dict) and 'set_sound' in a: + pass + else: + raise InvalidRuleException("Unrecognised action") + + return (conditions, actions) + + def priority_class_from_spec(self, spec): + map = { + 'underride': 0, + 'sender': 1, + 'room': 2, + 'content': 3, + 'override': 4 + } + + if spec['template'] not in map.keys(): + raise InvalidRuleException("Unknown template: %s" % (spec['kind'])) + pc = map[spec['template']] + + if spec['scope'] == 'device': + pc += 5 + + return pc + + @defer.inlineCallbacks + def on_PUT(self, request): + spec = self.rule_spec_from_path(request.postpath) + try: + priority_class = self.priority_class_from_spec(spec) + except InvalidRuleException as e: + raise SynapseError(400, e.message) + + user = yield self.auth.get_user_by_req(request) + + content = _parse_json(request) + + try: + (conditions, actions) = self.rule_tuple_from_request_object( + spec['template'], + spec['rule_id'], + content + ) + except InvalidRuleException as e: + raise SynapseError(400, e.message) + + before = request.args.get("before", None) + if before and len(before): + before = before[0] + after = request.args.get("after", None) + if after and len(after): + after = after[0] + + try: + yield self.hs.get_datastore().add_push_rule( + user_name=user.to_string(), + rule_id=spec['rule_id'], + priority_class=priority_class, + conditions=conditions, + actions=actions, + before=before, + after=after + ) + except InconsistentRuleException as e: + raise SynapseError(400, e.message) + except RuleNotFoundException: + raise SynapseError(400, "before/after rule not found") + + defer.returnValue((200, {})) + + def on_OPTIONS(self, _): + return 200, {} + + +class InvalidRuleException(Exception): + pass + + +# XXX: C+ped from rest/room.py - surely this should be common? +def _parse_json(request): + try: + content = json.loads(request.content.read()) + if type(content) != dict: + raise SynapseError(400, "Content must be a JSON object.", + errcode=Codes.NOT_JSON) + return content + except ValueError: + raise SynapseError(400, "Content not JSON.", errcode=Codes.NOT_JSON) + + +def register_servlets(hs, http_server): + PushRuleRestServlet(hs).register(http_server) diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py new file mode 100644 index 0000000000..76c4557600 --- /dev/null +++ b/synapse/storage/push_rule.py @@ -0,0 +1,196 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections + +from ._base import SQLBaseStore, Table +from twisted.internet import defer + +import logging +import copy +import json + +logger = logging.getLogger(__name__) + + +class PushRuleStore(SQLBaseStore): + @defer.inlineCallbacks + def get_push_rules_for_user_name(self, user_name): + sql = ( + "SELECT "+",".join(PushRuleTable.fields)+ + "FROM pushers " + "WHERE user_name = ?" + ) + + rows = yield self._execute(None, sql, user_name) + + dicts = [] + for r in rows: + d = {} + for i, f in enumerate(PushRuleTable.fields): + d[f] = r[i] + dicts.append(d) + + defer.returnValue(dicts) + + @defer.inlineCallbacks + def add_push_rule(self, **kwargs): + vals = copy.copy(kwargs) + if 'conditions' in vals: + vals['conditions'] = json.dumps(vals['conditions']) + if 'actions' in vals: + vals['actions'] = json.dumps(vals['actions']) + # we could check the rest of the keys are valid column names + # but sqlite will do that anyway so I think it's just pointless. + if 'id' in vals: + del vals['id'] + + if 'after' in kwargs or 'before' in kwargs: + ret = yield self.runInteraction( + "_add_push_rule_relative_txn", + self._add_push_rule_relative_txn, + **vals + ) + defer.returnValue(ret) + else: + ret = yield self.runInteraction( + "_add_push_rule_highest_priority_txn", + self._add_push_rule_highest_priority_txn, + **vals + ) + defer.returnValue(ret) + + def _add_push_rule_relative_txn(self, txn, user_name, **kwargs): + after = None + relative_to_rule = None + if 'after' in kwargs and kwargs['after']: + after = kwargs['after'] + relative_to_rule = after + if 'before' in kwargs and kwargs['before']: + relative_to_rule = kwargs['before'] + + # get the priority of the rule we're inserting after/before + sql = ( + "SELECT priority_class, priority FROM "+PushRuleTable.table_name+ + " WHERE user_name = ? and rule_id = ?" + ) + txn.execute(sql, (user_name, relative_to_rule)) + res = txn.fetchall() + if not res: + raise RuleNotFoundException() + (priority_class, base_rule_priority) = res[0] + + if 'priority_class' in kwargs and kwargs['priority_class'] != priority_class: + raise InconsistentRuleException( + "Given priority class does not match class of relative rule" + ) + + new_rule = copy.copy(kwargs) + if 'before' in new_rule: + del new_rule['before'] + if 'after' in new_rule: + del new_rule['after'] + new_rule['priority_class'] = priority_class + new_rule['user_name'] = user_name + + # check if the priority before/after is free + new_rule_priority = base_rule_priority + if after: + new_rule_priority -= 1 + else: + new_rule_priority += 1 + + new_rule['priority'] = new_rule_priority + + sql = ( + "SELECT COUNT(*) FROM "+PushRuleTable.table_name+ + " WHERE user_name = ? AND priority_class = ? AND priority = ?" + ) + txn.execute(sql, (user_name, priority_class, new_rule_priority)) + res = txn.fetchall() + num_conflicting = res[0][0] + + # if there are conflicting rules, bump everything + if num_conflicting: + sql = "UPDATE "+PushRuleTable.table_name+" SET priority = priority " + if after: + sql += "-1" + else: + sql += "+1" + sql += " WHERE user_name = ? AND priority_class = ? AND priority " + if after: + sql += "<= ?" + else: + sql += ">= ?" + + txn.execute(sql, (user_name, priority_class, new_rule_priority)) + + # now insert the new rule + sql = "INSERT OR REPLACE INTO "+PushRuleTable.table_name+" (" + sql += ",".join(new_rule.keys())+") VALUES (" + sql += ", ".join(["?" for _ in new_rule.keys()])+")" + + txn.execute(sql, new_rule.values()) + + def _add_push_rule_highest_priority_txn(self, txn, user_name, priority_class, **kwargs): + # find the highest priority rule in that class + sql = ( + "SELECT COUNT(*), MAX(priority) FROM "+PushRuleTable.table_name+ + " WHERE user_name = ? and priority_class = ?" + ) + txn.execute(sql, (user_name, priority_class)) + res = txn.fetchall() + (how_many, highest_prio) = res[0] + + new_prio = 0 + if how_many > 0: + new_prio = highest_prio + 1 + + # and insert the new rule + new_rule = copy.copy(kwargs) + if 'id' in new_rule: + del new_rule['id'] + new_rule['user_name'] = user_name + new_rule['priority_class'] = priority_class + new_rule['priority'] = new_prio + + sql = "INSERT OR REPLACE INTO "+PushRuleTable.table_name+" (" + sql += ",".join(new_rule.keys())+") VALUES (" + sql += ", ".join(["?" for _ in new_rule.keys()])+")" + + txn.execute(sql, new_rule.values()) + +class RuleNotFoundException(Exception): + pass + + +class InconsistentRuleException(Exception): + pass + + +class PushRuleTable(Table): + table_name = "push_rules" + + fields = [ + "id", + "user_name", + "rule_id", + "priority_class", + "priority", + "conditions", + "actions", + ] + + EntryType = collections.namedtuple("PushRuleEntry", fields) \ No newline at end of file From ede491b4e0c14d44ce43dd5b152abf148b54b9ed Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 22 Jan 2015 17:38:53 +0000 Subject: [PATCH 025/180] Oops: second part of commit dc938606 --- synapse/api/errors.py | 12 ++++++++++++ synapse/http/server.py | 8 ++------ synapse/rest/__init__.py | 3 ++- synapse/storage/__init__.py | 3 +++ synapse/storage/schema/delta/v10.sql | 13 +++++++++++++ synapse/storage/schema/pusher.sql | 13 +++++++++++++ 6 files changed, 45 insertions(+), 7 deletions(-) diff --git a/synapse/api/errors.py b/synapse/api/errors.py index a4155aebae..55181fe77e 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -21,6 +21,7 @@ logger = logging.getLogger(__name__) class Codes(object): + UNRECOGNIZED = "M_UNRECOGNIZED" UNAUTHORIZED = "M_UNAUTHORIZED" FORBIDDEN = "M_FORBIDDEN" BAD_JSON = "M_BAD_JSON" @@ -82,6 +83,17 @@ class RegistrationError(SynapseError): pass +class UnrecognizedRequestError(SynapseError): + """An error indicating we don't understand the request you're trying to make""" + def __init__(self, *args, **kwargs): + if "errcode" not in kwargs: + kwargs["errcode"] = Codes.NOT_FOUND + super(UnrecognizedRequestError, self).__init__( + 400, + "Unrecognized request", + **kwargs + ) + class AuthError(SynapseError): """An error raised when there was a problem authorising an event.""" diff --git a/synapse/http/server.py b/synapse/http/server.py index 8015a22edf..0f6539e1be 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -16,7 +16,7 @@ from synapse.http.agent_name import AGENT_NAME from synapse.api.errors import ( - cs_exception, SynapseError, CodeMessageException + cs_exception, SynapseError, CodeMessageException, UnrecognizedRequestError ) from synapse.util.logcontext import LoggingContext @@ -139,11 +139,7 @@ class JsonResource(HttpServer, resource.Resource): return # Huh. No one wanted to handle that? Fiiiiiine. Send 400. - self._send_response( - request, - 400, - {"error": "Unrecognized request"} - ) + raise UnrecognizedRequestError() except CodeMessageException as e: if isinstance(e, SynapseError): logger.info("%s SynapseError: %s - %s", request, e.code, e.msg) diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index 59521d0c77..8e5877cf3f 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -16,7 +16,7 @@ from . import ( room, events, register, login, profile, presence, initial_sync, directory, - voip, admin, pusher, + voip, admin, pusher, push_rule ) @@ -46,3 +46,4 @@ class RestServletFactory(object): voip.register_servlets(hs, client_resource) admin.register_servlets(hs, client_resource) pusher.register_servlets(hs, client_resource) + push_rule.register_servlets(hs, client_resource) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 191fe462a5..11706676d0 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -30,6 +30,7 @@ from .transactions import TransactionStore from .keys import KeyStore from .event_federation import EventFederationStore from .pusher import PusherStore +from .push_rule import PushRuleStore from .media_repository import MediaRepositoryStore from .state import StateStore @@ -62,6 +63,7 @@ SCHEMAS = [ "event_edges", "event_signatures", "pusher", + "push_rules", "media_repository", ] @@ -85,6 +87,7 @@ class DataStore(RoomMemberStore, RoomStore, EventFederationStore, MediaRepositoryStore, PusherStore, + PushRuleStore ): def __init__(self, hs): diff --git a/synapse/storage/schema/delta/v10.sql b/synapse/storage/schema/delta/v10.sql index b84ce20ef3..8c4dfd5c1b 100644 --- a/synapse/storage/schema/delta/v10.sql +++ b/synapse/storage/schema/delta/v10.sql @@ -31,3 +31,16 @@ CREATE TABLE IF NOT EXISTS pushers ( FOREIGN KEY(user_name) REFERENCES users(name), UNIQUE (app_id, pushkey) ); + +CREATE TABLE IF NOT EXISTS push_rules ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_name TEXT NOT NULL, + rule_id TEXT NOT NULL, + priority_class TINYINT NOT NULL, + priority INTEGER NOT NULL DEFAULT 0, + conditions TEXT NOT NULL, + actions TEXT NOT NULL, + UNIQUE(user_name, rule_id) +); + +CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); diff --git a/synapse/storage/schema/pusher.sql b/synapse/storage/schema/pusher.sql index b84ce20ef3..8c4dfd5c1b 100644 --- a/synapse/storage/schema/pusher.sql +++ b/synapse/storage/schema/pusher.sql @@ -31,3 +31,16 @@ CREATE TABLE IF NOT EXISTS pushers ( FOREIGN KEY(user_name) REFERENCES users(name), UNIQUE (app_id, pushkey) ); + +CREATE TABLE IF NOT EXISTS push_rules ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_name TEXT NOT NULL, + rule_id TEXT NOT NULL, + priority_class TINYINT NOT NULL, + priority INTEGER NOT NULL DEFAULT 0, + conditions TEXT NOT NULL, + actions TEXT NOT NULL, + UNIQUE(user_name, rule_id) +); + +CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); From 7ecb49ef25937558f1a19a8fe47879d4b9116316 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 22 Jan 2015 17:53:30 +0000 Subject: [PATCH 026/180] Insufficient newlines --- synapse/storage/push_rule.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index 76c4557600..dbbb35b2ab 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -172,6 +172,7 @@ class PushRuleStore(SQLBaseStore): txn.execute(sql, new_rule.values()) + class RuleNotFoundException(Exception): pass From 673773b21701c91997512d568bcc8d49a5586b3a Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 22 Jan 2015 18:27:07 +0000 Subject: [PATCH 027/180] oops, this is not its own schema file --- synapse/storage/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 11706676d0..8f56d90d95 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -63,7 +63,6 @@ SCHEMAS = [ "event_edges", "event_signatures", "pusher", - "push_rules", "media_repository", ] From 8a850573c9cf50dd83ba47c033b28fe2bbbaf9d4 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 22 Jan 2015 19:32:17 +0000 Subject: [PATCH 028/180] As yet fairly untested GET API for push rules --- synapse/api/errors.py | 14 ++- synapse/rest/client/v1/push_rule.py | 138 ++++++++++++++++++++++++++-- synapse/storage/push_rule.py | 8 +- 3 files changed, 145 insertions(+), 15 deletions(-) diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 55181fe77e..01207282d6 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -87,13 +87,25 @@ class UnrecognizedRequestError(SynapseError): """An error indicating we don't understand the request you're trying to make""" def __init__(self, *args, **kwargs): if "errcode" not in kwargs: - kwargs["errcode"] = Codes.NOT_FOUND + kwargs["errcode"] = Codes.UNRECOGNIZED super(UnrecognizedRequestError, self).__init__( 400, "Unrecognized request", **kwargs ) + +class NotFoundError(SynapseError): + """An error indicating we can't find the thing you asked for""" + def __init__(self, *args, **kwargs): + if "errcode" not in kwargs: + kwargs["errcode"] = Codes.NOT_FOUND + super(UnrecognizedRequestError, self).__init__( + 404, + "Not found", + **kwargs + ) + class AuthError(SynapseError): """An error raised when there was a problem authorising an event.""" diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index b5e74479cf..2803c1f071 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -15,7 +15,7 @@ from twisted.internet import defer -from synapse.api.errors import SynapseError, Codes, UnrecognizedRequestError +from synapse.api.errors import SynapseError, Codes, UnrecognizedRequestError, NotFoundError from base import RestServlet, client_path_pattern from synapse.storage.push_rule import InconsistentRuleException, RuleNotFoundException @@ -24,6 +24,14 @@ import json class PushRuleRestServlet(RestServlet): PATTERN = client_path_pattern("/pushrules/.*$") + PRIORITY_CLASS_MAP = { + 'underride': 0, + 'sender': 1, + 'room': 2, + 'content': 3, + 'override': 4 + } + PRIORITY_CLASS_INVERSE_MAP = {v: k for k,v in PRIORITY_CLASS_MAP.items()} def rule_spec_from_path(self, path): if len(path) < 2: @@ -109,15 +117,7 @@ class PushRuleRestServlet(RestServlet): return (conditions, actions) def priority_class_from_spec(self, spec): - map = { - 'underride': 0, - 'sender': 1, - 'room': 2, - 'content': 3, - 'override': 4 - } - - if spec['template'] not in map.keys(): + if spec['template'] not in PushRuleRestServlet.PRIORITY_CLASS_MAP.keys(): raise InvalidRuleException("Unknown template: %s" % (spec['kind'])) pc = map[spec['template']] @@ -171,10 +171,128 @@ class PushRuleRestServlet(RestServlet): defer.returnValue((200, {})) + @defer.inlineCallbacks + def on_GET(self, request): + user = yield self.auth.get_user_by_req(request) + + # we build up the full structure and then decide which bits of it + # to send which means doing unnecessary work sometimes but is + # is probably not going to make a whole lot of difference + rawrules = yield self.hs.get_datastore().get_push_rules_for_user_name(user.to_string()) + + rules = {'global': {}, 'device': {}} + + rules['global'] = _add_empty_priority_class_arrays(rules['global']) + + for r in rawrules: + rulearray = None + + r["conditions"] = json.loads(r["conditions"]) + r["actions"] = json.loads(r["actions"]) + + template_name = _priority_class_to_template_name(r['priority_class']) + + if r['priority_class'] > PushRuleRestServlet.PRIORITY_CLASS_MAP['override']: + # per-device rule + instance_handle = _instance_handle_from_conditions(r["conditions"]) + if not instance_handle: + continue + if instance_handle not in rules['device']: + rules['device'][instance_handle] = [] + rules['device'][instance_handle] = \ + _add_empty_priority_class_arrays(rules['device'][instance_handle]) + + rulearray = rules['device'][instance_handle] + else: + rulearray = rules['global'][template_name] + + template_rule = _rule_to_template(r) + if template_rule: + rulearray.append(template_rule) + + path = request.postpath[1:] + if path == []: + defer.returnValue((200, rules)) + + if path[0] == 'global': + path = path[1:] + result = _filter_ruleset_with_path(rules['global'], path) + defer.returnValue((200, result)) + elif path[0] == 'device': + path = path[1:] + if path == []: + raise UnrecognizedRequestError + instance_handle = path[0] + if instance_handle not in rules['device']: + ret = {} + ret = _add_empty_priority_class_arrays(ret) + defer.returnValue((200, ret)) + ruleset = rules['device'][instance_handle] + result = _filter_ruleset_with_path(ruleset, path) + defer.returnValue((200, result)) + else: + raise UnrecognizedRequestError() + + def on_OPTIONS(self, _): return 200, {} +def _add_empty_priority_class_arrays(d): + for pc in PushRuleRestServlet.PRIORITY_CLASS_MAP.keys(): + d[pc] = [] + return d + +def _instance_handle_from_conditions(conditions): + """ + Given a list of conditions, return the instance handle of the + device rule if there is one + """ + for c in conditions: + if c['kind'] == 'device': + return c['instance_handle'] + return None + +def _filter_ruleset_with_path(ruleset, path): + if path == []: + return ruleset + template_kind = path[0] + if template_kind not in ruleset: + raise UnrecognizedRequestError() + path = path[1:] + if path == []: + return ruleset[template_kind] + rule_id = path[0] + for r in ruleset[template_kind]: + if r['rule_id'] == rule_id: + return r + raise NotFoundError + +def _priority_class_to_template_name(pc): + if pc > PushRuleRestServlet.PRIORITY_CLASS_MAP['override']: + # per-device + prio_class_index = pc - PushRuleRestServlet.PRIORITY_CLASS_MAP['override'] + return PushRuleRestServlet.PRIORITY_CLASS_INVERSE_MAP[prio_class_index] + else: + return PushRuleRestServlet.PRIORITY_CLASS_INVERSE_MAP[pc] + +def _rule_to_template(rule): + template_name = _priority_class_to_template_name(rule['priority_class']) + if template_name in ['override', 'underride']: + return {k:rule[k] for k in ["rule_id", "conditions", "actions"]} + elif template_name in ["sender", "room"]: + return {k:rule[k] for k in ["rule_id", "actions"]} + elif template_name == 'content': + if len(rule["conditions"]) != 1: + return None + thecond = rule["conditions"][0] + if "pattern" not in thecond: + return None + ret = {k:rule[k] for k in ["rule_id", "actions"]} + ret["pattern"] = thecond["pattern"] + return ret + + class InvalidRuleException(Exception): pass diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index dbbb35b2ab..d087257ffc 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -29,11 +29,11 @@ class PushRuleStore(SQLBaseStore): @defer.inlineCallbacks def get_push_rules_for_user_name(self, user_name): sql = ( - "SELECT "+",".join(PushRuleTable.fields)+ - "FROM pushers " - "WHERE user_name = ?" + "SELECT "+",".join(PushRuleTable.fields)+" " + "FROM "+PushRuleTable.table_name+" " + "WHERE user_name = ? " + "ORDER BY priority_class DESC, priority DESC" ) - rows = yield self._execute(None, sql, user_name) dicts = [] From 6927b6b19783d7134eba3461c4fafe4efdec40f1 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 23 Jan 2015 10:21:47 +0000 Subject: [PATCH 029/180] This really serves me right for ever making a map called 'map'. --- synapse/rest/client/v1/push_rule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 2803c1f071..7df3fc7f09 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -119,7 +119,7 @@ class PushRuleRestServlet(RestServlet): def priority_class_from_spec(self, spec): if spec['template'] not in PushRuleRestServlet.PRIORITY_CLASS_MAP.keys(): raise InvalidRuleException("Unknown template: %s" % (spec['kind'])) - pc = map[spec['template']] + pc = PushRuleRestServlet.PRIORITY_CLASS_MAP[spec['template']] if spec['scope'] == 'device': pc += 5 From bcd48b9636071543fa64e7fb066275d1c9c1e363 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 23 Jan 2015 10:28:25 +0000 Subject: [PATCH 030/180] Fix adding rules without before/after & add the rule that we couldn't find to the error --- synapse/rest/client/v1/push_rule.py | 4 ++-- synapse/storage/push_rule.py | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 7df3fc7f09..77a0772479 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -166,8 +166,8 @@ class PushRuleRestServlet(RestServlet): ) except InconsistentRuleException as e: raise SynapseError(400, e.message) - except RuleNotFoundException: - raise SynapseError(400, "before/after rule not found") + except RuleNotFoundException as e: + raise SynapseError(400, e.message) defer.returnValue((200, {})) diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index d087257ffc..2366090e09 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -46,7 +46,7 @@ class PushRuleStore(SQLBaseStore): defer.returnValue(dicts) @defer.inlineCallbacks - def add_push_rule(self, **kwargs): + def add_push_rule(self, before, after, **kwargs): vals = copy.copy(kwargs) if 'conditions' in vals: vals['conditions'] = json.dumps(vals['conditions']) @@ -57,10 +57,12 @@ class PushRuleStore(SQLBaseStore): if 'id' in vals: del vals['id'] - if 'after' in kwargs or 'before' in kwargs: + if before or after: ret = yield self.runInteraction( "_add_push_rule_relative_txn", self._add_push_rule_relative_txn, + before=before, + after=after, **vals ) defer.returnValue(ret) @@ -89,7 +91,7 @@ class PushRuleStore(SQLBaseStore): txn.execute(sql, (user_name, relative_to_rule)) res = txn.fetchall() if not res: - raise RuleNotFoundException() + raise RuleNotFoundException("before/after rule not found: %s" % (relative_to_rule)) (priority_class, base_rule_priority) = res[0] if 'priority_class' in kwargs and kwargs['priority_class'] != priority_class: From f87586e661101849a90f9d106b207a529e4cf689 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 23 Jan 2015 10:32:40 +0000 Subject: [PATCH 031/180] right super() param --- synapse/api/errors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 01207282d6..4f59e1742c 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -100,7 +100,7 @@ class NotFoundError(SynapseError): def __init__(self, *args, **kwargs): if "errcode" not in kwargs: kwargs["errcode"] = Codes.NOT_FOUND - super(UnrecognizedRequestError, self).__init__( + super(NotFoundError, self).__init__( 404, "Not found", **kwargs From 49fe31792bc0cf709248e592baefb8f34606236a Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 23 Jan 2015 11:19:02 +0000 Subject: [PATCH 032/180] Add slightly pedantic trailing slash error. --- synapse/api/errors.py | 7 ++++++- synapse/rest/client/v1/push_rule.py | 17 ++++++++++++++--- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 4f59e1742c..5872e82d0f 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -88,9 +88,14 @@ class UnrecognizedRequestError(SynapseError): def __init__(self, *args, **kwargs): if "errcode" not in kwargs: kwargs["errcode"] = Codes.UNRECOGNIZED + message = None + if len(args) == 0: + message = "Unrecognized request" + else: + message = args[0] super(UnrecognizedRequestError, self).__init__( 400, - "Unrecognized request", + message, **kwargs ) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 77a0772479..6f108431b2 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -32,6 +32,8 @@ class PushRuleRestServlet(RestServlet): 'override': 4 } PRIORITY_CLASS_INVERSE_MAP = {v: k for k,v in PRIORITY_CLASS_MAP.items()} + SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR =\ + "Unrecognised request: You probably wanted a trailing slash" def rule_spec_from_path(self, path): if len(path) < 2: @@ -211,10 +213,14 @@ class PushRuleRestServlet(RestServlet): rulearray.append(template_rule) path = request.postpath[1:] - if path == []: - defer.returnValue((200, rules)) - if path[0] == 'global': + if path == []: + # we're a reference impl: pedantry is our job. + raise UnrecognizedRequestError(PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR) + + if path[0] == '': + defer.returnValue((200, rules)) + elif path[0] == 'global': path = path[1:] result = _filter_ruleset_with_path(rules['global'], path) defer.returnValue((200, result)) @@ -255,12 +261,17 @@ def _instance_handle_from_conditions(conditions): def _filter_ruleset_with_path(ruleset, path): if path == []: + raise UnrecognizedRequestError(PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR) + + if path[0] == '': return ruleset template_kind = path[0] if template_kind not in ruleset: raise UnrecognizedRequestError() path = path[1:] if path == []: + raise UnrecognizedRequestError(PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR) + if path[0] == '': return ruleset[template_kind] rule_id = path[0] for r in ruleset[template_kind]: From 6188c4f69c2f902410b43bc50c0ae8d488b4d93c Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 23 Jan 2015 13:23:10 +0000 Subject: [PATCH 033/180] make per-device rules work --- synapse/rest/client/v1/push_rule.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 6f108431b2..417fd368d7 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -73,7 +73,7 @@ class PushRuleRestServlet(RestServlet): spec['device'] = device return spec - def rule_tuple_from_request_object(self, rule_template, rule_id, req_obj): + def rule_tuple_from_request_object(self, rule_template, rule_id, req_obj, device=None): if rule_template in ['override', 'underride']: if 'conditions' not in req_obj: raise InvalidRuleException("Missing 'conditions'") @@ -104,6 +104,12 @@ class PushRuleRestServlet(RestServlet): else: raise InvalidRuleException("Unknown rule template: %s" % (rule_template)) + if device: + conditions.append({ + 'kind': 'device', + 'instance_handle': device + }) + if 'actions' not in req_obj: raise InvalidRuleException("No actions found") actions = req_obj['actions'] @@ -144,7 +150,8 @@ class PushRuleRestServlet(RestServlet): (conditions, actions) = self.rule_tuple_from_request_object( spec['template'], spec['rule_id'], - content + content, + device=spec['device'] if 'device' in spec else None ) except InvalidRuleException as e: raise SynapseError(400, e.message) @@ -200,11 +207,11 @@ class PushRuleRestServlet(RestServlet): if not instance_handle: continue if instance_handle not in rules['device']: - rules['device'][instance_handle] = [] + rules['device'][instance_handle] = {} rules['device'][instance_handle] = \ _add_empty_priority_class_arrays(rules['device'][instance_handle]) - rulearray = rules['device'][instance_handle] + rulearray = rules['device'][instance_handle][template_name] else: rulearray = rules['global'][template_name] @@ -227,7 +234,10 @@ class PushRuleRestServlet(RestServlet): elif path[0] == 'device': path = path[1:] if path == []: - raise UnrecognizedRequestError + raise UnrecognizedRequestError(PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR) + if path[0] == '': + defer.returnValue((200, rules['device'])) + instance_handle = path[0] if instance_handle not in rules['device']: ret = {} From 54c689c8199336b819c632a3e996120cb13007db Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 23 Jan 2015 13:25:14 +0000 Subject: [PATCH 034/180] stray space --- synapse/rest/client/v1/push_rule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 417fd368d7..9cb2494035 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -65,7 +65,7 @@ class PushRuleRestServlet(RestServlet): rule_id = path[0] spec = { - 'scope' : scope, + 'scope': scope, 'template': template, 'rule_id': rule_id } From 98e1080555965c650e09ed09bdac3b52daeda123 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 23 Jan 2015 13:25:36 +0000 Subject: [PATCH 035/180] redundant parens --- synapse/rest/client/v1/push_rule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 9cb2494035..46b8c3f625 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -122,7 +122,7 @@ class PushRuleRestServlet(RestServlet): else: raise InvalidRuleException("Unrecognised action") - return (conditions, actions) + return conditions, actions def priority_class_from_spec(self, spec): if spec['template'] not in PushRuleRestServlet.PRIORITY_CLASS_MAP.keys(): From d3e72b4d8788e64cd4b1d9668382639476730b4d Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 23 Jan 2015 13:25:58 +0000 Subject: [PATCH 036/180] Make string format tuple an actual tuple --- synapse/rest/client/v1/push_rule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 46b8c3f625..35ffcba3df 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -102,7 +102,7 @@ class PushRuleRestServlet(RestServlet): 'pattern': req_obj['pattern'] }] else: - raise InvalidRuleException("Unknown rule template: %s" % (rule_template)) + raise InvalidRuleException("Unknown rule template: %s" % (rule_template,)) if device: conditions.append({ From b3f66ea6fb5871d7eeb71b466740adb61a89c0d2 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 23 Jan 2015 13:28:00 +0000 Subject: [PATCH 037/180] more pep8 --- synapse/rest/client/v1/push_rule.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 35ffcba3df..ce2f0febf4 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -249,7 +249,6 @@ class PushRuleRestServlet(RestServlet): else: raise UnrecognizedRequestError() - def on_OPTIONS(self, _): return 200, {} @@ -259,6 +258,7 @@ def _add_empty_priority_class_arrays(d): d[pc] = [] return d + def _instance_handle_from_conditions(conditions): """ Given a list of conditions, return the instance handle of the @@ -289,6 +289,7 @@ def _filter_ruleset_with_path(ruleset, path): return r raise NotFoundError + def _priority_class_to_template_name(pc): if pc > PushRuleRestServlet.PRIORITY_CLASS_MAP['override']: # per-device @@ -297,12 +298,13 @@ def _priority_class_to_template_name(pc): else: return PushRuleRestServlet.PRIORITY_CLASS_INVERSE_MAP[pc] + def _rule_to_template(rule): template_name = _priority_class_to_template_name(rule['priority_class']) if template_name in ['override', 'underride']: return {k:rule[k] for k in ["rule_id", "conditions", "actions"]} elif template_name in ["sender", "room"]: - return {k:rule[k] for k in ["rule_id", "actions"]} + return {k: rule[k] for k in ["rule_id", "actions"]} elif template_name == 'content': if len(rule["conditions"]) != 1: return None From fc7a05c443c169b07b917bb2692bdd7c738824fb Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 23 Jan 2015 13:36:01 +0000 Subject: [PATCH 038/180] more pep8 suggestions --- synapse/push/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 3ee652f3bc..47da31e500 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -74,7 +74,6 @@ class Pusher(object): defer.returnValue(ctx) - @defer.inlineCallbacks def start(self): if not self.last_token: @@ -116,7 +115,7 @@ class Pusher(object): processed = False if self._should_notify_for_event(single_event): rejected = yield self.dispatch_push(single_event) - if not rejected == False: + if not rejected is False: processed = True for pk in rejected: if pk != self.pushkey: From 3b9cc882a50c886afd7d2cf1eaa7e02e8b0d0d51 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 23 Jan 2015 15:42:52 +0000 Subject: [PATCH 039/180] Add storage method have_events --- synapse/storage/__init__.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 015fcc8775..4f09909607 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -422,6 +422,35 @@ class DataStore(RoomMemberStore, RoomStore, ], ) + def have_events(self, event_ids): + """Given a list of event ids, check if we have already processed them. + + Returns: + dict: Has an entry for each event id we already have seen. Maps to + the rejected reason string if we rejected the event, else maps to + None. + """ + def f(txn): + sql = ( + "SELECT e.event_id, reason FROM events as e " + "LEFT JOIN rejections as r ON e.event_id = r.event_id " + "WHERE event_id = ?" + ) + + res = {} + for event_id in event_ids: + txn.execute(sql, (event_id,)) + row = txn.fetchone() + if row: + _, rejected = row + res[event_id] = rejected + + return res + + return self.runInteraction( + "have_events", f, + ) + def schema_path(schema): """ Get a filesystem path for the named database schema From f21f9fa3c51db49212c42adfe6972025e1d27a15 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 23 Jan 2015 17:07:06 +0000 Subject: [PATCH 040/180] Use push settings! --- synapse/push/__init__.py | 91 ++++++++++++++++++++++++++--- synapse/push/httppusher.py | 9 +-- synapse/rest/client/v1/push_rule.py | 43 +++++++++----- 3 files changed, 117 insertions(+), 26 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 47da31e500..53d3319699 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -21,6 +21,8 @@ from synapse.types import StreamToken import synapse.util.async import logging +import fnmatch +import json logger = logging.getLogger(__name__) @@ -29,6 +31,7 @@ class Pusher(object): INITIAL_BACKOFF = 1000 MAX_BACKOFF = 60 * 60 * 1000 GIVE_UP_AFTER = 24 * 60 * 60 * 1000 + DEFAULT_ACTIONS = ['notify'] def __init__(self, _hs, instance_handle, user_name, app_id, app_display_name, device_display_name, pushkey, pushkey_ts, @@ -37,7 +40,7 @@ class Pusher(object): self.evStreamHandler = self.hs.get_handlers().event_stream_handler self.store = self.hs.get_datastore() self.clock = self.hs.get_clock() - self.instance_handle = instance_handle, + self.instance_handle = instance_handle self.user_name = user_name self.app_id = app_id self.app_display_name = app_display_name @@ -51,7 +54,8 @@ class Pusher(object): self.failing_since = failing_since self.alive = True - def _should_notify_for_event(self, ev): + @defer.inlineCallbacks + def _actions_for_event(self, ev): """ This should take into account notification settings that the user has configured both globally and per-room when we have the ability @@ -59,8 +63,47 @@ class Pusher(object): """ if ev['user_id'] == self.user_name: # let's assume you probably know about messages you sent yourself + defer.returnValue(['dont_notify']) + + rules = yield self.store.get_push_rules_for_user_name(self.user_name) + + for r in rules: + matches = True + + conditions = json.loads(r['conditions']) + actions = json.loads(r['actions']) + + for c in conditions: + matches &= self._event_fulfills_condition(ev, c) + # ignore rules with no actions (we have an explict 'dont_notify' + if len(actions) == 0: + logger.warn( + "Ignoring rule id %s with no actions for user %s" % + (r['rule_id'], r['user_name']) + ) + continue + if matches: + defer.returnValue(actions) + + defer.returnValue(Pusher.DEFAULT_ACTIONS) + + def _event_fulfills_condition(self, ev, condition): + if condition['kind'] == 'event_match': + if 'pattern' not in condition: + logger.warn("event_match condition with no pattern") + return False + pat = condition['pattern'] + + val = _value_for_dotted_key(condition['key'], ev) + if fnmatch.fnmatch(val, pat): + return True return False - return True + elif condition['kind'] == 'device': + if 'instance_handle' not in condition: + return True + return condition['instance_handle'] == self.instance_handle + else: + return True @defer.inlineCallbacks def get_context_for_event(self, ev): @@ -113,8 +156,23 @@ class Pusher(object): continue processed = False - if self._should_notify_for_event(single_event): - rejected = yield self.dispatch_push(single_event) + actions = yield self._actions_for_event(single_event) + tweaks = _tweaks_for_actions(actions) + + if len(actions) == 0: + logger.warn("Empty actions! Using default action.") + actions = Pusher.DEFAULT_ACTIONS + if 'notify' not in actions and 'dont_notify' not in actions: + logger.warn("Neither notify nor dont_notify in actions: adding default") + actions.extend(Pusher.DEFAULT_ACTIONS) + if 'dont_notify' in actions: + logger.debug( + "%s for %s: dont_notify", + single_event['event_id'], self.user_name + ) + processed = True + else: + rejected = yield self.dispatch_push(single_event, tweaks) if not rejected is False: processed = True for pk in rejected: @@ -133,8 +191,6 @@ class Pusher(object): yield self.hs.get_pusherpool().remove_pusher( self.app_id, pk ) - else: - processed = True if not self.alive: continue @@ -202,7 +258,7 @@ class Pusher(object): def stop(self): self.alive = False - def dispatch_push(self, p): + def dispatch_push(self, p, tweaks): """ Overridden by implementing classes to actually deliver the notification :param p: The event to notify for as a single event from the event stream @@ -214,6 +270,25 @@ class Pusher(object): pass +def _value_for_dotted_key(dotted_key, event): + parts = dotted_key.split(".") + val = event + while len(parts) > 0: + if parts[0] not in val: + return None + val = val[parts[0]] + parts = parts[1:] + return val + +def _tweaks_for_actions(actions): + tweaks = {} + for a in actions: + if not isinstance(a, dict): + continue + if 'set_sound' in a: + tweaks['sound'] = a['set_sound'] + return tweaks + class PusherConfigException(Exception): def __init__(self, msg): super(PusherConfigException, self).__init__(msg) \ No newline at end of file diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 46433ad4a9..25db1dded5 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -52,7 +52,7 @@ class HttpPusher(Pusher): del self.data_minus_url['url'] @defer.inlineCallbacks - def _build_notification_dict(self, event): + def _build_notification_dict(self, event, tweaks): # we probably do not want to push for every presence update # (we may want to be able to set up notifications when specific # people sign in, but we'd want to only deliver the pertinent ones) @@ -83,7 +83,8 @@ class HttpPusher(Pusher): 'app_id': self.app_id, 'pushkey': self.pushkey, 'pushkey_ts': long(self.pushkey_ts / 1000), - 'data': self.data_minus_url + 'data': self.data_minus_url, + 'tweaks': tweaks } ] } @@ -97,8 +98,8 @@ class HttpPusher(Pusher): defer.returnValue(d) @defer.inlineCallbacks - def dispatch_push(self, event): - notification_dict = yield self._build_notification_dict(event) + def dispatch_push(self, event, tweaks): + notification_dict = yield self._build_notification_dict(event, tweaks) if not notification_dict: defer.returnValue([]) try: diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index ce2f0febf4..9dc2c0e11e 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -96,10 +96,15 @@ class PushRuleRestServlet(RestServlet): elif rule_template == 'content': if 'pattern' not in req_obj: raise InvalidRuleException("Content rule missing 'pattern'") + pat = req_obj['pattern'] + if pat.strip("*?[]") == pat: + # no special glob characters so we assume the user means + # 'contains this string' rather than 'is this string' + pat = "*%s*" % (pat) conditions = [{ 'kind': 'event_match', 'key': 'content.body', - 'pattern': req_obj['pattern'] + 'pattern': pat }] else: raise InvalidRuleException("Unknown rule template: %s" % (rule_template,)) @@ -115,7 +120,7 @@ class PushRuleRestServlet(RestServlet): actions = req_obj['actions'] for a in actions: - if a in ['notify', 'dont-notify', 'coalesce']: + if a in ['notify', 'dont_notify', 'coalesce']: pass elif isinstance(a, dict) and 'set_sound' in a: pass @@ -124,21 +129,11 @@ class PushRuleRestServlet(RestServlet): return conditions, actions - def priority_class_from_spec(self, spec): - if spec['template'] not in PushRuleRestServlet.PRIORITY_CLASS_MAP.keys(): - raise InvalidRuleException("Unknown template: %s" % (spec['kind'])) - pc = PushRuleRestServlet.PRIORITY_CLASS_MAP[spec['template']] - - if spec['scope'] == 'device': - pc += 5 - - return pc - @defer.inlineCallbacks def on_PUT(self, request): spec = self.rule_spec_from_path(request.postpath) try: - priority_class = self.priority_class_from_spec(spec) + priority_class = _priority_class_from_spec(spec) except InvalidRuleException as e: raise SynapseError(400, e.message) @@ -204,6 +199,7 @@ class PushRuleRestServlet(RestServlet): if r['priority_class'] > PushRuleRestServlet.PRIORITY_CLASS_MAP['override']: # per-device rule instance_handle = _instance_handle_from_conditions(r["conditions"]) + r = _strip_device_condition(r) if not instance_handle: continue if instance_handle not in rules['device']: @@ -239,6 +235,7 @@ class PushRuleRestServlet(RestServlet): defer.returnValue((200, rules['device'])) instance_handle = path[0] + path = path[1:] if instance_handle not in rules['device']: ret = {} ret = _add_empty_priority_class_arrays(ret) @@ -290,10 +287,21 @@ def _filter_ruleset_with_path(ruleset, path): raise NotFoundError +def _priority_class_from_spec(spec): + if spec['template'] not in PushRuleRestServlet.PRIORITY_CLASS_MAP.keys(): + raise InvalidRuleException("Unknown template: %s" % (spec['kind'])) + pc = PushRuleRestServlet.PRIORITY_CLASS_MAP[spec['template']] + + if spec['scope'] == 'device': + pc += len(PushRuleRestServlet.PRIORITY_CLASS_MAP) + + return pc + + def _priority_class_to_template_name(pc): if pc > PushRuleRestServlet.PRIORITY_CLASS_MAP['override']: # per-device - prio_class_index = pc - PushRuleRestServlet.PRIORITY_CLASS_MAP['override'] + prio_class_index = pc - len(PushRuleRestServlet.PRIORITY_CLASS_MAP) return PushRuleRestServlet.PRIORITY_CLASS_INVERSE_MAP[prio_class_index] else: return PushRuleRestServlet.PRIORITY_CLASS_INVERSE_MAP[pc] @@ -316,6 +324,13 @@ def _rule_to_template(rule): return ret +def _strip_device_condition(rule): + for i,c in enumerate(rule['conditions']): + if c['kind'] == 'device': + del rule['conditions'][i] + return rule + + class InvalidRuleException(Exception): pass From 5f84ba8ea1991dff279f0135f474d9debfd1419a Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 23 Jan 2015 17:49:37 +0000 Subject: [PATCH 041/180] Add API to delete push rules. --- synapse/rest/client/v1/push_rule.py | 41 ++++++++++++++++++++++++++++- synapse/storage/push_rule.py | 9 +++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 9dc2c0e11e..50bf5b9008 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -15,7 +15,8 @@ from twisted.internet import defer -from synapse.api.errors import SynapseError, Codes, UnrecognizedRequestError, NotFoundError +from synapse.api.errors import SynapseError, Codes, UnrecognizedRequestError, NotFoundError, \ + StoreError from base import RestServlet, client_path_pattern from synapse.storage.push_rule import InconsistentRuleException, RuleNotFoundException @@ -175,6 +176,44 @@ class PushRuleRestServlet(RestServlet): defer.returnValue((200, {})) + @defer.inlineCallbacks + def on_DELETE(self, request): + spec = self.rule_spec_from_path(request.postpath) + try: + priority_class = _priority_class_from_spec(spec) + except InvalidRuleException as e: + raise SynapseError(400, e.message) + + user = yield self.auth.get_user_by_req(request) + + if 'device' in spec: + rules = yield self.hs.get_datastore().get_push_rules_for_user_name( + user.to_string() + ) + + for r in rules: + conditions = json.loads(r['conditions']) + ih = _instance_handle_from_conditions(conditions) + if ih == spec['device'] and r['priority_class'] == priority_class: + yield self.hs.get_datastore().delete_push_rule( + user.to_string(), spec['rule_id'] + ) + defer.returnValue((200, {})) + raise NotFoundError() + else: + try: + yield self.hs.get_datastore().delete_push_rule( + user.to_string(), spec['rule_id'], + priority_class=priority_class + ) + defer.returnValue((200, {})) + except StoreError as e: + if e.code == 404: + raise NotFoundError() + else: + raise + + @defer.inlineCallbacks def on_GET(self, request): user = yield self.auth.get_user_by_req(request) diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index 2366090e09..ca04f2ccee 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -174,6 +174,15 @@ class PushRuleStore(SQLBaseStore): txn.execute(sql, new_rule.values()) + @defer.inlineCallbacks + def delete_push_rule(self, user_name, rule_id): + yield self._simple_delete_one( + PushRuleTable.table_name, + { + 'user_name': user_name, + 'rule_id': rule_id + } + ) class RuleNotFoundException(Exception): pass From e26340cee7049b6c36f4c3451ec7524fa6b80d1c Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 23 Jan 2015 18:31:29 +0000 Subject: [PATCH 042/180] Start implementing the v2_alpha sync API --- synapse/api/urls.py | 1 + synapse/http/servlet.py | 57 +++++++++ synapse/rest/client/v2_alpha/__init__.py | 33 ++++++ synapse/rest/client/v2_alpha/_base.py | 38 ++++++ synapse/rest/client/v2_alpha/sync.py | 143 +++++++++++++++++++++++ 5 files changed, 272 insertions(+) create mode 100644 synapse/rest/client/v2_alpha/__init__.py create mode 100644 synapse/rest/client/v2_alpha/_base.py create mode 100644 synapse/rest/client/v2_alpha/sync.py diff --git a/synapse/api/urls.py b/synapse/api/urls.py index a299392049..693c0efda6 100644 --- a/synapse/api/urls.py +++ b/synapse/api/urls.py @@ -16,6 +16,7 @@ """Contains the URL paths to prefix various aspects of the server with. """ CLIENT_PREFIX = "/_matrix/client/api/v1" +CLIENT_V2_ALPHA_PREFIX = "/_matrix/client/v2_alpha" FEDERATION_PREFIX = "/_matrix/federation/v1" WEB_CLIENT_PREFIX = "/_matrix/client" CONTENT_REPO_PREFIX = "/_matrix/content" diff --git a/synapse/http/servlet.py b/synapse/http/servlet.py index d5ccf2742f..a4eb6c817c 100644 --- a/synapse/http/servlet.py +++ b/synapse/http/servlet.py @@ -15,6 +15,8 @@ """ This module contains base REST classes for constructing REST servlets. """ +from synapse.api.errors import SynapseError + import logging @@ -54,3 +56,58 @@ class RestServlet(object): http_server.register_path(method, pattern, method_handler) else: raise NotImplementedError("RestServlet must register something.") + + @staticmethod + def parse_integer(request, name, default=None, required=False): + if name in request.args: + try: + return int(request.args[name][0]) + except: + message = "Query parameter %r must be an integer" % (name,) + raise SynapseError(400, message) + else: + if required: + message = "Missing integer query parameter %r" % (name,) + raise SynapseError(400, message) + else: + return default + + @staticmethod + def parse_boolean(request, name, default=None, required=False): + if name in request.args: + try: + return { + "true": True, + "false": False, + }[request.args[name][0]] + except: + message = ( + "Boolean query parameter %r must be one of" + " ['true', 'false']" + ) % (name,) + raise SynapseError(400, message) + else: + if required: + message = "Missing boolean query parameter %r" % (name,) + raise SynapseError(400, message) + else: + return default + + @staticmethod + def parse_string(request, name, default=None, required=False, + allowed_values=None, param_type="string"): + if name in request.args: + value = request.args[name][0] + if allowed_values is not None and value not in allowed_values: + message = "Query parameter %r must be one of [%s]" % ( + name, ", ".join(repr(v) for v in allowed_values) + ) + raise SynapseError(message) + else: + return value + else: + if required: + message = "Missing %s query parameter %r" % (param_type, name) + raise SynapseError(400, message) + else: + return default diff --git a/synapse/rest/client/v2_alpha/__init__.py b/synapse/rest/client/v2_alpha/__init__.py new file mode 100644 index 0000000000..e75f9d250f --- /dev/null +++ b/synapse/rest/client/v2_alpha/__init__.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from . import ( + sync, +) + +from synapse.http.server import JsonResource + + +class ClientV2AlphaRestResource(JsonResource): + """A resource for version 2 alpha of the matrix client API.""" + + def __init__(self, hs): + JsonResource.__init__(self) + self.register_servlets(self, hs) + + @staticmethod + def register_servlets(client_resource, hs): + sync.register_servlets(hs, client_resource) diff --git a/synapse/rest/client/v2_alpha/_base.py b/synapse/rest/client/v2_alpha/_base.py new file mode 100644 index 0000000000..22dc5cb862 --- /dev/null +++ b/synapse/rest/client/v2_alpha/_base.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This module contains base REST classes for constructing client v1 servlets. +""" + +from synapse.api.urls import CLIENT_V2_ALPHA_PREFIX +import re + +import logging + + +logger = logging.getLogger(__name__) + + +def client_v2_pattern(path_regex): + """Creates a regex compiled client path with the correct client path + prefix. + + Args: + path_regex (str): The regex string to match. This should NOT have a ^ + as this will be prefixed. + Returns: + SRE_Pattern + """ + return re.compile("^" + CLIENT_V2_ALPHA_PREFIX + path_regex) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py new file mode 100644 index 0000000000..39bb5ec8e9 --- /dev/null +++ b/synapse/rest/client/v2_alpha/sync.py @@ -0,0 +1,143 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from synapse.http.servlet import RestServlet +from ._base import client_v2_pattern + +import logging + +logger = logging.getLogger(__name__) + + +class SyncRestServlet(RestServlet): + """ + + GET parameters:: + timeout(int): How long to wait for new events in milliseconds. + limit(int): Maxiumum number of events per room to return. + gap(bool): Create gaps the message history if limit is exceeded to + ensure that the client has the most recent messages. Defaults to + "true". + sort(str,str): tuple of sort key (e.g. "timeline") and direction + (e.g. "asc", "desc"). Defaults to "timeline,asc". + since(batch_token): Batch token when asking for incremental deltas. + set_presence(str): What state the device presence should be set to. + default is "online". + backfill(bool): Should the HS request message history from other + servers. This may take a long time making it unsuitable for clients + expecting a prompt response. Defaults to "true". + filter(filter_id): A filter to apply to the events returned. + filter_*: Filter override parameters. + + Response JSON:: + { + "next_batch": // batch token for the next /sync + "private_user_data": // private events for this user. + "public_user_data": // public events for all users including the + // public events for this user. + "rooms": [{ // List of rooms with updates. + "room_id": // Id of the room being updated + "limited": // Was the per-room event limit exceeded? + "published": // Is the room published by our HS? + "event_map": // Map of EventID -> event JSON. + "events": { // The recent events in the room if gap is "true" + // otherwise the next events in the room. + "batch": [] // list of EventIDs in the "event_map". + "prev_batch": // back token for getting previous events. + } + "state": [] // list of EventIDs updating the current state to + // be what it should be at the end of the batch. + }] + } + """ + + + PATTERN = client_v2_pattern("/sync$") + ALLOWED_SORT = set(["timeline,asc", "timeline,desc"]) + ALLOWED_PRESENCE = set(["online", "offline", "idle"]) + + def __init__(self, hs): + super(SyncRestServlet, self).__init__() + self.auth = hs.get_auth() + #self.sync_handler = hs.get_handlers().sync_hanlder + + @defer.inlineCallbacks + def on_GET(self, request): + user = yield self.auth.get_user_by_req(request) + + timeout = self.parse_integer(request, "timeout", default=0) + limit = self.parse_integer(request, "limit", default=None) + gap = self.parse_boolean(request, "gap", default=True) + sort = self.parse_string( + request, "sort", default="timeline,asc", + allowed_values=self.ALLOWED_SORT + ) + since = self.parse_string(request, "since") + set_presence = self.parse_string( + request, "set_presence", default="online", + allowed_values=self.ALLOWED_PRESENCE + ) + backfill = self.parse_boolean(request, "backfill", default=True) + filter_id = self.parse_string(request, "filter", default=None) + + logger.info( + "/sync: user=%r, timeout=%r, limit=%r, gap=%r, sort=%r, since=%r," + " set_presence=%r, backfill=%r, filter_id=%r" % ( + user, timeout, limit, gap, sort, since, set_presence, + backfill, filter_id + ) + ) + + # TODO(mjark): Load filter and apply overrides. + # filter = self.filters.load_fitler(filter_id_str) + # filter = filter.apply_overrides(http_request) + # if filter.matches(event): + # # stuff + + # if timeout != 0: + # register for updates from the event stream + + #rooms = [] + + if gap: + pass + # now_stream_token = get_current_stream_token + # for room_id in get_rooms_for_user(user, filter=filter): + # state, events, start, end, limited, published = updates_for_room( + # from=since, to=now_stream_token, limit=limit, + # anchor_to_start=False + # ) + # rooms[room_id] = (state, events, start, limited, published) + # next_stream_token = now. + else: + pass + # now_stream_token = get_current_stream_token + # for room_id in get_rooms_for_user(user, filter=filter) + # state, events, start, end, limited, published = updates_for_room( + # from=since, to=now_stream_token, limit=limit, + # anchor_to_start=False + # ) + # next_stream_token = min(next_stream_token, end) + + + response_content = {} + + defer.returnValue((200, response_content)) + + +def register_servlets(hs, http_server): + SyncRestServlet(hs).register(http_server) From 2b1799883db3facf63b7982a0302542082414974 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 23 Jan 2015 18:49:05 +0000 Subject: [PATCH 043/180] Add client v2_alpha resource to synapse server resource tree --- synapse/app/homeserver.py | 7 ++++++- synapse/server.py | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index fabe8ddacb..40d28dcbdc 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -32,12 +32,13 @@ from synapse.http.server_key_resource import LocalKey from synapse.http.matrixfederationclient import MatrixFederationHttpClient from synapse.api.urls import ( CLIENT_PREFIX, FEDERATION_PREFIX, WEB_CLIENT_PREFIX, CONTENT_REPO_PREFIX, - SERVER_KEY_PREFIX, MEDIA_PREFIX + SERVER_KEY_PREFIX, MEDIA_PREFIX, CLIENT_V2_ALPHA_PREFIX, ) from synapse.config.homeserver import HomeServerConfig from synapse.crypto import context_factory from synapse.util.logcontext import LoggingContext from synapse.rest.client.v1 import ClientV1RestResource +from synapse.rest.client.v2_alpha import ClientV2AlphaRestResource from daemonize import Daemonize import twisted.manhole.telnet @@ -62,6 +63,9 @@ class SynapseHomeServer(HomeServer): def build_resource_for_client(self): return ClientV1RestResource(self) + def build_resource_for_client_v2_alpha(self): + return ClientV2AlphaRestResource(self) + def build_resource_for_federation(self): return JsonResource() @@ -105,6 +109,7 @@ class SynapseHomeServer(HomeServer): # [ ("/aaa/bbb/cc", Resource1), ("/aaa/dummy", Resource2) ] desired_tree = [ (CLIENT_PREFIX, self.get_resource_for_client()), + (CLIENT_V2_ALPHA_PREFIX, self.get_resource_for_client_v2_alpha()), (FEDERATION_PREFIX, self.get_resource_for_federation()), (CONTENT_REPO_PREFIX, self.get_resource_for_content_repo()), (SERVER_KEY_PREFIX, self.get_resource_for_server_key()), diff --git a/synapse/server.py b/synapse/server.py index 32013b1a91..92ed2c5e32 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -70,6 +70,7 @@ class BaseHomeServer(object): 'notifier', 'distributor', 'resource_for_client', + 'resource_for_client_v2_alpha', 'resource_for_federation', 'resource_for_web_client', 'resource_for_content_repo', From 7b8861924130821c1bbd05ce65260209a993f759 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 26 Jan 2015 10:45:24 +0000 Subject: [PATCH 044/180] Split up replication_layer module into client, server and transaction queue --- synapse/federation/federation_client.py | 293 ++++++++++++ synapse/federation/federation_server.py | 345 ++++++++++++++ synapse/federation/replication.py | 608 +----------------------- synapse/federation/transaction_queue.py | 9 +- synapse/storage/__init__.py | 2 +- 5 files changed, 654 insertions(+), 603 deletions(-) create mode 100644 synapse/federation/federation_client.py create mode 100644 synapse/federation/federation_server.py diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py new file mode 100644 index 0000000000..c80f4c61bc --- /dev/null +++ b/synapse/federation/federation_client.py @@ -0,0 +1,293 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from twisted.internet import defer + +from .units import Edu + +from synapse.util.logutils import log_function +from synapse.events import FrozenEvent + +import logging + + +logger = logging.getLogger(__name__) + + +class FederationClient(object): + @log_function + def send_pdu(self, pdu, destinations): + """Informs the replication layer about a new PDU generated within the + home server that should be transmitted to others. + + TODO: Figure out when we should actually resolve the deferred. + + Args: + pdu (Pdu): The new Pdu. + + Returns: + Deferred: Completes when we have successfully processed the PDU + and replicated it to any interested remote home servers. + """ + order = self._order + self._order += 1 + + logger.debug("[%s] transaction_layer.enqueue_pdu... ", pdu.event_id) + + # TODO, add errback, etc. + self._transaction_queue.enqueue_pdu(pdu, destinations, order) + + logger.debug( + "[%s] transaction_layer.enqueue_pdu... done", + pdu.event_id + ) + + @log_function + def send_edu(self, destination, edu_type, content): + edu = Edu( + origin=self.server_name, + destination=destination, + edu_type=edu_type, + content=content, + ) + + # TODO, add errback, etc. + self._transaction_queue.enqueue_edu(edu) + return defer.succeed(None) + + @log_function + def send_failure(self, failure, destination): + self._transaction_queue.enqueue_failure(failure, destination) + return defer.succeed(None) + + @log_function + def make_query(self, destination, query_type, args, + retry_on_dns_fail=True): + """Sends a federation Query to a remote homeserver of the given type + and arguments. + + Args: + destination (str): Domain name of the remote homeserver + query_type (str): Category of the query type; should match the + handler name used in register_query_handler(). + args (dict): Mapping of strings to strings containing the details + of the query request. + + Returns: + a Deferred which will eventually yield a JSON object from the + response + """ + return self.transport_layer.make_query( + destination, query_type, args, retry_on_dns_fail=retry_on_dns_fail + ) + + @defer.inlineCallbacks + @log_function + def backfill(self, dest, context, limit, extremities): + """Requests some more historic PDUs for the given context from the + given destination server. + + Args: + dest (str): The remote home server to ask. + context (str): The context to backfill. + limit (int): The maximum number of PDUs to return. + extremities (list): List of PDU id and origins of the first pdus + we have seen from the context + + Returns: + Deferred: Results in the received PDUs. + """ + logger.debug("backfill extrem=%s", extremities) + + # If there are no extremeties then we've (probably) reached the start. + if not extremities: + return + + transaction_data = yield self.transport_layer.backfill( + dest, context, extremities, limit) + + logger.debug("backfill transaction_data=%s", repr(transaction_data)) + + pdus = [ + self.event_from_pdu_json(p, outlier=False) + for p in transaction_data["pdus"] + ] + + defer.returnValue(pdus) + + @defer.inlineCallbacks + @log_function + def get_pdu(self, destinations, event_id, outlier=False): + """Requests the PDU with given origin and ID from the remote home + servers. + + Will attempt to get the PDU from each destination in the list until + one succeeds. + + This will persist the PDU locally upon receipt. + + Args: + destinations (list): Which home servers to query + pdu_origin (str): The home server that originally sent the pdu. + event_id (str) + outlier (bool): Indicates whether the PDU is an `outlier`, i.e. if + it's from an arbitary point in the context as opposed to part + of the current block of PDUs. Defaults to `False` + + Returns: + Deferred: Results in the requested PDU. + """ + + # TODO: Rate limit the number of times we try and get the same event. + + pdu = None + for destination in destinations: + try: + transaction_data = yield self.transport_layer.get_event( + destination, event_id + ) + except Exception as e: + logger.info( + "Failed to get PDU %s from %s because %s", + event_id, destination, e, + ) + continue + + logger.debug("transaction_data %r", transaction_data) + + pdu_list = [ + self.event_from_pdu_json(p, outlier=outlier) + for p in transaction_data["pdus"] + ] + + if pdu_list: + pdu = pdu_list[0] + # TODO: We need to check signatures here + break + + defer.returnValue(pdu) + + @defer.inlineCallbacks + @log_function + def get_state_for_room(self, destination, room_id, event_id): + """Requests all of the `current` state PDUs for a given room from + a remote home server. + + Args: + destination (str): The remote homeserver to query for the state. + room_id (str): The id of the room we're interested in. + event_id (str): The id of the event we want the state at. + + Returns: + Deferred: Results in a list of PDUs. + """ + + result = yield self.transport_layer.get_room_state( + destination, room_id, event_id=event_id, + ) + + pdus = [ + self.event_from_pdu_json(p, outlier=True) for p in result["pdus"] + ] + + auth_chain = [ + self.event_from_pdu_json(p, outlier=True) + for p in result.get("auth_chain", []) + ] + + defer.returnValue((pdus, auth_chain)) + + @defer.inlineCallbacks + @log_function + def get_event_auth(self, destination, room_id, event_id): + res = yield self.transport_layer.get_event_auth( + destination, room_id, event_id, + ) + + auth_chain = [ + self.event_from_pdu_json(p, outlier=True) + for p in res["auth_chain"] + ] + + auth_chain.sort(key=lambda e: e.depth) + + defer.returnValue(auth_chain) + + @defer.inlineCallbacks + def make_join(self, destination, room_id, user_id): + ret = yield self.transport_layer.make_join( + destination, room_id, user_id + ) + + pdu_dict = ret["event"] + + logger.debug("Got response to make_join: %s", pdu_dict) + + defer.returnValue(self.event_from_pdu_json(pdu_dict)) + + @defer.inlineCallbacks + def send_join(self, destination, pdu): + time_now = self._clock.time_msec() + _, content = yield self.transport_layer.send_join( + destination=destination, + room_id=pdu.room_id, + event_id=pdu.event_id, + content=pdu.get_pdu_json(time_now), + ) + + logger.debug("Got content: %s", content) + + state = [ + self.event_from_pdu_json(p, outlier=True) + for p in content.get("state", []) + ] + + auth_chain = [ + self.event_from_pdu_json(p, outlier=True) + for p in content.get("auth_chain", []) + ] + + auth_chain.sort(key=lambda e: e.depth) + + defer.returnValue({ + "state": state, + "auth_chain": auth_chain, + }) + + @defer.inlineCallbacks + def send_invite(self, destination, room_id, event_id, pdu): + time_now = self._clock.time_msec() + code, content = yield self.transport_layer.send_invite( + destination=destination, + room_id=room_id, + event_id=event_id, + content=pdu.get_pdu_json(time_now), + ) + + pdu_dict = content["event"] + + logger.debug("Got response to send_invite: %s", pdu_dict) + + defer.returnValue(self.event_from_pdu_json(pdu_dict)) + + def event_from_pdu_json(self, pdu_json, outlier=False): + event = FrozenEvent( + pdu_json + ) + + event.internal_metadata.outlier = outlier + + return event diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py new file mode 100644 index 0000000000..0597725ce7 --- /dev/null +++ b/synapse/federation/federation_server.py @@ -0,0 +1,345 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from twisted.internet import defer + +from .units import Transaction, Edu + +from synapse.util.logutils import log_function +from synapse.util.logcontext import PreserveLoggingContext +from synapse.events import FrozenEvent + +import logging + + +logger = logging.getLogger(__name__) + + +class FederationServer(object): + def set_handler(self, handler): + """Sets the handler that the replication layer will use to communicate + receipt of new PDUs from other home servers. The required methods are + documented on :py:class:`.ReplicationHandler`. + """ + self.handler = handler + + def register_edu_handler(self, edu_type, handler): + if edu_type in self.edu_handlers: + raise KeyError("Already have an EDU handler for %s" % (edu_type,)) + + self.edu_handlers[edu_type] = handler + + def register_query_handler(self, query_type, handler): + """Sets the handler callable that will be used to handle an incoming + federation Query of the given type. + + Args: + query_type (str): Category name of the query, which should match + the string used by make_query. + handler (callable): Invoked to handle incoming queries of this type + + handler is invoked as: + result = handler(args) + + where 'args' is a dict mapping strings to strings of the query + arguments. It should return a Deferred that will eventually yield an + object to encode as JSON. + """ + if query_type in self.query_handlers: + raise KeyError( + "Already have a Query handler for %s" % (query_type,) + ) + + self.query_handlers[query_type] = handler + + @defer.inlineCallbacks + @log_function + def on_backfill_request(self, origin, room_id, versions, limit): + pdus = yield self.handler.on_backfill_request( + origin, room_id, versions, limit + ) + + defer.returnValue((200, self._transaction_from_pdus(pdus).get_dict())) + + @defer.inlineCallbacks + @log_function + def on_incoming_transaction(self, transaction_data): + transaction = Transaction(**transaction_data) + + for p in transaction.pdus: + if "unsigned" in p: + unsigned = p["unsigned"] + if "age" in unsigned: + p["age"] = unsigned["age"] + if "age" in p: + p["age_ts"] = int(self._clock.time_msec()) - int(p["age"]) + del p["age"] + + pdu_list = [ + self.event_from_pdu_json(p) for p in transaction.pdus + ] + + logger.debug("[%s] Got transaction", transaction.transaction_id) + + response = yield self.transaction_actions.have_responded(transaction) + + if response: + logger.debug("[%s] We've already responed to this request", + transaction.transaction_id) + defer.returnValue(response) + return + + logger.debug("[%s] Transaction is new", transaction.transaction_id) + + with PreserveLoggingContext(): + dl = [] + for pdu in pdu_list: + dl.append(self._handle_new_pdu(transaction.origin, pdu)) + + if hasattr(transaction, "edus"): + for edu in [Edu(**x) for x in transaction.edus]: + self.received_edu( + transaction.origin, + edu.edu_type, + edu.content + ) + + results = yield defer.DeferredList(dl) + + ret = [] + for r in results: + if r[0]: + ret.append({}) + else: + logger.exception(r[1]) + ret.append({"error": str(r[1])}) + + logger.debug("Returning: %s", str(ret)) + + yield self.transaction_actions.set_response( + transaction, + 200, response + ) + defer.returnValue((200, response)) + + def received_edu(self, origin, edu_type, content): + if edu_type in self.edu_handlers: + self.edu_handlers[edu_type](origin, content) + else: + logger.warn("Received EDU of type %s with no handler", edu_type) + + @defer.inlineCallbacks + @log_function + def on_context_state_request(self, origin, room_id, event_id): + if event_id: + pdus = yield self.handler.get_state_for_pdu( + origin, room_id, event_id, + ) + auth_chain = yield self.store.get_auth_chain( + [pdu.event_id for pdu in pdus] + ) + else: + raise NotImplementedError("Specify an event") + + defer.returnValue((200, { + "pdus": [pdu.get_pdu_json() for pdu in pdus], + "auth_chain": [pdu.get_pdu_json() for pdu in auth_chain], + })) + + @defer.inlineCallbacks + @log_function + def on_pdu_request(self, origin, event_id): + pdu = yield self._get_persisted_pdu(origin, event_id) + + if pdu: + defer.returnValue( + (200, self._transaction_from_pdus([pdu]).get_dict()) + ) + else: + defer.returnValue((404, "")) + + @defer.inlineCallbacks + @log_function + def on_pull_request(self, origin, versions): + raise NotImplementedError("Pull transactions not implemented") + + @defer.inlineCallbacks + def on_query_request(self, query_type, args): + if query_type in self.query_handlers: + response = yield self.query_handlers[query_type](args) + defer.returnValue((200, response)) + else: + defer.returnValue( + (404, "No handler for Query type '%s'" % (query_type,)) + ) + + @defer.inlineCallbacks + def on_make_join_request(self, room_id, user_id): + pdu = yield self.handler.on_make_join_request(room_id, user_id) + time_now = self._clock.time_msec() + defer.returnValue({"event": pdu.get_pdu_json(time_now)}) + + @defer.inlineCallbacks + def on_invite_request(self, origin, content): + pdu = self.event_from_pdu_json(content) + ret_pdu = yield self.handler.on_invite_request(origin, pdu) + time_now = self._clock.time_msec() + defer.returnValue((200, {"event": ret_pdu.get_pdu_json(time_now)})) + + @defer.inlineCallbacks + def on_send_join_request(self, origin, content): + logger.debug("on_send_join_request: content: %s", content) + pdu = self.event_from_pdu_json(content) + logger.debug("on_send_join_request: pdu sigs: %s", pdu.signatures) + res_pdus = yield self.handler.on_send_join_request(origin, pdu) + time_now = self._clock.time_msec() + defer.returnValue((200, { + "state": [p.get_pdu_json(time_now) for p in res_pdus["state"]], + "auth_chain": [ + p.get_pdu_json(time_now) for p in res_pdus["auth_chain"] + ], + })) + + @defer.inlineCallbacks + def on_event_auth(self, origin, room_id, event_id): + time_now = self._clock.time_msec() + auth_pdus = yield self.handler.on_event_auth(event_id) + defer.returnValue((200, { + "auth_chain": [a.get_pdu_json(time_now) for a in auth_pdus], + })) + + @log_function + def _get_persisted_pdu(self, origin, event_id, do_auth=True): + """ Get a PDU from the database with given origin and id. + + Returns: + Deferred: Results in a `Pdu`. + """ + return self.handler.get_persisted_pdu( + origin, event_id, do_auth=do_auth + ) + + def _transaction_from_pdus(self, pdu_list): + """Returns a new Transaction containing the given PDUs suitable for + transmission. + """ + time_now = self._clock.time_msec() + pdus = [p.get_pdu_json(time_now) for p in pdu_list] + return Transaction( + origin=self.server_name, + pdus=pdus, + origin_server_ts=int(time_now), + destination=None, + ) + + @defer.inlineCallbacks + @log_function + def _handle_new_pdu(self, origin, pdu, max_recursion=10): + # We reprocess pdus when we have seen them only as outliers + existing = yield self._get_persisted_pdu( + origin, pdu.event_id, do_auth=False + ) + + already_seen = ( + existing and ( + not existing.internal_metadata.is_outlier() + or pdu.internal_metadata.is_outlier() + ) + ) + if already_seen: + logger.debug("Already seen pdu %s", pdu.event_id) + defer.returnValue({}) + return + + state = None + + auth_chain = [] + + have_seen = yield self.store.have_events( + [e for e, _ in pdu.prev_events] + ) + + # Get missing pdus if necessary. + if not pdu.internal_metadata.is_outlier(): + # We only backfill backwards to the min depth. + min_depth = yield self.handler.get_min_depth_for_context( + pdu.room_id + ) + + logger.debug( + "_handle_new_pdu min_depth for %s: %d", + pdu.room_id, min_depth + ) + + if min_depth and pdu.depth > min_depth and max_recursion > 0: + for event_id, hashes in pdu.prev_events: + if event_id not in have_seen: + logger.debug( + "_handle_new_pdu requesting pdu %s", + event_id + ) + + try: + new_pdu = yield self.federation_client.get_pdu( + [origin, pdu.origin], + event_id=event_id, + ) + + if new_pdu: + yield self._handle_new_pdu( + origin, + new_pdu, + max_recursion=max_recursion-1 + ) + + logger.debug("Processed pdu %s", event_id) + else: + logger.warn("Failed to get PDU %s", event_id) + except: + # TODO(erikj): Do some more intelligent retries. + logger.exception("Failed to get PDU") + else: + # We need to get the state at this event, since we have reached + # a backward extremity edge. + logger.debug( + "_handle_new_pdu getting state for %s", + pdu.room_id + ) + state, auth_chain = yield self.get_state_for_room( + origin, pdu.room_id, pdu.event_id, + ) + + ret = yield self.handler.on_receive_pdu( + origin, + pdu, + backfilled=False, + state=state, + auth_chain=auth_chain, + ) + + defer.returnValue(ret) + + def __str__(self): + return "" % self.server_name + + def event_from_pdu_json(self, pdu_json, outlier=False): + event = FrozenEvent( + pdu_json + ) + + event.internal_metadata.outlier = outlier + + return event diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index accf95e406..9ef4834927 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -17,15 +17,12 @@ a given transport. """ -from twisted.internet import defer +from .federation_client import FederationClient +from .federation_server import FederationServer -from .units import Transaction, Edu -from .persistence import TransactionActions from .transaction_queue import TransactionQueue -from synapse.util.logutils import log_function -from synapse.util.logcontext import PreserveLoggingContext -from synapse.events import FrozenEvent +from .persistence import TransactionActions import logging @@ -33,7 +30,7 @@ import logging logger = logging.getLogger(__name__) -class ReplicationLayer(object): +class ReplicationLayer(FederationClient, FederationServer): """This layer is responsible for replicating with remote home servers over the given transport. I.e., does the sending and receiving of PDUs to remote home servers. @@ -58,607 +55,20 @@ class ReplicationLayer(object): self.transport_layer.register_received_handler(self) self.transport_layer.register_request_handler(self) - self.store = hs.get_datastore() - # self.pdu_actions = PduActions(self.store) - self.transaction_actions = TransactionActions(self.store) + self.federation_client = self - self._transaction_queue = TransactionQueue( - hs, self.transaction_actions, transport_layer - ) + self.store = hs.get_datastore() self.handler = None self.edu_handlers = {} self.query_handlers = {} - self._order = 0 - self._clock = hs.get_clock() - self.event_builder_factory = hs.get_event_builder_factory() + self.transaction_actions = TransactionActions(self.store) + self._transaction_queue = TransactionQueue(hs, transport_layer) - def set_handler(self, handler): - """Sets the handler that the replication layer will use to communicate - receipt of new PDUs from other home servers. The required methods are - documented on :py:class:`.ReplicationHandler`. - """ - self.handler = handler - - def register_edu_handler(self, edu_type, handler): - if edu_type in self.edu_handlers: - raise KeyError("Already have an EDU handler for %s" % (edu_type,)) - - self.edu_handlers[edu_type] = handler - - def register_query_handler(self, query_type, handler): - """Sets the handler callable that will be used to handle an incoming - federation Query of the given type. - - Args: - query_type (str): Category name of the query, which should match - the string used by make_query. - handler (callable): Invoked to handle incoming queries of this type - - handler is invoked as: - result = handler(args) - - where 'args' is a dict mapping strings to strings of the query - arguments. It should return a Deferred that will eventually yield an - object to encode as JSON. - """ - if query_type in self.query_handlers: - raise KeyError( - "Already have a Query handler for %s" % (query_type,) - ) - - self.query_handlers[query_type] = handler - - @log_function - def send_pdu(self, pdu, destinations): - """Informs the replication layer about a new PDU generated within the - home server that should be transmitted to others. - - TODO: Figure out when we should actually resolve the deferred. - - Args: - pdu (Pdu): The new Pdu. - - Returns: - Deferred: Completes when we have successfully processed the PDU - and replicated it to any interested remote home servers. - """ - order = self._order - self._order += 1 - - logger.debug("[%s] transaction_layer.enqueue_pdu... ", pdu.event_id) - - # TODO, add errback, etc. - self._transaction_queue.enqueue_pdu(pdu, destinations, order) - - logger.debug( - "[%s] transaction_layer.enqueue_pdu... done", - pdu.event_id - ) - - @log_function - def send_edu(self, destination, edu_type, content): - edu = Edu( - origin=self.server_name, - destination=destination, - edu_type=edu_type, - content=content, - ) - - # TODO, add errback, etc. - self._transaction_queue.enqueue_edu(edu) - return defer.succeed(None) - - @log_function - def send_failure(self, failure, destination): - self._transaction_queue.enqueue_failure(failure, destination) - return defer.succeed(None) - - @log_function - def make_query(self, destination, query_type, args, - retry_on_dns_fail=True): - """Sends a federation Query to a remote homeserver of the given type - and arguments. - - Args: - destination (str): Domain name of the remote homeserver - query_type (str): Category of the query type; should match the - handler name used in register_query_handler(). - args (dict): Mapping of strings to strings containing the details - of the query request. - - Returns: - a Deferred which will eventually yield a JSON object from the - response - """ - return self.transport_layer.make_query( - destination, query_type, args, retry_on_dns_fail=retry_on_dns_fail - ) - - @defer.inlineCallbacks - @log_function - def backfill(self, dest, context, limit, extremities): - """Requests some more historic PDUs for the given context from the - given destination server. - - Args: - dest (str): The remote home server to ask. - context (str): The context to backfill. - limit (int): The maximum number of PDUs to return. - extremities (list): List of PDU id and origins of the first pdus - we have seen from the context - - Returns: - Deferred: Results in the received PDUs. - """ - logger.debug("backfill extrem=%s", extremities) - - # If there are no extremeties then we've (probably) reached the start. - if not extremities: - return - - transaction_data = yield self.transport_layer.backfill( - dest, context, extremities, limit) - - logger.debug("backfill transaction_data=%s", repr(transaction_data)) - - transaction = Transaction(**transaction_data) - - pdus = [ - self.event_from_pdu_json(p, outlier=False) - for p in transaction.pdus - ] - for pdu in pdus: - yield self._handle_new_pdu(dest, pdu, backfilled=True) - - defer.returnValue(pdus) - - @defer.inlineCallbacks - @log_function - def get_pdu(self, destination, event_id, outlier=False): - """Requests the PDU with given origin and ID from the remote home - server. - - This will persist the PDU locally upon receipt. - - Args: - destination (str): Which home server to query - pdu_origin (str): The home server that originally sent the pdu. - event_id (str) - outlier (bool): Indicates whether the PDU is an `outlier`, i.e. if - it's from an arbitary point in the context as opposed to part - of the current block of PDUs. Defaults to `False` - - Returns: - Deferred: Results in the requested PDU. - """ - - transaction_data = yield self.transport_layer.get_event( - destination, event_id - ) - - transaction = Transaction(**transaction_data) - - pdu_list = [ - self.event_from_pdu_json(p, outlier=outlier) - for p in transaction.pdus - ] - - pdu = None - if pdu_list: - pdu = pdu_list[0] - yield self._handle_new_pdu(destination, pdu) - - defer.returnValue(pdu) - - @defer.inlineCallbacks - @log_function - def get_state_for_room(self, destination, room_id, event_id): - """Requests all of the `current` state PDUs for a given room from - a remote home server. - - Args: - destination (str): The remote homeserver to query for the state. - room_id (str): The id of the room we're interested in. - event_id (str): The id of the event we want the state at. - - Returns: - Deferred: Results in a list of PDUs. - """ - - result = yield self.transport_layer.get_room_state( - destination, room_id, event_id=event_id, - ) - - pdus = [ - self.event_from_pdu_json(p, outlier=True) for p in result["pdus"] - ] - - auth_chain = [ - self.event_from_pdu_json(p, outlier=True) - for p in result.get("auth_chain", []) - ] - - defer.returnValue((pdus, auth_chain)) - - @defer.inlineCallbacks - @log_function - def get_event_auth(self, destination, room_id, event_id): - res = yield self.transport_layer.get_event_auth( - destination, room_id, event_id, - ) - - auth_chain = [ - self.event_from_pdu_json(p, outlier=True) - for p in res["auth_chain"] - ] - - auth_chain.sort(key=lambda e: e.depth) - - defer.returnValue(auth_chain) - - @defer.inlineCallbacks - @log_function - def on_backfill_request(self, origin, room_id, versions, limit): - pdus = yield self.handler.on_backfill_request( - origin, room_id, versions, limit - ) - - defer.returnValue((200, self._transaction_from_pdus(pdus).get_dict())) - - @defer.inlineCallbacks - @log_function - def on_incoming_transaction(self, transaction_data): - transaction = Transaction(**transaction_data) - - for p in transaction.pdus: - if "unsigned" in p: - unsigned = p["unsigned"] - if "age" in unsigned: - p["age"] = unsigned["age"] - if "age" in p: - p["age_ts"] = int(self._clock.time_msec()) - int(p["age"]) - del p["age"] - - pdu_list = [ - self.event_from_pdu_json(p) for p in transaction.pdus - ] - - logger.debug("[%s] Got transaction", transaction.transaction_id) - - response = yield self.transaction_actions.have_responded(transaction) - - if response: - logger.debug("[%s] We've already responed to this request", - transaction.transaction_id) - defer.returnValue(response) - return - - logger.debug("[%s] Transaction is new", transaction.transaction_id) - - with PreserveLoggingContext(): - dl = [] - for pdu in pdu_list: - dl.append(self._handle_new_pdu(transaction.origin, pdu)) - - if hasattr(transaction, "edus"): - for edu in [Edu(**x) for x in transaction.edus]: - self.received_edu( - transaction.origin, - edu.edu_type, - edu.content - ) - - results = yield defer.DeferredList(dl) - - ret = [] - for r in results: - if r[0]: - ret.append({}) - else: - logger.exception(r[1]) - ret.append({"error": str(r[1])}) - - logger.debug("Returning: %s", str(ret)) - - yield self.transaction_actions.set_response( - transaction, - 200, response - ) - defer.returnValue((200, response)) - - def received_edu(self, origin, edu_type, content): - if edu_type in self.edu_handlers: - self.edu_handlers[edu_type](origin, content) - else: - logger.warn("Received EDU of type %s with no handler", edu_type) - - @defer.inlineCallbacks - @log_function - def on_context_state_request(self, origin, room_id, event_id): - if event_id: - pdus = yield self.handler.get_state_for_pdu( - origin, room_id, event_id, - ) - auth_chain = yield self.store.get_auth_chain( - [pdu.event_id for pdu in pdus] - ) - else: - raise NotImplementedError("Specify an event") - - defer.returnValue((200, { - "pdus": [pdu.get_pdu_json() for pdu in pdus], - "auth_chain": [pdu.get_pdu_json() for pdu in auth_chain], - })) - - @defer.inlineCallbacks - @log_function - def on_pdu_request(self, origin, event_id): - pdu = yield self._get_persisted_pdu(origin, event_id) - - if pdu: - defer.returnValue( - (200, self._transaction_from_pdus([pdu]).get_dict()) - ) - else: - defer.returnValue((404, "")) - - @defer.inlineCallbacks - @log_function - def on_pull_request(self, origin, versions): - raise NotImplementedError("Pull transactions not implemented") - - @defer.inlineCallbacks - def on_query_request(self, query_type, args): - if query_type in self.query_handlers: - response = yield self.query_handlers[query_type](args) - defer.returnValue((200, response)) - else: - defer.returnValue( - (404, "No handler for Query type '%s'" % (query_type,)) - ) - - @defer.inlineCallbacks - def on_make_join_request(self, room_id, user_id): - pdu = yield self.handler.on_make_join_request(room_id, user_id) - time_now = self._clock.time_msec() - defer.returnValue({"event": pdu.get_pdu_json(time_now)}) - - @defer.inlineCallbacks - def on_invite_request(self, origin, content): - pdu = self.event_from_pdu_json(content) - ret_pdu = yield self.handler.on_invite_request(origin, pdu) - time_now = self._clock.time_msec() - defer.returnValue((200, {"event": ret_pdu.get_pdu_json(time_now)})) - - @defer.inlineCallbacks - def on_send_join_request(self, origin, content): - logger.debug("on_send_join_request: content: %s", content) - pdu = self.event_from_pdu_json(content) - logger.debug("on_send_join_request: pdu sigs: %s", pdu.signatures) - res_pdus = yield self.handler.on_send_join_request(origin, pdu) - time_now = self._clock.time_msec() - defer.returnValue((200, { - "state": [p.get_pdu_json(time_now) for p in res_pdus["state"]], - "auth_chain": [ - p.get_pdu_json(time_now) for p in res_pdus["auth_chain"] - ], - })) - - @defer.inlineCallbacks - def on_event_auth(self, origin, room_id, event_id): - time_now = self._clock.time_msec() - auth_pdus = yield self.handler.on_event_auth(event_id) - defer.returnValue((200, { - "auth_chain": [a.get_pdu_json(time_now) for a in auth_pdus], - })) - - @defer.inlineCallbacks - def make_join(self, destination, room_id, user_id): - ret = yield self.transport_layer.make_join( - destination, room_id, user_id - ) - - pdu_dict = ret["event"] - - logger.debug("Got response to make_join: %s", pdu_dict) - - defer.returnValue(self.event_from_pdu_json(pdu_dict)) - - @defer.inlineCallbacks - def send_join(self, destination, pdu): - time_now = self._clock.time_msec() - _, content = yield self.transport_layer.send_join( - destination=destination, - room_id=pdu.room_id, - event_id=pdu.event_id, - content=pdu.get_pdu_json(time_now), - ) - - logger.debug("Got content: %s", content) - - state = [ - self.event_from_pdu_json(p, outlier=True) - for p in content.get("state", []) - ] - - auth_chain = [ - self.event_from_pdu_json(p, outlier=True) - for p in content.get("auth_chain", []) - ] - - auth_chain.sort(key=lambda e: e.depth) - - defer.returnValue({ - "state": state, - "auth_chain": auth_chain, - }) - - @defer.inlineCallbacks - def send_invite(self, destination, room_id, event_id, pdu): - time_now = self._clock.time_msec() - code, content = yield self.transport_layer.send_invite( - destination=destination, - room_id=room_id, - event_id=event_id, - content=pdu.get_pdu_json(time_now), - ) - - pdu_dict = content["event"] - - logger.debug("Got response to send_invite: %s", pdu_dict) - - defer.returnValue(self.event_from_pdu_json(pdu_dict)) - - @log_function - def _get_persisted_pdu(self, origin, event_id, do_auth=True): - """ Get a PDU from the database with given origin and id. - - Returns: - Deferred: Results in a `Pdu`. - """ - return self.handler.get_persisted_pdu( - origin, event_id, do_auth=do_auth - ) - - def _transaction_from_pdus(self, pdu_list): - """Returns a new Transaction containing the given PDUs suitable for - transmission. - """ - time_now = self._clock.time_msec() - pdus = [p.get_pdu_json(time_now) for p in pdu_list] - return Transaction( - origin=self.server_name, - pdus=pdus, - origin_server_ts=int(time_now), - destination=None, - ) - - @defer.inlineCallbacks - @log_function - def _handle_new_pdu(self, origin, pdu, backfilled=False): - # We reprocess pdus when we have seen them only as outliers - existing = yield self._get_persisted_pdu( - origin, pdu.event_id, do_auth=False - ) - - already_seen = ( - existing and ( - not existing.internal_metadata.is_outlier() - or pdu.internal_metadata.is_outlier() - ) - ) - if already_seen: - logger.debug("Already seen pdu %s", pdu.event_id) - defer.returnValue({}) - return - - state = None - - auth_chain = [] - - # We need to make sure we have all the auth events. - # for e_id, _ in pdu.auth_events: - # exists = yield self._get_persisted_pdu( - # origin, - # e_id, - # do_auth=False - # ) - # - # if not exists: - # try: - # logger.debug( - # "_handle_new_pdu fetch missing auth event %s from %s", - # e_id, - # origin, - # ) - # - # yield self.get_pdu( - # origin, - # event_id=e_id, - # outlier=True, - # ) - # - # logger.debug("Processed pdu %s", e_id) - # except: - # logger.warn( - # "Failed to get auth event %s from %s", - # e_id, - # origin - # ) - - # Get missing pdus if necessary. - if not pdu.internal_metadata.is_outlier(): - # We only backfill backwards to the min depth. - min_depth = yield self.handler.get_min_depth_for_context( - pdu.room_id - ) - - logger.debug( - "_handle_new_pdu min_depth for %s: %d", - pdu.room_id, min_depth - ) - - if min_depth and pdu.depth > min_depth: - for event_id, hashes in pdu.prev_events: - exists = yield self._get_persisted_pdu( - origin, - event_id, - do_auth=False - ) - - if not exists: - logger.debug( - "_handle_new_pdu requesting pdu %s", - event_id - ) - - try: - yield self.get_pdu( - origin, - event_id=event_id, - ) - logger.debug("Processed pdu %s", event_id) - except: - # TODO(erikj): Do some more intelligent retries. - logger.exception("Failed to get PDU") - else: - # We need to get the state at this event, since we have reached - # a backward extremity edge. - logger.debug( - "_handle_new_pdu getting state for %s", - pdu.room_id - ) - state, auth_chain = yield self.get_state_for_room( - origin, pdu.room_id, pdu.event_id, - ) - - if not backfilled: - ret = yield self.handler.on_receive_pdu( - origin, - pdu, - backfilled=backfilled, - state=state, - auth_chain=auth_chain, - ) - else: - ret = None - - # yield self.pdu_actions.mark_as_processed(pdu) - - defer.returnValue(ret) + self._order = 0 def __str__(self): return "" % self.server_name - - def event_from_pdu_json(self, pdu_json, outlier=False): - event = FrozenEvent( - pdu_json - ) - - event.internal_metadata.outlier = outlier - - return event diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index c2cb4a1c49..9d4f2c09a2 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -16,6 +16,7 @@ from twisted.internet import defer +from .persistence import TransactionActions from .units import Transaction from synapse.util.logutils import log_function @@ -34,13 +35,15 @@ class TransactionQueue(object): It batches pending PDUs into single transactions. """ - def __init__(self, hs, transaction_actions, transport_layer): + def __init__(self, hs, transport_layer): self.server_name = hs.hostname - self.transaction_actions = transaction_actions + + self.store = hs.get_datastore() + self.transaction_actions = TransactionActions(self.store) + self.transport_layer = transport_layer self._clock = hs.get_clock() - self.store = hs.get_datastore() # Is a mapping from destinations -> deferreds. Used to keep track # of which destinations have transactions in flight and when they are diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 4f09909607..27d835db79 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -434,7 +434,7 @@ class DataStore(RoomMemberStore, RoomStore, sql = ( "SELECT e.event_id, reason FROM events as e " "LEFT JOIN rejections as r ON e.event_id = r.event_id " - "WHERE event_id = ?" + "WHERE e.event_id = ?" ) res = {} From c92d64a6c35713aabaed11e8ef1e62d2fb84a875 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 26 Jan 2015 14:33:11 +0000 Subject: [PATCH 045/180] Make it the responsibility of the replication layer to check signature and hashes. --- synapse/federation/federation_client.py | 108 +++++++++++++++++++++--- synapse/federation/federation_server.py | 89 ++++++++++++++++--- synapse/federation/replication.py | 2 + 3 files changed, 173 insertions(+), 26 deletions(-) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index c80f4c61bc..91b44cd8b3 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -20,6 +20,13 @@ from .units import Edu from synapse.util.logutils import log_function from synapse.events import FrozenEvent +from synapse.events.utils import prune_event + +from syutil.jsonutil import encode_canonical_json + +from synapse.crypto.event_signing import check_event_content_hash + +from synapse.api.errors import SynapseError import logging @@ -126,6 +133,11 @@ class FederationClient(object): for p in transaction_data["pdus"] ] + for i, pdu in enumerate(pdus): + pdus[i] = yield self._check_sigs_and_hash(pdu) + + # FIXME: We should handle signature failures more gracefully. + defer.returnValue(pdus) @defer.inlineCallbacks @@ -159,6 +171,22 @@ class FederationClient(object): transaction_data = yield self.transport_layer.get_event( destination, event_id ) + + logger.debug("transaction_data %r", transaction_data) + + pdu_list = [ + self.event_from_pdu_json(p, outlier=outlier) + for p in transaction_data["pdus"] + ] + + if pdu_list: + pdu = pdu_list[0] + + # Check signatures are correct. + pdu = yield self._check_sigs_and_hash(pdu) + + break + except Exception as e: logger.info( "Failed to get PDU %s from %s because %s", @@ -166,18 +194,6 @@ class FederationClient(object): ) continue - logger.debug("transaction_data %r", transaction_data) - - pdu_list = [ - self.event_from_pdu_json(p, outlier=outlier) - for p in transaction_data["pdus"] - ] - - if pdu_list: - pdu = pdu_list[0] - # TODO: We need to check signatures here - break - defer.returnValue(pdu) @defer.inlineCallbacks @@ -208,6 +224,16 @@ class FederationClient(object): for p in result.get("auth_chain", []) ] + for i, pdu in enumerate(pdus): + pdus[i] = yield self._check_sigs_and_hash(pdu) + + # FIXME: We should handle signature failures more gracefully. + + for i, pdu in enumerate(auth_chain): + auth_chain[i] = yield self._check_sigs_and_hash(pdu) + + # FIXME: We should handle signature failures more gracefully. + defer.returnValue((pdus, auth_chain)) @defer.inlineCallbacks @@ -222,6 +248,11 @@ class FederationClient(object): for p in res["auth_chain"] ] + for i, pdu in enumerate(auth_chain): + auth_chain[i] = yield self._check_sigs_and_hash(pdu) + + # FIXME: We should handle signature failures more gracefully. + auth_chain.sort(key=lambda e: e.depth) defer.returnValue(auth_chain) @@ -260,6 +291,16 @@ class FederationClient(object): for p in content.get("auth_chain", []) ] + for i, pdu in enumerate(state): + state[i] = yield self._check_sigs_and_hash(pdu) + + # FIXME: We should handle signature failures more gracefully. + + for i, pdu in enumerate(auth_chain): + auth_chain[i] = yield self._check_sigs_and_hash(pdu) + + # FIXME: We should handle signature failures more gracefully. + auth_chain.sort(key=lambda e: e.depth) defer.returnValue({ @@ -281,7 +322,14 @@ class FederationClient(object): logger.debug("Got response to send_invite: %s", pdu_dict) - defer.returnValue(self.event_from_pdu_json(pdu_dict)) + pdu = self.event_from_pdu_json(pdu_dict) + + # Check signatures are correct. + pdu = yield self._check_sigs_and_hash(pdu) + + # FIXME: We should handle signature failures more gracefully. + + defer.returnValue(pdu) def event_from_pdu_json(self, pdu_json, outlier=False): event = FrozenEvent( @@ -291,3 +339,37 @@ class FederationClient(object): event.internal_metadata.outlier = outlier return event + + @defer.inlineCallbacks + def _check_sigs_and_hash(self, pdu): + """Throws a SynapseError if the PDU does not have the correct + signatures. + + Returns: + FrozenEvent: Either the given event or it redacted if it failed the + content hash check. + """ + # Check signatures are correct. + redacted_event = prune_event(pdu) + redacted_pdu_json = redacted_event.get_pdu_json() + + try: + yield self.keyring.verify_json_for_server( + pdu.origin, redacted_pdu_json + ) + except SynapseError: + logger.warn( + "Signature check failed for %s redacted to %s", + encode_canonical_json(pdu.get_pdu_json()), + encode_canonical_json(redacted_pdu_json), + ) + raise + + if not check_event_content_hash(pdu): + logger.warn( + "Event content has been tampered, redacting %s, %s", + pdu.event_id, encode_canonical_json(pdu.get_dict()) + ) + defer.returnValue(redacted_event) + + defer.returnValue(pdu) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 0597725ce7..fc5342afaa 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -21,6 +21,13 @@ from .units import Transaction, Edu from synapse.util.logutils import log_function from synapse.util.logcontext import PreserveLoggingContext from synapse.events import FrozenEvent +from synapse.events.utils import prune_event + +from syutil.jsonutil import encode_canonical_json + +from synapse.crypto.event_signing import check_event_content_hash + +from synapse.api.errors import FederationError, SynapseError import logging @@ -97,8 +104,10 @@ class FederationServer(object): response = yield self.transaction_actions.have_responded(transaction) if response: - logger.debug("[%s] We've already responed to this request", - transaction.transaction_id) + logger.debug( + "[%s] We've already responed to this request", + transaction.transaction_id + ) defer.returnValue(response) return @@ -253,6 +262,9 @@ class FederationServer(object): origin, pdu.event_id, do_auth=False ) + # FIXME: Currently we fetch an event again when we already have it + # if it has been marked as an outlier. + already_seen = ( existing and ( not existing.internal_metadata.is_outlier() @@ -264,14 +276,27 @@ class FederationServer(object): defer.returnValue({}) return + # Check signature. + try: + pdu = yield self._check_sigs_and_hash(pdu) + except SynapseError as e: + raise FederationError( + "ERROR", + e.code, + e.msg, + affected=pdu.event_id, + ) + state = None auth_chain = [] have_seen = yield self.store.have_events( - [e for e, _ in pdu.prev_events] + [ev for ev, _ in pdu.prev_events] ) + fetch_state = False + # Get missing pdus if necessary. if not pdu.internal_metadata.is_outlier(): # We only backfill backwards to the min depth. @@ -311,16 +336,20 @@ class FederationServer(object): except: # TODO(erikj): Do some more intelligent retries. logger.exception("Failed to get PDU") - else: - # We need to get the state at this event, since we have reached - # a backward extremity edge. - logger.debug( - "_handle_new_pdu getting state for %s", - pdu.room_id - ) - state, auth_chain = yield self.get_state_for_room( - origin, pdu.room_id, pdu.event_id, - ) + fetch_state = True + else: + fetch_state = True + + if fetch_state: + # We need to get the state at this event, since we haven't + # processed all the prev events. + logger.debug( + "_handle_new_pdu getting state for %s", + pdu.room_id + ) + state, auth_chain = yield self.get_state_for_room( + origin, pdu.room_id, pdu.event_id, + ) ret = yield self.handler.on_receive_pdu( origin, @@ -343,3 +372,37 @@ class FederationServer(object): event.internal_metadata.outlier = outlier return event + + @defer.inlineCallbacks + def _check_sigs_and_hash(self, pdu): + """Throws a SynapseError if the PDU does not have the correct + signatures. + + Returns: + FrozenEvent: Either the given event or it redacted if it failed the + content hash check. + """ + # Check signatures are correct. + redacted_event = prune_event(pdu) + redacted_pdu_json = redacted_event.get_pdu_json() + + try: + yield self.keyring.verify_json_for_server( + pdu.origin, redacted_pdu_json + ) + except SynapseError: + logger.warn( + "Signature check failed for %s redacted to %s", + encode_canonical_json(pdu.get_pdu_json()), + encode_canonical_json(redacted_pdu_json), + ) + raise + + if not check_event_content_hash(pdu): + logger.warn( + "Event content has been tampered, redacting %s, %s", + pdu.event_id, encode_canonical_json(pdu.get_dict()) + ) + defer.returnValue(redacted_event) + + defer.returnValue(pdu) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 9ef4834927..e442c6c5d5 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -51,6 +51,8 @@ class ReplicationLayer(FederationClient, FederationServer): def __init__(self, hs, transport_layer): self.server_name = hs.hostname + self.keyring = hs.get_keyring() + self.transport_layer = transport_layer self.transport_layer.register_received_handler(self) self.transport_layer.register_request_handler(self) From 8d7accb28fdf2b8936eb865a716593a2ffcea0ed Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Fri, 23 Jan 2015 19:52:30 +0000 Subject: [PATCH 046/180] Initial minimal attempt at /user/:user_id/filter API - in-memory storage, no actual filter implementation --- synapse/rest/client/v2_alpha/__init__.py | 7 +- synapse/rest/client/v2_alpha/filter.py | 103 +++++++++++++++++++++++ 2 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 synapse/rest/client/v2_alpha/filter.py diff --git a/synapse/rest/client/v2_alpha/__init__.py b/synapse/rest/client/v2_alpha/__init__.py index bb740e2803..4349579ab7 100644 --- a/synapse/rest/client/v2_alpha/__init__.py +++ b/synapse/rest/client/v2_alpha/__init__.py @@ -14,6 +14,11 @@ # limitations under the License. +from . import ( + filter +) + + from synapse.http.server import JsonResource @@ -26,4 +31,4 @@ class ClientV2AlphaRestResource(JsonResource): @staticmethod def register_servlets(client_resource, hs): - pass + filter.register_servlets(hs, client_resource) diff --git a/synapse/rest/client/v2_alpha/filter.py b/synapse/rest/client/v2_alpha/filter.py new file mode 100644 index 0000000000..a9a180ec04 --- /dev/null +++ b/synapse/rest/client/v2_alpha/filter.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from synapse.api.errors import AuthError, SynapseError +from synapse.http.servlet import RestServlet +from synapse.types import UserID + +from ._base import client_v2_pattern + +import json +import logging + + +logger = logging.getLogger(__name__) + + +# TODO(paul) +_filters_for_user = {} + + +class GetFilterRestServlet(RestServlet): + PATTERN = client_v2_pattern("/user/(?P[^/]*)/filter/(?P[^/]*)") + + def __init__(self, hs): + super(GetFilterRestServlet, self).__init__() + self.hs = hs + self.auth = hs.get_auth() + + @defer.inlineCallbacks + def on_GET(self, request, user_id, filter_id): + target_user = UserID.from_string(user_id) + auth_user = yield self.auth.get_user_by_req(request) + + if target_user != auth_user: + raise AuthError(403, "Cannot get filters for other users") + + if not self.hs.is_mine(target_user): + raise SynapseError(400, "Can only get filters for local users") + + try: + filter_id = int(filter_id) + except: + raise SynapseError(400, "Invalid filter_id") + + filters = _filters_for_user.get(target_user.localpart, None) + + if not filters or filter_id >= len(filters): + raise SynapseError(400, "No such filter") + + defer.returnValue((200, filters[filter_id])) + + +class CreateFilterRestServlet(RestServlet): + PATTERN = client_v2_pattern("/user/(?P[^/]*)/filter") + + def __init__(self, hs): + super(CreateFilterRestServlet, self).__init__() + self.hs = hs + self.auth = hs.get_auth() + + @defer.inlineCallbacks + def on_POST(self, request, user_id): + target_user = UserID.from_string(user_id) + auth_user = yield self.auth.get_user_by_req(request) + + if target_user != auth_user: + raise AuthError(403, "Cannot create filters for other users") + + if not self.hs.is_mine(target_user): + raise SynapseError(400, "Can only create filters for local users") + + try: + content = json.loads(request.content.read()) + + # TODO(paul): check for required keys and invalid keys + except: + raise SynapseError(400, "Invalid filter definition") + + filters = _filters_for_user.setdefault(target_user.localpart, []) + + filter_id = len(filters) + filters.append(content) + + defer.returnValue((200, {"filter_id": str(filter_id)})) + + +def register_servlets(hs, http_server): + GetFilterRestServlet(hs).register(http_server) + CreateFilterRestServlet(hs).register(http_server) From efac71d6caeac9ef39dab521467932892b324e99 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 26 Jan 2015 14:37:14 +0000 Subject: [PATCH 047/180] Pushers should only try & look for rejected devices in something that's a list or tuple. --- synapse/push/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 53d3319699..0c51d2dd8f 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -173,7 +173,7 @@ class Pusher(object): processed = True else: rejected = yield self.dispatch_push(single_event, tweaks) - if not rejected is False: + if isinstance(rejected, list) or isinstance(rejected, tuple): processed = True for pk in rejected: if pk != self.pushkey: From 37b8a71f1086b394874fb13ee63dcbeb2b2334d2 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 26 Jan 2015 15:27:40 +0000 Subject: [PATCH 048/180] Initial trivial REST test of v2_alpha filter API --- tests/rest/client/v2_alpha/test_filter.py | 74 +++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 tests/rest/client/v2_alpha/test_filter.py diff --git a/tests/rest/client/v2_alpha/test_filter.py b/tests/rest/client/v2_alpha/test_filter.py new file mode 100644 index 0000000000..1d1273ab9a --- /dev/null +++ b/tests/rest/client/v2_alpha/test_filter.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from tests import unittest +from twisted.internet import defer + +from mock import Mock + +from ....utils import MockHttpResource, MockKey + +from synapse.server import HomeServer +from synapse.rest.client.v2_alpha import filter +from synapse.types import UserID + + +myid = "@apple:test" +PATH_PREFIX = "/_matrix/client/v2_alpha" + + +class FilterTestCase(unittest.TestCase): + + def setUp(self): + self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) + + mock_config = Mock() + mock_config.signing_key = [MockKey()] + + hs = HomeServer("test", + db_pool=None, + datastore=Mock(spec=[ + "insert_client_ip", + ]), + http_client=None, + resource_for_client=self.mock_resource, + resource_for_federation=self.mock_resource, + config=mock_config, + ) + + def _get_user_by_token(token=None): + return { + "user": UserID.from_string(myid), + "admin": False, + "device_id": None, + } + hs.get_auth().get_user_by_token = _get_user_by_token + + filter.register_servlets(hs, self.mock_resource) + + @defer.inlineCallbacks + def test_filter(self): + (code, response) = yield self.mock_resource.trigger("POST", + "/user/%s/filter" % (myid), + '{"type": ["m.*"]}' + ) + self.assertEquals(200, code) + self.assertEquals({"filter_id": "0"}, response) + + (code, response) = yield self.mock_resource.trigger("GET", + "/user/%s/filter/0" % (myid), None + ) + self.assertEquals(200, code) + self.assertEquals({"type": ["m.*"]}, response) From 0cfb4591a7754bcc08edddd17629006b5096d94d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 26 Jan 2015 15:46:31 +0000 Subject: [PATCH 049/180] Add handler for /sync API --- synapse/handlers/sync.py | 110 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 synapse/handlers/sync.py diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py new file mode 100644 index 0000000000..ec20ea4890 --- /dev/null +++ b/synapse/handlers/sync.py @@ -0,0 +1,110 @@ +import collections + + +SyncConfig = collections.namedtuple("SyncConfig", [ + "user", + "device", + "since", + "limit", + "gap", + "sort" + "backfill" + "filter", +) + + +RoomSyncResult = collections.namedtuple("RoomSyncResult", [ + "room_id", + "limited", + "published", + "prev_batch", + "events", + "state", + "event_map", +]) + + +class SyncResult(collections.namedtuple("SyncResult", [ + "next_batch", # Token for the next sync + "private_user_data", # List of private events for the user. + "public_user_data", # List of public events for all users. + "rooms", # RoomSyncResult for each room. +])): + __slots__ = [] + + def __nonzero__(self): + return self.private_user_data or self.public_user_data or self.rooms + + +class SyncHandler(BaseHandler): + + def __init__(self, hs): + super(SyncHandler, self).__init__(hs) + self.event_sources = hs.get_event_sources() + + def wait_for_sync_for_user(self, sync_config, since_token=None, timeout=0): + if timeout == 0: + return self.current_sync_for_user(sync_config, since) + else: + def current_sync_callback(since_token): + return self.current_sync_for_user( + self, since_token, sync_config + ) + return self.notifier.wait_for_events( + sync_config.filter, since_token, current_sync_callback + ) + defer.returnValue(result) + + def current_sync_for_user(self, sync_config, since_token=None): + if since_token is None: + return self.inital_sync(sync_config) + else: + return self.incremental_sync(sync_config) + + @defer.inlineCallbacks + def initial_sync(self, sync_config): + now_token = yield self.event_sources.get_current_token() + + presence_stream = self.event_sources.sources["presence"] + # TODO (markjh): This looks wrong, shouldn't we be getting the presence + # UP to the present rather than after the present? + pagination_config = PaginationConfig(from_token=now_token) + presence, _ = yield presence_stream.get_pagination_rows( + user, pagination_config.get_source_config("presence"), None + ) + room_list = yield self.store.get_rooms_for_user_where_membership_is( + user_id=user_id, + membership_list=[Membership.INVITE, Membership.JOIN] + ) + + # TODO (markjh): Does public mean "published"? + published_rooms = yield self.store.get_rooms(is_public=True) + published_room_ids = set(r["room_id"] for r in public_rooms) + + for event in room_list: + + messages, token = yield self.store.get_recent_events_for_room( + event.room_id, + limit=sync_config.limit, + end_token=now_token.room_key, + ) + prev_batch_token = now_token.copy_and_replace("room_key", token[0]) + current_state = yield self.state_handler.get_current_state( + event.room_id + ) + + rooms.append(RoomSyncResult( + room_id=event.room_id, + published=event.room_id in published_room_ids, + + + + + + @defer.inlineCallbacks + def incremental_sync(self, sync_config): + + + + + From 69a75b7ebebb393c1ce84ff949f3480a6af0a782 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 26 Jan 2015 16:52:47 +0000 Subject: [PATCH 050/180] Add brackets to make get room name / alias work --- synapse/storage/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 8f56d90d95..2534d109fd 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -402,8 +402,8 @@ class DataStore(RoomMemberStore, RoomStore, "redacted": del_sql, } - sql += " AND (s.type = 'm.room.name' AND s.state_key = '')" - sql += " OR s.type = 'm.room.aliases'" + sql += " AND ((s.type = 'm.room.name' AND s.state_key = '')" + sql += " OR s.type = 'm.room.aliases')" args = (room_id,) results = yield self._execute_and_decode(sql, *args) From b481889117ceb8b31dd2e491bca53b914d306312 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 26 Jan 2015 17:27:28 +0000 Subject: [PATCH 051/180] Support membership events and more camelcase/underscores --- synapse/push/__init__.py | 4 ++++ synapse/push/httppusher.py | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 0c51d2dd8f..b6d01a82a0 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -65,6 +65,10 @@ class Pusher(object): # let's assume you probably know about messages you sent yourself defer.returnValue(['dont_notify']) + if ev['type'] == 'm.room.member': + if ev['state_key'] != self.user_name: + defer.returnValue(['dont_notify']) + rules = yield self.store.get_push_rules_for_user_name(self.user_name) for r in rules: diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 25db1dded5..22532fcc6a 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -65,8 +65,6 @@ class HttpPusher(Pusher): d = { 'notification': { - 'transition': 'new', - # everything is new for now: we don't have read receipts 'id': event['event_id'], 'type': event['type'], 'from': event['user_id'], @@ -89,11 +87,13 @@ class HttpPusher(Pusher): ] } } + if event['type'] == 'm.room.member': + d['notification']['membership'] = event['content']['membership'] if len(ctx['aliases']): - d['notification']['roomAlias'] = ctx['aliases'][0] + d['notification']['room_alias'] = ctx['aliases'][0] if 'name' in ctx: - d['notification']['roomName'] = ctx['name'] + d['notification']['room_name'] = ctx['name'] defer.returnValue(d) From 436513068de73ab47d9ba9a32046420be3d86588 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 26 Jan 2015 18:53:31 +0000 Subject: [PATCH 052/180] Start implementing the non-incremental sync portion of the v2 /sync API --- synapse/events/utils.py | 6 +- synapse/handlers/__init__.py | 2 + synapse/handlers/sync.py | 87 +++++++++++++++------- synapse/rest/client/v2_alpha/sync.py | 103 ++++++++++++++++++++------- 4 files changed, 145 insertions(+), 53 deletions(-) diff --git a/synapse/events/utils.py b/synapse/events/utils.py index e391aca4cc..b7f1ad4b40 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -89,7 +89,7 @@ def prune_event(event): return type(event)(allowed_fields) -def serialize_event(e, time_now_ms, client_event=True): +def serialize_event(e, time_now_ms, client_event=True, strip_ids=False): # FIXME(erikj): To handle the case of presence events and the like if not isinstance(e, EventBase): return e @@ -138,4 +138,8 @@ def serialize_event(e, time_now_ms, client_event=True): d.pop("unsigned", None) d.pop("origin", None) + if strip_ids: + d.pop("room_id", None) + d.pop("event_id", None) + return d diff --git a/synapse/handlers/__init__.py b/synapse/handlers/__init__.py index fe071a4bc2..a32eab9316 100644 --- a/synapse/handlers/__init__.py +++ b/synapse/handlers/__init__.py @@ -26,6 +26,7 @@ from .presence import PresenceHandler from .directory import DirectoryHandler from .typing import TypingNotificationHandler from .admin import AdminHandler +from .sync import SyncHandler class Handlers(object): @@ -51,3 +52,4 @@ class Handlers(object): self.directory_handler = DirectoryHandler(hs) self.typing_notification_handler = TypingNotificationHandler(hs) self.admin_handler = AdminHandler(hs) + self.sync_handler = SyncHandler(hs) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index ec20ea4890..bbabaf3df1 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -1,26 +1,49 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ._base import BaseHandler + +from synapse.streams.config import PaginationConfig +from synapse.api.constants import Membership + +from twisted.internet import defer + import collections +import logging + +logger = logging.getLogger(__name__) SyncConfig = collections.namedtuple("SyncConfig", [ "user", "device", - "since", "limit", "gap", - "sort" - "backfill" + "sort", + "backfill", "filter", -) +]) RoomSyncResult = collections.namedtuple("RoomSyncResult", [ "room_id", "limited", "published", - "prev_batch", - "events", + "events", # dict of event "state", - "event_map", + "prev_batch", ]) @@ -41,10 +64,11 @@ class SyncHandler(BaseHandler): def __init__(self, hs): super(SyncHandler, self).__init__(hs) self.event_sources = hs.get_event_sources() + self.clock = hs.get_clock() def wait_for_sync_for_user(self, sync_config, since_token=None, timeout=0): if timeout == 0: - return self.current_sync_for_user(sync_config, since) + return self.current_sync_for_user(sync_config, since_token) else: def current_sync_callback(since_token): return self.current_sync_for_user( @@ -53,58 +77,71 @@ class SyncHandler(BaseHandler): return self.notifier.wait_for_events( sync_config.filter, since_token, current_sync_callback ) - defer.returnValue(result) def current_sync_for_user(self, sync_config, since_token=None): if since_token is None: - return self.inital_sync(sync_config) + return self.initial_sync(sync_config) else: return self.incremental_sync(sync_config) @defer.inlineCallbacks def initial_sync(self, sync_config): + if sync_config.sort == "timeline,desc": + # TODO(mjark): Handle going through events in reverse order?. + # What does "most recent events" mean when applying the limits mean + # in this case? + raise NotImplementedError() + now_token = yield self.event_sources.get_current_token() presence_stream = self.event_sources.sources["presence"] - # TODO (markjh): This looks wrong, shouldn't we be getting the presence + # TODO (mjark): This looks wrong, shouldn't we be getting the presence # UP to the present rather than after the present? pagination_config = PaginationConfig(from_token=now_token) presence, _ = yield presence_stream.get_pagination_rows( - user, pagination_config.get_source_config("presence"), None + user=sync_config.user, + pagination_config=pagination_config.get_source_config("presence"), + key=None ) room_list = yield self.store.get_rooms_for_user_where_membership_is( - user_id=user_id, + user_id=sync_config.user.to_string(), membership_list=[Membership.INVITE, Membership.JOIN] ) - # TODO (markjh): Does public mean "published"? + # TODO (mjark): Does public mean "published"? published_rooms = yield self.store.get_rooms(is_public=True) - published_room_ids = set(r["room_id"] for r in public_rooms) + published_room_ids = set(r["room_id"] for r in published_rooms) + rooms = [] for event in room_list: - - messages, token = yield self.store.get_recent_events_for_room( + #TODO (mjark): Apply the event filter in sync_config. + recent_events, token = yield self.store.get_recent_events_for_room( event.room_id, limit=sync_config.limit, end_token=now_token.room_key, ) prev_batch_token = now_token.copy_and_replace("room_key", token[0]) - current_state = yield self.state_handler.get_current_state( + current_state_events = yield self.state_handler.get_current_state( event.room_id ) rooms.append(RoomSyncResult( room_id=event.room_id, published=event.room_id in published_room_ids, + events=recent_events, + prev_batch=prev_batch_token, + state=current_state_events, + limited=True, + )) - - + defer.returnValue(SyncResult( + public_user_data=presence, + private_user_data=[], + rooms=rooms, + next_batch=now_token, + )) @defer.inlineCallbacks def incremental_sync(self, sync_config): - - - - - + pass diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 39bb5ec8e9..cc667ebafc 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd +# Copyright 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,6 +16,9 @@ from twisted.internet import defer from synapse.http.servlet import RestServlet +from synapse.handlers.sync import SyncConfig +from synapse.types import StreamToken +from synapse.events.utils import serialize_event from ._base import client_v2_pattern import logging @@ -73,14 +76,15 @@ class SyncRestServlet(RestServlet): def __init__(self, hs): super(SyncRestServlet, self).__init__() self.auth = hs.get_auth() - #self.sync_handler = hs.get_handlers().sync_hanlder + self.sync_handler = hs.get_handlers().sync_handler + self.clock = hs.get_clock() @defer.inlineCallbacks def on_GET(self, request): user = yield self.auth.get_user_by_req(request) timeout = self.parse_integer(request, "timeout", default=0) - limit = self.parse_integer(request, "limit", default=None) + limit = self.parse_integer(request, "limit", required=True) gap = self.parse_boolean(request, "gap", default=True) sort = self.parse_string( request, "sort", default="timeline,asc", @@ -91,7 +95,7 @@ class SyncRestServlet(RestServlet): request, "set_presence", default="online", allowed_values=self.ALLOWED_PRESENCE ) - backfill = self.parse_boolean(request, "backfill", default=True) + backfill = self.parse_boolean(request, "backfill", default=False) filter_id = self.parse_string(request, "filter", default=None) logger.info( @@ -108,36 +112,81 @@ class SyncRestServlet(RestServlet): # if filter.matches(event): # # stuff - # if timeout != 0: - # register for updates from the event stream + sync_config = SyncConfig( + user=user, + device="TODO", # TODO(mjark) Get the device_id from access_token + gap=gap, + limit=limit, + sort=sort, + backfill=backfill, + filter="TODO", # TODO(mjark) Add the filter to the config. + ) - #rooms = [] - - if gap: - pass - # now_stream_token = get_current_stream_token - # for room_id in get_rooms_for_user(user, filter=filter): - # state, events, start, end, limited, published = updates_for_room( - # from=since, to=now_stream_token, limit=limit, - # anchor_to_start=False - # ) - # rooms[room_id] = (state, events, start, limited, published) - # next_stream_token = now. + if since is not None: + since_token = StreamToken.from_string(since) else: - pass - # now_stream_token = get_current_stream_token - # for room_id in get_rooms_for_user(user, filter=filter) - # state, events, start, end, limited, published = updates_for_room( - # from=since, to=now_stream_token, limit=limit, - # anchor_to_start=False - # ) - # next_stream_token = min(next_stream_token, end) + since_token = None + sync_result = yield self.sync_handler.wait_for_sync_for_user( + sync_config, since_token=since_token, timeout=timeout + ) - response_content = {} + time_now = self.clock.time_msec() + + response_content = { + "public_user_data": self.encode_events( + sync_result.public_user_data, filter, time_now + ), + "private_user_data": self.encode_events( + sync_result.private_user_data, filter, time_now + ), + "rooms": self.encode_rooms(sync_result.rooms, filter, time_now), + "next_batch": sync_result.next_batch.to_string(), + } defer.returnValue((200, response_content)) + def encode_events(self, events, filter, time_now): + return [self.encode_event(event, filter, time_now) for event in events] + + @staticmethod + def encode_event(event, filter, time_now): + # TODO(mjark): Respect formatting requirements in the filter. + return serialize_event(event, time_now) + + def encode_rooms(self, rooms, filter, time_now): + return [self.encode_room(room, filter, time_now) for room in rooms] + + @staticmethod + def encode_room(room, filter, time_now): + event_map = {} + state_event_ids = [] + recent_event_ids = [] + for event in room.state: + # TODO(mjark): Respect formatting requirements in the filter. + event_map[event.event_id] = serialize_event( + event, time_now, strip_ids=True + ) + state_event_ids.append(event.event_id) + + for event in room.events: + # TODO(mjark): Respect formatting requirements in the filter. + event_map[event.event_id] = serialize_event( + event, time_now, strip_ids=True + ) + recent_event_ids.append(event.event_id) + return { + "room_id": room.room_id, + "event_map": event_map, + "events": { + "batch": recent_event_ids, + "prev_batch": room.prev_batch.to_string(), + }, + "state": state_event_ids, + "limited": room.limited, + "published": room.published, + } + def register_servlets(hs, http_server): SyncRestServlet(hs).register(http_server) From 39c1892b22012e20ce6f43e92b01f6fad780081d Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 27 Jan 2015 13:03:31 +0000 Subject: [PATCH 053/180] Minor changes to v2_alpha filter REST test to allow the setUp method to be shareable --- tests/rest/client/v2_alpha/test_filter.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/rest/client/v2_alpha/test_filter.py b/tests/rest/client/v2_alpha/test_filter.py index 1d1273ab9a..91b19e88ff 100644 --- a/tests/rest/client/v2_alpha/test_filter.py +++ b/tests/rest/client/v2_alpha/test_filter.py @@ -25,11 +25,12 @@ from synapse.rest.client.v2_alpha import filter from synapse.types import UserID -myid = "@apple:test" PATH_PREFIX = "/_matrix/client/v2_alpha" class FilterTestCase(unittest.TestCase): + USER_ID = "@apple:test" + TO_REGISTER = [filter] def setUp(self): self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) @@ -50,25 +51,26 @@ class FilterTestCase(unittest.TestCase): def _get_user_by_token(token=None): return { - "user": UserID.from_string(myid), + "user": UserID.from_string(self.USER_ID), "admin": False, "device_id": None, } hs.get_auth().get_user_by_token = _get_user_by_token - filter.register_servlets(hs, self.mock_resource) + for r in self.TO_REGISTER: + r.register_servlets(hs, self.mock_resource) @defer.inlineCallbacks def test_filter(self): (code, response) = yield self.mock_resource.trigger("POST", - "/user/%s/filter" % (myid), + "/user/%s/filter" % (self.USER_ID), '{"type": ["m.*"]}' ) self.assertEquals(200, code) self.assertEquals({"filter_id": "0"}, response) (code, response) = yield self.mock_resource.trigger("GET", - "/user/%s/filter/0" % (myid), None + "/user/%s/filter/0" % (self.USER_ID), None ) self.assertEquals(200, code) self.assertEquals({"type": ["m.*"]}, response) From f9958f34043bf4fdf5923f471f193b40188e67bb Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 27 Jan 2015 13:17:25 +0000 Subject: [PATCH 054/180] Use new V2AlphaRestTestCase --- tests/rest/client/v2_alpha/test_filter.py | 40 ++--------------------- 1 file changed, 2 insertions(+), 38 deletions(-) diff --git a/tests/rest/client/v2_alpha/test_filter.py b/tests/rest/client/v2_alpha/test_filter.py index 91b19e88ff..8629a1aed6 100644 --- a/tests/rest/client/v2_alpha/test_filter.py +++ b/tests/rest/client/v2_alpha/test_filter.py @@ -13,53 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -from tests import unittest from twisted.internet import defer -from mock import Mock +from . import V2AlphaRestTestCase -from ....utils import MockHttpResource, MockKey - -from synapse.server import HomeServer from synapse.rest.client.v2_alpha import filter -from synapse.types import UserID -PATH_PREFIX = "/_matrix/client/v2_alpha" - - -class FilterTestCase(unittest.TestCase): +class FilterTestCase(V2AlphaRestTestCase): USER_ID = "@apple:test" TO_REGISTER = [filter] - def setUp(self): - self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) - - mock_config = Mock() - mock_config.signing_key = [MockKey()] - - hs = HomeServer("test", - db_pool=None, - datastore=Mock(spec=[ - "insert_client_ip", - ]), - http_client=None, - resource_for_client=self.mock_resource, - resource_for_federation=self.mock_resource, - config=mock_config, - ) - - def _get_user_by_token(token=None): - return { - "user": UserID.from_string(self.USER_ID), - "admin": False, - "device_id": None, - } - hs.get_auth().get_user_by_token = _get_user_by_token - - for r in self.TO_REGISTER: - r.register_servlets(hs, self.mock_resource) - @defer.inlineCallbacks def test_filter(self): (code, response) = yield self.mock_resource.trigger("POST", From 05c7cba73a050f19cc52129b65b0183eaa832a42 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 27 Jan 2015 14:28:56 +0000 Subject: [PATCH 055/180] Initial trivial implementation of an actual 'Filtering' object; move storage of user filters into there --- synapse/api/filtering.py | 41 ++++++++++++++++++++++++++ synapse/rest/client/v2_alpha/filter.py | 25 ++++++++-------- synapse/server.py | 5 ++++ 3 files changed, 58 insertions(+), 13 deletions(-) create mode 100644 synapse/api/filtering.py diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py new file mode 100644 index 0000000000..922c40004c --- /dev/null +++ b/synapse/api/filtering.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# TODO(paul) +_filters_for_user = {} + + +class Filtering(object): + + def __init__(self, hs): + super(Filtering, self).__init__() + self.hs = hs + + def get_user_filter(self, user_localpart, filter_id): + filters = _filters_for_user.get(user_localpart, None) + + if not filters or filter_id >= len(filters): + raise KeyError() + + return filters[filter_id] + + def add_user_filter(self, user_localpart, definition): + filters = _filters_for_user.setdefault(user_localpart, []) + + filter_id = len(filters) + filters.append(definition) + + return filter_id diff --git a/synapse/rest/client/v2_alpha/filter.py b/synapse/rest/client/v2_alpha/filter.py index a9a180ec04..585c8e02e8 100644 --- a/synapse/rest/client/v2_alpha/filter.py +++ b/synapse/rest/client/v2_alpha/filter.py @@ -28,10 +28,6 @@ import logging logger = logging.getLogger(__name__) -# TODO(paul) -_filters_for_user = {} - - class GetFilterRestServlet(RestServlet): PATTERN = client_v2_pattern("/user/(?P[^/]*)/filter/(?P[^/]*)") @@ -39,6 +35,7 @@ class GetFilterRestServlet(RestServlet): super(GetFilterRestServlet, self).__init__() self.hs = hs self.auth = hs.get_auth() + self.filtering = hs.get_filtering() @defer.inlineCallbacks def on_GET(self, request, user_id, filter_id): @@ -56,13 +53,14 @@ class GetFilterRestServlet(RestServlet): except: raise SynapseError(400, "Invalid filter_id") - filters = _filters_for_user.get(target_user.localpart, None) - - if not filters or filter_id >= len(filters): + try: + defer.returnValue((200, self.filtering.get_user_filter( + user_localpart=target_user.localpart, + filter_id=filter_id, + ))) + except KeyError: raise SynapseError(400, "No such filter") - defer.returnValue((200, filters[filter_id])) - class CreateFilterRestServlet(RestServlet): PATTERN = client_v2_pattern("/user/(?P[^/]*)/filter") @@ -71,6 +69,7 @@ class CreateFilterRestServlet(RestServlet): super(CreateFilterRestServlet, self).__init__() self.hs = hs self.auth = hs.get_auth() + self.filtering = hs.get_filtering() @defer.inlineCallbacks def on_POST(self, request, user_id): @@ -90,10 +89,10 @@ class CreateFilterRestServlet(RestServlet): except: raise SynapseError(400, "Invalid filter definition") - filters = _filters_for_user.setdefault(target_user.localpart, []) - - filter_id = len(filters) - filters.append(content) + filter_id = self.filtering.add_user_filter( + user_localpart=target_user.localpart, + definition=content, + ) defer.returnValue((200, {"filter_id": str(filter_id)})) diff --git a/synapse/server.py b/synapse/server.py index f09d5d581e..9b42079e05 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -32,6 +32,7 @@ from synapse.streams.events import EventSources from synapse.api.ratelimiting import Ratelimiter from synapse.crypto.keyring import Keyring from synapse.events.builder import EventBuilderFactory +from synapse.api.filtering import Filtering class BaseHomeServer(object): @@ -79,6 +80,7 @@ class BaseHomeServer(object): 'ratelimiter', 'keyring', 'event_builder_factory', + 'filtering', ] def __init__(self, hostname, **kwargs): @@ -197,3 +199,6 @@ class HomeServer(BaseHomeServer): clock=self.get_clock(), hostname=self.hostname, ) + + def build_filtering(self): + return Filtering(self) From b1503112ce77e573aa8cfb7581ca4a916c7d018c Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 27 Jan 2015 15:56:14 +0000 Subject: [PATCH 056/180] Initial trivial unittest of Filtering object --- tests/api/test_filtering.py | 67 +++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 tests/api/test_filtering.py diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py new file mode 100644 index 0000000000..c6c5317696 --- /dev/null +++ b/tests/api/test_filtering.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from tests import unittest +from twisted.internet import defer + +from mock import Mock, NonCallableMock +from tests.utils import ( + MockHttpResource, MockClock, DeferredMockCallable, SQLiteMemoryDbPool, + MockKey +) + +from synapse.server import HomeServer + + +user_localpart = "test_user" + +class FilteringTestCase(unittest.TestCase): + + @defer.inlineCallbacks + def setUp(self): + db_pool = SQLiteMemoryDbPool() + yield db_pool.prepare() + + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + + self.mock_federation_resource = MockHttpResource() + + self.mock_http_client = Mock(spec=[]) + self.mock_http_client.put_json = DeferredMockCallable() + + hs = HomeServer("test", + db_pool=db_pool, + handlers=None, + http_client=self.mock_http_client, + config=self.mock_config, + keyring=Mock(), + ) + + self.filtering = hs.get_filtering() + + def test_filter(self): + filter_id = self.filtering.add_user_filter( + user_localpart=user_localpart, + definition={"type": ["m.*"]}, + ) + self.assertEquals(filter_id, 0) + + filter = self.filtering.get_user_filter( + user_localpart=user_localpart, + filter_id=filter_id, + ) + self.assertEquals(filter, {"type": ["m.*"]}) From 1d779691248be2fcdd43e97352d13c2cef239c10 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 27 Jan 2015 15:58:27 +0000 Subject: [PATCH 057/180] Unbreak bad presence merge - don't add these blocks together with an and: they're different things. --- synapse/handlers/events.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index 75fb941008..f1a3e4a4ad 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -51,19 +51,21 @@ class EventStreamHandler(BaseHandler): auth_user = self.hs.parse_userid(auth_user_id) try: - if affect_presence and auth_user not in self._streams_per_user: - self._streams_per_user[auth_user] = 0 - if auth_user in self._stop_timer_per_user: - try: - self.clock.cancel_call_later( - self._stop_timer_per_user.pop(auth_user) + if affect_presence: + if auth_user not in self._streams_per_user: + self._streams_per_user[auth_user] = 0 + if auth_user in self._stop_timer_per_user: + try: + print "cancel",auth_user + self.clock.cancel_call_later( + self._stop_timer_per_user.pop(auth_user) + ) + except: + logger.exception("Failed to cancel event timer") + else: + yield self.distributor.fire( + "started_user_eventstream", auth_user ) - except: - logger.exception("Failed to cancel event timer") - else: - yield self.distributor.fire( - "started_user_eventstream", auth_user - ) self._streams_per_user[auth_user] += 1 if pagin_config.from_token is None: From eba89f093f0376b8499953b53bbd85f9b12b7852 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 27 Jan 2015 16:00:07 +0000 Subject: [PATCH 058/180] Need a defer.inlineCallbacks here as we yield in it: otherwise nothing in the cb gets executed. --- synapse/handlers/events.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index f1a3e4a4ad..851eebf600 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -99,6 +99,7 @@ class EventStreamHandler(BaseHandler): # 10 seconds of grace to allow the client to reconnect again # before we think they're gone + @defer.inlineCallbacks def _later(): logger.debug( "_later stopped_user_eventstream %s", auth_user From 5eacaeb4a73aa4c4ff450d7b23acabe79124171f Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 27 Jan 2015 16:05:23 +0000 Subject: [PATCH 059/180] or of course we could just return the deferred --- synapse/handlers/events.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index 851eebf600..48de3630e3 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -99,7 +99,6 @@ class EventStreamHandler(BaseHandler): # 10 seconds of grace to allow the client to reconnect again # before we think they're gone - @defer.inlineCallbacks def _later(): logger.debug( "_later stopped_user_eventstream %s", auth_user @@ -107,7 +106,7 @@ class EventStreamHandler(BaseHandler): self._stop_timer_per_user.pop(auth_user, None) - yield self.distributor.fire( + return self.distributor.fire( "stopped_user_eventstream", auth_user ) From f7c4daa8f94e5968277ba438f6d3da1f0f27ba4c Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 27 Jan 2015 16:08:47 +0000 Subject: [PATCH 060/180] Oops, remove debugging --- synapse/handlers/events.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index 48de3630e3..52ef07eaae 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -56,7 +56,6 @@ class EventStreamHandler(BaseHandler): self._streams_per_user[auth_user] = 0 if auth_user in self._stop_timer_per_user: try: - print "cancel",auth_user self.clock.cancel_call_later( self._stop_timer_per_user.pop(auth_user) ) From 059651efa19a88eb0823bce1d5beff2d95cb01c2 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 27 Jan 2015 16:17:56 +0000 Subject: [PATCH 061/180] Have the Filtering API return Deferreds, so we can do the Datastore implementation nicely --- synapse/api/filtering.py | 16 ++++++++++++++-- synapse/rest/client/v2_alpha/filter.py | 8 +++++--- tests/api/test_filtering.py | 5 +++-- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 922c40004c..014e2e1fc9 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from twisted.internet import defer + # TODO(paul) _filters_for_user = {} @@ -24,18 +26,28 @@ class Filtering(object): super(Filtering, self).__init__() self.hs = hs + @defer.inlineCallbacks def get_user_filter(self, user_localpart, filter_id): filters = _filters_for_user.get(user_localpart, None) if not filters or filter_id >= len(filters): raise KeyError() - return filters[filter_id] + # trivial yield to make it a generator so d.iC works + yield + defer.returnValue(filters[filter_id]) + @defer.inlineCallbacks def add_user_filter(self, user_localpart, definition): filters = _filters_for_user.setdefault(user_localpart, []) filter_id = len(filters) filters.append(definition) - return filter_id + # trivial yield, see above + yield + defer.returnValue(filter_id) + + # TODO(paul): surely we should probably add a delete_user_filter or + # replace_user_filter at some point? There's no REST API specified for + # them however diff --git a/synapse/rest/client/v2_alpha/filter.py b/synapse/rest/client/v2_alpha/filter.py index 585c8e02e8..09e44e8ae0 100644 --- a/synapse/rest/client/v2_alpha/filter.py +++ b/synapse/rest/client/v2_alpha/filter.py @@ -54,10 +54,12 @@ class GetFilterRestServlet(RestServlet): raise SynapseError(400, "Invalid filter_id") try: - defer.returnValue((200, self.filtering.get_user_filter( + filter = yield self.filtering.get_user_filter( user_localpart=target_user.localpart, filter_id=filter_id, - ))) + ) + + defer.returnValue((200, filter)) except KeyError: raise SynapseError(400, "No such filter") @@ -89,7 +91,7 @@ class CreateFilterRestServlet(RestServlet): except: raise SynapseError(400, "Invalid filter definition") - filter_id = self.filtering.add_user_filter( + filter_id = yield self.filtering.add_user_filter( user_localpart=target_user.localpart, definition=content, ) diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py index c6c5317696..fecadd1056 100644 --- a/tests/api/test_filtering.py +++ b/tests/api/test_filtering.py @@ -53,14 +53,15 @@ class FilteringTestCase(unittest.TestCase): self.filtering = hs.get_filtering() + @defer.inlineCallbacks def test_filter(self): - filter_id = self.filtering.add_user_filter( + filter_id = yield self.filtering.add_user_filter( user_localpart=user_localpart, definition={"type": ["m.*"]}, ) self.assertEquals(filter_id, 0) - filter = self.filtering.get_user_filter( + filter = yield self.filtering.get_user_filter( user_localpart=user_localpart, filter_id=filter_id, ) From a56008842b43089433768f569f35b2d14523ac39 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 27 Jan 2015 16:24:22 +0000 Subject: [PATCH 062/180] Start implementing incremental initial sync --- synapse/events/utils.py | 1 + synapse/handlers/sync.py | 233 ++++++++++++++++++++++++++++++++++---- synapse/storage/stream.py | 41 +++++-- 3 files changed, 241 insertions(+), 34 deletions(-) diff --git a/synapse/events/utils.py b/synapse/events/utils.py index b7f1ad4b40..42fb0371e5 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -137,6 +137,7 @@ def serialize_event(e, time_now_ms, client_event=True, strip_ids=False): d.pop("depth", None) d.pop("unsigned", None) d.pop("origin", None) + d.pop("prev_state", None) if strip_ids: d.pop("room_id", None) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index bbabaf3df1..f8629a588f 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -37,14 +37,18 @@ SyncConfig = collections.namedtuple("SyncConfig", [ ]) -RoomSyncResult = collections.namedtuple("RoomSyncResult", [ +class RoomSyncResult(collections.namedtuple("RoomSyncResult", [ "room_id", "limited", "published", - "events", # dict of event + "events", "state", "prev_batch", -]) +])): + __slots__ = [] + + def __nonzero__(self): + return bool(self.events or self.state) class SyncResult(collections.namedtuple("SyncResult", [ @@ -56,7 +60,9 @@ class SyncResult(collections.namedtuple("SyncResult", [ __slots__ = [] def __nonzero__(self): - return self.private_user_data or self.public_user_data or self.rooms + return bool( + self.private_user_data or self.public_user_data or self.rooms + ) class SyncHandler(BaseHandler): @@ -67,7 +73,13 @@ class SyncHandler(BaseHandler): self.clock = hs.get_clock() def wait_for_sync_for_user(self, sync_config, since_token=None, timeout=0): - if timeout == 0: + """Get the sync for a client if we have new data for it now. Otherwise + wait for new data to arrive on the server. If the timeout expires, then + return an empty sync result. + Returns: + A Deferred SyncResult. + """ + if timeout == 0 or since_token is None: return self.current_sync_for_user(sync_config, since_token) else: def current_sync_callback(since_token): @@ -79,13 +91,25 @@ class SyncHandler(BaseHandler): ) def current_sync_for_user(self, sync_config, since_token=None): + """Get the sync for client needed to match what the server has now. + Returns: + A Deferred SyncResult. + """ if since_token is None: return self.initial_sync(sync_config) else: - return self.incremental_sync(sync_config) + if sync_config.gap: + return self.incremental_sync_with_gap(sync_config, since_token) + else: + #TODO(mjark): Handle gapless sync + pass @defer.inlineCallbacks def initial_sync(self, sync_config): + """Get a sync for a client which is starting without any state + Returns: + A Deferred SyncResult. + """ if sync_config.sort == "timeline,desc": # TODO(mjark): Handle going through events in reverse order?. # What does "most recent events" mean when applying the limits mean @@ -114,25 +138,86 @@ class SyncHandler(BaseHandler): rooms = [] for event in room_list: - #TODO (mjark): Apply the event filter in sync_config. - recent_events, token = yield self.store.get_recent_events_for_room( - event.room_id, - limit=sync_config.limit, - end_token=now_token.room_key, - ) - prev_batch_token = now_token.copy_and_replace("room_key", token[0]) - current_state_events = yield self.state_handler.get_current_state( - event.room_id + room_sync = yield self.initial_sync_for_room( + event.room_id, sync_config, now_token, published_room_ids ) + rooms.append(room_sync) - rooms.append(RoomSyncResult( - room_id=event.room_id, - published=event.room_id in published_room_ids, - events=recent_events, - prev_batch=prev_batch_token, - state=current_state_events, - limited=True, - )) + defer.returnValue(SyncResult( + public_user_data=presence, + private_user_data=[], + rooms=rooms, + next_batch=now_token, + )) + + @defer.inlineCallbacks + def intial_sync_for_room(self, room_id, sync_config, now_token, + published_room_ids): + """Sync a room for a client which is starting without any state + Returns: + A Deferred RoomSyncResult. + """ + recent_events, token = yield self.store.get_recent_events_for_room( + room_id, + limit=sync_config.limit, + end_token=now_token.room_key, + ) + prev_batch_token = now_token.copy_and_replace("room_key", token[0]) + current_state_events = yield self.state_handler.get_current_state( + room_id + ) + + defer.returnValue(RoomSyncResult( + room_id=room_id, + published=room_id in published_room_ids, + events=recent_events, + prev_batch=prev_batch_token, + state=current_state_events, + limited=True, + )) + + + @defer.inlineCallbacks + def incremental_sync_with_gap(self, sync_config, since_token): + """ Get the incremental delta needed to bring the client up to + date with the server. + Returns: + A Deferred SyncResult. + """ + if sync_config.sort == "timeline,desc": + # TODO(mjark): Handle going through events in reverse order?. + # What does "most recent events" mean when applying the limits mean + # in this case? + raise NotImplementedError() + + now_token = yield self.event_sources.get_current_token() + + presence_stream = self.event_sources.sources["presence"] + pagination_config = PaginationConfig( + from_token=since_token, to_token=now_token + ) + presence, _ = yield presence_stream.get_pagination_rows( + user=sync_config.user, + pagination_config=pagination_config.get_source_config("presence"), + key=None + ) + room_list = yield self.store.get_rooms_for_user_where_membership_is( + user_id=sync_config.user.to_string(), + membership_list=[Membership.INVITE, Membership.JOIN] + ) + + # TODO (mjark): Does public mean "published"? + published_rooms = yield self.store.get_rooms(is_public=True) + published_room_ids = set(r["room_id"] for r in published_rooms) + + rooms = [] + for event in room_list: + room_sync = yield self.incremental_sync_with_gap_for_room( + event.room_id, sync_config, since_token, now_token, + published_room_ids + ) + if room_sync: + rooms.append(room_sync) defer.returnValue(SyncResult( public_user_data=presence, @@ -143,5 +228,103 @@ class SyncHandler(BaseHandler): @defer.inlineCallbacks - def incremental_sync(self, sync_config): - pass + def incremental_sync_with_gap_for_room(self, room_id, sync_config, + since_token, now_token, + published_room_ids): + """ Get the incremental delta needed to bring the client up to date for + the room. Gives the client the most recent events and the changes to + state. + Returns: + A Deferred RoomSyncResult + """ + # TODO(mjark): Check if they have joined the room between + # the previous sync and this one. + # TODO(mjark): Apply the event filter in sync_config + # TODO(mjark): Check for redactions we might have missed. + # TODO(mjark): Typing notifications. + recents, token = yield self.store.get_recent_events_for_room( + room_id, + limit=sync_config.limit + 1, + from_token=since_token.room_key, + end_token=now_token.room_key, + ) + + logging.debug("Recents %r", recents) + + if len(recents) > sync_config.limit: + limited = True + recents = recents[1:] + else: + limited = False + + prev_batch_token = now_token.copy_and_replace("room_key", token[0]) + + # TODO(mjark): This seems racy since this isn't being passed a + # token to indicate what point in the stream this is + current_state_events = yield self.state_handler.get_current_state( + room_id + ) + + state_at_previous_sync = yield self.get_state_at_previous_sync( + room_id, since_token=since_token + ) + + state_events_delta = yield self.compute_state_delta( + since_token=since_token, + previous_state=state_at_previous_sync, + current_state=current_state_events, + ) + + room_sync = RoomSyncResult( + room_id=room_id, + published=room_id in published_room_ids, + events=recents, + prev_batch=prev_batch_token, + state=state_events_delta, + limited=limited, + ) + + logging.debug("Room sync: %r", room_sync) + + defer.returnValue(room_sync) + + @defer.inlineCallbacks + def get_state_at_previous_sync(self, room_id, since_token): + """ Get the room state at the previous sync the client made. + Returns: + A Deferred list of Events. + """ + last_events, token = yield self.store.get_recent_events_for_room( + room_id, end_token=since_token.room_key, limit=1, + ) + + if last_events: + last_event = last_events[0] + last_context = yield self.state_handler.compute_event_context( + last_event + ) + if last_event.is_state(): + state = [last_event] + last_context.current_state.values() + else: + state = last_context.current_state.values() + else: + state = () + defer.returnValue(state) + + + def compute_state_delta(self, since_token, previous_state, current_state): + """ Works out the differnce in state between the current state and the + state the client got when it last performed a sync. + Returns: + A list of events. + """ + # TODO(mjark) Check if the state events were received by the server + # after the previous sync, since we need to include those state + # updates even if they occured logically before the previous event. + # TODO(mjark) Check for new redactions in the state events. + previous_dict = {event.event_id:event for event in previous_state} + state_delta = [] + for event in current_state: + if event.event_id not in previous_dict: + state_delta.append(event) + return state_delta diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 8ac2adab05..06aca1a4e5 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -265,17 +265,38 @@ class StreamStore(SQLBaseStore): return self.runInteraction("paginate_room_events", f) def get_recent_events_for_room(self, room_id, limit, end_token, - with_feedback=False): + with_feedback=False, from_token=None): # TODO (erikj): Handle compressed feedback - sql = ( - "SELECT stream_ordering, topological_ordering, event_id FROM events " - "WHERE room_id = ? AND stream_ordering <= ? AND outlier = 0 " - "ORDER BY topological_ordering DESC, stream_ordering DESC LIMIT ? " - ) + end_token = _StreamToken.parse_stream_token(end_token) - def f(txn): - txn.execute(sql, (room_id, end_token, limit,)) + if from_token is None: + sql = ( + "SELECT stream_ordering, topological_ordering, event_id" + " FROM events" + " WHERE room_id = ? AND stream_ordering <= ? AND outlier = 0" + " ORDER BY topological_ordering DESC, stream_ordering DESC" + " LIMIT ?" + ) + else: + from_token = _StreamToken.parse_stream_token(from_token) + sql = ( + "SELECT stream_ordering, topological_ordering, event_id" + " FROM events" + " WHERE room_id = ? AND stream_ordering > ?" + " AND stream_ordering <= ? AND outlier = 0" + " ORDER BY topological_ordering DESC, stream_ordering DESC" + " LIMIT ?" + ) + + + def get_recent_events_for_room_txn(txn): + if from_token is None: + txn.execute(sql, (room_id, end_token.stream, limit,)) + else: + txn.execute(sql, ( + room_id, from_token.stream, end_token.stream, limit + )) rows = self.cursor_to_dict(txn) @@ -303,7 +324,9 @@ class StreamStore(SQLBaseStore): return events, token - return self.runInteraction("get_recent_events_for_room", f) + return self.runInteraction( + "get_recent_events_for_room", get_recent_events_for_room_txn + ) def get_room_events_max_id(self): return self.runInteraction( From 54e513b4e6b5c644b9a2aeb02cef8258e87ae26a Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 27 Jan 2015 17:48:13 +0000 Subject: [PATCH 063/180] Move storage of user filters into real datastore layer; now have to mock it out in the REST-level tests --- synapse/api/filtering.py | 27 ++----------- synapse/storage/__init__.py | 3 +- synapse/storage/filtering.py | 46 +++++++++++++++++++++++ tests/rest/client/v2_alpha/__init__.py | 9 +++-- tests/rest/client/v2_alpha/test_filter.py | 21 +++++++++++ 5 files changed, 79 insertions(+), 27 deletions(-) create mode 100644 synapse/storage/filtering.py diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 014e2e1fc9..20b6951d47 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -16,37 +16,18 @@ from twisted.internet import defer -# TODO(paul) -_filters_for_user = {} - - class Filtering(object): def __init__(self, hs): super(Filtering, self).__init__() - self.hs = hs + self.store = hs.get_datastore() - @defer.inlineCallbacks def get_user_filter(self, user_localpart, filter_id): - filters = _filters_for_user.get(user_localpart, None) + return self.store.get_user_filter(user_localpart, filter_id) - if not filters or filter_id >= len(filters): - raise KeyError() - - # trivial yield to make it a generator so d.iC works - yield - defer.returnValue(filters[filter_id]) - - @defer.inlineCallbacks def add_user_filter(self, user_localpart, definition): - filters = _filters_for_user.setdefault(user_localpart, []) - - filter_id = len(filters) - filters.append(definition) - - # trivial yield, see above - yield - defer.returnValue(filter_id) + # TODO(paul): implement sanity checking of the definition + return self.store.add_user_filter(user_localpart, definition) # TODO(paul): surely we should probably add a delete_user_filter or # replace_user_filter at some point? There's no REST API specified for diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 4beb951b9f..efa63031bd 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -30,9 +30,9 @@ from .transactions import TransactionStore from .keys import KeyStore from .event_federation import EventFederationStore from .media_repository import MediaRepositoryStore - from .state import StateStore from .signatures import SignatureStore +from .filtering import FilteringStore from syutil.base64util import decode_base64 from syutil.jsonutil import encode_canonical_json @@ -82,6 +82,7 @@ class DataStore(RoomMemberStore, RoomStore, DirectoryStore, KeyStore, StateStore, SignatureStore, EventFederationStore, MediaRepositoryStore, + FilteringStore, ): def __init__(self, hs): diff --git a/synapse/storage/filtering.py b/synapse/storage/filtering.py new file mode 100644 index 0000000000..18e0e7c298 --- /dev/null +++ b/synapse/storage/filtering.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from ._base import SQLBaseStore + + +# TODO(paul) +_filters_for_user = {} + + +class FilteringStore(SQLBaseStore): + @defer.inlineCallbacks + def get_user_filter(self, user_localpart, filter_id): + filters = _filters_for_user.get(user_localpart, None) + + if not filters or filter_id >= len(filters): + raise KeyError() + + # trivial yield to make it a generator so d.iC works + yield + defer.returnValue(filters[filter_id]) + + @defer.inlineCallbacks + def add_user_filter(self, user_localpart, definition): + filters = _filters_for_user.setdefault(user_localpart, []) + + filter_id = len(filters) + filters.append(definition) + + # trivial yield, see above + yield + defer.returnValue(filter_id) diff --git a/tests/rest/client/v2_alpha/__init__.py b/tests/rest/client/v2_alpha/__init__.py index f59745e13c..3fe62d5ac6 100644 --- a/tests/rest/client/v2_alpha/__init__.py +++ b/tests/rest/client/v2_alpha/__init__.py @@ -39,9 +39,7 @@ class V2AlphaRestTestCase(unittest.TestCase): hs = HomeServer("test", db_pool=None, - datastore=Mock(spec=[ - "insert_client_ip", - ]), + datastore=self.make_datastore_mock(), http_client=None, resource_for_client=self.mock_resource, resource_for_federation=self.mock_resource, @@ -58,3 +56,8 @@ class V2AlphaRestTestCase(unittest.TestCase): for r in self.TO_REGISTER: r.register_servlets(hs, self.mock_resource) + + def make_datastore_mock(self): + return Mock(spec=[ + "insert_client_ip", + ]) diff --git a/tests/rest/client/v2_alpha/test_filter.py b/tests/rest/client/v2_alpha/test_filter.py index 8629a1aed6..1add727e6b 100644 --- a/tests/rest/client/v2_alpha/test_filter.py +++ b/tests/rest/client/v2_alpha/test_filter.py @@ -15,6 +15,8 @@ from twisted.internet import defer +from mock import Mock + from . import V2AlphaRestTestCase from synapse.rest.client.v2_alpha import filter @@ -24,6 +26,25 @@ class FilterTestCase(V2AlphaRestTestCase): USER_ID = "@apple:test" TO_REGISTER = [filter] + def make_datastore_mock(self): + datastore = super(FilterTestCase, self).make_datastore_mock() + + self._user_filters = {} + + def add_user_filter(user_localpart, definition): + filters = self._user_filters.setdefault(user_localpart, []) + filter_id = len(filters) + filters.append(definition) + return defer.succeed(filter_id) + datastore.add_user_filter = add_user_filter + + def get_user_filter(user_localpart, filter_id): + filters = self._user_filters[user_localpart] + return defer.succeed(filters[filter_id]) + datastore.get_user_filter = get_user_filter + + return datastore + @defer.inlineCallbacks def test_filter(self): (code, response) = yield self.mock_resource.trigger("POST", From 0c14a699bb4e103a1845b0808821138cfea99552 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 27 Jan 2015 18:07:21 +0000 Subject: [PATCH 064/180] More unit-testing of REST errors --- tests/rest/client/v2_alpha/test_filter.py | 36 ++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/tests/rest/client/v2_alpha/test_filter.py b/tests/rest/client/v2_alpha/test_filter.py index 1add727e6b..80ddabf818 100644 --- a/tests/rest/client/v2_alpha/test_filter.py +++ b/tests/rest/client/v2_alpha/test_filter.py @@ -21,6 +21,8 @@ from . import V2AlphaRestTestCase from synapse.rest.client.v2_alpha import filter +from synapse.api.errors import StoreError + class FilterTestCase(V2AlphaRestTestCase): USER_ID = "@apple:test" @@ -39,14 +41,18 @@ class FilterTestCase(V2AlphaRestTestCase): datastore.add_user_filter = add_user_filter def get_user_filter(user_localpart, filter_id): + if user_localpart not in self._user_filters: + raise StoreError(404, "No user") filters = self._user_filters[user_localpart] + if filter_id >= len(filters): + raise StoreError(404, "No filter") return defer.succeed(filters[filter_id]) datastore.get_user_filter = get_user_filter return datastore @defer.inlineCallbacks - def test_filter(self): + def test_add_filter(self): (code, response) = yield self.mock_resource.trigger("POST", "/user/%s/filter" % (self.USER_ID), '{"type": ["m.*"]}' @@ -54,8 +60,36 @@ class FilterTestCase(V2AlphaRestTestCase): self.assertEquals(200, code) self.assertEquals({"filter_id": "0"}, response) + self.assertIn("apple", self._user_filters) + self.assertEquals(len(self._user_filters["apple"]), 1) + self.assertEquals({"type": ["m.*"]}, self._user_filters["apple"][0]) + + @defer.inlineCallbacks + def test_get_filter(self): + self._user_filters["apple"] = [ + {"type": ["m.*"]} + ] + (code, response) = yield self.mock_resource.trigger("GET", "/user/%s/filter/0" % (self.USER_ID), None ) self.assertEquals(200, code) self.assertEquals({"type": ["m.*"]}, response) + + @defer.inlineCallbacks + def test_get_filter_no_id(self): + self._user_filters["apple"] = [ + {"type": ["m.*"]} + ] + + (code, response) = yield self.mock_resource.trigger("GET", + "/user/%s/filter/2" % (self.USER_ID), None + ) + self.assertEquals(404, code) + + @defer.inlineCallbacks + def test_get_filter_no_user(self): + (code, response) = yield self.mock_resource.trigger("GET", + "/user/%s/filter/0" % (self.USER_ID), None + ) + self.assertEquals(404, code) From 06cc1470129d443f71bfc81ba716f63b9505467d Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 27 Jan 2015 18:46:03 +0000 Subject: [PATCH 065/180] Initial stab at real SQL storage implementation of user filter definitions --- synapse/storage/__init__.py | 1 + synapse/storage/filtering.py | 49 ++++++++++++++++++++-------- synapse/storage/schema/filtering.sql | 24 ++++++++++++++ tests/api/test_filtering.py | 19 ++++++++++- 4 files changed, 78 insertions(+), 15 deletions(-) create mode 100644 synapse/storage/schema/filtering.sql diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index efa63031bd..7c5631d014 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -61,6 +61,7 @@ SCHEMAS = [ "event_edges", "event_signatures", "media_repository", + "filtering", ] diff --git a/synapse/storage/filtering.py b/synapse/storage/filtering.py index 18e0e7c298..e98eaf8032 100644 --- a/synapse/storage/filtering.py +++ b/synapse/storage/filtering.py @@ -17,6 +17,8 @@ from twisted.internet import defer from ._base import SQLBaseStore +import json + # TODO(paul) _filters_for_user = {} @@ -25,22 +27,41 @@ _filters_for_user = {} class FilteringStore(SQLBaseStore): @defer.inlineCallbacks def get_user_filter(self, user_localpart, filter_id): - filters = _filters_for_user.get(user_localpart, None) + def_json = yield self._simple_select_one_onecol( + table="user_filters", + keyvalues={ + "user_id": user_localpart, + "filter_id": filter_id, + }, + retcol="definition", + allow_none=False, + ) - if not filters or filter_id >= len(filters): - raise KeyError() + defer.returnValue(json.loads(def_json)) - # trivial yield to make it a generator so d.iC works - yield - defer.returnValue(filters[filter_id]) - - @defer.inlineCallbacks def add_user_filter(self, user_localpart, definition): - filters = _filters_for_user.setdefault(user_localpart, []) + def_json = json.dumps(definition) - filter_id = len(filters) - filters.append(definition) + # Need an atomic transaction to SELECT the maximal ID so far then + # INSERT a new one + def _do_txn(txn): + sql = ( + "SELECT MAX(filter_id) FROM user_filters " + "WHERE user_id = ?" + ) + txn.execute(sql, (user_localpart,)) + max_id = txn.fetchone()[0] + if max_id is None: + filter_id = 0 + else: + filter_id = max_id + 1 - # trivial yield, see above - yield - defer.returnValue(filter_id) + sql = ( + "INSERT INTO user_filters (user_id, filter_id, definition)" + "VALUES(?, ?, ?)" + ) + txn.execute(sql, (user_localpart, filter_id, def_json)) + + return filter_id + + return self.runInteraction("add_user_filter", _do_txn) diff --git a/synapse/storage/schema/filtering.sql b/synapse/storage/schema/filtering.sql new file mode 100644 index 0000000000..795aca4afd --- /dev/null +++ b/synapse/storage/schema/filtering.sql @@ -0,0 +1,24 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +CREATE TABLE IF NOT EXISTS user_filters( + user_id TEXT, + filter_id INTEGER, + definition TEXT, + FOREIGN KEY(user_id) REFERENCES users(id) +); + +CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( + user_id, filter_id +); diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py index fecadd1056..149948374d 100644 --- a/tests/api/test_filtering.py +++ b/tests/api/test_filtering.py @@ -53,16 +53,33 @@ class FilteringTestCase(unittest.TestCase): self.filtering = hs.get_filtering() + self.datastore = hs.get_datastore() + @defer.inlineCallbacks - def test_filter(self): + def test_add_filter(self): filter_id = yield self.filtering.add_user_filter( user_localpart=user_localpart, definition={"type": ["m.*"]}, ) + self.assertEquals(filter_id, 0) + self.assertEquals({"type": ["m.*"]}, + (yield self.datastore.get_user_filter( + user_localpart=user_localpart, + filter_id=0, + )) + ) + + @defer.inlineCallbacks + def test_get_filter(self): + filter_id = yield self.datastore.add_user_filter( + user_localpart=user_localpart, + definition={"type": ["m.*"]}, + ) filter = yield self.filtering.get_user_filter( user_localpart=user_localpart, filter_id=filter_id, ) + self.assertEquals(filter, {"type": ["m.*"]}) From 8398f19bcea8fb0134b37efa303dc65b017d75ce Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 27 Jan 2015 19:00:09 +0000 Subject: [PATCH 066/180] Created schema delta --- synapse/storage/__init__.py | 2 +- synapse/storage/schema/delta/v12.sql | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 synapse/storage/schema/delta/v12.sql diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 7c5631d014..00a04f565d 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -67,7 +67,7 @@ SCHEMAS = [ # Remember to update this number every time an incompatible change is made to # database schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 11 +SCHEMA_VERSION = 12 class _RollbackButIsFineException(Exception): diff --git a/synapse/storage/schema/delta/v12.sql b/synapse/storage/schema/delta/v12.sql new file mode 100644 index 0000000000..795aca4afd --- /dev/null +++ b/synapse/storage/schema/delta/v12.sql @@ -0,0 +1,24 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +CREATE TABLE IF NOT EXISTS user_filters( + user_id TEXT, + filter_id INTEGER, + definition TEXT, + FOREIGN KEY(user_id) REFERENCES users(id) +); + +CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( + user_id, filter_id +); From b19cf6a105ad87954e15cf01e60e14fec280db6d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 27 Jan 2015 20:09:52 +0000 Subject: [PATCH 067/180] Wait for events if the incremental sync is empty and a timeout is given --- demo/start.sh | 2 +- synapse/handlers/sync.py | 19 ++++++++++------ synapse/notifier.py | 48 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 8 deletions(-) diff --git a/demo/start.sh b/demo/start.sh index bb2248770d..a7502e7a8d 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -16,7 +16,7 @@ if [ $# -eq 1 ]; then fi fi -for port in 8080 8081 8082; do +for port in 8080; do echo "Starting server on port $port... " https_port=$((port + 400)) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index f8629a588f..9f5f73eab6 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -72,6 +72,7 @@ class SyncHandler(BaseHandler): self.event_sources = hs.get_event_sources() self.clock = hs.get_clock() + @defer.inlineCallbacks def wait_for_sync_for_user(self, sync_config, since_token=None, timeout=0): """Get the sync for a client if we have new data for it now. Otherwise wait for new data to arrive on the server. If the timeout expires, then @@ -80,15 +81,19 @@ class SyncHandler(BaseHandler): A Deferred SyncResult. """ if timeout == 0 or since_token is None: - return self.current_sync_for_user(sync_config, since_token) + result = yield self.current_sync_for_user(sync_config, since_token) + defer.returnValue(result) else: - def current_sync_callback(since_token): - return self.current_sync_for_user( - self, since_token, sync_config - ) - return self.notifier.wait_for_events( - sync_config.filter, since_token, current_sync_callback + def current_sync_callback(): + return self.current_sync_for_user(sync_config, since_token) + + rm_handler = self.hs.get_handlers().room_member_handler + room_ids = yield rm_handler.get_rooms_for_user(sync_config.user) + result = yield self.notifier.wait_for_events( + sync_config.user, room_ids, + sync_config.filter, timeout, current_sync_callback ) + defer.returnValue(result) def current_sync_for_user(self, sync_config, since_token=None): """Get the sync for client needed to match what the server has now. diff --git a/synapse/notifier.py b/synapse/notifier.py index 3aec1d4af2..922bf064d0 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -18,6 +18,7 @@ from twisted.internet import defer from synapse.util.logutils import log_function from synapse.util.logcontext import PreserveLoggingContext from synapse.util.async import run_on_reactor +from synapse.types import StreamToken import logging @@ -205,6 +206,53 @@ class Notifier(object): [notify(l).addErrback(eb) for l in listeners] ) + @defer.inlineCallbacks + def wait_for_events(self, user, rooms, filter, timeout, callback): + """Wait until the callback returns a non empty response or the + timeout fires. + """ + + deferred = defer.Deferred() + + from_token=StreamToken("s0","0","0") + + listener = [_NotificationListener( + user=user, + rooms=rooms, + from_token=from_token, + limit=1, + timeout=timeout, + deferred=deferred, + )] + + if timeout: + self._register_with_keys(listener[0]) + + result = yield callback() + if timeout: + timed_out = [False] + def _timeout_listener(): + timed_out[0] = True + listener[0].notify(self, [], from_token, from_token) + + self.clock.call_later(timeout/1000., _timeout_listener) + while not result and not timed_out[0]: + yield deferred + deferred = defer.Deferred() + listener[0] = _NotificationListener( + user=user, + rooms=rooms, + from_token=from_token, + limit=1, + timeout=timeout, + deferred=deferred, + ) + self._register_with_keys(listener[0]) + result = yield callback() + + defer.returnValue(result) + + def get_events_for(self, user, rooms, pagination_config, timeout): """ For the given user and rooms, return any new events for them. If there are no new events wait for up to `timeout` milliseconds for any From e020574d65a994858ac53c45070ae5016090d2f3 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 27 Jan 2015 20:19:36 +0000 Subject: [PATCH 068/180] Fix Formatting --- synapse/handlers/sync.py | 13 +++++-------- synapse/notifier.py | 4 ++-- synapse/rest/client/v2_alpha/sync.py | 5 ++--- synapse/storage/stream.py | 1 - 4 files changed, 9 insertions(+), 14 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 9f5f73eab6..82a2c6986a 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -52,10 +52,10 @@ class RoomSyncResult(collections.namedtuple("RoomSyncResult", [ class SyncResult(collections.namedtuple("SyncResult", [ - "next_batch", # Token for the next sync - "private_user_data", # List of private events for the user. - "public_user_data", # List of public events for all users. - "rooms", # RoomSyncResult for each room. + "next_batch", # Token for the next sync + "private_user_data", # List of private events for the user. + "public_user_data", # List of public events for all users. + "rooms", # RoomSyncResult for each room. ])): __slots__ = [] @@ -181,7 +181,6 @@ class SyncHandler(BaseHandler): limited=True, )) - @defer.inlineCallbacks def incremental_sync_with_gap(self, sync_config, since_token): """ Get the incremental delta needed to bring the client up to @@ -231,7 +230,6 @@ class SyncHandler(BaseHandler): next_batch=now_token, )) - @defer.inlineCallbacks def incremental_sync_with_gap_for_room(self, room_id, sync_config, since_token, now_token, @@ -316,7 +314,6 @@ class SyncHandler(BaseHandler): state = () defer.returnValue(state) - def compute_state_delta(self, since_token, previous_state, current_state): """ Works out the differnce in state between the current state and the state the client got when it last performed a sync. @@ -327,7 +324,7 @@ class SyncHandler(BaseHandler): # after the previous sync, since we need to include those state # updates even if they occured logically before the previous event. # TODO(mjark) Check for new redactions in the state events. - previous_dict = {event.event_id:event for event in previous_state} + previous_dict = {event.event_id: event for event in previous_state} state_delta = [] for event in current_state: if event.event_id not in previous_dict: diff --git a/synapse/notifier.py b/synapse/notifier.py index 922bf064d0..e3b6ead620 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -214,7 +214,7 @@ class Notifier(object): deferred = defer.Deferred() - from_token=StreamToken("s0","0","0") + from_token = StreamToken("s0", "0", "0") listener = [_NotificationListener( user=user, @@ -231,6 +231,7 @@ class Notifier(object): result = yield callback() if timeout: timed_out = [False] + def _timeout_listener(): timed_out[0] = True listener[0].notify(self, [], from_token, from_token) @@ -252,7 +253,6 @@ class Notifier(object): defer.returnValue(result) - def get_events_for(self, user, rooms, pagination_config, timeout): """ For the given user and rooms, return any new events for them. If there are no new events wait for up to `timeout` milliseconds for any diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index cc667ebafc..0c17208cd3 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -68,7 +68,6 @@ class SyncRestServlet(RestServlet): } """ - PATTERN = client_v2_pattern("/sync$") ALLOWED_SORT = set(["timeline,asc", "timeline,desc"]) ALLOWED_PRESENCE = set(["online", "offline", "idle"]) @@ -114,12 +113,12 @@ class SyncRestServlet(RestServlet): sync_config = SyncConfig( user=user, - device="TODO", # TODO(mjark) Get the device_id from access_token + device="TODO", # TODO(mjark) Get the device_id from access_token gap=gap, limit=limit, sort=sort, backfill=backfill, - filter="TODO", # TODO(mjark) Add the filter to the config. + filter="TODO", # TODO(mjark) Add the filter to the config. ) if since is not None: diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 06aca1a4e5..db1816ea84 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -289,7 +289,6 @@ class StreamStore(SQLBaseStore): " LIMIT ?" ) - def get_recent_events_for_room_txn(txn): if from_token is None: txn.execute(sql, (room_id, end_token.stream, limit,)) From e32ded7b3e290067f95497ceee0ac04842ba9cdf Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 10:09:54 +0000 Subject: [PATCH 069/180] Add matrix.org as a trusted ID server because it's now passed through on ports 80/443 and the web client defaults to that now. Fixes email validation (including signing up with an email address). --- synapse/handlers/register.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 732652c228..66a89c10b2 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -163,7 +163,7 @@ class RegistrationHandler(BaseHandler): # each request httpCli = SimpleHttpClient(self.hs) # XXX: make this configurable! - trustedIdServers = ['matrix.org:8090'] + trustedIdServers = ['matrix.org:8090', 'matrix.org'] if not creds['idServer'] in trustedIdServers: logger.warn('%s is not a trusted ID server: rejecting 3pid ' + 'credentials', creds['idServer']) From 273b12729b99addf4474c9092f44ff300fd8153b Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 11:55:49 +0000 Subject: [PATCH 070/180] Reset badge count to zero when last active time is bumped --- synapse/handlers/presence.py | 5 +++++ synapse/push/__init__.py | 19 ++++++++++++++++++ synapse/push/httppusher.py | 38 +++++++++++++++++++++++++++++++++--- synapse/push/pusherpool.py | 17 ++++++++++++++++ 4 files changed, 76 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 8aeed99274..24d901b51f 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -86,6 +86,10 @@ class PresenceHandler(BaseHandler): "changed_presencelike_data", self.changed_presencelike_data ) + # outbound signal from the presence module to advertise when a user's + # presence has changed + distributor.declare("user_presence_changed") + self.distributor = distributor self.federation = hs.get_replication_layer() @@ -603,6 +607,7 @@ class PresenceHandler(BaseHandler): room_ids=room_ids, statuscache=statuscache, ) + yield self.distributor.fire("user_presence_changed", user, statuscache) @defer.inlineCallbacks def _push_presence_remote(self, user, destination, state=None): diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index b6d01a82a0..4862d0de27 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -54,6 +54,9 @@ class Pusher(object): self.failing_since = failing_since self.alive = True + # The last value of last_active_time that we saw + self.last_last_active_time = 0 + @defer.inlineCallbacks def _actions_for_event(self, ev): """ @@ -273,6 +276,22 @@ class Pusher(object): """ pass + def reset_badge_count(self): + pass + + def presence_changed(self, state): + """ + We clear badge counts whenever a user's last_active time is bumped + This is by no means perfect but I think it's the best we can do + without read receipts. + """ + if 'last_active' in state.state: + last_active = state.state['last_active'] + if last_active > self.last_last_active_time: + logger.info("Resetting badge count for %s", self.user_name) + self.reset_badge_count() + self.last_last_active_time = last_active + def _value_for_dotted_key(dotted_key, event): parts = dotted_key.split(".") diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 22532fcc6a..d592bc2fd2 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -71,11 +71,12 @@ class HttpPusher(Pusher): # we may have to fetch this over federation and we # can't trust it anyway: is it worth it? #'from_display_name': 'Steve Stevington' - #'counts': { -- we don't mark messages as read yet so + 'counts': { #-- we don't mark messages as read yet so # we have no way of knowing - # 'unread': 1, + # Just set the badge to 1 until we have read receipts + 'unread': 1, # 'missed_calls': 2 - # }, + }, 'devices': [ { 'app_id': self.app_id, @@ -111,3 +112,34 @@ class HttpPusher(Pusher): if 'rejected' in resp: rejected = resp['rejected'] defer.returnValue(rejected) + + @defer.inlineCallbacks + def reset_badge_count(self): + d = { + 'notification': { + 'id': '', + 'type': None, + 'from': '', + 'counts': { + 'unread': 0, + 'missed_calls': 0 + }, + 'devices': [ + { + 'app_id': self.app_id, + 'pushkey': self.pushkey, + 'pushkey_ts': long(self.pushkey_ts / 1000), + 'data': self.data_minus_url, + } + ] + } + } + try: + resp = yield self.httpCli.post_json_get_json(self.url, d) + except: + logger.exception("Failed to push %s ", self.url) + defer.returnValue(False) + rejected = [] + if 'rejected' in resp: + rejected = resp['rejected'] + defer.returnValue(rejected) \ No newline at end of file diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 2dfecf178b..65ab4f46e1 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -18,6 +18,7 @@ from twisted.internet import defer from httppusher import HttpPusher from synapse.push import PusherConfigException +from synapse.api.constants import PresenceState import logging import json @@ -32,6 +33,22 @@ class PusherPool: self.pushers = {} self.last_pusher_started = -1 + distributor = self.hs.get_distributor() + distributor.observe( + "user_presence_changed", self.user_presence_changed + ) + + @defer.inlineCallbacks + def user_presence_changed(self, user, state): + user_name = user.to_string() + + # until we have read receipts, pushers use this to reset a user's + # badge counters to zero + for p in self.pushers.values(): + if p.user_name == user_name: + yield p.presence_changed(state) + + @defer.inlineCallbacks def start(self): pushers = yield self.store.get_all_pushers() From fb532d84250e7d112461c62b3ec2443cc61d1348 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 13:06:09 +0000 Subject: [PATCH 071/180] Unused import --- synapse/push/pusherpool.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 65ab4f46e1..e69ee2cf17 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -18,7 +18,6 @@ from twisted.internet import defer from httppusher import HttpPusher from synapse.push import PusherConfigException -from synapse.api.constants import PresenceState import logging import json From ca7240a2f06b6f6797c857b5cc4d8998a1a46557 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 13:17:55 +0000 Subject: [PATCH 072/180] Update copyright --- synapse/push/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 4862d0de27..fd8d6d4bbd 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 6df6f5e0840e6a2b878f62c1656834bfbd5bcd62 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 13:56:35 +0000 Subject: [PATCH 073/180] Redundant bracketing & missed space --- synapse/push/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index fd8d6d4bbd..8ddc833577 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -187,8 +187,8 @@ class Pusher(object): # for sanity, we only remove the pushkey if it # was the one we actually sent... logger.warn( - ("Ignoring rejected pushkey %s because we" + - "didn't send it"), (pk,) + ("Ignoring rejected pushkey %s because we " + + "didn't send it"), pk ) else: logger.info( From e1ca0f1396425db5c80a975bb83596c7384484ef Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 13:58:32 +0000 Subject: [PATCH 074/180] Brackets rather than slashes at end --- synapse/push/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 8ddc833577..8a3e8fd2a3 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -226,9 +226,9 @@ class Pusher(object): self.failing_since ) - if self.failing_since and \ - self.failing_since < \ - self.clock.time_msec() - Pusher.GIVE_UP_AFTER: + if (self.failing_since and + self.failing_since < + self.clock.time_msec() - Pusher.GIVE_UP_AFTER): # we really only give up so that if the URL gets # fixed, we don't suddenly deliver a load # of old notifications. From 03149ad23ace1f200993a74b998a6986d70c9420 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:01:24 +0000 Subject: [PATCH 075/180] More code style things --- synapse/push/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 8a3e8fd2a3..b79f2d4b27 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -187,7 +187,7 @@ class Pusher(object): # for sanity, we only remove the pushkey if it # was the one we actually sent... logger.warn( - ("Ignoring rejected pushkey %s because we " + + ("Ignoring rejected pushkey %s because we " "didn't send it"), pk ) else: @@ -234,7 +234,8 @@ class Pusher(object): # of old notifications. logger.warn("Giving up on a notification to user %s, " "pushkey %s", - self.user_name, self.pushkey) + self.user_name, self.pushkey + ) self.backoff_delay = Pusher.INITIAL_BACKOFF self.last_token = chunk['end'] self.store.update_pusher_last_token( @@ -256,7 +257,7 @@ class Pusher(object): self.user_name, self.clock.time_msec() - self.failing_since, self.backoff_delay - ) + ) yield synapse.util.async.sleep(self.backoff_delay / 1000.0) self.backoff_delay *= 2 if self.backoff_delay > Pusher.MAX_BACKOFF: From 20c47383dca7d68608c903fe1f856756fd3da057 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:10:46 +0000 Subject: [PATCH 076/180] Oops, bad merge: needed to change the base class of the rest servlets too. --- synapse/push/__init__.py | 5 +++-- synapse/rest/client/v1/push_rule.py | 4 ++-- synapse/rest/client/v1/pusher.py | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index b79f2d4b27..1cac5fff4e 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -269,8 +269,9 @@ class Pusher(object): def dispatch_push(self, p, tweaks): """ Overridden by implementing classes to actually deliver the notification - :param p: The event to notify for as a single event from the event stream - :return: If the notification was delivered, an array containing any + Args: + p The event to notify for as a single event from the event stream + Returns: If the notification was delivered, an array containing any pushkeys that were rejected by the push gateway. False if the notification could not be delivered (ie. should be retried). diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 50bf5b9008..00fed42f44 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -17,13 +17,13 @@ from twisted.internet import defer from synapse.api.errors import SynapseError, Codes, UnrecognizedRequestError, NotFoundError, \ StoreError -from base import RestServlet, client_path_pattern +from .base import ClientV1RestServlet, client_path_pattern from synapse.storage.push_rule import InconsistentRuleException, RuleNotFoundException import json -class PushRuleRestServlet(RestServlet): +class PushRuleRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/pushrules/.*$") PRIORITY_CLASS_MAP = { 'underride': 0, diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py index 4659c9b1d9..80a11890a3 100644 --- a/synapse/rest/client/v1/pusher.py +++ b/synapse/rest/client/v1/pusher.py @@ -17,12 +17,12 @@ from twisted.internet import defer from synapse.api.errors import SynapseError, Codes from synapse.push import PusherConfigException -from base import RestServlet, client_path_pattern +from .base import ClientV1RestServlet, client_path_pattern import json -class PusherRestServlet(RestServlet): +class PusherRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/pushers/set$") @defer.inlineCallbacks From 5f2665320fbe4392c9f487820c697691463cf709 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:11:45 +0000 Subject: [PATCH 077/180] It is 2015 --- synapse/push/httppusher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index d592bc2fd2..e12b946727 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 6fde707addc29b6717fda9617a127093fb5b08ae Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:14:49 +0000 Subject: [PATCH 078/180] doc style fix --- synapse/push/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 1cac5fff4e..10ac890482 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -270,7 +270,7 @@ class Pusher(object): """ Overridden by implementing classes to actually deliver the notification Args: - p The event to notify for as a single event from the event stream + p: The event to notify for as a single event from the event stream Returns: If the notification was delivered, an array containing any pushkeys that were rejected by the push gateway. False if the notification could not be delivered (ie. From dd3abbd61fba7d3ea9fe300a309202f0eb4c5fbd Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:22:39 +0000 Subject: [PATCH 079/180] 2015 --- synapse/push/pusherpool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index e69ee2cf17..716d9514cd 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 30fbba168b0f9bf8b38594b3badb9af463167485 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:23:16 +0000 Subject: [PATCH 080/180] Easy on the newlines --- synapse/push/pusherpool.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 716d9514cd..856defedac 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -47,7 +47,6 @@ class PusherPool: if p.user_name == user_name: yield p.presence_changed(state) - @defer.inlineCallbacks def start(self): pushers = yield self.store.get_all_pushers() From 4fbf2328c29ca8f8a7918f6763b91787e55f37f8 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:24:28 +0000 Subject: [PATCH 081/180] Unnecessary new line --- synapse/rest/client/v1/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/rest/client/v1/__init__.py b/synapse/rest/client/v1/__init__.py index 96a9a474f1..d8d01cdd16 100644 --- a/synapse/rest/client/v1/__init__.py +++ b/synapse/rest/client/v1/__init__.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - from . import ( room, events, register, login, profile, presence, initial_sync, directory, voip, admin, pusher, push_rule From 6741c3dbd90b6406d7eba8445868cdd8fd2ec6a5 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:26:03 +0000 Subject: [PATCH 082/180] Brackets are nicer --- synapse/rest/client/v1/push_rule.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 00fed42f44..b03d804d82 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -33,8 +33,8 @@ class PushRuleRestServlet(ClientV1RestServlet): 'override': 4 } PRIORITY_CLASS_INVERSE_MAP = {v: k for k,v in PRIORITY_CLASS_MAP.items()} - SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR =\ - "Unrecognised request: You probably wanted a trailing slash" + SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR = ( + "Unrecognised request: You probably wanted a trailing slash") def rule_spec_from_path(self, path): if len(path) < 2: From d93ce29a86b79478bfb2011c9b8d3c8ec7d0bdd0 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:27:01 +0000 Subject: [PATCH 083/180] Ah, the comma of doom. --- synapse/rest/client/v1/push_rule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index b03d804d82..c6085370a4 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -101,7 +101,7 @@ class PushRuleRestServlet(ClientV1RestServlet): if pat.strip("*?[]") == pat: # no special glob characters so we assume the user means # 'contains this string' rather than 'is this string' - pat = "*%s*" % (pat) + pat = "*%s*" % (pat,) conditions = [{ 'kind': 'event_match', 'key': 'content.body', From 032f8d4ed3a5b4015087b7a97295da9b580a95b3 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:33:15 +0000 Subject: [PATCH 084/180] Another superfluous newline --- synapse/rest/client/v1/push_rule.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index c6085370a4..3a08bdd9af 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -213,7 +213,6 @@ class PushRuleRestServlet(ClientV1RestServlet): else: raise - @defer.inlineCallbacks def on_GET(self, request): user = yield self.auth.get_user_by_req(request) From 8807f4170eaf2515407b656b7eb6fe7e8fc93796 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:35:00 +0000 Subject: [PATCH 085/180] Better style --- synapse/rest/client/v1/push_rule.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 3a08bdd9af..52f2b19bbe 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -242,8 +242,11 @@ class PushRuleRestServlet(ClientV1RestServlet): continue if instance_handle not in rules['device']: rules['device'][instance_handle] = {} - rules['device'][instance_handle] = \ - _add_empty_priority_class_arrays(rules['device'][instance_handle]) + rules['device'][instance_handle] = ( + _add_empty_priority_class_arrays( + rules['device'][instance_handle] + ) + ) rulearray = rules['device'][instance_handle][template_name] else: From 3cb5b73c0dd10ba9020547a81864846c70d4e709 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:37:55 +0000 Subject: [PATCH 086/180] Unnecessary newline. --- synapse/rest/client/v1/push_rule.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 52f2b19bbe..550554c18c 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -290,7 +290,6 @@ class PushRuleRestServlet(ClientV1RestServlet): def on_OPTIONS(self, _): return 200, {} - def _add_empty_priority_class_arrays(d): for pc in PushRuleRestServlet.PRIORITY_CLASS_MAP.keys(): d[pc] = [] From 289a2498743caae7e0c3366a1c1f7855d48d9a8e Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:39:03 +0000 Subject: [PATCH 087/180] Unnecessary newlines. --- synapse/rest/client/v1/push_rule.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 550554c18c..dbcac3d4e1 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -295,7 +295,6 @@ def _add_empty_priority_class_arrays(d): d[pc] = [] return d - def _instance_handle_from_conditions(conditions): """ Given a list of conditions, return the instance handle of the @@ -326,7 +325,6 @@ def _filter_ruleset_with_path(ruleset, path): return r raise NotFoundError - def _priority_class_from_spec(spec): if spec['template'] not in PushRuleRestServlet.PRIORITY_CLASS_MAP.keys(): raise InvalidRuleException("Unknown template: %s" % (spec['kind'])) @@ -337,7 +335,6 @@ def _priority_class_from_spec(spec): return pc - def _priority_class_to_template_name(pc): if pc > PushRuleRestServlet.PRIORITY_CLASS_MAP['override']: # per-device @@ -346,7 +343,6 @@ def _priority_class_to_template_name(pc): else: return PushRuleRestServlet.PRIORITY_CLASS_INVERSE_MAP[pc] - def _rule_to_template(rule): template_name = _priority_class_to_template_name(rule['priority_class']) if template_name in ['override', 'underride']: @@ -363,7 +359,6 @@ def _rule_to_template(rule): ret["pattern"] = thecond["pattern"] return ret - def _strip_device_condition(rule): for i,c in enumerate(rule['conditions']): if c['kind'] == 'device': From 2cfdfee572dc037b65571ab72efcb3223c7d8d11 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:41:51 +0000 Subject: [PATCH 088/180] spaces --- synapse/rest/client/v1/push_rule.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index dbcac3d4e1..2b33bdac08 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -346,7 +346,7 @@ def _priority_class_to_template_name(pc): def _rule_to_template(rule): template_name = _priority_class_to_template_name(rule['priority_class']) if template_name in ['override', 'underride']: - return {k:rule[k] for k in ["rule_id", "conditions", "actions"]} + return {k: rule[k] for k in ["rule_id", "conditions", "actions"]} elif template_name in ["sender", "room"]: return {k: rule[k] for k in ["rule_id", "actions"]} elif template_name == 'content': @@ -355,7 +355,7 @@ def _rule_to_template(rule): thecond = rule["conditions"][0] if "pattern" not in thecond: return None - ret = {k:rule[k] for k in ["rule_id", "actions"]} + ret = {k: rule[k] for k in ["rule_id", "actions"]} ret["pattern"] = thecond["pattern"] return ret From 0cbb6b0f5235e4501a0fb360e881d152644a17cd Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:44:41 +0000 Subject: [PATCH 089/180] Google doc style --- synapse/storage/_base.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 4f172d3967..809c81f47f 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -195,10 +195,11 @@ class SQLBaseStore(object): def _simple_upsert(self, table, keyvalues, values): """ - :param table: The table to upsert into - :param keyvalues: Dict of the unique key tables and their new values - :param values: Dict of all the nonunique columns and their new values - :return: A deferred + Args: + table (str): The table to upsert into + keyvalues (dict): The unique key tables and their new values + values (dict): The nonunique columns and their new values + Returns: A deferred """ return self.runInteraction( "_simple_upsert", From fb0928097a0dc1606aebb9aed8f070bcea304178 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:48:07 +0000 Subject: [PATCH 090/180] More magic commas (including the place I copied it from...) --- synapse/storage/_base.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 809c81f47f..9261c999cb 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -210,8 +210,8 @@ class SQLBaseStore(object): # Try to update sql = "UPDATE %s SET %s WHERE %s" % ( table, - ", ".join("%s = ?" % (k) for k in values), - " AND ".join("%s = ?" % (k) for k in keyvalues) + ", ".join("%s = ?" % (k,) for k in values), + " AND ".join("%s = ?" % (k,) for k in keyvalues) ) sqlargs = values.values() + keyvalues.values() logger.debug( @@ -390,8 +390,8 @@ class SQLBaseStore(object): if updatevalues: update_sql = "UPDATE %s SET %s WHERE %s" % ( table, - ", ".join("%s = ?" % (k) for k in updatevalues), - " AND ".join("%s = ?" % (k) for k in keyvalues) + ", ".join("%s = ?" % (k,) for k in updatevalues), + " AND ".join("%s = ?" % (k,) for k in keyvalues) ) def func(txn): From 6d485dd1c727e7ecfe3991066bd058794ae05051 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:48:42 +0000 Subject: [PATCH 091/180] unnecessary newlines --- synapse/storage/_base.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 9261c999cb..4e8bd3faa9 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -237,8 +237,6 @@ class SQLBaseStore(object): ) txn.execute(sql, allvalues.values()) - - def _simple_select_one(self, table, keyvalues, retcols, allow_none=False): """Executes a SELECT query on the named table, which is expected to From 445ad9941ea2e4038846aa6fed456e3250ae49b1 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:49:59 +0000 Subject: [PATCH 092/180] Redundant parens --- synapse/storage/push_rule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index ca04f2ccee..f5a736be44 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -92,7 +92,7 @@ class PushRuleStore(SQLBaseStore): res = txn.fetchall() if not res: raise RuleNotFoundException("before/after rule not found: %s" % (relative_to_rule)) - (priority_class, base_rule_priority) = res[0] + priority_class, base_rule_priority = res[0] if 'priority_class' in kwargs and kwargs['priority_class'] != priority_class: raise InconsistentRuleException( From 93aac9bb7b3023e6c82961b1cdd655a48ec567fb Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:51:01 +0000 Subject: [PATCH 093/180] Newline --- synapse/storage/push_rule.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index f5a736be44..48105234f6 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -184,6 +184,7 @@ class PushRuleStore(SQLBaseStore): } ) + class RuleNotFoundException(Exception): pass From e78dd332928c111c8a62985bce0a3c1c5631244e Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:52:58 +0000 Subject: [PATCH 094/180] Use %s instead of + --- synapse/storage/push_rule.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index 48105234f6..0342996ed1 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -85,8 +85,8 @@ class PushRuleStore(SQLBaseStore): # get the priority of the rule we're inserting after/before sql = ( - "SELECT priority_class, priority FROM "+PushRuleTable.table_name+ - " WHERE user_name = ? and rule_id = ?" + "SELECT priority_class, priority FROM ? " + "WHERE user_name = ? and rule_id = ?" % (PushRuleTable.table_name,) ) txn.execute(sql, (user_name, relative_to_rule)) res = txn.fetchall() From c59bcabf0b5c0ab78c0f89da75b031993c4660d9 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 28 Jan 2015 15:36:21 +0000 Subject: [PATCH 095/180] Return the device_id from get_auth_by_req --- synapse/api/auth.py | 7 ++++-- synapse/rest/client/v1/admin.py | 2 +- synapse/rest/client/v1/directory.py | 4 ++-- synapse/rest/client/v1/events.py | 4 ++-- synapse/rest/client/v1/initial_sync.py | 2 +- synapse/rest/client/v1/presence.py | 8 +++---- synapse/rest/client/v1/profile.py | 4 ++-- synapse/rest/client/v1/room.py | 24 ++++++++++----------- synapse/rest/client/v1/voip.py | 2 +- synapse/rest/media/v0/content_repository.py | 2 +- synapse/rest/media/v1/upload_resource.py | 2 +- tests/rest/client/v1/test_presence.py | 2 +- tests/rest/client/v1/test_profile.py | 2 +- 13 files changed, 34 insertions(+), 31 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index a342a0e0da..292e9e2a80 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -290,7 +290,9 @@ class Auth(object): Args: request - An HTTP request with an access_token query parameter. Returns: - UserID : User ID object of the user making the request + Tuple of UserID and device string: + User ID object of the user making the request + Device ID string of the device the user is using Raises: AuthError if no user by that token exists or the token is invalid. """ @@ -299,6 +301,7 @@ class Auth(object): access_token = request.args["access_token"][0] user_info = yield self.get_user_by_token(access_token) user = user_info["user"] + device_id = user_info["device_id"] ip_addr = self.hs.get_ip_from_request(request) user_agent = request.requestHeaders.getRawHeaders( @@ -314,7 +317,7 @@ class Auth(object): user_agent=user_agent ) - defer.returnValue(user) + defer.returnValue((user, device_id)) except KeyError: raise AuthError(403, "Missing access token.") diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py index 1051d96f96..6cfce1a479 100644 --- a/synapse/rest/client/v1/admin.py +++ b/synapse/rest/client/v1/admin.py @@ -31,7 +31,7 @@ class WhoisRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): target_user = UserID.from_string(user_id) - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) is_admin = yield self.auth.is_server_admin(auth_user) if not is_admin and target_user != auth_user: diff --git a/synapse/rest/client/v1/directory.py b/synapse/rest/client/v1/directory.py index 15ae8749b8..ef853af411 100644 --- a/synapse/rest/client/v1/directory.py +++ b/synapse/rest/client/v1/directory.py @@ -45,7 +45,7 @@ class ClientDirectoryServer(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, room_alias): - user = yield self.auth.get_user_by_req(request) + user, device_id = yield self.auth.get_user_by_req(request) content = _parse_json(request) if not "room_id" in content: @@ -85,7 +85,7 @@ class ClientDirectoryServer(ClientV1RestServlet): @defer.inlineCallbacks def on_DELETE(self, request, room_alias): - user = yield self.auth.get_user_by_req(request) + user, device_id = yield self.auth.get_user_by_req(request) is_admin = yield self.auth.is_server_admin(user) if not is_admin: diff --git a/synapse/rest/client/v1/events.py b/synapse/rest/client/v1/events.py index a0d051227b..e58ee46fcd 100644 --- a/synapse/rest/client/v1/events.py +++ b/synapse/rest/client/v1/events.py @@ -34,7 +34,7 @@ class EventStreamRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request): - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) try: handler = self.handlers.event_stream_handler pagin_config = PaginationConfig.from_request(request) @@ -71,7 +71,7 @@ class EventRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, event_id): - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) handler = self.handlers.event_handler event = yield handler.get_event(auth_user, event_id) diff --git a/synapse/rest/client/v1/initial_sync.py b/synapse/rest/client/v1/initial_sync.py index 357fa845b4..78d30abbf8 100644 --- a/synapse/rest/client/v1/initial_sync.py +++ b/synapse/rest/client/v1/initial_sync.py @@ -25,7 +25,7 @@ class InitialSyncRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request): - user = yield self.auth.get_user_by_req(request) + user, device_id = yield self.auth.get_user_by_req(request) with_feedback = "feedback" in request.args as_client_event = "raw" not in request.args pagination_config = PaginationConfig.from_request(request) diff --git a/synapse/rest/client/v1/presence.py b/synapse/rest/client/v1/presence.py index b6c207e662..74669274a7 100644 --- a/synapse/rest/client/v1/presence.py +++ b/synapse/rest/client/v1/presence.py @@ -32,7 +32,7 @@ class PresenceStatusRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) state = yield self.handlers.presence_handler.get_state( @@ -42,7 +42,7 @@ class PresenceStatusRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) state = {} @@ -77,7 +77,7 @@ class PresenceListRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) if not self.hs.is_mine(user): @@ -97,7 +97,7 @@ class PresenceListRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request, user_id): - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) if not self.hs.is_mine(user): diff --git a/synapse/rest/client/v1/profile.py b/synapse/rest/client/v1/profile.py index 24f8d56952..f04abb2c26 100644 --- a/synapse/rest/client/v1/profile.py +++ b/synapse/rest/client/v1/profile.py @@ -37,7 +37,7 @@ class ProfileDisplaynameRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) try: @@ -70,7 +70,7 @@ class ProfileAvatarURLRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) try: diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 58b09b6fc1..c8c34b4801 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -62,7 +62,7 @@ class RoomCreateRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request): - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) room_config = self.get_room_config(request) info = yield self.make_room(room_config, auth_user, None) @@ -125,7 +125,7 @@ class RoomStateEventRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id, event_type, state_key): - user = yield self.auth.get_user_by_req(request) + user, device_id = yield self.auth.get_user_by_req(request) msg_handler = self.handlers.message_handler data = yield msg_handler.get_room_data( @@ -143,7 +143,7 @@ class RoomStateEventRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, room_id, event_type, state_key): - user = yield self.auth.get_user_by_req(request) + user, device_id = yield self.auth.get_user_by_req(request) content = _parse_json(request) @@ -173,7 +173,7 @@ class RoomSendEventRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request, room_id, event_type): - user = yield self.auth.get_user_by_req(request) + user, device_id = yield self.auth.get_user_by_req(request) content = _parse_json(request) msg_handler = self.handlers.message_handler @@ -216,7 +216,7 @@ class JoinRoomAliasServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request, room_identifier): - user = yield self.auth.get_user_by_req(request) + user, device_id = yield self.auth.get_user_by_req(request) # the identifier could be a room alias or a room id. Try one then the # other if it fails to parse, without swallowing other valid @@ -283,7 +283,7 @@ class RoomMemberListRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): # TODO support Pagination stream API (limit/tokens) - user = yield self.auth.get_user_by_req(request) + user, device_id = yield self.auth.get_user_by_req(request) handler = self.handlers.room_member_handler members = yield handler.get_room_members_as_pagination_chunk( room_id=room_id, @@ -311,7 +311,7 @@ class RoomMessageListRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): - user = yield self.auth.get_user_by_req(request) + user, device_id = yield self.auth.get_user_by_req(request) pagination_config = PaginationConfig.from_request( request, default_limit=10, ) @@ -335,7 +335,7 @@ class RoomStateRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): - user = yield self.auth.get_user_by_req(request) + user, device_id = yield self.auth.get_user_by_req(request) handler = self.handlers.message_handler # Get all the current state for this room events = yield handler.get_state_events( @@ -351,7 +351,7 @@ class RoomInitialSyncRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): - user = yield self.auth.get_user_by_req(request) + user, device_id = yield self.auth.get_user_by_req(request) pagination_config = PaginationConfig.from_request(request) content = yield self.handlers.message_handler.room_initial_sync( room_id=room_id, @@ -396,7 +396,7 @@ class RoomMembershipRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request, room_id, membership_action): - user = yield self.auth.get_user_by_req(request) + user, device_id = yield self.auth.get_user_by_req(request) content = _parse_json(request) @@ -445,7 +445,7 @@ class RoomRedactEventRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request, room_id, event_id): - user = yield self.auth.get_user_by_req(request) + user, device_id = yield self.auth.get_user_by_req(request) content = _parse_json(request) msg_handler = self.handlers.message_handler @@ -483,7 +483,7 @@ class RoomTypingRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, room_id, user_id): - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) room_id = urllib.unquote(room_id) target_user = UserID.from_string(urllib.unquote(user_id)) diff --git a/synapse/rest/client/v1/voip.py b/synapse/rest/client/v1/voip.py index 822d863ce6..42d8e30bab 100644 --- a/synapse/rest/client/v1/voip.py +++ b/synapse/rest/client/v1/voip.py @@ -28,7 +28,7 @@ class VoipRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request): - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) turnUris = self.hs.config.turn_uris turnSecret = self.hs.config.turn_shared_secret diff --git a/synapse/rest/media/v0/content_repository.py b/synapse/rest/media/v0/content_repository.py index 79ae0e3d74..311ab89edb 100644 --- a/synapse/rest/media/v0/content_repository.py +++ b/synapse/rest/media/v0/content_repository.py @@ -66,7 +66,7 @@ class ContentRepoResource(resource.Resource): @defer.inlineCallbacks def map_request_to_name(self, request): # auth the user - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) # namespace all file uploads on the user prefix = base64.urlsafe_b64encode( diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py index b1718a630b..6bed8a8efa 100644 --- a/synapse/rest/media/v1/upload_resource.py +++ b/synapse/rest/media/v1/upload_resource.py @@ -42,7 +42,7 @@ class UploadResource(BaseMediaResource): @defer.inlineCallbacks def _async_render_POST(self, request): try: - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) # TODO: The checks here are a bit late. The content will have # already been uploaded to a tmp file at this point content_length = request.getHeader("Content-Length") diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py index 65d5cc4916..a4f2abf213 100644 --- a/tests/rest/client/v1/test_presence.py +++ b/tests/rest/client/v1/test_presence.py @@ -282,7 +282,7 @@ class PresenceEventStreamTestCase(unittest.TestCase): hs.get_clock().time_msec.return_value = 1000000 def _get_user_by_req(req=None): - return UserID.from_string(myid) + return (UserID.from_string(myid), "") hs.get_auth().get_user_by_req = _get_user_by_req diff --git a/tests/rest/client/v1/test_profile.py b/tests/rest/client/v1/test_profile.py index 39cd68d829..6a2085276a 100644 --- a/tests/rest/client/v1/test_profile.py +++ b/tests/rest/client/v1/test_profile.py @@ -58,7 +58,7 @@ class ProfileTestCase(unittest.TestCase): ) def _get_user_by_req(request=None): - return UserID.from_string(myid) + return (UserID.from_string(myid), "") hs.get_auth().get_user_by_req = _get_user_by_req From 60b143a52e69751a406ea83cdab58f4045cdd9d4 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 15:48:28 +0000 Subject: [PATCH 096/180] Move pushers delta to v12 and bump schema version --- synapse/storage/__init__.py | 2 +- synapse/storage/schema/delta/{v10.sql => v12.sql} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename synapse/storage/schema/delta/{v10.sql => v12.sql} (100%) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 2534d109fd..277581b4e2 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -69,7 +69,7 @@ SCHEMAS = [ # Remember to update this number every time an incompatible change is made to # database schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 11 +SCHEMA_VERSION = 12 class _RollbackButIsFineException(Exception): diff --git a/synapse/storage/schema/delta/v10.sql b/synapse/storage/schema/delta/v12.sql similarity index 100% rename from synapse/storage/schema/delta/v10.sql rename to synapse/storage/schema/delta/v12.sql From 0ef5bfd6a9eaaae14e199997658b3d0006abd854 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 28 Jan 2015 16:16:53 +0000 Subject: [PATCH 097/180] Start implementing auth conflict res --- synapse/api/auth.py | 38 +++-- synapse/api/constants.py | 6 + synapse/federation/federation_client.py | 39 +++++ synapse/handlers/federation.py | 211 ++++++++++++++++++++---- synapse/storage/rejections.py | 10 ++ synapse/storage/schema/im.sql | 1 + 6 files changed, 253 insertions(+), 52 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index a342a0e0da..461faa8c78 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -353,9 +353,23 @@ class Auth(object): def add_auth_events(self, builder, context): yield run_on_reactor() - if builder.type == EventTypes.Create: - builder.auth_events = [] - return + auth_ids = self.compute_auth_events(builder, context) + + auth_events_entries = yield self.store.add_event_hashes( + auth_ids + ) + + builder.auth_events = auth_events_entries + + context.auth_events = { + k: v + for k, v in context.current_state.items() + if v.event_id in auth_ids + } + + def compute_auth_events(self, event, context): + if event.type == EventTypes.Create: + return [] auth_ids = [] @@ -368,7 +382,7 @@ class Auth(object): key = (EventTypes.JoinRules, "", ) join_rule_event = context.current_state.get(key) - key = (EventTypes.Member, builder.user_id, ) + key = (EventTypes.Member, event.user_id, ) member_event = context.current_state.get(key) key = (EventTypes.Create, "", ) @@ -382,8 +396,8 @@ class Auth(object): else: is_public = False - if builder.type == EventTypes.Member: - e_type = builder.content["membership"] + if event.type == EventTypes.Member: + e_type = event.content["membership"] if e_type in [Membership.JOIN, Membership.INVITE]: if join_rule_event: auth_ids.append(join_rule_event.event_id) @@ -398,17 +412,7 @@ class Auth(object): if member_event.content["membership"] == Membership.JOIN: auth_ids.append(member_event.event_id) - auth_events_entries = yield self.store.add_event_hashes( - auth_ids - ) - - builder.auth_events = auth_events_entries - - context.auth_events = { - k: v - for k, v in context.current_state.items() - if v.event_id in auth_ids - } + return auth_ids @log_function def _can_send_event(self, event, auth_events): diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 7ee6dcc46e..0d3fc629af 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -74,3 +74,9 @@ class EventTypes(object): Message = "m.room.message" Topic = "m.room.topic" Name = "m.room.name" + + +class RejectedReason(object): + AUTH_ERROR = "auth_error" + REPLACED = "replaced" + NOT_ANCESTOR = "not_ancestor" diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 91b44cd8b3..ebcd593506 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -331,6 +331,45 @@ class FederationClient(object): defer.returnValue(pdu) + @defer.inlineCallbacks + def query_auth(self, destination, room_id, event_id, local_auth): + """ + Params: + destination (str) + event_it (str) + local_auth (list) + """ + time_now = self._clock.time_msec() + + send_content = { + "auth_chain": [e.get_pdu_json(time_now) for e in local_auth], + } + + code, content = yield self.transport_layer.send_invite( + destination=destination, + room_id=room_id, + event_id=event_id, + content=send_content, + ) + + auth_chain = [ + (yield self._check_sigs_and_hash(self.event_from_pdu_json(e))) + for e in content["auth_chain"] + ] + + missing = [ + (yield self._check_sigs_and_hash(self.event_from_pdu_json(e))) + for e in content.get("missing", []) + ] + + ret = { + "auth_chain": auth_chain, + "rejects": content.get("rejects", []), + "missing": missing, + } + + defer.returnValue(ret) + def event_from_pdu_json(self, pdu_json, outlier=False): event = FrozenEvent( pdu_json diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index bcdcc90a18..97e3c503b9 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -17,19 +17,16 @@ from ._base import BaseHandler -from synapse.events.utils import prune_event from synapse.api.errors import ( - AuthError, FederationError, SynapseError, StoreError, + AuthError, FederationError, StoreError, ) -from synapse.api.constants import EventTypes, Membership +from synapse.api.constants import EventTypes, Membership, RejectedReason from synapse.util.logutils import log_function from synapse.util.async import run_on_reactor from synapse.crypto.event_signing import ( - compute_event_signature, check_event_content_hash, - add_hashes_and_signatures, + compute_event_signature, add_hashes_and_signatures, ) from synapse.types import UserID -from syutil.jsonutil import encode_canonical_json from twisted.internet import defer @@ -113,33 +110,6 @@ class FederationHandler(BaseHandler): logger.debug("Processing event: %s", event.event_id) - redacted_event = prune_event(event) - - redacted_pdu_json = redacted_event.get_pdu_json() - try: - yield self.keyring.verify_json_for_server( - event.origin, redacted_pdu_json - ) - except SynapseError as e: - logger.warn( - "Signature check failed for %s redacted to %s", - encode_canonical_json(pdu.get_pdu_json()), - encode_canonical_json(redacted_pdu_json), - ) - raise FederationError( - "ERROR", - e.code, - e.msg, - affected=event.event_id, - ) - - if not check_event_content_hash(event): - logger.warn( - "Event content has been tampered, redacting %s, %s", - event.event_id, encode_canonical_json(event.get_dict()) - ) - event = redacted_event - logger.debug("Event: %s", event) # FIXME (erikj): Awful hack to make the case where we are not currently @@ -180,7 +150,6 @@ class FederationHandler(BaseHandler): if state: for e in state: - logging.info("A :) %r", e) e.internal_metadata.outlier = True try: yield self._handle_new_event(e) @@ -747,7 +716,20 @@ class FederationHandler(BaseHandler): event.event_id, event.signatures, ) - self.auth.check(event, auth_events=context.auth_events) + try: + self.auth.check(event, auth_events=context.auth_events) + except AuthError: + # TODO: Store rejection. + context.rejected = RejectedReason.AUTH_ERROR + + yield self.store.persist_event( + event, + context=context, + backfilled=backfilled, + is_new_state=False, + current_state=current_state, + ) + raise logger.debug( "_handle_new_event: Before persist_event: %s, sigs: %s", @@ -768,3 +750,162 @@ class FederationHandler(BaseHandler): ) defer.returnValue(context) + + @defer.inlineCallbacks + def do_auth(self, origin, event, context): + for e_id, _ in event.auth_events: + pass + + auth_events = set(e_id for e_id, _ in event.auth_events) + current_state = set(e.event_id for e in context.auth_events.values()) + + missing_auth = auth_events - current_state + + if missing_auth: + # Do auth conflict res. + + # 1. Get what we think is the auth chain. + auth_ids = self.auth.compute_auth_events(event, context) + local_auth_chain = yield self.store.get_auth_chain(auth_ids) + + # 2. Get remote difference. + result = yield self.replication_layer.query_auth( + origin, + event.room_id, + event.event_id, + local_auth_chain, + ) + + # 3. Process any remote auth chain events we haven't seen. + for e in result.get("missing", []): + # TODO. + pass + + # 4. Look at rejects and their proofs. + # TODO. + + try: + self.auth.check(event, auth_events=context.auth_events) + except AuthError: + raise + + @defer.inlineCallbacks + def construct_auth_difference(self, local_auth, remote_auth): + """ Given a local and remote auth chain, find the differences. This + assumes that we have already processed all events in remote_auth + + Params: + local_auth (list) + remote_auth (list) + + Returns: + dict + """ + + # TODO: Make sure we are OK with local_auth or remote_auth having more + # auth events in them than strictly necessary. + + def sort_fun(ev): + return ev.depth, ev.event_id + + # We find the differences by starting at the "bottom" of each list + # and iterating up on both lists. The lists are ordered by depth and + # then event_id, we iterate up both lists until we find the event ids + # don't match. Then we look at depth/event_id to see which side is + # missing that event, and iterate only up that list. Repeat. + + remote_list = list(remote_auth) + remote_list.sort(key=sort_fun) + + local_list = list(local_auth) + local_list.sort(key=sort_fun) + + local_iter = iter(local_list) + remote_iter = iter(remote_list) + + current_local = local_iter.next() + current_remote = remote_iter.next() + + def get_next(it, opt=None): + return it.next() if it.has_next() else opt + + missing_remotes = [] + missing_locals = [] + while current_local and current_remote: + if current_remote is None: + missing_locals.append(current_local) + current_local = get_next(local_iter) + continue + + if current_local is None: + missing_remotes.append(current_remote) + current_remote = get_next(remote_iter) + continue + + if current_local.event_id == current_remote.event_id: + current_local = get_next(local_iter) + current_remote = get_next(remote_iter) + continue + + if current_local.depth < current_remote.depth: + missing_locals.append(current_local) + current_local = get_next(local_iter) + continue + + if current_local.depth > current_remote.depth: + missing_remotes.append(current_remote) + current_remote = get_next(remote_iter) + continue + + # They have the same depth, so we fall back to the event_id order + if current_local.event_id < current_remote.event_id: + missing_locals.append(current_local) + current_local = get_next(local_iter) + + if current_local.event_id > current_remote.event_id: + missing_remotes.append(current_remote) + current_remote = get_next(remote_iter) + continue + + # missing locals should be sent to the server + # We should find why we are missing remotes, as they will have been + # rejected. + + # Remove events from missing_remotes if they are referencing a missing + # remote. We only care about the "root" rejected ones. + missing_remote_ids = [e.event_id for e in missing_remotes] + base_remote_rejected = list(missing_remotes) + for e in missing_remotes: + for e_id, _ in e.auth_events: + if e_id in missing_remote_ids: + base_remote_rejected.remove(e) + + reason_map = {} + + for e in base_remote_rejected: + reason = yield self.store.get_rejection_reason(e.event_id) + if reason is None: + # FIXME: ERRR?! + raise RuntimeError("") + + reason_map[e.event_id] = reason + + if reason == RejectedReason.AUTH_ERROR: + pass + elif reason == RejectedReason.REPLACED: + # TODO: Get proof + pass + elif reason == RejectedReason.NOT_ANCESTOR: + # TODO: Get proof. + pass + + defer.returnValue({ + "rejects": { + e.event_id: { + "reason": reason_map[e.event_id], + "proof": None, + } + for e in base_remote_rejected + }, + "missing": missing_locals, + }) diff --git a/synapse/storage/rejections.py b/synapse/storage/rejections.py index 7d38b31f44..b7249700d7 100644 --- a/synapse/storage/rejections.py +++ b/synapse/storage/rejections.py @@ -31,3 +31,13 @@ class RejectionsStore(SQLBaseStore): "last_failure": self._clock.time_msec(), } ) + + def get_rejection_reason(self, event_id): + self._simple_select_one_onecol( + table="rejections", + retcol="reason", + keyvalues={ + "event_id": event_id, + }, + allow_none=True, + ) diff --git a/synapse/storage/schema/im.sql b/synapse/storage/schema/im.sql index bc7c6b6ed5..5866a387f6 100644 --- a/synapse/storage/schema/im.sql +++ b/synapse/storage/schema/im.sql @@ -128,5 +128,6 @@ CREATE TABLE IF NOT EXISTS rejections( event_id TEXT NOT NULL, reason TEXT NOT NULL, last_check TEXT NOT NULL, + root_rejected TEXT, CONSTRAINT ev_id UNIQUE (event_id) ON CONFLICT REPLACE ); From c23e3db544eb940d95a092b661e3872480f3bf30 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 28 Jan 2015 16:45:18 +0000 Subject: [PATCH 098/180] Add filter JSON sanity checks. --- synapse/api/filtering.py | 109 ++++++++++++++++++++++++- synapse/rest/client/v2_alpha/filter.py | 2 +- synapse/storage/filtering.py | 4 +- tests/api/test_filtering.py | 24 +++++- 4 files changed, 128 insertions(+), 11 deletions(-) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 20b6951d47..6c7a73b6d5 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -13,7 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from twisted.internet import defer +from synapse.api.errors import SynapseError +from synapse.types import UserID, RoomID class Filtering(object): @@ -25,10 +26,110 @@ class Filtering(object): def get_user_filter(self, user_localpart, filter_id): return self.store.get_user_filter(user_localpart, filter_id) - def add_user_filter(self, user_localpart, definition): - # TODO(paul): implement sanity checking of the definition - return self.store.add_user_filter(user_localpart, definition) + def add_user_filter(self, user_localpart, user_filter): + self._check_valid_filter(user_filter) + return self.store.add_user_filter(user_localpart, user_filter) # TODO(paul): surely we should probably add a delete_user_filter or # replace_user_filter at some point? There's no REST API specified for # them however + + def _check_valid_filter(self, user_filter): + """Check if the provided filter is valid. + + This inspects all definitions contained within the filter. + + Args: + user_filter(dict): The filter + Raises: + SynapseError: If the filter is not valid. + """ + # NB: Filters are the complete json blobs. "Definitions" are an + # individual top-level key e.g. public_user_data. Filters are made of + # many definitions. + + top_level_definitions = [ + "public_user_data", "private_user_data", "server_data" + ] + + room_level_definitions = [ + "state", "events", "ephemeral" + ] + + for key in top_level_definitions: + if key in user_filter: + self._check_definition(user_filter[key]) + + if "room" in user_filter: + for key in room_level_definitions: + if key in user_filter["room"]: + self._check_definition(user_filter["room"][key]) + + + def _check_definition(self, definition): + """Check if the provided definition is valid. + + This inspects not only the types but also the values to make sure they + make sense. + + Args: + definition(dict): The filter definition + Raises: + SynapseError: If there was a problem with this definition. + """ + # NB: Filters are the complete json blobs. "Definitions" are an + # individual top-level key e.g. public_user_data. Filters are made of + # many definitions. + if type(definition) != dict: + raise SynapseError( + 400, "Expected JSON object, not %s" % (definition,) + ) + + # check rooms are valid room IDs + room_id_keys = ["rooms", "not_rooms"] + for key in room_id_keys: + if key in definition: + if type(definition[key]) != list: + raise SynapseError(400, "Expected %s to be a list." % key) + for room_id in definition[key]: + RoomID.from_string(room_id) + + # check senders are valid user IDs + user_id_keys = ["senders", "not_senders"] + for key in user_id_keys: + if key in definition: + if type(definition[key]) != list: + raise SynapseError(400, "Expected %s to be a list." % key) + for user_id in definition[key]: + UserID.from_string(user_id) + + # TODO: We don't limit event type values but we probably should... + # check types are valid event types + event_keys = ["types", "not_types"] + for key in event_keys: + if key in definition: + if type(definition[key]) != list: + raise SynapseError(400, "Expected %s to be a list." % key) + for event_type in definition[key]: + if not isinstance(event_type, basestring): + raise SynapseError(400, "Event type should be a string") + + try: + event_format = definition["format"] + if event_format not in ["federation", "events"]: + raise SynapseError(400, "Invalid format: %s" % (event_format,)) + except KeyError: + pass # format is optional + + try: + event_select_list = definition["select"] + for select_key in event_select_list: + if select_key not in ["event_id", "origin_server_ts", + "thread_id", "content", "content.body"]: + raise SynapseError(400, "Bad select: %s" % (select_key,)) + except KeyError: + pass # select is optional + + if ("bundle_updates" in definition and + type(definition["bundle_updates"]) != bool): + raise SynapseError(400, "Bad bundle_updates: expected bool.") diff --git a/synapse/rest/client/v2_alpha/filter.py b/synapse/rest/client/v2_alpha/filter.py index 09e44e8ae0..81a3e95155 100644 --- a/synapse/rest/client/v2_alpha/filter.py +++ b/synapse/rest/client/v2_alpha/filter.py @@ -93,7 +93,7 @@ class CreateFilterRestServlet(RestServlet): filter_id = yield self.filtering.add_user_filter( user_localpart=target_user.localpart, - definition=content, + user_filter=content, ) defer.returnValue((200, {"filter_id": str(filter_id)})) diff --git a/synapse/storage/filtering.py b/synapse/storage/filtering.py index e98eaf8032..bab68a9eef 100644 --- a/synapse/storage/filtering.py +++ b/synapse/storage/filtering.py @@ -39,8 +39,8 @@ class FilteringStore(SQLBaseStore): defer.returnValue(json.loads(def_json)) - def add_user_filter(self, user_localpart, definition): - def_json = json.dumps(definition) + def add_user_filter(self, user_localpart, user_filter): + def_json = json.dumps(user_filter) # Need an atomic transaction to SELECT the maximal ID so far then # INSERT a new one diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py index 149948374d..188fbfb91e 100644 --- a/tests/api/test_filtering.py +++ b/tests/api/test_filtering.py @@ -57,13 +57,21 @@ class FilteringTestCase(unittest.TestCase): @defer.inlineCallbacks def test_add_filter(self): + user_filter = { + "room": { + "state": { + "types": ["m.*"] + } + } + } + filter_id = yield self.filtering.add_user_filter( user_localpart=user_localpart, - definition={"type": ["m.*"]}, + user_filter=user_filter, ) self.assertEquals(filter_id, 0) - self.assertEquals({"type": ["m.*"]}, + self.assertEquals(user_filter, (yield self.datastore.get_user_filter( user_localpart=user_localpart, filter_id=0, @@ -72,9 +80,17 @@ class FilteringTestCase(unittest.TestCase): @defer.inlineCallbacks def test_get_filter(self): + user_filter = { + "room": { + "state": { + "types": ["m.*"] + } + } + } + filter_id = yield self.datastore.add_user_filter( user_localpart=user_localpart, - definition={"type": ["m.*"]}, + user_filter=user_filter, ) filter = yield self.filtering.get_user_filter( @@ -82,4 +98,4 @@ class FilteringTestCase(unittest.TestCase): filter_id=filter_id, ) - self.assertEquals(filter, {"type": ["m.*"]}) + self.assertEquals(filter, user_filter) From 388581e087a3658c1b70d2aa1d17a132953350ca Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 28 Jan 2015 16:58:23 +0000 Subject: [PATCH 099/180] Extract the id token of the token when authing users, include the token and device_id in the internal meta data for the event along with the transaction id when sending events --- synapse/api/auth.py | 8 ++- synapse/handlers/message.py | 12 +++- synapse/rest/client/v1/admin.py | 2 +- synapse/rest/client/v1/directory.py | 4 +- synapse/rest/client/v1/events.py | 4 +- synapse/rest/client/v1/initial_sync.py | 2 +- synapse/rest/client/v1/presence.py | 8 +-- synapse/rest/client/v1/profile.py | 4 +- synapse/rest/client/v1/room.py | 64 ++++++++++++--------- synapse/rest/client/v1/voip.py | 2 +- synapse/rest/media/v0/content_repository.py | 2 +- synapse/rest/media/v1/upload_resource.py | 2 +- synapse/storage/registration.py | 3 +- synapse/types.py | 3 + tests/rest/client/v1/test_presence.py | 2 + tests/rest/client/v1/test_rooms.py | 7 +++ tests/rest/client/v1/test_typing.py | 1 + tests/storage/test_registration.py | 10 +++- 18 files changed, 92 insertions(+), 48 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 292e9e2a80..3959e06a8b 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -21,7 +21,7 @@ from synapse.api.constants import EventTypes, Membership, JoinRules from synapse.api.errors import AuthError, StoreError, Codes, SynapseError from synapse.util.logutils import log_function from synapse.util.async import run_on_reactor -from synapse.types import UserID +from synapse.types import UserID, ClientID import logging @@ -292,7 +292,7 @@ class Auth(object): Returns: Tuple of UserID and device string: User ID object of the user making the request - Device ID string of the device the user is using + Client ID object of the client instance the user is using Raises: AuthError if no user by that token exists or the token is invalid. """ @@ -302,6 +302,7 @@ class Auth(object): user_info = yield self.get_user_by_token(access_token) user = user_info["user"] device_id = user_info["device_id"] + token_id = user_info["token_id"] ip_addr = self.hs.get_ip_from_request(request) user_agent = request.requestHeaders.getRawHeaders( @@ -317,7 +318,7 @@ class Auth(object): user_agent=user_agent ) - defer.returnValue((user, device_id)) + defer.returnValue((user, ClientID(device_id, token_id))) except KeyError: raise AuthError(403, "Missing access token.") @@ -342,6 +343,7 @@ class Auth(object): "admin": bool(ret.get("admin", False)), "device_id": ret.get("device_id"), "user": UserID.from_string(ret.get("name")), + "token_id": ret.get("token_id", None), } defer.returnValue(user_info) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 9c3271fe88..6fbd2af4ab 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -114,7 +114,8 @@ class MessageHandler(BaseHandler): defer.returnValue(chunk) @defer.inlineCallbacks - def create_and_send_event(self, event_dict, ratelimit=True): + def create_and_send_event(self, event_dict, ratelimit=True, + client=None, txn_id=None): """ Given a dict from a client, create and handle a new event. Creates an FrozenEvent object, filling out auth_events, prev_events, @@ -148,6 +149,15 @@ class MessageHandler(BaseHandler): builder.content ) + if client is not None: + if client.token_id is not None: + builder.internal_metadata.token_id = client.token_id + if client.device_id is not None: + builder.internal_metadata.device_id = client.device_id + + if txn_id is not None: + builder.internal_metadata.txn_id = txn_id + event, context = yield self._create_new_client_event( builder=builder, ) diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py index 6cfce1a479..2ce754b028 100644 --- a/synapse/rest/client/v1/admin.py +++ b/synapse/rest/client/v1/admin.py @@ -31,7 +31,7 @@ class WhoisRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): target_user = UserID.from_string(user_id) - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) is_admin = yield self.auth.is_server_admin(auth_user) if not is_admin and target_user != auth_user: diff --git a/synapse/rest/client/v1/directory.py b/synapse/rest/client/v1/directory.py index ef853af411..8f65efec5f 100644 --- a/synapse/rest/client/v1/directory.py +++ b/synapse/rest/client/v1/directory.py @@ -45,7 +45,7 @@ class ClientDirectoryServer(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, room_alias): - user, device_id = yield self.auth.get_user_by_req(request) + user, client = yield self.auth.get_user_by_req(request) content = _parse_json(request) if not "room_id" in content: @@ -85,7 +85,7 @@ class ClientDirectoryServer(ClientV1RestServlet): @defer.inlineCallbacks def on_DELETE(self, request, room_alias): - user, device_id = yield self.auth.get_user_by_req(request) + user, client = yield self.auth.get_user_by_req(request) is_admin = yield self.auth.is_server_admin(user) if not is_admin: diff --git a/synapse/rest/client/v1/events.py b/synapse/rest/client/v1/events.py index e58ee46fcd..77b7c25a03 100644 --- a/synapse/rest/client/v1/events.py +++ b/synapse/rest/client/v1/events.py @@ -34,7 +34,7 @@ class EventStreamRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) try: handler = self.handlers.event_stream_handler pagin_config = PaginationConfig.from_request(request) @@ -71,7 +71,7 @@ class EventRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, event_id): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) handler = self.handlers.event_handler event = yield handler.get_event(auth_user, event_id) diff --git a/synapse/rest/client/v1/initial_sync.py b/synapse/rest/client/v1/initial_sync.py index 78d30abbf8..4a259bba64 100644 --- a/synapse/rest/client/v1/initial_sync.py +++ b/synapse/rest/client/v1/initial_sync.py @@ -25,7 +25,7 @@ class InitialSyncRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request): - user, device_id = yield self.auth.get_user_by_req(request) + user, client = yield self.auth.get_user_by_req(request) with_feedback = "feedback" in request.args as_client_event = "raw" not in request.args pagination_config = PaginationConfig.from_request(request) diff --git a/synapse/rest/client/v1/presence.py b/synapse/rest/client/v1/presence.py index 74669274a7..7feb4aadb1 100644 --- a/synapse/rest/client/v1/presence.py +++ b/synapse/rest/client/v1/presence.py @@ -32,7 +32,7 @@ class PresenceStatusRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) state = yield self.handlers.presence_handler.get_state( @@ -42,7 +42,7 @@ class PresenceStatusRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) state = {} @@ -77,7 +77,7 @@ class PresenceListRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) if not self.hs.is_mine(user): @@ -97,7 +97,7 @@ class PresenceListRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request, user_id): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) if not self.hs.is_mine(user): diff --git a/synapse/rest/client/v1/profile.py b/synapse/rest/client/v1/profile.py index f04abb2c26..15d6f3fc6c 100644 --- a/synapse/rest/client/v1/profile.py +++ b/synapse/rest/client/v1/profile.py @@ -37,7 +37,7 @@ class ProfileDisplaynameRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) try: @@ -70,7 +70,7 @@ class ProfileAvatarURLRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) try: diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index c8c34b4801..410f19ccf6 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -62,7 +62,7 @@ class RoomCreateRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) room_config = self.get_room_config(request) info = yield self.make_room(room_config, auth_user, None) @@ -125,7 +125,7 @@ class RoomStateEventRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id, event_type, state_key): - user, device_id = yield self.auth.get_user_by_req(request) + user, client = yield self.auth.get_user_by_req(request) msg_handler = self.handlers.message_handler data = yield msg_handler.get_room_data( @@ -142,8 +142,8 @@ class RoomStateEventRestServlet(ClientV1RestServlet): defer.returnValue((200, data.get_dict()["content"])) @defer.inlineCallbacks - def on_PUT(self, request, room_id, event_type, state_key): - user, device_id = yield self.auth.get_user_by_req(request) + def on_PUT(self, request, room_id, event_type, state_key, txn_id=None): + user, client = yield self.auth.get_user_by_req(request) content = _parse_json(request) @@ -158,7 +158,9 @@ class RoomStateEventRestServlet(ClientV1RestServlet): event_dict["state_key"] = state_key msg_handler = self.handlers.message_handler - yield msg_handler.create_and_send_event(event_dict) + yield msg_handler.create_and_send_event( + event_dict, client=client, txn_id=txn_id, + ) defer.returnValue((200, {})) @@ -172,8 +174,8 @@ class RoomSendEventRestServlet(ClientV1RestServlet): register_txn_path(self, PATTERN, http_server, with_get=True) @defer.inlineCallbacks - def on_POST(self, request, room_id, event_type): - user, device_id = yield self.auth.get_user_by_req(request) + def on_POST(self, request, room_id, event_type, txn_id=None): + user, client = yield self.auth.get_user_by_req(request) content = _parse_json(request) msg_handler = self.handlers.message_handler @@ -183,7 +185,9 @@ class RoomSendEventRestServlet(ClientV1RestServlet): "content": content, "room_id": room_id, "sender": user.to_string(), - } + }, + client=client, + txn_id=txn_id, ) defer.returnValue((200, {"event_id": event.event_id})) @@ -200,7 +204,7 @@ class RoomSendEventRestServlet(ClientV1RestServlet): except KeyError: pass - response = yield self.on_POST(request, room_id, event_type) + response = yield self.on_POST(request, room_id, event_type, txn_id) self.txns.store_client_transaction(request, txn_id, response) defer.returnValue(response) @@ -215,8 +219,8 @@ class JoinRoomAliasServlet(ClientV1RestServlet): register_txn_path(self, PATTERN, http_server) @defer.inlineCallbacks - def on_POST(self, request, room_identifier): - user, device_id = yield self.auth.get_user_by_req(request) + def on_POST(self, request, room_identifier, txn_id=None): + user, client = yield self.auth.get_user_by_req(request) # the identifier could be a room alias or a room id. Try one then the # other if it fails to parse, without swallowing other valid @@ -245,7 +249,9 @@ class JoinRoomAliasServlet(ClientV1RestServlet): "room_id": identifier.to_string(), "sender": user.to_string(), "state_key": user.to_string(), - } + }, + client=client, + txn_id=txn_id, ) defer.returnValue((200, {"room_id": identifier.to_string()})) @@ -259,7 +265,7 @@ class JoinRoomAliasServlet(ClientV1RestServlet): except KeyError: pass - response = yield self.on_POST(request, room_identifier) + response = yield self.on_POST(request, room_identifier, txn_id) self.txns.store_client_transaction(request, txn_id, response) defer.returnValue(response) @@ -283,7 +289,7 @@ class RoomMemberListRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): # TODO support Pagination stream API (limit/tokens) - user, device_id = yield self.auth.get_user_by_req(request) + user, client = yield self.auth.get_user_by_req(request) handler = self.handlers.room_member_handler members = yield handler.get_room_members_as_pagination_chunk( room_id=room_id, @@ -311,7 +317,7 @@ class RoomMessageListRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): - user, device_id = yield self.auth.get_user_by_req(request) + user, client = yield self.auth.get_user_by_req(request) pagination_config = PaginationConfig.from_request( request, default_limit=10, ) @@ -335,7 +341,7 @@ class RoomStateRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): - user, device_id = yield self.auth.get_user_by_req(request) + user, client = yield self.auth.get_user_by_req(request) handler = self.handlers.message_handler # Get all the current state for this room events = yield handler.get_state_events( @@ -351,7 +357,7 @@ class RoomInitialSyncRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): - user, device_id = yield self.auth.get_user_by_req(request) + user, client = yield self.auth.get_user_by_req(request) pagination_config = PaginationConfig.from_request(request) content = yield self.handlers.message_handler.room_initial_sync( room_id=room_id, @@ -395,8 +401,8 @@ class RoomMembershipRestServlet(ClientV1RestServlet): register_txn_path(self, PATTERN, http_server) @defer.inlineCallbacks - def on_POST(self, request, room_id, membership_action): - user, device_id = yield self.auth.get_user_by_req(request) + def on_POST(self, request, room_id, membership_action, txn_id=None): + user, client = yield self.auth.get_user_by_req(request) content = _parse_json(request) @@ -418,7 +424,9 @@ class RoomMembershipRestServlet(ClientV1RestServlet): "room_id": room_id, "sender": user.to_string(), "state_key": state_key, - } + }, + client=client, + txn_id=txn_id, ) defer.returnValue((200, {})) @@ -432,7 +440,9 @@ class RoomMembershipRestServlet(ClientV1RestServlet): except KeyError: pass - response = yield self.on_POST(request, room_id, membership_action) + response = yield self.on_POST( + request, room_id, membership_action, txn_id + ) self.txns.store_client_transaction(request, txn_id, response) defer.returnValue(response) @@ -444,8 +454,8 @@ class RoomRedactEventRestServlet(ClientV1RestServlet): register_txn_path(self, PATTERN, http_server) @defer.inlineCallbacks - def on_POST(self, request, room_id, event_id): - user, device_id = yield self.auth.get_user_by_req(request) + def on_POST(self, request, room_id, event_id, txn_id=None): + user, client = yield self.auth.get_user_by_req(request) content = _parse_json(request) msg_handler = self.handlers.message_handler @@ -456,7 +466,9 @@ class RoomRedactEventRestServlet(ClientV1RestServlet): "room_id": room_id, "sender": user.to_string(), "redacts": event_id, - } + }, + client=client, + txn_id=txn_id, ) defer.returnValue((200, {"event_id": event.event_id})) @@ -470,7 +482,7 @@ class RoomRedactEventRestServlet(ClientV1RestServlet): except KeyError: pass - response = yield self.on_POST(request, room_id, event_id) + response = yield self.on_POST(request, room_id, event_id, txn_id) self.txns.store_client_transaction(request, txn_id, response) defer.returnValue(response) @@ -483,7 +495,7 @@ class RoomTypingRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, room_id, user_id): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) room_id = urllib.unquote(room_id) target_user = UserID.from_string(urllib.unquote(user_id)) diff --git a/synapse/rest/client/v1/voip.py b/synapse/rest/client/v1/voip.py index 42d8e30bab..11d08fbced 100644 --- a/synapse/rest/client/v1/voip.py +++ b/synapse/rest/client/v1/voip.py @@ -28,7 +28,7 @@ class VoipRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) turnUris = self.hs.config.turn_uris turnSecret = self.hs.config.turn_shared_secret diff --git a/synapse/rest/media/v0/content_repository.py b/synapse/rest/media/v0/content_repository.py index 311ab89edb..22e26e3cd5 100644 --- a/synapse/rest/media/v0/content_repository.py +++ b/synapse/rest/media/v0/content_repository.py @@ -66,7 +66,7 @@ class ContentRepoResource(resource.Resource): @defer.inlineCallbacks def map_request_to_name(self, request): # auth the user - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) # namespace all file uploads on the user prefix = base64.urlsafe_b64encode( diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py index 6bed8a8efa..b939a30e19 100644 --- a/synapse/rest/media/v1/upload_resource.py +++ b/synapse/rest/media/v1/upload_resource.py @@ -42,7 +42,7 @@ class UploadResource(BaseMediaResource): @defer.inlineCallbacks def _async_render_POST(self, request): try: - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) # TODO: The checks here are a bit late. The content will have # already been uploaded to a tmp file at this point content_length = request.getHeader("Content-Length") diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 75dffa4db2..029b07cc66 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -122,7 +122,8 @@ class RegistrationStore(SQLBaseStore): def _query_for_auth(self, txn, token): sql = ( - "SELECT users.name, users.admin, access_tokens.device_id" + "SELECT users.name, users.admin," + " access_tokens.device_id, access_tokens.id as token_id" " FROM users" " INNER JOIN access_tokens on users.id = access_tokens.user_id" " WHERE token = ?" diff --git a/synapse/types.py b/synapse/types.py index faac729ff2..46dbab5374 100644 --- a/synapse/types.py +++ b/synapse/types.py @@ -119,3 +119,6 @@ class StreamToken( d = self._asdict() d[key] = new_value return StreamToken(**d) + + +ClientID = namedtuple("ClientID", ("device_id", "token_id")) diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py index a4f2abf213..f849120a3e 100644 --- a/tests/rest/client/v1/test_presence.py +++ b/tests/rest/client/v1/test_presence.py @@ -75,6 +75,7 @@ class PresenceStateTestCase(unittest.TestCase): "user": UserID.from_string(myid), "admin": False, "device_id": None, + "token_id": 1, } hs.get_auth().get_user_by_token = _get_user_by_token @@ -165,6 +166,7 @@ class PresenceListTestCase(unittest.TestCase): "user": UserID.from_string(myid), "admin": False, "device_id": None, + "token_id": 1, } hs.handlers.room_member_handler = Mock( diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py index 76ed550b75..81ead10e76 100644 --- a/tests/rest/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -70,6 +70,7 @@ class RoomPermissionsTestCase(RestTestCase): "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, + "token_id": 1, } hs.get_auth().get_user_by_token = _get_user_by_token @@ -466,6 +467,7 @@ class RoomsMemberListTestCase(RestTestCase): "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, + "token_id": 1, } hs.get_auth().get_user_by_token = _get_user_by_token @@ -555,6 +557,7 @@ class RoomsCreateTestCase(RestTestCase): "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, + "token_id": 1, } hs.get_auth().get_user_by_token = _get_user_by_token @@ -657,6 +660,7 @@ class RoomTopicTestCase(RestTestCase): "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, + "token_id": 1, } hs.get_auth().get_user_by_token = _get_user_by_token @@ -773,6 +777,7 @@ class RoomMemberStateTestCase(RestTestCase): "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, + "token_id": 1, } hs.get_auth().get_user_by_token = _get_user_by_token @@ -909,6 +914,7 @@ class RoomMessagesTestCase(RestTestCase): "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, + "token_id": 1, } hs.get_auth().get_user_by_token = _get_user_by_token @@ -1013,6 +1019,7 @@ class RoomInitialSyncTestCase(RestTestCase): "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, + "token_id": 1, } hs.get_auth().get_user_by_token = _get_user_by_token diff --git a/tests/rest/client/v1/test_typing.py b/tests/rest/client/v1/test_typing.py index c89b37d004..c5d5b06da3 100644 --- a/tests/rest/client/v1/test_typing.py +++ b/tests/rest/client/v1/test_typing.py @@ -73,6 +73,7 @@ class RoomTypingTestCase(RestTestCase): "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, + "token_id": 1, } hs.get_auth().get_user_by_token = _get_user_by_token diff --git a/tests/storage/test_registration.py b/tests/storage/test_registration.py index 84bfde7568..6f8bea2f61 100644 --- a/tests/storage/test_registration.py +++ b/tests/storage/test_registration.py @@ -53,7 +53,10 @@ class RegistrationStoreTestCase(unittest.TestCase): ) self.assertEquals( - {"admin": 0, "device_id": None, "name": self.user_id}, + {"admin": 0, + "device_id": None, + "name": self.user_id, + "token_id": 1}, (yield self.store.get_user_by_token(self.tokens[0])) ) @@ -63,7 +66,10 @@ class RegistrationStoreTestCase(unittest.TestCase): yield self.store.add_access_token_to_user(self.user_id, self.tokens[1]) self.assertEquals( - {"admin": 0, "device_id": None, "name": self.user_id}, + {"admin": 0, + "device_id": None, + "name": self.user_id, + "token_id": 2}, (yield self.store.get_user_by_token(self.tokens[1])) ) From c18e551640994c8b2c509509bcf664748dd05724 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 28 Jan 2015 17:08:53 +0000 Subject: [PATCH 100/180] Add a : to the doc string after the type of the return value --- synapse/api/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 3959e06a8b..f08cb76159 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -290,7 +290,7 @@ class Auth(object): Args: request - An HTTP request with an access_token query parameter. Returns: - Tuple of UserID and device string: + tuple : of UserID and device string: User ID object of the user making the request Client ID object of the client instance the user is using Raises: From 3cca61e006d7e69b6643721c01ab7d81a8c2f373 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 28 Jan 2015 17:16:12 +0000 Subject: [PATCH 101/180] Rename ClientID to ClientInfo since it is a pair of IDs rather than a single identifier --- synapse/api/auth.py | 4 ++-- synapse/types.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index f08cb76159..9c03024512 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -21,7 +21,7 @@ from synapse.api.constants import EventTypes, Membership, JoinRules from synapse.api.errors import AuthError, StoreError, Codes, SynapseError from synapse.util.logutils import log_function from synapse.util.async import run_on_reactor -from synapse.types import UserID, ClientID +from synapse.types import UserID, ClientInfo import logging @@ -318,7 +318,7 @@ class Auth(object): user_agent=user_agent ) - defer.returnValue((user, ClientID(device_id, token_id))) + defer.returnValue((user, ClientInfo(device_id, token_id))) except KeyError: raise AuthError(403, "Missing access token.") diff --git a/synapse/types.py b/synapse/types.py index 46dbab5374..f6a1b0bbcf 100644 --- a/synapse/types.py +++ b/synapse/types.py @@ -121,4 +121,4 @@ class StreamToken( return StreamToken(**d) -ClientID = namedtuple("ClientID", ("device_id", "token_id")) +ClientInfo = namedtuple("ClientInfo", ("device_id", "token_id")) From c81a19552f12a16591ae695fa1ef660f9e38730e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 28 Jan 2015 17:32:41 +0000 Subject: [PATCH 102/180] Add ports back to demo/start.sh --- demo/start.sh | 2 +- synapse/rest/client/v2_alpha/sync.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/demo/start.sh b/demo/start.sh index a7502e7a8d..bb2248770d 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -16,7 +16,7 @@ if [ $# -eq 1 ]; then fi fi -for port in 8080; do +for port in 8080 8081 8082; do echo "Starting server on port $port... " https_port=$((port + 400)) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 0c17208cd3..a0ab9839a6 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -80,7 +80,7 @@ class SyncRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request): - user = yield self.auth.get_user_by_req(request) + user, client = yield self.auth.get_user_by_req(request) timeout = self.parse_integer(request, "timeout", default=0) limit = self.parse_integer(request, "limit", required=True) From 11634017f47779d784325da5513517ad76b0dbc1 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 28 Jan 2015 17:42:19 +0000 Subject: [PATCH 103/180] s/definition/filter_json/ since definition is now used to mean a component of the filter, rather than the complete json --- synapse/storage/filtering.py | 4 ++-- synapse/storage/schema/filtering.sql | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/storage/filtering.py b/synapse/storage/filtering.py index bab68a9eef..cb01c2040f 100644 --- a/synapse/storage/filtering.py +++ b/synapse/storage/filtering.py @@ -33,7 +33,7 @@ class FilteringStore(SQLBaseStore): "user_id": user_localpart, "filter_id": filter_id, }, - retcol="definition", + retcol="filter_json", allow_none=False, ) @@ -57,7 +57,7 @@ class FilteringStore(SQLBaseStore): filter_id = max_id + 1 sql = ( - "INSERT INTO user_filters (user_id, filter_id, definition)" + "INSERT INTO user_filters (user_id, filter_id, filter_json)" "VALUES(?, ?, ?)" ) txn.execute(sql, (user_localpart, filter_id, def_json)) diff --git a/synapse/storage/schema/filtering.sql b/synapse/storage/schema/filtering.sql index 795aca4afd..beb39ca201 100644 --- a/synapse/storage/schema/filtering.sql +++ b/synapse/storage/schema/filtering.sql @@ -15,7 +15,7 @@ CREATE TABLE IF NOT EXISTS user_filters( user_id TEXT, filter_id INTEGER, - definition TEXT, + filter_json TEXT, FOREIGN KEY(user_id) REFERENCES users(id) ); From 8552ed8df2990d79b0015e0e84dd98de25fd0a9d Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 18:04:40 +0000 Subject: [PATCH 104/180] Change uses of get_user_by_req because it returns a tuple now. --- synapse/rest/client/v1/push_rule.py | 6 +++--- synapse/rest/client/v1/pusher.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 2b33bdac08..64743a2f46 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -138,7 +138,7 @@ class PushRuleRestServlet(ClientV1RestServlet): except InvalidRuleException as e: raise SynapseError(400, e.message) - user = yield self.auth.get_user_by_req(request) + user, _ = yield self.auth.get_user_by_req(request) content = _parse_json(request) @@ -184,7 +184,7 @@ class PushRuleRestServlet(ClientV1RestServlet): except InvalidRuleException as e: raise SynapseError(400, e.message) - user = yield self.auth.get_user_by_req(request) + user, _ = yield self.auth.get_user_by_req(request) if 'device' in spec: rules = yield self.hs.get_datastore().get_push_rules_for_user_name( @@ -215,7 +215,7 @@ class PushRuleRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request): - user = yield self.auth.get_user_by_req(request) + user, _ = yield self.auth.get_user_by_req(request) # we build up the full structure and then decide which bits of it # to send which means doing unnecessary work sometimes but is diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py index 80a11890a3..72d5e9e476 100644 --- a/synapse/rest/client/v1/pusher.py +++ b/synapse/rest/client/v1/pusher.py @@ -27,7 +27,7 @@ class PusherRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request): - user = yield self.auth.get_user_by_req(request) + user, _ = yield self.auth.get_user_by_req(request) content = _parse_json(request) From d5bdf3c0c7958e6a080f9ec4b38a51428717d02a Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 18:06:04 +0000 Subject: [PATCH 105/180] Allow the push rule delete method to take more specifiers. --- synapse/storage/push_rule.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index 0342996ed1..c7b553292e 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -175,14 +175,17 @@ class PushRuleStore(SQLBaseStore): txn.execute(sql, new_rule.values()) @defer.inlineCallbacks - def delete_push_rule(self, user_name, rule_id): - yield self._simple_delete_one( - PushRuleTable.table_name, - { - 'user_name': user_name, - 'rule_id': rule_id - } - ) + def delete_push_rule(self, user_name, rule_id, **kwargs): + """ + Delete a push rule. Args specify the row to be deleted and can be + any of the columns in the push_rule table, but below are the + standard ones + + Args: + user_name (str): The matrix ID of the push rule owner + rule_id (str): The rule_id of the rule to be deleted + """ + yield self._simple_delete_one(PushRuleTable.table_name, kwargs) class RuleNotFoundException(Exception): From b0b80074e04dcb2b70dff56b0368060a19c065d3 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 01:48:36 +0000 Subject: [PATCH 106/180] SYN-252: Supply the stream and topological parts in the correct order to the constructor --- synapse/storage/stream.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 8ac2adab05..062ca06fb3 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -82,10 +82,10 @@ class _StreamToken(namedtuple("_StreamToken", "topological stream")): def parse(cls, string): try: if string[0] == 's': - return cls(None, int(string[1:])) + return cls(topological=None, stream=int(string[1:])) if string[0] == 't': parts = string[1:].split('-', 1) - return cls(int(parts[1]), int(parts[0])) + return cls(topological=int(parts[0]), stream=int(parts[1])) except: pass raise SynapseError(400, "Invalid token %r" % (string,)) @@ -94,7 +94,7 @@ class _StreamToken(namedtuple("_StreamToken", "topological stream")): def parse_stream_token(cls, string): try: if string[0] == 's': - return cls(None, int(string[1:])) + return cls(topological=None, stream=int(string[1:])) except: pass raise SynapseError(400, "Invalid token %r" % (string,)) From 1b4a164c026ac027b7c84422f6c743c36896a0e7 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 02:34:35 +0000 Subject: [PATCH 107/180] Add support for formatting events in the way a v2 client expects --- synapse/events/utils.py | 91 +++++++++++++++++++++++------------------ 1 file changed, 52 insertions(+), 39 deletions(-) diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 42fb0371e5..dd7f3d6f42 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -89,7 +89,46 @@ def prune_event(event): return type(event)(allowed_fields) -def serialize_event(e, time_now_ms, client_event=True, strip_ids=False): +def format_event_raw(d): + return d + + +def format_event_for_client_v1(d): + d["user_id"] = d.pop("sender", None) + + move_keys = ("age", "redacted_because", "replaces_state", "prev_content") + for key in move_keys: + if key in d["unsigned"]: + d[key] = d["unsigned"][key] + + drop_keys = ( + "auth_events", "prev_events", "hashes", "signatures", "depth", + "unsigned", "origin" + ) + for key in drop_keys: + d.pop(key, None) + return d + + +def format_event_for_client_v2(d): + drop_keys = ( + "auth_events", "prev_events", "hashes", "signatures", "depth", "origin" + ) + for key in drop_keys: + d.pop(key, None) + return d + + +def format_event_for_client_v2_without_event_id(d): + d = format_event_for_client_v2(d) + d.pop("room_id", None) + d.pop("event_id", None) + return d + + +def serialize_event(e, time_now_ms, as_client_event=True, + event_format=format_event_for_client_v1, + token_id=None): # FIXME(erikj): To handle the case of presence events and the like if not isinstance(e, EventBase): return e @@ -99,48 +138,22 @@ def serialize_event(e, time_now_ms, client_event=True, strip_ids=False): # Should this strip out None's? d = {k: v for k, v in e.get_dict().items()} - if not client_event: - # set the age and keep all other keys - if "age_ts" in d["unsigned"]: - d["unsigned"]["age"] = time_now_ms - d["unsigned"]["age_ts"] - return d - if "age_ts" in d["unsigned"]: - d["age"] = time_now_ms - d["unsigned"]["age_ts"] - del d["unsigned"]["age_ts"] - - d["user_id"] = d.pop("sender", None) + d["unsigned"]["age"] = time_now_ms - d["unsigned"]["age_ts"] + d["unsigned"]["age_ts"] if "redacted_because" in e.unsigned: - d["redacted_because"] = serialize_event( + d["unsigned"]["redacted_because"] = serialize_event( e.unsigned["redacted_because"], time_now_ms ) - del d["unsigned"]["redacted_because"] + if token_id is not None: + if token_id == e.internal_metadata["token_id"]: + txn_id = e.internal_metadata.get("txn_id", None) + if txn_id is not None: + d["unsigned"]["transaction_id"] = txn_id - if "redacted_by" in e.unsigned: - d["redacted_by"] = e.unsigned["redacted_by"] - del d["unsigned"]["redacted_by"] - - if "replaces_state" in e.unsigned: - d["replaces_state"] = e.unsigned["replaces_state"] - del d["unsigned"]["replaces_state"] - - if "prev_content" in e.unsigned: - d["prev_content"] = e.unsigned["prev_content"] - del d["unsigned"]["prev_content"] - - del d["auth_events"] - del d["prev_events"] - del d["hashes"] - del d["signatures"] - d.pop("depth", None) - d.pop("unsigned", None) - d.pop("origin", None) - d.pop("prev_state", None) - - if strip_ids: - d.pop("room_id", None) - d.pop("event_id", None) - - return d + if as_client_event: + return event_format(d) + else: + return d From b9c442c85c9f8c2aa7fcb57d6132dec9b85b4e60 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 02:45:33 +0000 Subject: [PATCH 108/180] Include transaction ids in unsigned section of events in the sync results for the clients that made those requests --- synapse/events/utils.py | 11 ++++++----- synapse/rest/client/v2_alpha/sync.py | 23 ++++++++++++++++------- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/synapse/events/utils.py b/synapse/events/utils.py index dd7f3d6f42..21316cc125 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -103,7 +103,7 @@ def format_event_for_client_v1(d): drop_keys = ( "auth_events", "prev_events", "hashes", "signatures", "depth", - "unsigned", "origin" + "unsigned", "origin", "prev_state" ) for key in drop_keys: d.pop(key, None) @@ -112,7 +112,8 @@ def format_event_for_client_v1(d): def format_event_for_client_v2(d): drop_keys = ( - "auth_events", "prev_events", "hashes", "signatures", "depth", "origin" + "auth_events", "prev_events", "hashes", "signatures", "depth", + "origin", "prev_state", ) for key in drop_keys: d.pop(key, None) @@ -140,7 +141,7 @@ def serialize_event(e, time_now_ms, as_client_event=True, if "age_ts" in d["unsigned"]: d["unsigned"]["age"] = time_now_ms - d["unsigned"]["age_ts"] - d["unsigned"]["age_ts"] + del d["unsigned"]["age_ts"] if "redacted_because" in e.unsigned: d["unsigned"]["redacted_because"] = serialize_event( @@ -148,8 +149,8 @@ def serialize_event(e, time_now_ms, as_client_event=True, ) if token_id is not None: - if token_id == e.internal_metadata["token_id"]: - txn_id = e.internal_metadata.get("txn_id", None) + if token_id == getattr(e.internal_metadata, "token_id", None): + txn_id = getattr(e.internal_metadata, "txn_id", None) if txn_id is not None: d["unsigned"]["transaction_id"] = txn_id diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index a0ab9839a6..4d950f9956 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -18,7 +18,9 @@ from twisted.internet import defer from synapse.http.servlet import RestServlet from synapse.handlers.sync import SyncConfig from synapse.types import StreamToken -from synapse.events.utils import serialize_event +from synapse.events.utils import ( + serialize_event, format_event_for_client_v2_without_event_id, +) from ._base import client_v2_pattern import logging @@ -139,7 +141,9 @@ class SyncRestServlet(RestServlet): "private_user_data": self.encode_events( sync_result.private_user_data, filter, time_now ), - "rooms": self.encode_rooms(sync_result.rooms, filter, time_now), + "rooms": self.encode_rooms( + sync_result.rooms, filter, time_now, client.token_id + ), "next_batch": sync_result.next_batch.to_string(), } @@ -153,25 +157,30 @@ class SyncRestServlet(RestServlet): # TODO(mjark): Respect formatting requirements in the filter. return serialize_event(event, time_now) - def encode_rooms(self, rooms, filter, time_now): - return [self.encode_room(room, filter, time_now) for room in rooms] + def encode_rooms(self, rooms, filter, time_now, token_id): + return [ + self.encode_room(room, filter, time_now, token_id) + for room in rooms + ] @staticmethod - def encode_room(room, filter, time_now): + def encode_room(room, filter, time_now, token_id): event_map = {} state_event_ids = [] recent_event_ids = [] for event in room.state: # TODO(mjark): Respect formatting requirements in the filter. event_map[event.event_id] = serialize_event( - event, time_now, strip_ids=True + event, time_now, token_id=token_id, + event_format=format_event_for_client_v2_without_event_id, ) state_event_ids.append(event.event_id) for event in room.events: # TODO(mjark): Respect formatting requirements in the filter. event_map[event.event_id] = serialize_event( - event, time_now, strip_ids=True + event, time_now, token_id=token_id, + event_format=format_event_for_client_v2_without_event_id, ) recent_event_ids.append(event.event_id) return { From 3dbce6f4a59fde2a67e563ce338f510feda2dd1a Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 03:33:51 +0000 Subject: [PATCH 109/180] Add typing notifications to sync --- synapse/handlers/sync.py | 32 +++++++++++++++++++--------- synapse/rest/client/v2_alpha/sync.py | 18 +++++++--------- 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 82a2c6986a..b86e783e14 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -44,11 +44,12 @@ class RoomSyncResult(collections.namedtuple("RoomSyncResult", [ "events", "state", "prev_batch", + "typing", ])): __slots__ = [] def __nonzero__(self): - return bool(self.events or self.state) + return bool(self.events or self.state or self.typing) class SyncResult(collections.namedtuple("SyncResult", [ @@ -196,15 +197,25 @@ class SyncHandler(BaseHandler): now_token = yield self.event_sources.get_current_token() - presence_stream = self.event_sources.sources["presence"] - pagination_config = PaginationConfig( - from_token=since_token, to_token=now_token - ) - presence, _ = yield presence_stream.get_pagination_rows( + presence_source = self.event_sources.sources["presence"] + presence, presence_key = yield presence_source.get_new_events_for_user( user=sync_config.user, - pagination_config=pagination_config.get_source_config("presence"), - key=None + from_key=since_token.presence_key, + limit=sync_config.limit, ) + now_token = now_token.copy_and_replace("presence_key", presence_key) + + typing_source = self.event_sources.sources["typing"] + typing, typing_key = yield typing_source.get_new_events_for_user( + user=sync_config.user, + from_key=since_token.typing_key, + limit=sync_config.limit, + ) + now_token = now_token.copy_and_replace("typing_key", typing_key) + + typing_by_room = {event["room_id"]: event for event in typing} + logger.debug("Typing %r", typing_by_room) + room_list = yield self.store.get_rooms_for_user_where_membership_is( user_id=sync_config.user.to_string(), membership_list=[Membership.INVITE, Membership.JOIN] @@ -218,7 +229,7 @@ class SyncHandler(BaseHandler): for event in room_list: room_sync = yield self.incremental_sync_with_gap_for_room( event.room_id, sync_config, since_token, now_token, - published_room_ids + published_room_ids, typing_by_room ) if room_sync: rooms.append(room_sync) @@ -233,7 +244,7 @@ class SyncHandler(BaseHandler): @defer.inlineCallbacks def incremental_sync_with_gap_for_room(self, room_id, sync_config, since_token, now_token, - published_room_ids): + published_room_ids, typing_by_room): """ Get the incremental delta needed to bring the client up to date for the room. Gives the client the most recent events and the changes to state. @@ -285,6 +296,7 @@ class SyncHandler(BaseHandler): prev_batch=prev_batch_token, state=state_events_delta, limited=limited, + typing=typing_by_room.get(room_id, None) ) logging.debug("Room sync: %r", room_sync) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 4d950f9956..76489e27c8 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -135,10 +135,10 @@ class SyncRestServlet(RestServlet): time_now = self.clock.time_msec() response_content = { - "public_user_data": self.encode_events( + "public_user_data": self.encode_user_data( sync_result.public_user_data, filter, time_now ), - "private_user_data": self.encode_events( + "private_user_data": self.encode_user_data( sync_result.private_user_data, filter, time_now ), "rooms": self.encode_rooms( @@ -149,13 +149,8 @@ class SyncRestServlet(RestServlet): defer.returnValue((200, response_content)) - def encode_events(self, events, filter, time_now): - return [self.encode_event(event, filter, time_now) for event in events] - - @staticmethod - def encode_event(event, filter, time_now): - # TODO(mjark): Respect formatting requirements in the filter. - return serialize_event(event, time_now) + def encode_user_data(self, events, filter, time_now): + return events def encode_rooms(self, rooms, filter, time_now, token_id): return [ @@ -183,7 +178,7 @@ class SyncRestServlet(RestServlet): event_format=format_event_for_client_v2_without_event_id, ) recent_event_ids.append(event.event_id) - return { + result = { "room_id": room.room_id, "event_map": event_map, "events": { @@ -194,6 +189,9 @@ class SyncRestServlet(RestServlet): "limited": room.limited, "published": room.published, } + if room.typing is not None: + result["typing"] = room.typing + return result def register_servlets(hs, http_server): From e3e72b8c5c004c403929406fc952141fd0f1b8ae Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 03:35:25 +0000 Subject: [PATCH 110/180] Remove typing TODO --- synapse/handlers/sync.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index b86e783e14..9e1188da56 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -255,7 +255,6 @@ class SyncHandler(BaseHandler): # the previous sync and this one. # TODO(mjark): Apply the event filter in sync_config # TODO(mjark): Check for redactions we might have missed. - # TODO(mjark): Typing notifications. recents, token = yield self.store.get_recent_events_for_room( room_id, limit=sync_config.limit + 1, From 3773759c0f1e25e6905e23368f770da99ceb3ea0 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 29 Jan 2015 09:15:33 +0000 Subject: [PATCH 111/180] Also edit the filter column on the delta SQL --- synapse/storage/schema/delta/v12.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/schema/delta/v12.sql b/synapse/storage/schema/delta/v12.sql index 795aca4afd..beb39ca201 100644 --- a/synapse/storage/schema/delta/v12.sql +++ b/synapse/storage/schema/delta/v12.sql @@ -15,7 +15,7 @@ CREATE TABLE IF NOT EXISTS user_filters( user_id TEXT, filter_id INTEGER, - definition TEXT, + filter_json TEXT, FOREIGN KEY(user_id) REFERENCES users(id) ); From 2a4fda7b88cf91db8de2e524df162153d3f27094 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 29 Jan 2015 09:27:16 +0000 Subject: [PATCH 112/180] Add filtering.filter_events function, with stub passes_filter function. --- synapse/api/filtering.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 6c7a73b6d5..d7ba6510ee 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -34,6 +34,21 @@ class Filtering(object): # replace_user_filter at some point? There's no REST API specified for # them however + def passes_filter(self, filter_json, event): + """Check if the event passes through the filter. + + Args: + filter_json(dict): The filter specification + event(Event): The event to check + Returns: + True if the event passes through the filter. + """ + return True + + def filter_events(self, events, user, filter_id): + filter_json = self.get_user_filter(user, filter_id) + return [e for e in events if self.passes_filter(filter_json, e)] + def _check_valid_filter(self, user_filter): """Check if the provided filter is valid. From 50de1eaad94715a1dda470f44f379683e5fa552b Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 29 Jan 2015 10:24:57 +0000 Subject: [PATCH 113/180] Add filtering public API; outline filtering algorithm. --- synapse/api/filtering.py | 60 ++++++++++++++++++++++++++++++++++------ 1 file changed, 52 insertions(+), 8 deletions(-) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index d7ba6510ee..21fe72d6c2 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -30,25 +30,69 @@ class Filtering(object): self._check_valid_filter(user_filter) return self.store.add_user_filter(user_localpart, user_filter) + def filter_public_user_data(self, events, user, filter_id): + return self._filter_on_key( + events, user, filter_id, ["public_user_data"] + ) + + def filter_private_user_data(self, events, user, filter_id): + return self._filter_on_key( + events, user, filter_id, ["private_user_data"] + ) + + def filter_room_state(self, events, user, filter_id): + return self._filter_on_key( + events, user, filter_id, ["room", "state"] + ) + + def filter_room_events(self, events, user, filter_id): + return self._filter_on_key( + events, user, filter_id, ["room", "events"] + ) + + def filter_room_ephemeral(self, events, user, filter_id): + return self._filter_on_key( + events, user, filter_id, ["room", "ephemeral"] + ) + # TODO(paul): surely we should probably add a delete_user_filter or # replace_user_filter at some point? There's no REST API specified for # them however - def passes_filter(self, filter_json, event): - """Check if the event passes through the filter. + def _filter_on_key(self, events, user, filter_id, keys): + filter_json = self.get_user_filter(user.localpart, filter_id) + if not filter_json: + return events + + try: + # extract the right definition from the filter + definition = filter_json + for key in keys: + definition = definition[key] + return self._filter_with_definition(events, definition) + except KeyError: + return events # return all events if definition isn't specified. + + def _filter_with_definition(self, events, definition): + return [e for e in events if self._passes_definition(definition, e)] + + def _passes_definition(self, definition, event): + """Check if the event passes through the given definition. Args: - filter_json(dict): The filter specification - event(Event): The event to check + definition(dict): The definition to check against. + event(Event): The event to check. Returns: True if the event passes through the filter. """ + # Algorithm notes: + # For each key in the definition, check the event meets the criteria: + # * For types: Literal match or prefix match (if ends with wildcard) + # * For senders/rooms: Literal match only + # * "not_" checks take presedence (e.g. if "m.*" is in both 'types' + # and 'not_types' then it is treated as only being in 'not_types') return True - def filter_events(self, events, user, filter_id): - filter_json = self.get_user_filter(user, filter_id) - return [e for e in events if self.passes_filter(filter_json, e)] - def _check_valid_filter(self, user_filter): """Check if the provided filter is valid. From 777d9914b537d06ebba91948a26d74d3a04b7284 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 29 Jan 2015 11:38:06 +0000 Subject: [PATCH 114/180] Implement filter algorithm. Add basic event type unit tests to assert it works. --- synapse/api/filtering.py | 49 +++++++++++++++++++++++++++++++++++++ tests/api/test_filtering.py | 45 +++++++++++++++++++++++++++++++++- 2 files changed, 93 insertions(+), 1 deletion(-) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 21fe72d6c2..8bc95aa394 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -91,8 +91,57 @@ class Filtering(object): # * For senders/rooms: Literal match only # * "not_" checks take presedence (e.g. if "m.*" is in both 'types' # and 'not_types' then it is treated as only being in 'not_types') + + # room checks + if hasattr(event, "room_id"): + room_id = event.room_id + allow_rooms = definition["rooms"] if "rooms" in definition else None + reject_rooms = ( + definition["not_rooms"] if "not_rooms" in definition else None + ) + if reject_rooms and room_id in reject_rooms: + return False + if allow_rooms and room_id not in allow_rooms: + return False + + # sender checks + if hasattr(event, "sender"): + # Should we be including event.state_key for some event types? + sender = event.sender + allow_senders = ( + definition["senders"] if "senders" in definition else None + ) + reject_senders = ( + definition["not_senders"] if "not_senders" in definition else None + ) + if reject_senders and sender in reject_senders: + return False + if allow_senders and sender not in allow_senders: + return False + + # type checks + if "not_types" in definition: + for def_type in definition["not_types"]: + if self._event_matches_type(event, def_type): + return False + if "types" in definition: + included = False + for def_type in definition["types"]: + if self._event_matches_type(event, def_type): + included = True + break + if not included: + return False + return True + def _event_matches_type(self, event, def_type): + if def_type.endswith("*"): + type_prefix = def_type[:-1] + return event.type.startswith(type_prefix) + else: + return event.type == def_type + def _check_valid_filter(self, user_filter): """Check if the provided filter is valid. diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py index 188fbfb91e..4d40d88b00 100644 --- a/tests/api/test_filtering.py +++ b/tests/api/test_filtering.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - +from collections import namedtuple from tests import unittest from twisted.internet import defer @@ -27,6 +27,7 @@ from synapse.server import HomeServer user_localpart = "test_user" +MockEvent = namedtuple("MockEvent", "sender type room_id") class FilteringTestCase(unittest.TestCase): @@ -55,6 +56,48 @@ class FilteringTestCase(unittest.TestCase): self.datastore = hs.get_datastore() + def test_definition_include_literal_types(self): + definition = { + "types": ["m.room.message", "org.matrix.foo.bar"] + } + event = MockEvent( + sender="@foo:bar", + type="m.room.message", + room_id="!foo:bar" + ) + + self.assertTrue( + self.filtering._passes_definition(definition, event) + ) + + def test_definition_include_wildcard_types(self): + definition = { + "types": ["m.*", "org.matrix.foo.bar"] + } + event = MockEvent( + sender="@foo:bar", + type="m.room.message", + room_id="!foo:bar" + ) + + self.assertTrue( + self.filtering._passes_definition(definition, event) + ) + + def test_definition_exclude_unknown_types(self): + definition = { + "types": ["m.room.message", "org.matrix.foo.bar"] + } + event = MockEvent( + sender="@foo:bar", + type="now.for.something.completely.different", + room_id="!foo:bar" + ) + + self.assertFalse( + self.filtering._passes_definition(definition, event) + ) + @defer.inlineCallbacks def test_add_filter(self): user_filter = { From 5561a879205316ae2c4cd0106cfd99d4fe35bceb Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 29 Jan 2015 12:06:16 +0000 Subject: [PATCH 115/180] Add more unit tests for the filter algorithm. --- tests/api/test_filtering.py | 264 +++++++++++++++++++++++++++++++++++- 1 file changed, 259 insertions(+), 5 deletions(-) diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py index 4d40d88b00..380dd97937 100644 --- a/tests/api/test_filtering.py +++ b/tests/api/test_filtering.py @@ -56,7 +56,7 @@ class FilteringTestCase(unittest.TestCase): self.datastore = hs.get_datastore() - def test_definition_include_literal_types(self): + def test_definition_types_works_with_literals(self): definition = { "types": ["m.room.message", "org.matrix.foo.bar"] } @@ -65,12 +65,11 @@ class FilteringTestCase(unittest.TestCase): type="m.room.message", room_id="!foo:bar" ) - self.assertTrue( self.filtering._passes_definition(definition, event) ) - def test_definition_include_wildcard_types(self): + def test_definition_types_works_with_wildcards(self): definition = { "types": ["m.*", "org.matrix.foo.bar"] } @@ -79,12 +78,11 @@ class FilteringTestCase(unittest.TestCase): type="m.room.message", room_id="!foo:bar" ) - self.assertTrue( self.filtering._passes_definition(definition, event) ) - def test_definition_exclude_unknown_types(self): + def test_definition_types_works_with_unknowns(self): definition = { "types": ["m.room.message", "org.matrix.foo.bar"] } @@ -93,7 +91,263 @@ class FilteringTestCase(unittest.TestCase): type="now.for.something.completely.different", room_id="!foo:bar" ) + self.assertFalse( + self.filtering._passes_definition(definition, event) + ) + def test_definition_not_types_works_with_literals(self): + definition = { + "not_types": ["m.room.message", "org.matrix.foo.bar"] + } + event = MockEvent( + sender="@foo:bar", + type="m.room.message", + room_id="!foo:bar" + ) + self.assertFalse( + self.filtering._passes_definition(definition, event) + ) + + def test_definition_not_types_works_with_wildcards(self): + definition = { + "not_types": ["m.room.message", "org.matrix.*"] + } + event = MockEvent( + sender="@foo:bar", + type="org.matrix.custom.event", + room_id="!foo:bar" + ) + self.assertFalse( + self.filtering._passes_definition(definition, event) + ) + + def test_definition_not_types_works_with_unknowns(self): + definition = { + "not_types": ["m.*", "org.*"] + } + event = MockEvent( + sender="@foo:bar", + type="com.nom.nom.nom", + room_id="!foo:bar" + ) + self.assertTrue( + self.filtering._passes_definition(definition, event) + ) + + def test_definition_not_types_takes_priority_over_types(self): + definition = { + "not_types": ["m.*", "org.*"], + "types": ["m.room.message", "m.room.topic"] + } + event = MockEvent( + sender="@foo:bar", + type="m.room.topic", + room_id="!foo:bar" + ) + self.assertFalse( + self.filtering._passes_definition(definition, event) + ) + + def test_definition_senders_works_with_literals(self): + definition = { + "senders": ["@flibble:wibble"] + } + event = MockEvent( + sender="@flibble:wibble", + type="com.nom.nom.nom", + room_id="!foo:bar" + ) + self.assertTrue( + self.filtering._passes_definition(definition, event) + ) + + def test_definition_senders_works_with_unknowns(self): + definition = { + "senders": ["@flibble:wibble"] + } + event = MockEvent( + sender="@challenger:appears", + type="com.nom.nom.nom", + room_id="!foo:bar" + ) + self.assertFalse( + self.filtering._passes_definition(definition, event) + ) + + def test_definition_not_senders_works_with_literals(self): + definition = { + "not_senders": ["@flibble:wibble"] + } + event = MockEvent( + sender="@flibble:wibble", + type="com.nom.nom.nom", + room_id="!foo:bar" + ) + self.assertFalse( + self.filtering._passes_definition(definition, event) + ) + + def test_definition_not_senders_works_with_unknowns(self): + definition = { + "not_senders": ["@flibble:wibble"] + } + event = MockEvent( + sender="@challenger:appears", + type="com.nom.nom.nom", + room_id="!foo:bar" + ) + self.assertTrue( + self.filtering._passes_definition(definition, event) + ) + + def test_definition_not_senders_takes_priority_over_senders(self): + definition = { + "not_senders": ["@misspiggy:muppets"], + "senders": ["@kermit:muppets", "@misspiggy:muppets"] + } + event = MockEvent( + sender="@misspiggy:muppets", + type="m.room.topic", + room_id="!foo:bar" + ) + self.assertFalse( + self.filtering._passes_definition(definition, event) + ) + + def test_definition_rooms_works_with_literals(self): + definition = { + "rooms": ["!secretbase:unknown"] + } + event = MockEvent( + sender="@foo:bar", + type="m.room.message", + room_id="!secretbase:unknown" + ) + self.assertTrue( + self.filtering._passes_definition(definition, event) + ) + + def test_definition_rooms_works_with_unknowns(self): + definition = { + "rooms": ["!secretbase:unknown"] + } + event = MockEvent( + sender="@foo:bar", + type="m.room.message", + room_id="!anothersecretbase:unknown" + ) + self.assertFalse( + self.filtering._passes_definition(definition, event) + ) + + def test_definition_not_rooms_works_with_literals(self): + definition = { + "not_rooms": ["!anothersecretbase:unknown"] + } + event = MockEvent( + sender="@foo:bar", + type="m.room.message", + room_id="!anothersecretbase:unknown" + ) + self.assertFalse( + self.filtering._passes_definition(definition, event) + ) + + def test_definition_not_rooms_works_with_unknowns(self): + definition = { + "not_rooms": ["!secretbase:unknown"] + } + event = MockEvent( + sender="@foo:bar", + type="m.room.message", + room_id="!anothersecretbase:unknown" + ) + self.assertTrue( + self.filtering._passes_definition(definition, event) + ) + + def test_definition_not_rooms_takes_priority_over_rooms(self): + definition = { + "not_rooms": ["!secretbase:unknown"], + "rooms": ["!secretbase:unknown"] + } + event = MockEvent( + sender="@foo:bar", + type="m.room.message", + room_id="!secretbase:unknown" + ) + self.assertFalse( + self.filtering._passes_definition(definition, event) + ) + + def test_definition_combined_event(self): + definition = { + "not_senders": ["@misspiggy:muppets"], + "senders": ["@kermit:muppets"], + "rooms": ["!stage:unknown"], + "not_rooms": ["!piggyshouse:muppets"], + "types": ["m.room.message", "muppets.kermit.*"], + "not_types": ["muppets.misspiggy.*"] + } + event = MockEvent( + sender="@kermit:muppets", # yup + type="m.room.message", # yup + room_id="!stage:unknown" # yup + ) + self.assertTrue( + self.filtering._passes_definition(definition, event) + ) + + def test_definition_combined_event_bad_sender(self): + definition = { + "not_senders": ["@misspiggy:muppets"], + "senders": ["@kermit:muppets"], + "rooms": ["!stage:unknown"], + "not_rooms": ["!piggyshouse:muppets"], + "types": ["m.room.message", "muppets.kermit.*"], + "not_types": ["muppets.misspiggy.*"] + } + event = MockEvent( + sender="@misspiggy:muppets", # nope + type="m.room.message", # yup + room_id="!stage:unknown" # yup + ) + self.assertFalse( + self.filtering._passes_definition(definition, event) + ) + + def test_definition_combined_event_bad_room(self): + definition = { + "not_senders": ["@misspiggy:muppets"], + "senders": ["@kermit:muppets"], + "rooms": ["!stage:unknown"], + "not_rooms": ["!piggyshouse:muppets"], + "types": ["m.room.message", "muppets.kermit.*"], + "not_types": ["muppets.misspiggy.*"] + } + event = MockEvent( + sender="@kermit:muppets", # yup + type="m.room.message", # yup + room_id="!piggyshouse:muppets" # nope + ) + self.assertFalse( + self.filtering._passes_definition(definition, event) + ) + + def test_definition_combined_event_bad_type(self): + definition = { + "not_senders": ["@misspiggy:muppets"], + "senders": ["@kermit:muppets"], + "rooms": ["!stage:unknown"], + "not_rooms": ["!piggyshouse:muppets"], + "types": ["m.room.message", "muppets.kermit.*"], + "not_types": ["muppets.misspiggy.*"] + } + event = MockEvent( + sender="@kermit:muppets", # yup + type="muppets.misspiggy.kisses", # nope + room_id="!stage:unknown" # yup + ) self.assertFalse( self.filtering._passes_definition(definition, event) ) From 83172487b05d7d99ccae0b353daee2f242445011 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 29 Jan 2015 12:20:59 +0000 Subject: [PATCH 116/180] Add basic filtering public API unit tests. Use defers in the right places. --- synapse/api/filtering.py | 11 +++++--- tests/api/test_filtering.py | 54 ++++++++++++++++++++++++++++++++++++- 2 files changed, 60 insertions(+), 5 deletions(-) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 8bc95aa394..7e239138b7 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from twisted.internet import defer from synapse.api.errors import SynapseError from synapse.types import UserID, RoomID @@ -59,19 +60,21 @@ class Filtering(object): # replace_user_filter at some point? There's no REST API specified for # them however + @defer.inlineCallbacks def _filter_on_key(self, events, user, filter_id, keys): - filter_json = self.get_user_filter(user.localpart, filter_id) + filter_json = yield self.get_user_filter(user.localpart, filter_id) if not filter_json: - return events + defer.returnValue(events) try: # extract the right definition from the filter definition = filter_json for key in keys: definition = definition[key] - return self._filter_with_definition(events, definition) + defer.returnValue(self._filter_with_definition(events, definition)) except KeyError: - return events # return all events if definition isn't specified. + # return all events if definition isn't specified. + defer.returnValue(events) def _filter_with_definition(self, events, definition): return [e for e in events if self._passes_definition(definition, e)] diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py index 380dd97937..97fb9758e9 100644 --- a/tests/api/test_filtering.py +++ b/tests/api/test_filtering.py @@ -24,7 +24,7 @@ from tests.utils import ( ) from synapse.server import HomeServer - +from synapse.types import UserID user_localpart = "test_user" MockEvent = namedtuple("MockEvent", "sender type room_id") @@ -352,6 +352,58 @@ class FilteringTestCase(unittest.TestCase): self.filtering._passes_definition(definition, event) ) + @defer.inlineCallbacks + def test_filter_public_user_data_match(self): + user_filter = { + "public_user_data": { + "types": ["m.*"] + } + } + user = UserID.from_string("@" + user_localpart + ":test") + filter_id = yield self.datastore.add_user_filter( + user_localpart=user_localpart, + user_filter=user_filter, + ) + event = MockEvent( + sender="@foo:bar", + type="m.profile", + room_id="!foo:bar" + ) + events = [event] + + results = yield self.filtering.filter_public_user_data( + events=events, + user=user, + filter_id=filter_id + ) + self.assertEquals(events, results) + + @defer.inlineCallbacks + def test_filter_public_user_data_no_match(self): + user_filter = { + "public_user_data": { + "types": ["m.*"] + } + } + user = UserID.from_string("@" + user_localpart + ":test") + filter_id = yield self.datastore.add_user_filter( + user_localpart=user_localpart, + user_filter=user_filter, + ) + event = MockEvent( + sender="@foo:bar", + type="custom.avatar.3d.crazy", + room_id="!foo:bar" + ) + events = [event] + + results = yield self.filtering.filter_public_user_data( + events=events, + user=user, + filter_id=filter_id + ) + self.assertEquals([], results) + @defer.inlineCallbacks def test_add_filter(self): user_filter = { From c183cec8f61f5b3488973f01ca5203183a00e6d1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 29 Jan 2015 13:44:52 +0000 Subject: [PATCH 117/180] Add post_json(...) method to federation client --- synapse/http/matrixfederationclient.py | 37 ++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 1dda3ba2c7..b1b2916fd3 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -244,6 +244,43 @@ class MatrixFederationHttpClient(object): defer.returnValue((response.code, body)) + @defer.inlineCallbacks + def post_json(self, destination, path, data={}): + """ Sends the specifed json data using POST + + Args: + destination (str): The remote server to send the HTTP request + to. + path (str): The HTTP path. + data (dict): A dict containing the data that will be used as + the request body. This will be encoded as JSON. + + Returns: + Deferred: Succeeds when we get a 2xx HTTP response. The result + will be the decoded JSON body. On a 4xx or 5xx error response a + CodeMessageException is raised. + """ + + def body_callback(method, url_bytes, headers_dict): + self.sign_request( + destination, method, url_bytes, headers_dict, data + ) + return None + + response = yield self._create_request( + destination.encode("ascii"), + "POST", + path.encode("ascii"), + body_callback=body_callback, + headers_dict={"Content-Type": ["application/json"]}, + ) + + logger.debug("Getting resp body") + body = yield readBody(response) + logger.debug("Got resp body") + + defer.returnValue((response.code, body)) + @defer.inlineCallbacks def get_json(self, destination, path, args={}, retry_on_dns_fail=True): """ GETs some json from the given host homeserver and path From 5a3a15f5c186af6b818f2b2d3a4dafeee48b4e33 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 29 Jan 2015 13:58:22 +0000 Subject: [PATCH 118/180] Make post_json(...) actually send data. --- synapse/http/matrixfederationclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index b1b2916fd3..c7bf1b47b8 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -265,7 +265,7 @@ class MatrixFederationHttpClient(object): self.sign_request( destination, method, url_bytes, headers_dict, data ) - return None + return _JsonProducer(data) response = yield self._create_request( destination.encode("ascii"), From 38b27bd2cbf38141938d6170c41e1d1dac9928cd Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 29 Jan 2015 14:28:34 +0000 Subject: [PATCH 119/180] Add filter_room_state unit tests. --- tests/api/test_filtering.py | 56 +++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py index 97fb9758e9..aa93616a9f 100644 --- a/tests/api/test_filtering.py +++ b/tests/api/test_filtering.py @@ -404,6 +404,62 @@ class FilteringTestCase(unittest.TestCase): ) self.assertEquals([], results) + @defer.inlineCallbacks + def test_filter_room_state_match(self): + user_filter = { + "room": { + "state": { + "types": ["m.*"] + } + } + } + user = UserID.from_string("@" + user_localpart + ":test") + filter_id = yield self.datastore.add_user_filter( + user_localpart=user_localpart, + user_filter=user_filter, + ) + event = MockEvent( + sender="@foo:bar", + type="m.room.topic", + room_id="!foo:bar" + ) + events = [event] + + results = yield self.filtering.filter_room_state( + events=events, + user=user, + filter_id=filter_id + ) + self.assertEquals(events, results) + + @defer.inlineCallbacks + def test_filter_room_state_no_match(self): + user_filter = { + "room": { + "state": { + "types": ["m.*"] + } + } + } + user = UserID.from_string("@" + user_localpart + ":test") + filter_id = yield self.datastore.add_user_filter( + user_localpart=user_localpart, + user_filter=user_filter, + ) + event = MockEvent( + sender="@foo:bar", + type="org.matrix.custom.event", + room_id="!foo:bar" + ) + events = [event] + + results = yield self.filtering.filter_room_state( + events=events, + user=user, + filter_id=filter_id + ) + self.assertEquals([], results) + @defer.inlineCallbacks def test_add_filter(self): user_filter = { From e016f4043b81ffdedf71c4459772f66757386e44 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 14:40:28 +0000 Subject: [PATCH 120/180] Use get_room_events_stream to get changes to the rooms if the number of changes is small --- synapse/handlers/sync.py | 56 +++++++++++++++++++++++++++++++-------- synapse/storage/stream.py | 7 +++++ 2 files changed, 52 insertions(+), 11 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 9e1188da56..e93dfe005d 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -216,23 +216,57 @@ class SyncHandler(BaseHandler): typing_by_room = {event["room_id"]: event for event in typing} logger.debug("Typing %r", typing_by_room) - room_list = yield self.store.get_rooms_for_user_where_membership_is( - user_id=sync_config.user.to_string(), - membership_list=[Membership.INVITE, Membership.JOIN] - ) + rm_handler = self.hs.get_handlers().room_member_handler + room_ids = yield rm_handler.get_rooms_for_user(sync_config.user) # TODO (mjark): Does public mean "published"? published_rooms = yield self.store.get_rooms(is_public=True) published_room_ids = set(r["room_id"] for r in published_rooms) + room_events, _ = yield self.store.get_room_events_stream( + sync_config.user.to_string(), + from_key=since_token.room_key, + to_key=now_token.room_key, + room_id=None, + limit=sync_config.limit + 1, + ) + rooms = [] - for event in room_list: - room_sync = yield self.incremental_sync_with_gap_for_room( - event.room_id, sync_config, since_token, now_token, - published_room_ids, typing_by_room - ) - if room_sync: - rooms.append(room_sync) + if len(room_events) <= sync_config.limit: + # There is no gap in any of the rooms. Therefore we can just + # partition the new events by room and return them. + events_by_room_id = {} + for event in room_events: + events_by_room_id.setdefault(event.room_id, []).append(event) + + for room_id in room_ids: + recents = events_by_room_id.get(room_id, []) + state = [event for event in recents if event.is_state()] + if recents: + prev_batch = now_token.copy_and_replace( + "room_key", recents[0].internal_metadata.before + ) + else: + prev_batch = now_token + room_sync = RoomSyncResult( + room_id=room_id, + published=room_id in published_room_ids, + events=recents, + prev_batch=prev_batch, + state=state, + limited=False, + typing=typing_by_room.get(room_id, None) + ) + if room_sync is not None: + rooms.append(room_sync) + else: + for room_id in room_ids: + room_sync = yield self.incremental_sync_with_gap_for_room( + room_id, sync_config, since_token, now_token, + published_room_ids, typing_by_room + ) + if room_sync: + rooms.append(room_sync) defer.returnValue(SyncResult( public_user_data=presence, diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index db1816ea84..93ccfd8c10 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -181,6 +181,13 @@ class StreamStore(SQLBaseStore): get_prev_content=True ) + for event, row in zip(ret, rows): + stream = row["stream_ordering"] + topo = event.depth + internal = event.internal_metadata + internal.before = str(_StreamToken(topo, stream - 1)) + internal.after = str(_StreamToken(topo, stream)) + if rows: key = "s%d" % max([r["stream_ordering"] for r in rows]) else: From e4f50fa0aa3426a272b1526072c4c42802989ba4 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 14:53:18 +0000 Subject: [PATCH 121/180] Move bump schema delta --- synapse/storage/schema/delta/{v12.sql => v13.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename synapse/storage/schema/delta/{v12.sql => v13.sql} (100%) diff --git a/synapse/storage/schema/delta/v12.sql b/synapse/storage/schema/delta/v13.sql similarity index 100% rename from synapse/storage/schema/delta/v12.sql rename to synapse/storage/schema/delta/v13.sql From 33391db5f8d9d0d365607ca50ba59ce72c90cda0 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 15:54:54 +0000 Subject: [PATCH 122/180] Merge in auth changes from develop --- synapse/rest/client/v2_alpha/filter.py | 4 ++-- tests/rest/client/v2_alpha/__init__.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/synapse/rest/client/v2_alpha/filter.py b/synapse/rest/client/v2_alpha/filter.py index 81a3e95155..cee06ccaca 100644 --- a/synapse/rest/client/v2_alpha/filter.py +++ b/synapse/rest/client/v2_alpha/filter.py @@ -40,7 +40,7 @@ class GetFilterRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id, filter_id): target_user = UserID.from_string(user_id) - auth_user = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) if target_user != auth_user: raise AuthError(403, "Cannot get filters for other users") @@ -76,7 +76,7 @@ class CreateFilterRestServlet(RestServlet): @defer.inlineCallbacks def on_POST(self, request, user_id): target_user = UserID.from_string(user_id) - auth_user = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) if target_user != auth_user: raise AuthError(403, "Cannot create filters for other users") diff --git a/tests/rest/client/v2_alpha/__init__.py b/tests/rest/client/v2_alpha/__init__.py index 3fe62d5ac6..fa70575c57 100644 --- a/tests/rest/client/v2_alpha/__init__.py +++ b/tests/rest/client/v2_alpha/__init__.py @@ -51,6 +51,7 @@ class V2AlphaRestTestCase(unittest.TestCase): "user": UserID.from_string(self.USER_ID), "admin": False, "device_id": None, + "token_id": 1, } hs.get_auth().get_user_by_token = _get_user_by_token From 9150a0d62ed4195b41834cea8a836332e74fb96b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 16:01:14 +0000 Subject: [PATCH 123/180] Fix code-style --- synapse/api/filtering.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 7e239138b7..e16c0e559f 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -74,7 +74,7 @@ class Filtering(object): defer.returnValue(self._filter_with_definition(events, definition)) except KeyError: # return all events if definition isn't specified. - defer.returnValue(events) + defer.returnValue(events) def _filter_with_definition(self, events, definition): return [e for e in events if self._passes_definition(definition, e)] @@ -94,14 +94,12 @@ class Filtering(object): # * For senders/rooms: Literal match only # * "not_" checks take presedence (e.g. if "m.*" is in both 'types' # and 'not_types' then it is treated as only being in 'not_types') - + # room checks if hasattr(event, "room_id"): room_id = event.room_id - allow_rooms = definition["rooms"] if "rooms" in definition else None - reject_rooms = ( - definition["not_rooms"] if "not_rooms" in definition else None - ) + allow_rooms = definition.get("rooms", None) + reject_rooms = definition.get("not_rooms", None) if reject_rooms and room_id in reject_rooms: return False if allow_rooms and room_id not in allow_rooms: @@ -111,12 +109,8 @@ class Filtering(object): if hasattr(event, "sender"): # Should we be including event.state_key for some event types? sender = event.sender - allow_senders = ( - definition["senders"] if "senders" in definition else None - ) - reject_senders = ( - definition["not_senders"] if "not_senders" in definition else None - ) + allow_senders = definition.get("senders", None) + reject_senders = definition.get("not_senders", None) if reject_senders and sender in reject_senders: return False if allow_senders and sender not in allow_senders: @@ -176,7 +170,6 @@ class Filtering(object): if key in user_filter["room"]: self._check_definition(user_filter["room"][key]) - def _check_definition(self, definition): """Check if the provided definition is valid. From 8b1dd9f57f8afb2d602d3b533ab89dbe1df6b465 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 29 Jan 2015 16:10:01 +0000 Subject: [PATCH 124/180] Only send a badge-reset if the user actually has unread notifications. --- synapse/push/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 10ac890482..fa967c5a5d 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -56,6 +56,7 @@ class Pusher(object): # The last value of last_active_time that we saw self.last_last_active_time = 0 + self.has_unread = True @defer.inlineCallbacks def _actions_for_event(self, ev): @@ -180,6 +181,7 @@ class Pusher(object): processed = True else: rejected = yield self.dispatch_push(single_event, tweaks) + self.has_unread = True if isinstance(rejected, list) or isinstance(rejected, tuple): processed = True for pk in rejected: @@ -290,9 +292,12 @@ class Pusher(object): if 'last_active' in state.state: last_active = state.state['last_active'] if last_active > self.last_last_active_time: - logger.info("Resetting badge count for %s", self.user_name) - self.reset_badge_count() self.last_last_active_time = last_active + if self.has_unread: + logger.info("Resetting badge count for %s", self.user_name) + self.reset_badge_count() + self.has_unread = False + def _value_for_dotted_key(dotted_key, event): From acb68a39e02f405c116135400e33a3b1940a07f8 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 16:10:35 +0000 Subject: [PATCH 125/180] Code style fixes. --- synapse/api/errors.py | 1 + synapse/push/__init__.py | 15 +++++++-------- synapse/push/httppusher.py | 8 ++++---- synapse/push/pusherpool.py | 2 +- synapse/rest/__init__.py | 2 +- synapse/rest/client/v1/push_rule.py | 29 ++++++++++++++++++++++------- synapse/storage/push_rule.py | 9 +++++---- synapse/storage/pusher.py | 2 +- 8 files changed, 42 insertions(+), 26 deletions(-) diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 5872e82d0f..ad478aa6b7 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -111,6 +111,7 @@ class NotFoundError(SynapseError): **kwargs ) + class AuthError(SynapseError): """An error raised when there was a problem authorising an event.""" diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index fa967c5a5d..472ede5480 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -189,8 +189,8 @@ class Pusher(object): # for sanity, we only remove the pushkey if it # was the one we actually sent... logger.warn( - ("Ignoring rejected pushkey %s because we " - "didn't send it"), pk + ("Ignoring rejected pushkey %s because we" + " didn't send it"), pk ) else: logger.info( @@ -236,8 +236,7 @@ class Pusher(object): # of old notifications. logger.warn("Giving up on a notification to user %s, " "pushkey %s", - self.user_name, self.pushkey - ) + self.user_name, self.pushkey) self.backoff_delay = Pusher.INITIAL_BACKOFF self.last_token = chunk['end'] self.store.update_pusher_last_token( @@ -258,8 +257,7 @@ class Pusher(object): "Trying again in %dms", self.user_name, self.clock.time_msec() - self.failing_since, - self.backoff_delay - ) + self.backoff_delay) yield synapse.util.async.sleep(self.backoff_delay / 1000.0) self.backoff_delay *= 2 if self.backoff_delay > Pusher.MAX_BACKOFF: @@ -299,7 +297,6 @@ class Pusher(object): self.has_unread = False - def _value_for_dotted_key(dotted_key, event): parts = dotted_key.split(".") val = event @@ -310,6 +307,7 @@ def _value_for_dotted_key(dotted_key, event): parts = parts[1:] return val + def _tweaks_for_actions(actions): tweaks = {} for a in actions: @@ -319,6 +317,7 @@ def _tweaks_for_actions(actions): tweaks['sound'] = a['set_sound'] return tweaks + class PusherConfigException(Exception): def __init__(self, msg): - super(PusherConfigException, self).__init__(msg) \ No newline at end of file + super(PusherConfigException, self).__init__(msg) diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index e12b946727..ab128e31e5 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -71,11 +71,11 @@ class HttpPusher(Pusher): # we may have to fetch this over federation and we # can't trust it anyway: is it worth it? #'from_display_name': 'Steve Stevington' - 'counts': { #-- we don't mark messages as read yet so - # we have no way of knowing + 'counts': { # -- we don't mark messages as read yet so + # we have no way of knowing # Just set the badge to 1 until we have read receipts 'unread': 1, - # 'missed_calls': 2 + # 'missed_calls': 2 }, 'devices': [ { @@ -142,4 +142,4 @@ class HttpPusher(Pusher): rejected = [] if 'rejected' in resp: rejected = resp['rejected'] - defer.returnValue(rejected) \ No newline at end of file + defer.returnValue(rejected) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 856defedac..4892c21e7b 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -149,4 +149,4 @@ class PusherPool: logger.info("Stopping pusher %s", fullid) self.pushers[fullid].stop() del self.pushers[fullid] - yield self.store.delete_pusher_by_app_id_pushkey(app_id, pushkey) \ No newline at end of file + yield self.store.delete_pusher_by_app_id_pushkey(app_id, pushkey) diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index 90afd93333..1a84d94cd9 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -11,4 +11,4 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. \ No newline at end of file +# limitations under the License. diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 64743a2f46..2b1e930326 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -30,9 +30,9 @@ class PushRuleRestServlet(ClientV1RestServlet): 'sender': 1, 'room': 2, 'content': 3, - 'override': 4 + 'override': 4, } - PRIORITY_CLASS_INVERSE_MAP = {v: k for k,v in PRIORITY_CLASS_MAP.items()} + PRIORITY_CLASS_INVERSE_MAP = {v: k for k, v in PRIORITY_CLASS_MAP.items()} SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR = ( "Unrecognised request: You probably wanted a trailing slash") @@ -260,7 +260,9 @@ class PushRuleRestServlet(ClientV1RestServlet): if path == []: # we're a reference impl: pedantry is our job. - raise UnrecognizedRequestError(PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR) + raise UnrecognizedRequestError( + PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR + ) if path[0] == '': defer.returnValue((200, rules)) @@ -271,7 +273,9 @@ class PushRuleRestServlet(ClientV1RestServlet): elif path[0] == 'device': path = path[1:] if path == []: - raise UnrecognizedRequestError(PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR) + raise UnrecognizedRequestError( + PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR + ) if path[0] == '': defer.returnValue((200, rules['device'])) @@ -290,11 +294,13 @@ class PushRuleRestServlet(ClientV1RestServlet): def on_OPTIONS(self, _): return 200, {} + def _add_empty_priority_class_arrays(d): for pc in PushRuleRestServlet.PRIORITY_CLASS_MAP.keys(): d[pc] = [] return d + def _instance_handle_from_conditions(conditions): """ Given a list of conditions, return the instance handle of the @@ -305,9 +311,12 @@ def _instance_handle_from_conditions(conditions): return c['instance_handle'] return None + def _filter_ruleset_with_path(ruleset, path): if path == []: - raise UnrecognizedRequestError(PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR) + raise UnrecognizedRequestError( + PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR + ) if path[0] == '': return ruleset @@ -316,7 +325,9 @@ def _filter_ruleset_with_path(ruleset, path): raise UnrecognizedRequestError() path = path[1:] if path == []: - raise UnrecognizedRequestError(PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR) + raise UnrecognizedRequestError( + PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR + ) if path[0] == '': return ruleset[template_kind] rule_id = path[0] @@ -325,6 +336,7 @@ def _filter_ruleset_with_path(ruleset, path): return r raise NotFoundError + def _priority_class_from_spec(spec): if spec['template'] not in PushRuleRestServlet.PRIORITY_CLASS_MAP.keys(): raise InvalidRuleException("Unknown template: %s" % (spec['kind'])) @@ -335,6 +347,7 @@ def _priority_class_from_spec(spec): return pc + def _priority_class_to_template_name(pc): if pc > PushRuleRestServlet.PRIORITY_CLASS_MAP['override']: # per-device @@ -343,6 +356,7 @@ def _priority_class_to_template_name(pc): else: return PushRuleRestServlet.PRIORITY_CLASS_INVERSE_MAP[pc] + def _rule_to_template(rule): template_name = _priority_class_to_template_name(rule['priority_class']) if template_name in ['override', 'underride']: @@ -359,8 +373,9 @@ def _rule_to_template(rule): ret["pattern"] = thecond["pattern"] return ret + def _strip_device_condition(rule): - for i,c in enumerate(rule['conditions']): + for i, c in enumerate(rule['conditions']): if c['kind'] == 'device': del rule['conditions'][i] return rule diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index c7b553292e..27502d2399 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -117,7 +117,7 @@ class PushRuleStore(SQLBaseStore): new_rule['priority'] = new_rule_priority sql = ( - "SELECT COUNT(*) FROM "+PushRuleTable.table_name+ + "SELECT COUNT(*) FROM " + PushRuleTable.table_name + " WHERE user_name = ? AND priority_class = ? AND priority = ?" ) txn.execute(sql, (user_name, priority_class, new_rule_priority)) @@ -146,10 +146,11 @@ class PushRuleStore(SQLBaseStore): txn.execute(sql, new_rule.values()) - def _add_push_rule_highest_priority_txn(self, txn, user_name, priority_class, **kwargs): + def _add_push_rule_highest_priority_txn(self, txn, user_name, + priority_class, **kwargs): # find the highest priority rule in that class sql = ( - "SELECT COUNT(*), MAX(priority) FROM "+PushRuleTable.table_name+ + "SELECT COUNT(*), MAX(priority) FROM " + PushRuleTable.table_name + " WHERE user_name = ? and priority_class = ?" ) txn.execute(sql, (user_name, priority_class)) @@ -209,4 +210,4 @@ class PushRuleTable(Table): "actions", ] - EntryType = collections.namedtuple("PushRuleEntry", fields) \ No newline at end of file + EntryType = collections.namedtuple("PushRuleEntry", fields) diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index 113cdc8a8e..f253c9e2c3 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -170,4 +170,4 @@ class PushersTable(Table): "failing_since" ] - EntryType = collections.namedtuple("PusherEntry", fields) \ No newline at end of file + EntryType = collections.namedtuple("PusherEntry", fields) From 4d9dd9bdc02ca7b524c90ae527687e7a838acee4 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 16:23:03 +0000 Subject: [PATCH 126/180] Fix v2 initial sync --- synapse/handlers/sync.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index e93dfe005d..cfe2944917 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -157,7 +157,7 @@ class SyncHandler(BaseHandler): )) @defer.inlineCallbacks - def intial_sync_for_room(self, room_id, sync_config, now_token, + def initial_sync_for_room(self, room_id, sync_config, now_token, published_room_ids): """Sync a room for a client which is starting without any state Returns: @@ -180,6 +180,7 @@ class SyncHandler(BaseHandler): prev_batch=prev_batch_token, state=current_state_events, limited=True, + typing=None, )) @defer.inlineCallbacks From cc42d3f9076db1815a989bab7ef6004796b31b3c Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 16:27:38 +0000 Subject: [PATCH 127/180] Fix check for empty room update --- synapse/handlers/sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index cfe2944917..ec8beb4c6a 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -258,7 +258,7 @@ class SyncHandler(BaseHandler): limited=False, typing=typing_by_room.get(room_id, None) ) - if room_sync is not None: + if room_sync: rooms.append(room_sync) else: for room_id in room_ids: From 722b65f46131349c5afdcc7eb48297cdd9d9cbd6 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 16:41:21 +0000 Subject: [PATCH 128/180] Move typing notifs to an "emphermal" event list on the room object --- synapse/handlers/sync.py | 12 +++++++----- synapse/rest/client/v2_alpha/sync.py | 4 ++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index ec8beb4c6a..3860c3c95b 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -44,12 +44,12 @@ class RoomSyncResult(collections.namedtuple("RoomSyncResult", [ "events", "state", "prev_batch", - "typing", + "ephemeral", ])): __slots__ = [] def __nonzero__(self): - return bool(self.events or self.state or self.typing) + return bool(self.events or self.state or self.ephemeral) class SyncResult(collections.namedtuple("SyncResult", [ @@ -180,7 +180,7 @@ class SyncHandler(BaseHandler): prev_batch=prev_batch_token, state=current_state_events, limited=True, - typing=None, + ephemeral=[], )) @defer.inlineCallbacks @@ -214,7 +214,9 @@ class SyncHandler(BaseHandler): ) now_token = now_token.copy_and_replace("typing_key", typing_key) - typing_by_room = {event["room_id"]: event for event in typing} + typing_by_room = {event["room_id"]: [event] for event in typing} + for event in typing: + event.pop("room_id") logger.debug("Typing %r", typing_by_room) rm_handler = self.hs.get_handlers().room_member_handler @@ -256,7 +258,7 @@ class SyncHandler(BaseHandler): prev_batch=prev_batch, state=state, limited=False, - typing=typing_by_room.get(room_id, None) + ephemeral=typing_by_room.get(room_id, []) ) if room_sync: rooms.append(room_sync) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 76489e27c8..2ae2eec55d 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -66,6 +66,7 @@ class SyncRestServlet(RestServlet): } "state": [] // list of EventIDs updating the current state to // be what it should be at the end of the batch. + "ephemeral": [] }] } """ @@ -188,9 +189,8 @@ class SyncRestServlet(RestServlet): "state": state_event_ids, "limited": room.limited, "published": room.published, + "ephemeral": room.ephemeral, } - if room.typing is not None: - result["typing"] = room.typing return result From 4ad45f258245344218c5ac9ffcf20052bfdb5ba2 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 16:41:49 +0000 Subject: [PATCH 129/180] Fix indent --- synapse/handlers/sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 3860c3c95b..8c7681a48d 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -158,7 +158,7 @@ class SyncHandler(BaseHandler): @defer.inlineCallbacks def initial_sync_for_room(self, room_id, sync_config, now_token, - published_room_ids): + published_room_ids): """Sync a room for a client which is starting without any state Returns: A Deferred RoomSyncResult. From 78015948a7febb18e000651f72f8f58830a55b93 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 29 Jan 2015 16:50:23 +0000 Subject: [PATCH 130/180] Initial implementation of auth conflict resolution --- synapse/events/utils.py | 6 +- synapse/federation/federation_client.py | 2 +- synapse/federation/federation_server.py | 33 ++++ synapse/federation/transport/client.py | 16 ++ synapse/federation/transport/server.py | 21 ++- synapse/handlers/federation.py | 209 +++++++++++++++--------- synapse/storage/rejections.py | 4 +- tests/handlers/test_federation.py | 2 + 8 files changed, 211 insertions(+), 82 deletions(-) diff --git a/synapse/events/utils.py b/synapse/events/utils.py index bcb5457278..10a6b9f264 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -45,12 +45,14 @@ def prune_event(event): "membership", ] + event_dict = event.get_dict() + new_content = {} def add_fields(*fields): for field in fields: if field in event.content: - new_content[field] = event.content[field] + new_content[field] = event_dict["content"][field] if event_type == EventTypes.Member: add_fields("membership") @@ -75,7 +77,7 @@ def prune_event(event): allowed_fields = { k: v - for k, v in event.get_dict().items() + for k, v in event_dict.items() if k in allowed_keys } diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index ebcd593506..1173ca817b 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -345,7 +345,7 @@ class FederationClient(object): "auth_chain": [e.get_pdu_json(time_now) for e in local_auth], } - code, content = yield self.transport_layer.send_invite( + code, content = yield self.transport_layer.send_query_auth( destination=destination, room_id=room_id, event_id=event_id, diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index fc5342afaa..8cff4e6472 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -230,6 +230,39 @@ class FederationServer(object): "auth_chain": [a.get_pdu_json(time_now) for a in auth_pdus], })) + @defer.inlineCallbacks + def on_query_auth_request(self, origin, content, event_id): + auth_chain = [ + (yield self._check_sigs_and_hash(self.event_from_pdu_json(e))) + for e in content["auth_chain"] + ] + + missing = [ + (yield self._check_sigs_and_hash(self.event_from_pdu_json(e))) + for e in content.get("missing", []) + ] + + ret = yield self.handler.on_query_auth( + origin, event_id, auth_chain, content.get("rejects", []), missing + ) + + time_now = self._clock.time_msec() + send_content = { + "auth_chain": [ + e.get_pdu_json(time_now) + for e in ret["auth_chain"] + ], + "rejects": content.get("rejects", []), + "missing": [ + e.get_pdu_json(time_now) + for e in ret.get("missing", []) + ], + } + + defer.returnValue( + (200, send_content) + ) + @log_function def _get_persisted_pdu(self, origin, event_id, do_auth=True): """ Get a PDU from the database with given origin and id. diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index e634a3a213..4cb1dea2de 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -213,3 +213,19 @@ class TransportLayerClient(object): ) defer.returnValue(response) + + @defer.inlineCallbacks + @log_function + def send_query_auth(self, destination, room_id, event_id, content): + path = PREFIX + "/query_auth/%s/%s" % (room_id, event_id) + + code, content = yield self.client.post_json( + destination=destination, + path=path, + data=content, + ) + + if not 200 <= code < 300: + raise RuntimeError("Got %d from send_invite", code) + + defer.returnValue(json.loads(content)) diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index a380a6910b..9c9f8d525b 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -42,7 +42,7 @@ class TransportLayerServer(object): content = None origin = None - if request.method == "PUT": + if request.method in ["PUT", "POST"]: # TODO: Handle other method types? other content types? try: content_bytes = request.content.read() @@ -234,6 +234,16 @@ class TransportLayerServer(object): ) ) ) + self.server.register_path( + "POST", + re.compile("^" + PREFIX + "/query_auth/([^/]*)/([^/]*)$"), + self._with_authentication( + lambda origin, content, query, context, event_id: + self._on_query_auth_request( + origin, content, event_id, + ) + ) + ) @defer.inlineCallbacks @log_function @@ -325,3 +335,12 @@ class TransportLayerServer(object): ) defer.returnValue((200, content)) + + @defer.inlineCallbacks + @log_function + def _on_query_auth_request(self, origin, content, event_id): + new_content = yield self.request_handler.on_query_auth_request( + origin, content, event_id + ) + + defer.returnValue((200, new_content)) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 97e3c503b9..14c26d8cea 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -126,7 +126,7 @@ class FederationHandler(BaseHandler): if not state: state, auth_chain = yield replication.get_state_for_room( - origin, context=event.room_id, event_id=event.event_id, + origin, room_id=event.room_id, event_id=event.event_id, ) if not auth_chain: @@ -139,7 +139,7 @@ class FederationHandler(BaseHandler): for e in auth_chain: e.internal_metadata.outlier = True try: - yield self._handle_new_event(e, fetch_auth_from=origin) + yield self._handle_new_event(origin, e) except: logger.exception( "Failed to handle auth event %s", @@ -152,7 +152,7 @@ class FederationHandler(BaseHandler): for e in state: e.internal_metadata.outlier = True try: - yield self._handle_new_event(e) + yield self._handle_new_event(origin, e) except: logger.exception( "Failed to handle state event %s", @@ -161,6 +161,7 @@ class FederationHandler(BaseHandler): try: yield self._handle_new_event( + origin, event, state=state, backfilled=backfilled, @@ -363,7 +364,14 @@ class FederationHandler(BaseHandler): for e in auth_chain: e.internal_metadata.outlier = True try: - yield self._handle_new_event(e) + auth_ids = [e_id for e_id, _ in e.auth_events] + auth = { + (e.type, e.state_key): e for e in auth_chain + if e.event_id in auth_ids + } + yield self._handle_new_event( + target_host, e, auth_events=auth + ) except: logger.exception( "Failed to handle auth event %s", @@ -374,8 +382,13 @@ class FederationHandler(BaseHandler): # FIXME: Auth these. e.internal_metadata.outlier = True try: + auth_ids = [e_id for e_id, _ in e.auth_events] + auth = { + (e.type, e.state_key): e for e in auth_chain + if e.event_id in auth_ids + } yield self._handle_new_event( - e, fetch_auth_from=target_host + target_host, e, auth_events=auth ) except: logger.exception( @@ -384,6 +397,7 @@ class FederationHandler(BaseHandler): ) yield self._handle_new_event( + target_host, new_event, state=state, current_state=state, @@ -450,7 +464,7 @@ class FederationHandler(BaseHandler): event.internal_metadata.outlier = False - context = yield self._handle_new_event(event) + context = yield self._handle_new_event(origin, event) logger.debug( "on_send_join_request: After _handle_new_event: %s, sigs: %s", @@ -651,11 +665,12 @@ class FederationHandler(BaseHandler): waiters.pop().callback(None) @defer.inlineCallbacks - def _handle_new_event(self, event, state=None, backfilled=False, - current_state=None, fetch_auth_from=None): + @log_function + def _handle_new_event(self, origin, event, state=None, backfilled=False, + current_state=None, auth_events=None): logger.debug( - "_handle_new_event: Before annotate: %s, sigs: %s", + "_handle_new_event: %s, sigs: %s", event.event_id, event.signatures, ) @@ -663,62 +678,34 @@ class FederationHandler(BaseHandler): event, old_state=state ) + if not auth_events: + auth_events = context.auth_events + logger.debug( - "_handle_new_event: Before auth fetch: %s, sigs: %s", - event.event_id, event.signatures, + "_handle_new_event: %s, auth_events: %s", + event.event_id, auth_events, ) is_new_state = not event.internal_metadata.is_outlier() - known_ids = set( - [s.event_id for s in context.auth_events.values()] - ) - - for e_id, _ in event.auth_events: - if e_id not in known_ids: - e = yield self.store.get_event(e_id, allow_none=True) - - if not e and fetch_auth_from is not None: - # Grab the auth_chain over federation if we are missing - # auth events. - auth_chain = yield self.replication_layer.get_event_auth( - fetch_auth_from, event.event_id, event.room_id - ) - for auth_event in auth_chain: - yield self._handle_new_event(auth_event) - e = yield self.store.get_event(e_id, allow_none=True) - - if not e: - # TODO: Do some conflict res to make sure that we're - # not the ones who are wrong. - logger.info( - "Rejecting %s as %s not in db or %s", - event.event_id, e_id, known_ids, - ) - # FIXME: How does raising AuthError work with federation? - raise AuthError(403, "Cannot find auth event") - - context.auth_events[(e.type, e.state_key)] = e - - logger.debug( - "_handle_new_event: Before hack: %s, sigs: %s", - event.event_id, event.signatures, - ) - + # This is a hack to fix some old rooms where the initial join event + # didn't reference the create event in its auth events. if event.type == EventTypes.Member and not event.auth_events: if len(event.prev_events) == 1: c = yield self.store.get_event(event.prev_events[0][0]) if c.type == EventTypes.Create: - context.auth_events[(c.type, c.state_key)] = c - - logger.debug( - "_handle_new_event: Before auth check: %s, sigs: %s", - event.event_id, event.signatures, - ) + auth_events[(c.type, c.state_key)] = c try: - self.auth.check(event, auth_events=context.auth_events) - except AuthError: + yield self.do_auth( + origin, event, context, auth_events=auth_events + ) + except AuthError as e: + logger.warn( + "Rejecting %s because %s", + event.event_id, e.msg + ) + # TODO: Store rejection. context.rejected = RejectedReason.AUTH_ERROR @@ -731,11 +718,6 @@ class FederationHandler(BaseHandler): ) raise - logger.debug( - "_handle_new_event: Before persist_event: %s, sigs: %s", - event.event_id, event.signatures, - ) - yield self.store.persist_event( event, context=context, @@ -744,25 +726,73 @@ class FederationHandler(BaseHandler): current_state=current_state, ) - logger.debug( - "_handle_new_event: After persist_event: %s, sigs: %s", - event.event_id, event.signatures, - ) - defer.returnValue(context) @defer.inlineCallbacks - def do_auth(self, origin, event, context): - for e_id, _ in event.auth_events: - pass + def on_query_auth(self, origin, event_id, remote_auth_chain, rejects, + missing): + # Just go through and process each event in `remote_auth_chain`. We + # don't want to fall into the trap of `missing` being wrong. + for e in remote_auth_chain: + try: + yield self._handle_new_event(origin, e) + except AuthError: + pass - auth_events = set(e_id for e_id, _ in event.auth_events) - current_state = set(e.event_id for e in context.auth_events.values()) + # Now get the current auth_chain for the event. + local_auth_chain = yield self.store.get_auth_chain([event_id]) - missing_auth = auth_events - current_state + # TODO: Check if we would now reject event_id. If so we need to tell + # everyone. + + ret = yield self.construct_auth_difference( + local_auth_chain, remote_auth_chain + ) + + logger.debug("on_query_auth reutrning: %s", ret) + + defer.returnValue(ret) + + @defer.inlineCallbacks + @log_function + def do_auth(self, origin, event, context, auth_events): + # Check if we have all the auth events. + res = yield self.store.have_events( + [e_id for e_id, _ in event.auth_events] + ) + + event_auth_events = set(e_id for e_id, _ in event.auth_events) + seen_events = set(res.keys()) + + missing_auth = event_auth_events - seen_events if missing_auth: + logger.debug("Missing auth: %s", missing_auth) + # If we don't have all the auth events, we need to get them. + remote_auth_chain = yield self.replication_layer.get_event_auth( + origin, event.room_id, event.event_id + ) + + for e in remote_auth_chain: + try: + auth_ids = [e_id for e_id, _ in e.auth_events] + auth = { + (e.type, e.state_key): e for e in remote_auth_chain + if e.event_id in auth_ids + } + yield self._handle_new_event( + origin, e, auth_events=auth + ) + auth_events[(e.type, e.state_key)] = e + except AuthError: + pass + + current_state = set(e.event_id for e in auth_events.values()) + different_auth = event_auth_events - current_state + + if different_auth and not event.internal_metadata.is_outlier(): # Do auth conflict res. + logger.debug("Different auth: %s", different_auth) # 1. Get what we think is the auth chain. auth_ids = self.auth.compute_auth_events(event, context) @@ -778,14 +808,24 @@ class FederationHandler(BaseHandler): # 3. Process any remote auth chain events we haven't seen. for e in result.get("missing", []): - # TODO. - pass + try: + auth_ids = [e_id for e_id, _ in e.auth_events] + auth = { + (e.type, e.state_key): e for e in result["auth_chain"] + if e.event_id in auth_ids + } + yield self._handle_new_event( + origin, e, auth_events=auth + ) + auth_events[(e.type, e.state_key)] = e + except AuthError: + pass # 4. Look at rejects and their proofs. # TODO. try: - self.auth.check(event, auth_events=context.auth_events) + self.auth.check(event, auth_events=auth_events) except AuthError: raise @@ -802,12 +842,16 @@ class FederationHandler(BaseHandler): dict """ + logger.debug("construct_auth_difference Start!") + # TODO: Make sure we are OK with local_auth or remote_auth having more # auth events in them than strictly necessary. def sort_fun(ev): return ev.depth, ev.event_id + logger.debug("construct_auth_difference after sort_fun!") + # We find the differences by starting at the "bottom" of each list # and iterating up on both lists. The lists are ordered by depth and # then event_id, we iterate up both lists until we find the event ids @@ -823,11 +867,18 @@ class FederationHandler(BaseHandler): local_iter = iter(local_list) remote_iter = iter(remote_list) - current_local = local_iter.next() - current_remote = remote_iter.next() + logger.debug("construct_auth_difference before get_next!") def get_next(it, opt=None): - return it.next() if it.has_next() else opt + try: + return it.next() + except: + return opt + + current_local = get_next(local_iter) + current_remote = get_next(remote_iter) + + logger.debug("construct_auth_difference before while") missing_remotes = [] missing_locals = [] @@ -867,6 +918,8 @@ class FederationHandler(BaseHandler): current_remote = get_next(remote_iter) continue + logger.debug("construct_auth_difference after while") + # missing locals should be sent to the server # We should find why we are missing remotes, as they will have been # rejected. @@ -886,6 +939,7 @@ class FederationHandler(BaseHandler): reason = yield self.store.get_rejection_reason(e.event_id) if reason is None: # FIXME: ERRR?! + logger.warn("Could not find reason for %s", e.event_id) raise RuntimeError("") reason_map[e.event_id] = reason @@ -899,7 +953,10 @@ class FederationHandler(BaseHandler): # TODO: Get proof. pass + logger.debug("construct_auth_difference returning") + defer.returnValue({ + "auth_chain": local_auth, "rejects": { e.event_id: { "reason": reason_map[e.event_id], diff --git a/synapse/storage/rejections.py b/synapse/storage/rejections.py index b7249700d7..4e1a9a2783 100644 --- a/synapse/storage/rejections.py +++ b/synapse/storage/rejections.py @@ -28,12 +28,12 @@ class RejectionsStore(SQLBaseStore): values={ "event_id": event_id, "reason": reason, - "last_failure": self._clock.time_msec(), + "last_check": self._clock.time_msec(), } ) def get_rejection_reason(self, event_id): - self._simple_select_one_onecol( + return self._simple_select_one_onecol( table="rejections", retcol="reason", keyvalues={ diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index ed21defd13..44dbce6bea 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -52,6 +52,7 @@ class FederationTestCase(unittest.TestCase): "get_room", "get_destination_retry_timings", "set_destination_retry_timings", + "have_events", ]), resource_for_federation=NonCallableMock(), http_client=NonCallableMock(spec_set=[]), @@ -90,6 +91,7 @@ class FederationTestCase(unittest.TestCase): self.datastore.persist_event.return_value = defer.succeed(None) self.datastore.get_room.return_value = defer.succeed(True) self.auth.check_host_in_room.return_value = defer.succeed(True) + self.datastore.have_events.return_value = defer.succeed({}) def annotate(ev, old_state=None): context = Mock() From e0d2c6889bf31cf5f48e77334fed23352d19a75d Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 29 Jan 2015 17:04:31 +0000 Subject: [PATCH 131/180] Allow kind to be set to null to delete a pusher. --- synapse/rest/client/v1/pusher.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py index 72d5e9e476..353a4a6589 100644 --- a/synapse/rest/client/v1/pusher.py +++ b/synapse/rest/client/v1/pusher.py @@ -31,6 +31,16 @@ class PusherRestServlet(ClientV1RestServlet): content = _parse_json(request) + pusher_pool = self.hs.get_pusherpool() + + if ('pushkey' in content and 'app_id' in content + and 'kind' in content and + content['kind'] is None): + yield pusher_pool.remove_pusher( + content['app_id'], content['pushkey'] + ) + defer.returnValue((200, {})) + reqd = ['instance_handle', 'kind', 'app_id', 'app_display_name', 'device_display_name', 'pushkey', 'lang', 'data'] missing = [] @@ -41,7 +51,6 @@ class PusherRestServlet(ClientV1RestServlet): raise SynapseError(400, "Missing parameters: "+','.join(missing), errcode=Codes.MISSING_PARAM) - pusher_pool = self.hs.get_pusherpool() try: yield pusher_pool.add_pusher( user_name=user.to_string(), From 4bdfce30d70dddaa7c6de551fe3c9eed4a899d49 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 29 Jan 2015 17:12:11 +0000 Subject: [PATCH 132/180] Renumber priority classes so we can use 0 for defaults. --- synapse/rest/client/v1/push_rule.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 2b1e930326..0f78fa667c 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -26,11 +26,11 @@ import json class PushRuleRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/pushrules/.*$") PRIORITY_CLASS_MAP = { - 'underride': 0, - 'sender': 1, - 'room': 2, - 'content': 3, - 'override': 4, + 'underride': 1, + 'sender': 2, + 'room': 3, + 'content': 4, + 'override': 5, } PRIORITY_CLASS_INVERSE_MAP = {v: k for k, v in PRIORITY_CLASS_MAP.items()} SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR = ( From 93ed31dda2e23742c3d7f3eee6ac6839682f0ce9 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 17:41:48 +0000 Subject: [PATCH 133/180] Create a separate filter object to do the actual filtering, so that we can split the storage and management of filters from the actual filter code and don't have to load a filter from the db each time we filter an event --- synapse/api/filtering.py | 220 ++++++++++++------------- synapse/rest/client/v2_alpha/filter.py | 2 +- tests/api/test_filtering.py | 108 ++++++------ 3 files changed, 166 insertions(+), 164 deletions(-) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index e16c0e559f..b7e5d3222f 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -25,127 +25,25 @@ class Filtering(object): self.store = hs.get_datastore() def get_user_filter(self, user_localpart, filter_id): - return self.store.get_user_filter(user_localpart, filter_id) + result = self.store.get_user_filter(user_localpart, filter_id) + result.addCallback(Filter) + return result def add_user_filter(self, user_localpart, user_filter): self._check_valid_filter(user_filter) return self.store.add_user_filter(user_localpart, user_filter) - def filter_public_user_data(self, events, user, filter_id): - return self._filter_on_key( - events, user, filter_id, ["public_user_data"] - ) - - def filter_private_user_data(self, events, user, filter_id): - return self._filter_on_key( - events, user, filter_id, ["private_user_data"] - ) - - def filter_room_state(self, events, user, filter_id): - return self._filter_on_key( - events, user, filter_id, ["room", "state"] - ) - - def filter_room_events(self, events, user, filter_id): - return self._filter_on_key( - events, user, filter_id, ["room", "events"] - ) - - def filter_room_ephemeral(self, events, user, filter_id): - return self._filter_on_key( - events, user, filter_id, ["room", "ephemeral"] - ) - # TODO(paul): surely we should probably add a delete_user_filter or # replace_user_filter at some point? There's no REST API specified for # them however - @defer.inlineCallbacks - def _filter_on_key(self, events, user, filter_id, keys): - filter_json = yield self.get_user_filter(user.localpart, filter_id) - if not filter_json: - defer.returnValue(events) - - try: - # extract the right definition from the filter - definition = filter_json - for key in keys: - definition = definition[key] - defer.returnValue(self._filter_with_definition(events, definition)) - except KeyError: - # return all events if definition isn't specified. - defer.returnValue(events) - - def _filter_with_definition(self, events, definition): - return [e for e in events if self._passes_definition(definition, e)] - - def _passes_definition(self, definition, event): - """Check if the event passes through the given definition. - - Args: - definition(dict): The definition to check against. - event(Event): The event to check. - Returns: - True if the event passes through the filter. - """ - # Algorithm notes: - # For each key in the definition, check the event meets the criteria: - # * For types: Literal match or prefix match (if ends with wildcard) - # * For senders/rooms: Literal match only - # * "not_" checks take presedence (e.g. if "m.*" is in both 'types' - # and 'not_types' then it is treated as only being in 'not_types') - - # room checks - if hasattr(event, "room_id"): - room_id = event.room_id - allow_rooms = definition.get("rooms", None) - reject_rooms = definition.get("not_rooms", None) - if reject_rooms and room_id in reject_rooms: - return False - if allow_rooms and room_id not in allow_rooms: - return False - - # sender checks - if hasattr(event, "sender"): - # Should we be including event.state_key for some event types? - sender = event.sender - allow_senders = definition.get("senders", None) - reject_senders = definition.get("not_senders", None) - if reject_senders and sender in reject_senders: - return False - if allow_senders and sender not in allow_senders: - return False - - # type checks - if "not_types" in definition: - for def_type in definition["not_types"]: - if self._event_matches_type(event, def_type): - return False - if "types" in definition: - included = False - for def_type in definition["types"]: - if self._event_matches_type(event, def_type): - included = True - break - if not included: - return False - - return True - - def _event_matches_type(self, event, def_type): - if def_type.endswith("*"): - type_prefix = def_type[:-1] - return event.type.startswith(type_prefix) - else: - return event.type == def_type - - def _check_valid_filter(self, user_filter): + def _check_valid_filter(self, user_filter_json): """Check if the provided filter is valid. This inspects all definitions contained within the filter. Args: - user_filter(dict): The filter + user_filter_json(dict): The filter Raises: SynapseError: If the filter is not valid. """ @@ -162,13 +60,13 @@ class Filtering(object): ] for key in top_level_definitions: - if key in user_filter: - self._check_definition(user_filter[key]) + if key in user_filter_json: + self._check_definition(user_filter_json[key]) - if "room" in user_filter: + if "room" in user_filter_json: for key in room_level_definitions: - if key in user_filter["room"]: - self._check_definition(user_filter["room"][key]) + if key in user_filter_json["room"]: + self._check_definition(user_filter_json["room"][key]) def _check_definition(self, definition): """Check if the provided definition is valid. @@ -237,3 +135,101 @@ class Filtering(object): if ("bundle_updates" in definition and type(definition["bundle_updates"]) != bool): raise SynapseError(400, "Bad bundle_updates: expected bool.") + + +class Filter(object): + def __init__(self, filter_json): + self.filter_json = filter_json + + def filter_public_user_data(self, events): + return self._filter_on_key(events, ["public_user_data"]) + + def filter_private_user_data(self, events): + return self._filter_on_key(events, ["private_user_data"]) + + def filter_room_state(self, events): + return self._filter_on_key(events, ["room", "state"]) + + def filter_room_events(self, events): + return self._filter_on_key(events, ["room", "events"]) + + def filter_room_ephemeral(self, events): + return self._filter_on_key(events, ["room", "ephemeral"]) + + def _filter_on_key(self, events, keys): + filter_json = self.filter_json + if not filter_json: + return events + + try: + # extract the right definition from the filter + definition = filter_json + for key in keys: + definition = definition[key] + return self._filter_with_definition(events, definition) + except KeyError: + # return all events if definition isn't specified. + return events + + def _filter_with_definition(self, events, definition): + return [e for e in events if self._passes_definition(definition, e)] + + def _passes_definition(self, definition, event): + """Check if the event passes through the given definition. + + Args: + definition(dict): The definition to check against. + event(Event): The event to check. + Returns: + True if the event passes through the filter. + """ + # Algorithm notes: + # For each key in the definition, check the event meets the criteria: + # * For types: Literal match or prefix match (if ends with wildcard) + # * For senders/rooms: Literal match only + # * "not_" checks take presedence (e.g. if "m.*" is in both 'types' + # and 'not_types' then it is treated as only being in 'not_types') + + # room checks + if hasattr(event, "room_id"): + room_id = event.room_id + allow_rooms = definition.get("rooms", None) + reject_rooms = definition.get("not_rooms", None) + if reject_rooms and room_id in reject_rooms: + return False + if allow_rooms and room_id not in allow_rooms: + return False + + # sender checks + if hasattr(event, "sender"): + # Should we be including event.state_key for some event types? + sender = event.sender + allow_senders = definition.get("senders", None) + reject_senders = definition.get("not_senders", None) + if reject_senders and sender in reject_senders: + return False + if allow_senders and sender not in allow_senders: + return False + + # type checks + if "not_types" in definition: + for def_type in definition["not_types"]: + if self._event_matches_type(event, def_type): + return False + if "types" in definition: + included = False + for def_type in definition["types"]: + if self._event_matches_type(event, def_type): + included = True + break + if not included: + return False + + return True + + def _event_matches_type(self, event, def_type): + if def_type.endswith("*"): + type_prefix = def_type[:-1] + return event.type.startswith(type_prefix) + else: + return event.type == def_type diff --git a/synapse/rest/client/v2_alpha/filter.py b/synapse/rest/client/v2_alpha/filter.py index cee06ccaca..6ddc495d23 100644 --- a/synapse/rest/client/v2_alpha/filter.py +++ b/synapse/rest/client/v2_alpha/filter.py @@ -59,7 +59,7 @@ class GetFilterRestServlet(RestServlet): filter_id=filter_id, ) - defer.returnValue((200, filter)) + defer.returnValue((200, filter.filter_json)) except KeyError: raise SynapseError(400, "No such filter") diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py index aa93616a9f..babf4c37f1 100644 --- a/tests/api/test_filtering.py +++ b/tests/api/test_filtering.py @@ -25,6 +25,7 @@ from tests.utils import ( from synapse.server import HomeServer from synapse.types import UserID +from synapse.api.filtering import Filter user_localpart = "test_user" MockEvent = namedtuple("MockEvent", "sender type room_id") @@ -53,6 +54,7 @@ class FilteringTestCase(unittest.TestCase): ) self.filtering = hs.get_filtering() + self.filter = Filter({}) self.datastore = hs.get_datastore() @@ -66,7 +68,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertTrue( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_types_works_with_wildcards(self): @@ -79,7 +81,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertTrue( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_types_works_with_unknowns(self): @@ -92,7 +94,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertFalse( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_not_types_works_with_literals(self): @@ -105,7 +107,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertFalse( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_not_types_works_with_wildcards(self): @@ -118,7 +120,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertFalse( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_not_types_works_with_unknowns(self): @@ -131,7 +133,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertTrue( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_not_types_takes_priority_over_types(self): @@ -145,7 +147,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertFalse( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_senders_works_with_literals(self): @@ -158,7 +160,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertTrue( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_senders_works_with_unknowns(self): @@ -171,7 +173,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertFalse( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_not_senders_works_with_literals(self): @@ -184,7 +186,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertFalse( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_not_senders_works_with_unknowns(self): @@ -197,7 +199,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertTrue( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_not_senders_takes_priority_over_senders(self): @@ -211,7 +213,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertFalse( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_rooms_works_with_literals(self): @@ -224,7 +226,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!secretbase:unknown" ) self.assertTrue( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_rooms_works_with_unknowns(self): @@ -237,7 +239,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!anothersecretbase:unknown" ) self.assertFalse( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_not_rooms_works_with_literals(self): @@ -250,7 +252,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!anothersecretbase:unknown" ) self.assertFalse( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_not_rooms_works_with_unknowns(self): @@ -263,7 +265,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!anothersecretbase:unknown" ) self.assertTrue( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_not_rooms_takes_priority_over_rooms(self): @@ -277,7 +279,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!secretbase:unknown" ) self.assertFalse( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_combined_event(self): @@ -295,7 +297,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!stage:unknown" # yup ) self.assertTrue( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_combined_event_bad_sender(self): @@ -313,7 +315,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!stage:unknown" # yup ) self.assertFalse( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_combined_event_bad_room(self): @@ -331,7 +333,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!piggyshouse:muppets" # nope ) self.assertFalse( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_combined_event_bad_type(self): @@ -349,12 +351,12 @@ class FilteringTestCase(unittest.TestCase): room_id="!stage:unknown" # yup ) self.assertFalse( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) @defer.inlineCallbacks def test_filter_public_user_data_match(self): - user_filter = { + user_filter_json = { "public_user_data": { "types": ["m.*"] } @@ -362,7 +364,7 @@ class FilteringTestCase(unittest.TestCase): user = UserID.from_string("@" + user_localpart + ":test") filter_id = yield self.datastore.add_user_filter( user_localpart=user_localpart, - user_filter=user_filter, + user_filter=user_filter_json, ) event = MockEvent( sender="@foo:bar", @@ -371,16 +373,17 @@ class FilteringTestCase(unittest.TestCase): ) events = [event] - results = yield self.filtering.filter_public_user_data( - events=events, - user=user, - filter_id=filter_id + user_filter = yield self.filtering.get_user_filter( + user_localpart=user_localpart, + filter_id=filter_id, ) + + results = user_filter.filter_public_user_data(events=events) self.assertEquals(events, results) @defer.inlineCallbacks def test_filter_public_user_data_no_match(self): - user_filter = { + user_filter_json = { "public_user_data": { "types": ["m.*"] } @@ -388,7 +391,7 @@ class FilteringTestCase(unittest.TestCase): user = UserID.from_string("@" + user_localpart + ":test") filter_id = yield self.datastore.add_user_filter( user_localpart=user_localpart, - user_filter=user_filter, + user_filter=user_filter_json, ) event = MockEvent( sender="@foo:bar", @@ -397,16 +400,17 @@ class FilteringTestCase(unittest.TestCase): ) events = [event] - results = yield self.filtering.filter_public_user_data( - events=events, - user=user, - filter_id=filter_id + user_filter = yield self.filtering.get_user_filter( + user_localpart=user_localpart, + filter_id=filter_id, ) + + results = user_filter.filter_public_user_data(events=events) self.assertEquals([], results) @defer.inlineCallbacks def test_filter_room_state_match(self): - user_filter = { + user_filter_json = { "room": { "state": { "types": ["m.*"] @@ -416,7 +420,7 @@ class FilteringTestCase(unittest.TestCase): user = UserID.from_string("@" + user_localpart + ":test") filter_id = yield self.datastore.add_user_filter( user_localpart=user_localpart, - user_filter=user_filter, + user_filter=user_filter_json, ) event = MockEvent( sender="@foo:bar", @@ -425,16 +429,17 @@ class FilteringTestCase(unittest.TestCase): ) events = [event] - results = yield self.filtering.filter_room_state( - events=events, - user=user, - filter_id=filter_id + user_filter = yield self.filtering.get_user_filter( + user_localpart=user_localpart, + filter_id=filter_id, ) + + results = user_filter.filter_room_state(events=events) self.assertEquals(events, results) @defer.inlineCallbacks def test_filter_room_state_no_match(self): - user_filter = { + user_filter_json = { "room": { "state": { "types": ["m.*"] @@ -444,7 +449,7 @@ class FilteringTestCase(unittest.TestCase): user = UserID.from_string("@" + user_localpart + ":test") filter_id = yield self.datastore.add_user_filter( user_localpart=user_localpart, - user_filter=user_filter, + user_filter=user_filter_json, ) event = MockEvent( sender="@foo:bar", @@ -453,16 +458,17 @@ class FilteringTestCase(unittest.TestCase): ) events = [event] - results = yield self.filtering.filter_room_state( - events=events, - user=user, - filter_id=filter_id + user_filter = yield self.filtering.get_user_filter( + user_localpart=user_localpart, + filter_id=filter_id, ) + + results = user_filter.filter_room_state(events) self.assertEquals([], results) @defer.inlineCallbacks def test_add_filter(self): - user_filter = { + user_filter_json = { "room": { "state": { "types": ["m.*"] @@ -472,11 +478,11 @@ class FilteringTestCase(unittest.TestCase): filter_id = yield self.filtering.add_user_filter( user_localpart=user_localpart, - user_filter=user_filter, + user_filter=user_filter_json, ) self.assertEquals(filter_id, 0) - self.assertEquals(user_filter, + self.assertEquals(user_filter_json, (yield self.datastore.get_user_filter( user_localpart=user_localpart, filter_id=0, @@ -485,7 +491,7 @@ class FilteringTestCase(unittest.TestCase): @defer.inlineCallbacks def test_get_filter(self): - user_filter = { + user_filter_json = { "room": { "state": { "types": ["m.*"] @@ -495,7 +501,7 @@ class FilteringTestCase(unittest.TestCase): filter_id = yield self.datastore.add_user_filter( user_localpart=user_localpart, - user_filter=user_filter, + user_filter=user_filter_json, ) filter = yield self.filtering.get_user_filter( @@ -503,4 +509,4 @@ class FilteringTestCase(unittest.TestCase): filter_id=filter_id, ) - self.assertEquals(filter, user_filter) + self.assertEquals(filter.filter_json, user_filter_json) From 365a1867293b8f1d34a1f100d1bc8d2d39bb3d94 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 18:11:28 +0000 Subject: [PATCH 134/180] Add basic filtering support --- synapse/rest/client/v2_alpha/sync.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 2ae2eec55d..c1277d2675 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -21,6 +21,7 @@ from synapse.types import StreamToken from synapse.events.utils import ( serialize_event, format_event_for_client_v2_without_event_id, ) +from synapse.api.filtering import Filter from ._base import client_v2_pattern import logging @@ -80,6 +81,7 @@ class SyncRestServlet(RestServlet): self.auth = hs.get_auth() self.sync_handler = hs.get_handlers().sync_handler self.clock = hs.get_clock() + self.filtering = hs.get_filtering() @defer.inlineCallbacks def on_GET(self, request): @@ -109,9 +111,14 @@ class SyncRestServlet(RestServlet): ) # TODO(mjark): Load filter and apply overrides. - # filter = self.filters.load_fitler(filter_id_str) + try: + filter = yield self.filtering.get_user_filter( + user.localpart, filter_id + ) + except: + filter = Filter({}) # filter = filter.apply_overrides(http_request) - # if filter.matches(event): + #if filter.matches(event): # # stuff sync_config = SyncConfig( @@ -121,7 +128,7 @@ class SyncRestServlet(RestServlet): limit=limit, sort=sort, backfill=backfill, - filter="TODO", # TODO(mjark) Add the filter to the config. + filter=filter, ) if since is not None: @@ -162,9 +169,11 @@ class SyncRestServlet(RestServlet): @staticmethod def encode_room(room, filter, time_now, token_id): event_map = {} + state_events = filter.filter_room_state(room.state) + recent_events = filter.filter_room_events(room.events) state_event_ids = [] recent_event_ids = [] - for event in room.state: + for event in state_events: # TODO(mjark): Respect formatting requirements in the filter. event_map[event.event_id] = serialize_event( event, time_now, token_id=token_id, @@ -172,7 +181,7 @@ class SyncRestServlet(RestServlet): ) state_event_ids.append(event.event_id) - for event in room.events: + for event in recent_events: # TODO(mjark): Respect formatting requirements in the filter. event_map[event.event_id] = serialize_event( event, time_now, token_id=token_id, From ece828a7b72a25502d8daa15f6429c73a620b53c Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 18:15:24 +0000 Subject: [PATCH 135/180] Update todo for the filtering on sync --- synapse/handlers/sync.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 8c7681a48d..5768702192 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -290,7 +290,8 @@ class SyncHandler(BaseHandler): """ # TODO(mjark): Check if they have joined the room between # the previous sync and this one. - # TODO(mjark): Apply the event filter in sync_config + # TODO(mjark): Apply the event filter in sync_config taking care to get + # enough events to reach the limit # TODO(mjark): Check for redactions we might have missed. recents, token = yield self.store.get_recent_events_for_room( room_id, From 1235f7f383617a91b4b96e287cc1ad205b80a5de Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 29 Jan 2015 18:38:22 +0000 Subject: [PATCH 136/180] Add default push rules including setting a sound for messages mentioning your username / display name --- synapse/push/__init__.py | 47 ++++++++++++++++++++++++++++++++------- synapse/push/baserules.py | 35 +++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 8 deletions(-) create mode 100644 synapse/push/baserules.py diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 472ede5480..d19e13d644 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -16,9 +16,10 @@ from twisted.internet import defer from synapse.streams.config import PaginationConfig -from synapse.types import StreamToken +from synapse.types import StreamToken, UserID import synapse.util.async +import baserules import logging import fnmatch @@ -75,14 +76,34 @@ class Pusher(object): rules = yield self.store.get_push_rules_for_user_name(self.user_name) + for r in rules: + r['conditions'] = json.loads(r['conditions']) + r['actions'] = json.loads(r['actions']) + + user_name_localpart = UserID.from_string(self.user_name).localpart + + rules.extend(baserules.make_base_rules(user_name_localpart)) + + # get *our* member event for display name matching + member_events_for_room = yield self.store.get_current_state( + room_id=ev['room_id'], + event_type='m.room.member', + state_key=self.user_name + ) + my_display_name = None + if len(member_events_for_room) > 0: + my_display_name = member_events_for_room[0].content['displayname'] + for r in rules: matches = True - conditions = json.loads(r['conditions']) - actions = json.loads(r['actions']) + conditions = r['conditions'] + actions = r['actions'] for c in conditions: - matches &= self._event_fulfills_condition(ev, c) + matches &= self._event_fulfills_condition( + ev, c, display_name=my_display_name + ) # ignore rules with no actions (we have an explict 'dont_notify' if len(actions) == 0: logger.warn( @@ -95,7 +116,7 @@ class Pusher(object): defer.returnValue(Pusher.DEFAULT_ACTIONS) - def _event_fulfills_condition(self, ev, condition): + def _event_fulfills_condition(self, ev, condition, display_name): if condition['kind'] == 'event_match': if 'pattern' not in condition: logger.warn("event_match condition with no pattern") @@ -103,13 +124,23 @@ class Pusher(object): pat = condition['pattern'] val = _value_for_dotted_key(condition['key'], ev) - if fnmatch.fnmatch(val, pat): - return True - return False + if val is None: + return False + return fnmatch.fnmatch(val.upper(), pat.upper()) elif condition['kind'] == 'device': if 'instance_handle' not in condition: return True return condition['instance_handle'] == self.instance_handle + elif condition['kind'] == 'contains_display_name': + # This is special because display names can be different + # between rooms and so you can't really hard code it in a rule. + # Optimisation: we should cache these names and update them from + # the event stream. + if 'content' not in ev or 'body' not in ev['content']: + return False + return fnmatch.fnmatch( + ev['content']['body'].upper(), "*%s*" % (display_name.upper(),) + ) else: return True diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py new file mode 100644 index 0000000000..4caf7beed2 --- /dev/null +++ b/synapse/push/baserules.py @@ -0,0 +1,35 @@ +def make_base_rules(user_name): + """ + Nominally we reserve priority class 0 for these rules, although + in practice we just append them to the end so we don't actually need it. + """ + return [ + { + 'conditions': [ + { + 'kind': 'event_match', + 'key': 'content.body', + 'pattern': '*%s*' % (user_name,), # Matrix ID match + } + ], + 'actions': [ + 'notify', + { + 'set_sound': 'default' + } + ] + }, + { + 'conditions': [ + { + 'kind': 'contains_display_name' + } + ], + 'actions': [ + 'notify', + { + 'set_sound': 'default' + } + ] + }, + ] \ No newline at end of file From 0b1688639750e2401571263c127817d0b0a43644 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 29 Jan 2015 18:51:22 +0000 Subject: [PATCH 137/180] Change 'from' in notification pokes to 'sender' to match client API v2. Send sender display names where they exist. --- synapse/push/__init__.py | 10 ++++++++++ synapse/push/httppusher.py | 9 ++++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index d19e13d644..19478c72a2 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -154,6 +154,16 @@ class Pusher(object): if name_aliases[0] is not None: ctx['name'] = name_aliases[0] + their_member_events_for_room = yield self.store.get_current_state( + room_id=ev['room_id'], + event_type='m.room.member', + state_key=ev['user_id'] + ) + if len(their_member_events_for_room) > 0: + dn = their_member_events_for_room[0].content['displayname'] + if dn is not None: + ctx['sender_display_name'] = dn + defer.returnValue(ctx) @defer.inlineCallbacks diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index ab128e31e5..ac7c3148d7 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -67,10 +67,7 @@ class HttpPusher(Pusher): 'notification': { 'id': event['event_id'], 'type': event['type'], - 'from': event['user_id'], - # we may have to fetch this over federation and we - # can't trust it anyway: is it worth it? - #'from_display_name': 'Steve Stevington' + 'sender': event['user_id'], 'counts': { # -- we don't mark messages as read yet so # we have no way of knowing # Just set the badge to 1 until we have read receipts @@ -93,6 +90,8 @@ class HttpPusher(Pusher): if len(ctx['aliases']): d['notification']['room_alias'] = ctx['aliases'][0] + if 'sender_display_name' in ctx: + d['notification']['sender_display_name'] = ctx['sender_display_name'] if 'name' in ctx: d['notification']['room_name'] = ctx['name'] @@ -119,7 +118,7 @@ class HttpPusher(Pusher): 'notification': { 'id': '', 'type': None, - 'from': '', + 'sender': '', 'counts': { 'unread': 0, 'missed_calls': 0 From fc946f3b8da8c7f71a9c25bf542c04472147bc5b Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 29 Jan 2015 21:59:17 +0000 Subject: [PATCH 138/180] Include content in notification pokes --- synapse/push/httppusher.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index ac7c3148d7..d4c5f03b01 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -87,6 +87,8 @@ class HttpPusher(Pusher): } if event['type'] == 'm.room.member': d['notification']['membership'] = event['content']['membership'] + if 'content' in event: + d['notification']['content'] = event['content'] if len(ctx['aliases']): d['notification']['room_alias'] = ctx['aliases'][0] From c1c7b398270cc19fcf9410b9159f741092e0510b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 10:30:54 +0000 Subject: [PATCH 139/180] Fix bug where we changes in outlier in metadata dict propogated to other events --- synapse/events/__init__.py | 2 +- synapse/events/builder.py | 5 +++-- synapse/events/utils.py | 5 ++++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 4252e5ab5c..bf07951027 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -18,7 +18,7 @@ from synapse.util.frozenutils import freeze, unfreeze class _EventInternalMetadata(object): def __init__(self, internal_metadata_dict): - self.__dict__ = internal_metadata_dict + self.__dict__ = dict(internal_metadata_dict) def get_dict(self): return dict(self.__dict__) diff --git a/synapse/events/builder.py b/synapse/events/builder.py index a9b1b99a10..9d45bdb892 100644 --- a/synapse/events/builder.py +++ b/synapse/events/builder.py @@ -23,14 +23,15 @@ import copy class EventBuilder(EventBase): - def __init__(self, key_values={}): + def __init__(self, key_values={}, internal_metadata_dict={}): signatures = copy.deepcopy(key_values.pop("signatures", {})) unsigned = copy.deepcopy(key_values.pop("unsigned", {})) super(EventBuilder, self).__init__( key_values, signatures=signatures, - unsigned=unsigned + unsigned=unsigned, + internal_metadata_dict=internal_metadata_dict, ) def build(self): diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 10a6b9f264..08d6d6fa47 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -88,7 +88,10 @@ def prune_event(event): if "age_ts" in event.unsigned: allowed_fields["unsigned"]["age_ts"] = event.unsigned["age_ts"] - return type(event)(allowed_fields) + return type(event)( + allowed_fields, + internal_metadata_dict=event.internal_metadata.get_dict() + ) def serialize_event(hs, e, client_event=True): From c1d860870b418b9583078022d0babe557b73760f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 10:48:47 +0000 Subject: [PATCH 140/180] Fix regression where we no longer correctly handled the case of gaps in our event graph --- synapse/federation/federation_server.py | 3 +++ synapse/handlers/federation.py | 8 ++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 8cff4e6472..845a07a3a3 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -366,10 +366,13 @@ class FederationServer(object): logger.debug("Processed pdu %s", event_id) else: logger.warn("Failed to get PDU %s", event_id) + fetch_state = True except: # TODO(erikj): Do some more intelligent retries. logger.exception("Failed to get PDU") fetch_state = True + else: + fetch_state = True else: fetch_state = True diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 14c26d8cea..de47a97e6b 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -119,7 +119,7 @@ class FederationHandler(BaseHandler): event.room_id, self.server_name ) - if not is_in_room and not event.internal_metadata.outlier: + if not is_in_room and not event.internal_metadata.is_outlier(): logger.debug("Got event for room we're not in.") replication = self.replication_layer @@ -780,6 +780,7 @@ class FederationHandler(BaseHandler): (e.type, e.state_key): e for e in remote_auth_chain if e.event_id in auth_ids } + e.internal_metadata.outlier = True yield self._handle_new_event( origin, e, auth_events=auth ) @@ -787,6 +788,8 @@ class FederationHandler(BaseHandler): except AuthError: pass + # FIXME: Assumes we have and stored all the state for all the + # prev_events current_state = set(e.event_id for e in auth_events.values()) different_auth = event_auth_events - current_state @@ -814,6 +817,7 @@ class FederationHandler(BaseHandler): (e.type, e.state_key): e for e in result["auth_chain"] if e.event_id in auth_ids } + e.internal_metadata.outlier = True yield self._handle_new_event( origin, e, auth_events=auth ) @@ -882,7 +886,7 @@ class FederationHandler(BaseHandler): missing_remotes = [] missing_locals = [] - while current_local and current_remote: + while current_local or current_remote: if current_remote is None: missing_locals.append(current_local) current_local = get_next(local_iter) From 823999716eb506740b34d87fcdaaf3ebbe4f760c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 11:07:57 +0000 Subject: [PATCH 141/180] Fix bug in timeout handling in keyclient --- synapse/crypto/keyclient.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/synapse/crypto/keyclient.py b/synapse/crypto/keyclient.py index 9c910fa3fc..cdb6279764 100644 --- a/synapse/crypto/keyclient.py +++ b/synapse/crypto/keyclient.py @@ -61,9 +61,11 @@ class SynapseKeyClientProtocol(HTTPClient): def __init__(self): self.remote_key = defer.Deferred() + self.host = None def connectionMade(self): - logger.debug("Connected to %s", self.transport.getHost()) + self.host = self.transport.getHost() + logger.debug("Connected to %s", self.host) self.sendCommand(b"GET", b"/_matrix/key/v1/") self.endHeaders() self.timer = reactor.callLater( @@ -92,8 +94,7 @@ class SynapseKeyClientProtocol(HTTPClient): self.timer.cancel() def on_timeout(self): - logger.debug("Timeout waiting for response from %s", - self.transport.getHost()) + logger.debug("Timeout waiting for response from %s", self.host) self.remote_key.errback(IOError("Timeout waiting for response")) self.transport.abortConnection() From 0c2d245fdf5f24f99138f1c60c4e6bed52cf5d5a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 11:08:52 +0000 Subject: [PATCH 142/180] Update the current state of an event if we update auth events. --- synapse/handlers/federation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index de47a97e6b..cc22f21cd1 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -706,7 +706,6 @@ class FederationHandler(BaseHandler): event.event_id, e.msg ) - # TODO: Store rejection. context.rejected = RejectedReason.AUTH_ERROR yield self.store.persist_event( @@ -828,6 +827,9 @@ class FederationHandler(BaseHandler): # 4. Look at rejects and their proofs. # TODO. + context.current_state.update(auth_events) + context.state_group = None + try: self.auth.check(event, auth_events=auth_events) except AuthError: From bd03947c05f23e4adb6ce1a745d725a106fed66c Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 30 Jan 2015 11:13:42 +0000 Subject: [PATCH 143/180] We do need Twisted 14, not 15: we use internal Twisted things that have been removed in 15. --- setup.py | 2 +- synapse/python_dependencies.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 043cd044a7..3249e87a96 100755 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ setup( install_requires=[ "syutil==0.0.2", "matrix_angular_sdk==0.6.0", - "Twisted>=14.0.0", + "Twisted==14.0.2", "service_identity>=1.0.0", "pyopenssl>=0.14", "pyyaml", diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 4182ad990f..826a36f203 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -6,7 +6,7 @@ logger = logging.getLogger(__name__) REQUIREMENTS = { "syutil==0.0.2": ["syutil"], "matrix_angular_sdk==0.6.0": ["syweb>=0.6.0"], - "Twisted>=14.0.0": ["twisted>=14.0.0"], + "Twisted==14.0.2": ["twisted==14.0.2"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], "pyopenssl>=0.14": ["OpenSSL>=0.14"], "pyyaml": ["yaml"], From 2c9e136d5721d1797e153268f4562c27887a6201 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 11:14:33 +0000 Subject: [PATCH 144/180] Fix bad merge fo python_dependencies.py --- synapse/python_dependencies.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index ba9308803c..4182ad990f 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -6,7 +6,6 @@ logger = logging.getLogger(__name__) REQUIREMENTS = { "syutil==0.0.2": ["syutil"], "matrix_angular_sdk==0.6.0": ["syweb>=0.6.0"], - "matrix_angular_sdk>=0.6.0": ["syweb>=0.6.0"], "Twisted>=14.0.0": ["twisted>=14.0.0"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], "pyopenssl>=0.14": ["OpenSSL>=0.14"], From 0adf3e54454d499045d2bfd8f6bcfece4e8fe6ed Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 11:16:41 +0000 Subject: [PATCH 145/180] Revert accidental bumping of angluar_sdk dep --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 733cfa8318..043cd044a7 100755 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ setup( description="Reference Synapse Home Server", install_requires=[ "syutil==0.0.2", - "matrix_angular_sdk==0.6.1", + "matrix_angular_sdk==0.6.0", "Twisted>=14.0.0", "service_identity>=1.0.0", "pyopenssl>=0.14", @@ -47,7 +47,7 @@ setup( dependency_links=[ "https://github.com/matrix-org/syutil/tarball/v0.0.2#egg=syutil-0.0.2", "https://github.com/pyca/pynacl/tarball/d4d3175589b892f6ea7c22f466e0e223853516fa#egg=pynacl-0.3.0", - "https://github.com/matrix-org/matrix-angular-sdk/tarball/v0.6.0/#egg=matrix_angular_sdk-0.6.1", + "https://github.com/matrix-org/matrix-angular-sdk/tarball/v0.6.0/#egg=matrix_angular_sdk-0.6.0", ], setup_requires=[ "setuptools_trial", From 22dd1cde2d83a2448074816108b85d1957315236 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Jan 2015 11:32:35 +0000 Subject: [PATCH 146/180] Filter the recent events before applying the limit when doing an incremental sync with a gap --- synapse/api/filtering.py | 2 -- synapse/handlers/sync.py | 53 ++++++++++++++++++++-------- synapse/rest/client/v2_alpha/sync.py | 2 +- synapse/storage/stream.py | 21 +++++++---- 4 files changed, 54 insertions(+), 24 deletions(-) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index b7e5d3222f..fa4de2614d 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -12,8 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from twisted.internet import defer - from synapse.api.errors import SynapseError from synapse.types import UserID, RoomID diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 5768702192..0df1851b0e 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -278,6 +278,40 @@ class SyncHandler(BaseHandler): next_batch=now_token, )) + @defer.inlineCallbacks + def load_filtered_recents(self, room_id, sync_config, since_token, + now_token): + limited = True + recents = [] + filtering_factor = 2 + load_limit = max(sync_config.limit * filtering_factor, 100) + max_repeat = 3 # Only try a few times per room, otherwise + room_key = now_token.room_key + + while limited and len(recents) < sync_config.limit and max_repeat: + events, room_key = yield self.store.get_recent_events_for_room( + room_id, + limit=load_limit + 1, + from_token=since_token.room_key, + end_token=room_key, + ) + loaded_recents = sync_config.filter.filter_room_events(events) + loaded_recents.extend(recents) + recents = loaded_recents + if len(events) <= load_limit: + limited = False + max_repeat -= 1 + + if len(recents) > sync_config.limit: + recents = recents[-sync_config.limit:] + room_key = recents[0].internal_metadata.before + + prev_batch_token = now_token.copy_and_replace( + "room_key", room_key + ) + + defer.returnValue((recents, prev_batch_token, limited)) + @defer.inlineCallbacks def incremental_sync_with_gap_for_room(self, room_id, sync_config, since_token, now_token, @@ -288,28 +322,17 @@ class SyncHandler(BaseHandler): Returns: A Deferred RoomSyncResult """ + # TODO(mjark): Check if they have joined the room between # the previous sync and this one. - # TODO(mjark): Apply the event filter in sync_config taking care to get - # enough events to reach the limit # TODO(mjark): Check for redactions we might have missed. - recents, token = yield self.store.get_recent_events_for_room( - room_id, - limit=sync_config.limit + 1, - from_token=since_token.room_key, - end_token=now_token.room_key, + + recents, prev_batch_token, limited = self.load_filtered_recents( + room_id, sync_config, since_token, ) logging.debug("Recents %r", recents) - if len(recents) > sync_config.limit: - limited = True - recents = recents[1:] - else: - limited = False - - prev_batch_token = now_token.copy_and_replace("room_key", token[0]) - # TODO(mjark): This seems racy since this isn't being passed a # token to indicate what point in the stream this is current_state_events = yield self.state_handler.get_current_state( diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index c1277d2675..46ea50d118 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -116,7 +116,7 @@ class SyncRestServlet(RestServlet): user.localpart, filter_id ) except: - filter = Filter({}) + filter = Filter({}) # filter = filter.apply_overrides(http_request) #if filter.matches(event): # # stuff diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 2ea5e1a021..73504c8b52 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -181,15 +181,11 @@ class StreamStore(SQLBaseStore): get_prev_content=True ) - for event, row in zip(ret, rows): - stream = row["stream_ordering"] - topo = event.depth - internal = event.internal_metadata - internal.before = str(_StreamToken(topo, stream - 1)) - internal.after = str(_StreamToken(topo, stream)) + self._set_before_and_after(ret, rows) if rows: key = "s%d" % max([r["stream_ordering"] for r in rows]) + else: # Assume we didn't get anything because there was nothing to # get. @@ -267,6 +263,8 @@ class StreamStore(SQLBaseStore): get_prev_content=True ) + self._set_before_and_after(events, rows) + return events, next_token, return self.runInteraction("paginate_room_events", f) @@ -328,6 +326,8 @@ class StreamStore(SQLBaseStore): get_prev_content=True ) + self._set_before_and_after(events, rows) + return events, token return self.runInteraction( @@ -354,3 +354,12 @@ class StreamStore(SQLBaseStore): key = res[0]["m"] return "s%d" % (key,) + + @staticmethod + def _set_before_and_after(events, rows): + for event, row in zip(events, rows): + stream = row["stream_ordering"] + topo = event.depth + internal = event.internal_metadata + internal.before = str(_StreamToken(topo, stream - 1)) + internal.after = str(_StreamToken(topo, stream)) From e97de6d96a25912b34fa38cd3a587d798f6b676c Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Jan 2015 11:35:20 +0000 Subject: [PATCH 147/180] Filter the recent events before applying the limit when doing an initial sync --- synapse/handlers/sync.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 0df1851b0e..b83fcad655 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -163,12 +163,11 @@ class SyncHandler(BaseHandler): Returns: A Deferred RoomSyncResult. """ - recent_events, token = yield self.store.get_recent_events_for_room( - room_id, - limit=sync_config.limit, - end_token=now_token.room_key, + + recents, prev_batch_token, limited = self.load_filtered_recents( + room_id, sync_config, now_token, ) - prev_batch_token = now_token.copy_and_replace("room_key", token[0]) + current_state_events = yield self.state_handler.get_current_state( room_id ) @@ -176,10 +175,10 @@ class SyncHandler(BaseHandler): defer.returnValue(RoomSyncResult( room_id=room_id, published=room_id in published_room_ids, - events=recent_events, + events=recents, prev_batch=prev_batch_token, state=current_state_events, - limited=True, + limited=limited, ephemeral=[], )) @@ -279,8 +278,8 @@ class SyncHandler(BaseHandler): )) @defer.inlineCallbacks - def load_filtered_recents(self, room_id, sync_config, since_token, - now_token): + def load_filtered_recents(self, room_id, sync_config, now_token, + since_token=None): limited = True recents = [] filtering_factor = 2 @@ -292,7 +291,7 @@ class SyncHandler(BaseHandler): events, room_key = yield self.store.get_recent_events_for_room( room_id, limit=load_limit + 1, - from_token=since_token.room_key, + from_token=since_token.room_key if since_token else None, end_token=room_key, ) loaded_recents = sync_config.filter.filter_room_events(events) @@ -328,7 +327,7 @@ class SyncHandler(BaseHandler): # TODO(mjark): Check for redactions we might have missed. recents, prev_batch_token, limited = self.load_filtered_recents( - room_id, sync_config, since_token, + room_id, sync_config, now_token, since_token, ) logging.debug("Recents %r", recents) From 8498d348d818aa2d2cb9bb9bb2775103840f355d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Jan 2015 11:42:09 +0000 Subject: [PATCH 148/180] Fix token formatting --- synapse/handlers/sync.py | 6 +++--- synapse/storage/stream.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index b83fcad655..3c68e2a9ec 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -164,7 +164,7 @@ class SyncHandler(BaseHandler): A Deferred RoomSyncResult. """ - recents, prev_batch_token, limited = self.load_filtered_recents( + recents, prev_batch_token, limited = yield self.load_filtered_recents( room_id, sync_config, now_token, ) @@ -288,7 +288,7 @@ class SyncHandler(BaseHandler): room_key = now_token.room_key while limited and len(recents) < sync_config.limit and max_repeat: - events, room_key = yield self.store.get_recent_events_for_room( + events, (room_key,_) = yield self.store.get_recent_events_for_room( room_id, limit=load_limit + 1, from_token=since_token.room_key if since_token else None, @@ -326,7 +326,7 @@ class SyncHandler(BaseHandler): # the previous sync and this one. # TODO(mjark): Check for redactions we might have missed. - recents, prev_batch_token, limited = self.load_filtered_recents( + recents, prev_batch_token, limited = yield self.load_filtered_recents( room_id, sync_config, now_token, since_token, ) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 73504c8b52..3ccb6f8a61 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -316,9 +316,9 @@ class StreamStore(SQLBaseStore): toke = rows[0]["stream_ordering"] - 1 start_token = str(_StreamToken(topo, toke)) - token = (start_token, end_token) + token = (start_token, str(end_token)) else: - token = (end_token, end_token) + token = (str(end_token), str(end_token)) events = self._get_events_txn( txn, From c562f237f6236c981f2e7858ff2748f62bd63ad1 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Jan 2015 11:43:00 +0000 Subject: [PATCH 149/180] Unused import --- synapse/api/filtering.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index b7e5d3222f..fa4de2614d 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -12,8 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from twisted.internet import defer - from synapse.api.errors import SynapseError from synapse.types import UserID, RoomID From 4a67834bc84f604605c618049599f4638434c7cf Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Jan 2015 11:50:15 +0000 Subject: [PATCH 150/180] Pass client info to the sync_config --- synapse/handlers/sync.py | 5 +++-- synapse/rest/client/v2_alpha/sync.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 3c68e2a9ec..1a74a4c97c 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -28,7 +28,7 @@ logger = logging.getLogger(__name__) SyncConfig = collections.namedtuple("SyncConfig", [ "user", - "device", + "client_info", "limit", "gap", "sort", @@ -288,12 +288,13 @@ class SyncHandler(BaseHandler): room_key = now_token.room_key while limited and len(recents) < sync_config.limit and max_repeat: - events, (room_key,_) = yield self.store.get_recent_events_for_room( + events, keys = yield self.store.get_recent_events_for_room( room_id, limit=load_limit + 1, from_token=since_token.room_key if since_token else None, end_token=room_key, ) + (room_key, _) = keys loaded_recents = sync_config.filter.filter_room_events(events) loaded_recents.extend(recents) recents = loaded_recents diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 46ea50d118..81d5cf8ead 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -123,7 +123,7 @@ class SyncRestServlet(RestServlet): sync_config = SyncConfig( user=user, - device="TODO", # TODO(mjark) Get the device_id from access_token + client_info=client, gap=gap, limit=limit, sort=sort, From a70a801184814d116ed5b10a952e17c45df7bfc8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 13:34:01 +0000 Subject: [PATCH 151/180] Fix bug where we superfluously asked for current state. Change API of /query_auth/ so that we don't duplicate events in the response. --- synapse/api/auth.py | 2 + synapse/federation/federation_client.py | 7 +--- synapse/federation/federation_server.py | 12 +++--- synapse/handlers/federation.py | 51 +++++++++---------------- synapse/state.py | 20 ++++++++-- 5 files changed, 43 insertions(+), 49 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 3471afd7e7..37e31d2b6f 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -102,6 +102,8 @@ class Auth(object): def check_host_in_room(self, room_id, host): curr_state = yield self.state.get_current_state(room_id) + logger.debug("Got curr_state %s", curr_state) + for event in curr_state: if event.type == EventTypes.Member: try: diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 1173ca817b..e1539bd0e0 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -357,15 +357,10 @@ class FederationClient(object): for e in content["auth_chain"] ] - missing = [ - (yield self._check_sigs_and_hash(self.event_from_pdu_json(e))) - for e in content.get("missing", []) - ] - ret = { "auth_chain": auth_chain, "rejects": content.get("rejects", []), - "missing": missing, + "missing": content.get("missing", []), } defer.returnValue(ret) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 845a07a3a3..84ed0a0ba0 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -252,11 +252,8 @@ class FederationServer(object): e.get_pdu_json(time_now) for e in ret["auth_chain"] ], - "rejects": content.get("rejects", []), - "missing": [ - e.get_pdu_json(time_now) - for e in ret.get("missing", []) - ], + "rejects": ret.get("rejects", []), + "missing": ret.get("missing", []), } defer.returnValue( @@ -372,7 +369,10 @@ class FederationServer(object): logger.exception("Failed to get PDU") fetch_state = True else: - fetch_state = True + prevs = {e_id for e_id, _ in pdu.prev_events} + seen = set(have_seen.keys()) + if prevs - seen: + fetch_state = True else: fetch_state = True diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index cc22f21cd1..35cad4182a 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -121,38 +121,18 @@ class FederationHandler(BaseHandler): ) if not is_in_room and not event.internal_metadata.is_outlier(): logger.debug("Got event for room we're not in.") - - replication = self.replication_layer - - if not state: - state, auth_chain = yield replication.get_state_for_room( - origin, room_id=event.room_id, event_id=event.event_id, - ) - - if not auth_chain: - auth_chain = yield replication.get_event_auth( - origin, - context=event.room_id, - event_id=event.event_id, - ) - - for e in auth_chain: - e.internal_metadata.outlier = True - try: - yield self._handle_new_event(origin, e) - except: - logger.exception( - "Failed to handle auth event %s", - e.event_id, - ) - current_state = state - if state: + if state and auth_chain is not None: for e in state: e.internal_metadata.outlier = True try: - yield self._handle_new_event(origin, e) + auth_ids = [e_id for e_id, _ in e.auth_events] + auth = { + (e.type, e.state_key): e for e in auth_chain + if e.event_id in auth_ids + } + yield self._handle_new_event(origin, e, auth_events=auth) except: logger.exception( "Failed to handle state event %s", @@ -809,18 +789,23 @@ class FederationHandler(BaseHandler): ) # 3. Process any remote auth chain events we haven't seen. - for e in result.get("missing", []): + for missing_id in result.get("missing", []): try: - auth_ids = [e_id for e_id, _ in e.auth_events] + for e in result["auth_chain"]: + if e.event_id == missing_id: + ev = e + break + + auth_ids = [e_id for e_id, _ in ev.auth_events] auth = { (e.type, e.state_key): e for e in result["auth_chain"] if e.event_id in auth_ids } - e.internal_metadata.outlier = True + ev.internal_metadata.outlier = True yield self._handle_new_event( - origin, e, auth_events=auth + origin, ev, auth_events=auth ) - auth_events[(e.type, e.state_key)] = e + auth_events[(ev.type, ev.state_key)] = ev except AuthError: pass @@ -970,5 +955,5 @@ class FederationHandler(BaseHandler): } for e in base_remote_rejected }, - "missing": missing_locals, + "missing": [e.event_id for e in missing_locals], }) diff --git a/synapse/state.py b/synapse/state.py index d9fdfb34be..e6632978b5 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -166,10 +166,17 @@ class StateHandler(object): first is the name of a state group if one and only one is involved, otherwise `None`. """ + logger.debug("resolve_state_groups event_ids %s", event_ids) + state_groups = yield self.store.get_state_groups( event_ids ) + logger.debug( + "resolve_state_groups state_groups %s", + state_groups.keys() + ) + group_names = set(state_groups.keys()) if len(group_names) == 1: name, state_list = state_groups.items().pop() @@ -205,6 +212,15 @@ class StateHandler(object): if len(v.values()) > 1 } + logger.debug( + "resolve_state_groups Unconflicted state: %s", + unconflicted_state.values(), + ) + logger.debug( + "resolve_state_groups Conflicted state: %s", + conflicted_state.values(), + ) + if event_type: prev_states_events = conflicted_state.get( (event_type, state_key), [] @@ -240,10 +256,6 @@ class StateHandler(object): 1. power levels 2. memberships 3. other events. - - :param conflicted_state: - :param auth_events: - :return: """ resolved_state = {} power_key = (EventTypes.PowerLevels, "") From 8fe39a03119b3d092e84319d6d99b6ffc49a83f5 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Jan 2015 13:33:41 +0000 Subject: [PATCH 152/180] Check if the user has joined the room between incremental syncs --- synapse/handlers/sync.py | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 1a74a4c97c..9e07f30b24 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -16,7 +16,7 @@ from ._base import BaseHandler from synapse.streams.config import PaginationConfig -from synapse.api.constants import Membership +from synapse.api.constants import Membership, EventTypes from twisted.internet import defer @@ -250,6 +250,11 @@ class SyncHandler(BaseHandler): ) else: prev_batch = now_token + + state = yield self.check_joined_room( + sync_config, room_id, state + ) + room_sync = RoomSyncResult( room_id=room_id, published=room_id in published_room_ids, @@ -323,8 +328,6 @@ class SyncHandler(BaseHandler): A Deferred RoomSyncResult """ - # TODO(mjark): Check if they have joined the room between - # the previous sync and this one. # TODO(mjark): Check for redactions we might have missed. recents, prev_batch_token, limited = yield self.load_filtered_recents( @@ -349,6 +352,10 @@ class SyncHandler(BaseHandler): current_state=current_state_events, ) + state_events_delta = yield self.check_joined_room( + sync_config, room_id, state_events_delta + ) + room_sync = RoomSyncResult( room_id=room_id, published=room_id in published_room_ids, @@ -356,7 +363,7 @@ class SyncHandler(BaseHandler): prev_batch=prev_batch_token, state=state_events_delta, limited=limited, - typing=typing_by_room.get(room_id, None) + ephemeral=typing_by_room.get(room_id, None) ) logging.debug("Room sync: %r", room_sync) @@ -402,3 +409,19 @@ class SyncHandler(BaseHandler): if event.event_id not in previous_dict: state_delta.append(event) return state_delta + + @defer.inlineCallbacks + def check_joined_room(self, sync_config, room_id, state_delta): + joined = False + for event in state_delta: + if ( + event.type == EventTypes.Member + and event.state_key == sync_config.user.to_string() + ): + if event.content["membership"] == Membership.JOIN: + joined = True + + if joined: + state_delta = yield self.state_handler.get_current_state(room_id) + + defer.returnValue(state_delta) From 76d7fd39cd44393a5c712930e77c64e202df17cc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 13:52:02 +0000 Subject: [PATCH 153/180] Style changes. --- synapse/state.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/synapse/state.py b/synapse/state.py index d9fdfb34be..43bda35253 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -37,6 +37,9 @@ def _get_state_key_from_event(event): KeyStateTuple = namedtuple("KeyStateTuple", ("context", "type", "state_key")) +AuthEventTypes = (EventTypes.Create, EventTypes.Member, EventTypes.PowerLevels,) + + class StateHandler(object): """ Responsible for doing state conflict resolution. """ @@ -215,7 +218,7 @@ class StateHandler(object): auth_events = { k: e for k, e in unconflicted_state.items() - if k[0] in (EventTypes.Create, EventTypes.Member, EventTypes.PowerLevels,) + if k[0] in AuthEventTypes } try: @@ -240,10 +243,6 @@ class StateHandler(object): 1. power levels 2. memberships 3. other events. - - :param conflicted_state: - :param auth_events: - :return: """ resolved_state = {} power_key = (EventTypes.PowerLevels, "") @@ -305,4 +304,4 @@ class StateHandler(object): def key_func(e): return -int(e.depth), hashlib.sha1(e.event_id).hexdigest() - return sorted(events, key=key_func) \ No newline at end of file + return sorted(events, key=key_func) From 7a9f6f083e0998893eb9d837512009509d81c998 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 13:55:46 +0000 Subject: [PATCH 154/180] Remove commented line --- synapse/state.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/state.py b/synapse/state.py index 43bda35253..dff11711a6 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -46,7 +46,6 @@ class StateHandler(object): def __init__(self, hs): self.store = hs.get_datastore() - # self.auth = hs.get_auth() self.hs = hs @defer.inlineCallbacks From 3d7026e709b145378a6b8a37404910951ef682b3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 14:37:19 +0000 Subject: [PATCH 155/180] Add a slightly more helpful comment --- synapse/state.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/state.py b/synapse/state.py index dff11711a6..081bc31bb5 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -296,7 +296,8 @@ class StateHandler(object): except AuthError: pass - # Oh dear. + # Use the last event (the one with the least depth) if they all fail + # the auth check. return event def _ordered_events(self, events): From 322a047502c938bfe9a6acab47e370e69fefc522 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 30 Jan 2015 14:46:03 +0000 Subject: [PATCH 156/180] Add room member count condition and default rule to make a noise on rooms of only 2 people. --- synapse/push/__init__.py | 50 +++++++++++++++++++++++++++++++++---- synapse/push/baserules.py | 14 +++++++++++ synapse/storage/__init__.py | 5 +++- 3 files changed, 63 insertions(+), 6 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 19478c72a2..cc05278c8c 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -24,6 +24,7 @@ import baserules import logging import fnmatch import json +import re logger = logging.getLogger(__name__) @@ -34,6 +35,8 @@ class Pusher(object): GIVE_UP_AFTER = 24 * 60 * 60 * 1000 DEFAULT_ACTIONS = ['notify'] + INEQUALITY_EXPR = re.compile("^([=<>]*)([0-9]*)$") + def __init__(self, _hs, instance_handle, user_name, app_id, app_display_name, device_display_name, pushkey, pushkey_ts, data, last_token, last_success, failing_since): @@ -88,11 +91,21 @@ class Pusher(object): member_events_for_room = yield self.store.get_current_state( room_id=ev['room_id'], event_type='m.room.member', - state_key=self.user_name + state_key=None ) my_display_name = None - if len(member_events_for_room) > 0: - my_display_name = member_events_for_room[0].content['displayname'] + room_member_count = 0 + for mev in member_events_for_room: + if mev.content['membership'] != 'join': + continue + + # This loop does two things: + # 1) Find our current display name + if mev.state_key == self.user_name: + my_display_name = mev.content['displayname'] + + # and 2) Get the number of people in that room + room_member_count += 1 for r in rules: matches = True @@ -102,7 +115,8 @@ class Pusher(object): for c in conditions: matches &= self._event_fulfills_condition( - ev, c, display_name=my_display_name + ev, c, display_name=my_display_name, + room_member_count=room_member_count ) # ignore rules with no actions (we have an explict 'dont_notify' if len(actions) == 0: @@ -116,7 +130,7 @@ class Pusher(object): defer.returnValue(Pusher.DEFAULT_ACTIONS) - def _event_fulfills_condition(self, ev, condition, display_name): + def _event_fulfills_condition(self, ev, condition, display_name, room_member_count): if condition['kind'] == 'event_match': if 'pattern' not in condition: logger.warn("event_match condition with no pattern") @@ -138,9 +152,35 @@ class Pusher(object): # the event stream. if 'content' not in ev or 'body' not in ev['content']: return False + if not display_name: + return False return fnmatch.fnmatch( ev['content']['body'].upper(), "*%s*" % (display_name.upper(),) ) + elif condition['kind'] == 'room_member_count': + if 'is' not in condition: + return False + m = Pusher.INEQUALITY_EXPR.match(condition['is']) + if not m: + return False + ineq = m.group(1) + rhs = m.group(2) + if not rhs.isdigit(): + return False + rhs = int(rhs) + + if ineq == '' or ineq == '==': + return room_member_count == rhs + elif ineq == '<': + return room_member_count < rhs + elif ineq == '>': + return room_member_count > rhs + elif ineq == '>=': + return room_member_count >= rhs + elif ineq == '<=': + return room_member_count <= rhs + else: + return False else: return True diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index 4caf7beed2..bd162baade 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -32,4 +32,18 @@ def make_base_rules(user_name): } ] }, + { + 'conditions': [ + { + 'kind': 'room_member_count', + 'is': '2' + } + ], + 'actions': [ + 'notify', + { + 'set_sound': 'default' + } + ] + } ] \ No newline at end of file diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 277581b4e2..7b18acf421 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -375,9 +375,12 @@ class DataStore(RoomMemberStore, RoomStore, "redacted": del_sql, } - if event_type: + if event_type and state_key is not None: sql += " AND s.type = ? AND s.state_key = ? " args = (room_id, event_type, state_key) + elif event_type: + sql += " AND s.type = ?" + args = (room_id, event_type) else: args = (room_id, ) From 472cf532b7e64a30c5d7ecdeec9b8f89abf8276c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 14:48:03 +0000 Subject: [PATCH 157/180] Put CREATE rejections into seperate .sql --- synapse/storage/schema/im.sql | 7 ------- synapse/storage/schema/rejections.sql | 21 +++++++++++++++++++++ 2 files changed, 21 insertions(+), 7 deletions(-) create mode 100644 synapse/storage/schema/rejections.sql diff --git a/synapse/storage/schema/im.sql b/synapse/storage/schema/im.sql index bc7c6b6ed5..dd00c1cd2f 100644 --- a/synapse/storage/schema/im.sql +++ b/synapse/storage/schema/im.sql @@ -123,10 +123,3 @@ CREATE TABLE IF NOT EXISTS room_hosts( ); CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id); - -CREATE TABLE IF NOT EXISTS rejections( - event_id TEXT NOT NULL, - reason TEXT NOT NULL, - last_check TEXT NOT NULL, - CONSTRAINT ev_id UNIQUE (event_id) ON CONFLICT REPLACE -); diff --git a/synapse/storage/schema/rejections.sql b/synapse/storage/schema/rejections.sql new file mode 100644 index 0000000000..bd2a8b1bb5 --- /dev/null +++ b/synapse/storage/schema/rejections.sql @@ -0,0 +1,21 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS rejections( + event_id TEXT NOT NULL, + reason TEXT NOT NULL, + last_check TEXT NOT NULL, + CONSTRAINT ev_id UNIQUE (event_id) ON CONFLICT REPLACE +); From 2f4cb04f455d24d0086b37bc363137e995d908d5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 14:48:11 +0000 Subject: [PATCH 158/180] Be more specific in naming columns in selects. --- synapse/storage/_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 1f5e74a16a..b350fd61f1 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -504,7 +504,7 @@ class SQLBaseStore(object): def _get_event_txn(self, txn, event_id, check_redacted=True, get_prev_content=False, allow_rejected=False): sql = ( - "SELECT internal_metadata, json, r.event_id, reason " + "SELECT e.internal_metadata, e.json, r.event_id, rej.reason " "FROM event_json as e " "LEFT JOIN redactions as r ON e.event_id = r.redacts " "LEFT JOIN rejections as rej on rej.event_id = e.event_id " From e97f756a05519f9d5a8a6ff78182b691dd1355df Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Jan 2015 14:54:06 +0000 Subject: [PATCH 159/180] Use 'in' to test if the key exists, remove unused _filters_for_user --- synapse/api/filtering.py | 8 ++------ synapse/storage/filtering.py | 4 ---- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index fa4de2614d..4d570b74f8 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -114,21 +114,17 @@ class Filtering(object): if not isinstance(event_type, basestring): raise SynapseError(400, "Event type should be a string") - try: + if "format" in definition: event_format = definition["format"] if event_format not in ["federation", "events"]: raise SynapseError(400, "Invalid format: %s" % (event_format,)) - except KeyError: - pass # format is optional - try: + if "select" in definition: event_select_list = definition["select"] for select_key in event_select_list: if select_key not in ["event_id", "origin_server_ts", "thread_id", "content", "content.body"]: raise SynapseError(400, "Bad select: %s" % (select_key,)) - except KeyError: - pass # select is optional if ("bundle_updates" in definition and type(definition["bundle_updates"]) != bool): diff --git a/synapse/storage/filtering.py b/synapse/storage/filtering.py index cb01c2040f..e86eeced45 100644 --- a/synapse/storage/filtering.py +++ b/synapse/storage/filtering.py @@ -20,10 +20,6 @@ from ._base import SQLBaseStore import json -# TODO(paul) -_filters_for_user = {} - - class FilteringStore(SQLBaseStore): @defer.inlineCallbacks def get_user_filter(self, user_localpart, filter_id): From 4f7fe63b6df891c196698b9d896ca7893c7a8a8e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 14:57:53 +0000 Subject: [PATCH 160/180] Remember to add schema file to list --- synapse/storage/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index f233ff2a2a..d03ee80303 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -70,6 +70,7 @@ SCHEMAS = [ "pusher", "media_repository", "filtering", + "rejections", ] From 91015ad008b0d4538022fbddae7da397f7bd7000 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 14:58:54 +0000 Subject: [PATCH 161/180] Remove merge conflict --- synapse/storage/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index d03ee80303..f35ece6446 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -32,11 +32,8 @@ from .event_federation import EventFederationStore from .pusher import PusherStore from .push_rule import PushRuleStore from .media_repository import MediaRepositoryStore -<<<<<<< HEAD from .rejections import RejectionsStore -======= ->>>>>>> 471c47441d0c188e845b75c8f446c44899fdcfe7 from .state import StateStore from .signatures import SignatureStore from .filtering import FilteringStore From 4ffac34a646fa2e763ba9214199d7803d1174959 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 30 Jan 2015 15:02:21 +0000 Subject: [PATCH 162/180] Add glob asterisks when running rules. Means that now you can't do exact matches even in override rules, but I think we can live with that. Advantage is that you'll now always get back what was put in to the API. --- synapse/push/__init__.py | 5 +++++ synapse/rest/client/v1/push_rule.py | 5 +---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index cc05278c8c..6a302305fb 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -137,6 +137,11 @@ class Pusher(object): return False pat = condition['pattern'] + if pat.strip("*?[]") == pat: + # no special glob characters so we assume the user means + # 'contains this string' rather than 'is this string' + pat = "*%s*" % (pat,) + val = _value_for_dotted_key(condition['key'], ev) if val is None: return False diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 0f78fa667c..61e3bc8236 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -98,10 +98,7 @@ class PushRuleRestServlet(ClientV1RestServlet): if 'pattern' not in req_obj: raise InvalidRuleException("Content rule missing 'pattern'") pat = req_obj['pattern'] - if pat.strip("*?[]") == pat: - # no special glob characters so we assume the user means - # 'contains this string' rather than 'is this string' - pat = "*%s*" % (pat,) + conditions = [{ 'kind': 'event_match', 'key': 'content.body', From 017dfaef4c0d4550acd45d27cdcea1c20766684d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Jan 2015 15:52:05 +0000 Subject: [PATCH 163/180] Add doc string for __nonzero__ overrides for sync results, raise not implemented if the client attempts to do a gapless sync --- synapse/handlers/sync.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 9e07f30b24..dc69b3cfe7 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -49,6 +49,9 @@ class RoomSyncResult(collections.namedtuple("RoomSyncResult", [ __slots__ = [] def __nonzero__(self): + """Make the result appear empty if there are no updates. This is used + to tell if room needs to be part of the sync result. + """ return bool(self.events or self.state or self.ephemeral) @@ -61,6 +64,10 @@ class SyncResult(collections.namedtuple("SyncResult", [ __slots__ = [] def __nonzero__(self): + """Make the result appear empty if there are no updates. This is used + to tell if the notifier needs to wait for more events when polling for + events. + """ return bool( self.private_user_data or self.public_user_data or self.rooms ) @@ -108,7 +115,7 @@ class SyncHandler(BaseHandler): return self.incremental_sync_with_gap(sync_config, since_token) else: #TODO(mjark): Handle gapless sync - pass + raise NotImplementedError() @defer.inlineCallbacks def initial_sync(self, sync_config): From b4b892f4a3f9194289f57ed2d166eca26da41594 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 30 Jan 2015 15:54:29 +0000 Subject: [PATCH 164/180] Spit out server default rules too. --- synapse/push/baserules.py | 11 +++++------ synapse/rest/client/v1/push_rule.py | 16 ++++++++++++---- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index bd162baade..382de118e0 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -1,9 +1,5 @@ def make_base_rules(user_name): - """ - Nominally we reserve priority class 0 for these rules, although - in practice we just append them to the end so we don't actually need it. - """ - return [ + rules = [ { 'conditions': [ { @@ -46,4 +42,7 @@ def make_base_rules(user_name): } ] } - ] \ No newline at end of file + ] + for r in rules: + r['priority_class'] = 0 + return rules \ No newline at end of file diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 61e3bc8236..faa7919fbb 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -19,6 +19,7 @@ from synapse.api.errors import SynapseError, Codes, UnrecognizedRequestError, No StoreError from .base import ClientV1RestServlet, client_path_pattern from synapse.storage.push_rule import InconsistentRuleException, RuleNotFoundException +import synapse.push.baserules as baserules import json @@ -26,6 +27,7 @@ import json class PushRuleRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/pushrules/.*$") PRIORITY_CLASS_MAP = { + 'default': 0, 'underride': 1, 'sender': 2, 'room': 3, @@ -137,6 +139,9 @@ class PushRuleRestServlet(ClientV1RestServlet): user, _ = yield self.auth.get_user_by_req(request) + if spec['template'] == 'default': + raise SynapseError(403, "The default rules are immutable.") + content = _parse_json(request) try: @@ -218,6 +223,10 @@ class PushRuleRestServlet(ClientV1RestServlet): # to send which means doing unnecessary work sometimes but is # is probably not going to make a whole lot of difference rawrules = yield self.hs.get_datastore().get_push_rules_for_user_name(user.to_string()) + for r in rawrules: + r["conditions"] = json.loads(r["conditions"]) + r["actions"] = json.loads(r["actions"]) + rawrules.extend(baserules.make_base_rules(user.to_string())) rules = {'global': {}, 'device': {}} @@ -226,9 +235,6 @@ class PushRuleRestServlet(ClientV1RestServlet): for r in rawrules: rulearray = None - r["conditions"] = json.loads(r["conditions"]) - r["actions"] = json.loads(r["actions"]) - template_name = _priority_class_to_template_name(r['priority_class']) if r['priority_class'] > PushRuleRestServlet.PRIORITY_CLASS_MAP['override']: @@ -356,7 +362,9 @@ def _priority_class_to_template_name(pc): def _rule_to_template(rule): template_name = _priority_class_to_template_name(rule['priority_class']) - if template_name in ['override', 'underride']: + if template_name in ['default']: + return {k: rule[k] for k in ["conditions", "actions"]} + elif template_name in ['override', 'underride']: return {k: rule[k] for k in ["rule_id", "conditions", "actions"]} elif template_name in ["sender", "room"]: return {k: rule[k] for k in ["rule_id", "actions"]} From 7a1e881665ab72e29ff931f88ee2b65e8e690f74 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 15:56:32 +0000 Subject: [PATCH 165/180] Remove debug logging --- synapse/state.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/synapse/state.py b/synapse/state.py index 038e5eba11..8a056ee955 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -214,15 +214,6 @@ class StateHandler(object): if len(v.values()) > 1 } - logger.debug( - "resolve_state_groups Unconflicted state: %s", - unconflicted_state.values(), - ) - logger.debug( - "resolve_state_groups Conflicted state: %s", - conflicted_state.values(), - ) - if event_type: prev_states_events = conflicted_state.get( (event_type, state_key), [] From b724a809c46f782231e18b70a632ce1ea9c540da Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 15:57:53 +0000 Subject: [PATCH 166/180] Only auth_events with event if event in event.auth_events --- synapse/handlers/federation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 35cad4182a..40836e0c51 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -805,7 +805,9 @@ class FederationHandler(BaseHandler): yield self._handle_new_event( origin, ev, auth_events=auth ) - auth_events[(ev.type, ev.state_key)] = ev + + if ev.event_id in event_auth_events: + auth_events[(ev.type, ev.state_key)] = ev except AuthError: pass From 776ac820f9bb7f0e9c2fae9facbee05b0132079e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 15:58:28 +0000 Subject: [PATCH 167/180] Briefly doc structure of query_auth API. --- synapse/federation/federation_server.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 84ed0a0ba0..5fbd8b19de 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -232,6 +232,24 @@ class FederationServer(object): @defer.inlineCallbacks def on_query_auth_request(self, origin, content, event_id): + """ + Content is a dict with keys:: + auth_chain (list): A list of events that give the auth chain. + missing (list): A list of event_ids indicating what the other + side (`origin`) think we're missing. + rejects (dict): A mapping from event_id to a 2-tuple of reason + string and a proof (or None) of why the event was rejected. + The keys of this dict give the list of events the `origin` has + rejected. + + Args: + origin (str) + content (dict) + event_id (str) + + Returns: + Deferred: Results in `dict` with the same format as `content` + """ auth_chain = [ (yield self._check_sigs_and_hash(self.event_from_pdu_json(e))) for e in content["auth_chain"] From 88391bcdc328e9bfdae05b99f46ccdfcbe225382 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 16:09:30 +0000 Subject: [PATCH 168/180] Allow any greater version for webclient --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3249e87a96..abccca0f2c 100755 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ setup( description="Reference Synapse Home Server", install_requires=[ "syutil==0.0.2", - "matrix_angular_sdk==0.6.0", + "matrix_angular_sdk>=0.6.0", "Twisted==14.0.2", "service_identity>=1.0.0", "pyopenssl>=0.14", From 2cd29dbdd91a8f58f988eb3c417ad6a187fc7e87 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 16:51:58 +0000 Subject: [PATCH 169/180] Fix bug where accepting invite over federation didn't work. Add logging. --- synapse/handlers/federation.py | 59 ++++++++++++++++++++++++++++------ 1 file changed, 50 insertions(+), 9 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 40836e0c51..623c54d584 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -343,6 +343,10 @@ class FederationHandler(BaseHandler): for e in auth_chain: e.internal_metadata.outlier = True + + if e.event_id == event.event_id: + continue + try: auth_ids = [e_id for e_id, _ in e.auth_events] auth = { @@ -359,7 +363,9 @@ class FederationHandler(BaseHandler): ) for e in state: - # FIXME: Auth these. + if e.event_id == event.event_id: + continue + e.internal_metadata.outlier = True try: auth_ids = [e_id for e_id, _ in e.auth_events] @@ -376,11 +382,18 @@ class FederationHandler(BaseHandler): e.event_id, ) + auth_ids = [e_id for e_id, _ in event.auth_events] + auth_events = { + (e.type, e.state_key): e for e in auth_chain + if e.event_id in auth_ids + } + yield self._handle_new_event( target_host, new_event, state=state, current_state=state, + auth_events=auth_events, ) yield self.notifier.on_new_room_event( @@ -752,7 +765,17 @@ class FederationHandler(BaseHandler): origin, event.room_id, event.event_id ) + seen_remotes = yield self.store.have_events( + [e.event_id for e in remote_auth_chain] + ) + for e in remote_auth_chain: + if e.event_id in seen_remotes.keys(): + continue + + if e.event_id == event.event_id: + continue + try: auth_ids = [e_id for e_id, _ in e.auth_events] auth = { @@ -760,10 +783,17 @@ class FederationHandler(BaseHandler): if e.event_id in auth_ids } e.internal_metadata.outlier = True + + logger.debug( + "do_auth %s missing_auth: %s", + event.event_id, e.event_id + ) yield self._handle_new_event( origin, e, auth_events=auth ) - auth_events[(e.type, e.state_key)] = e + + if e.event_id in event_auth_events: + auth_events[(e.type, e.state_key)] = e except AuthError: pass @@ -788,20 +818,31 @@ class FederationHandler(BaseHandler): local_auth_chain, ) - # 3. Process any remote auth chain events we haven't seen. - for missing_id in result.get("missing", []): - try: - for e in result["auth_chain"]: - if e.event_id == missing_id: - ev = e - break + seen_remotes = yield self.store.have_events( + [e.event_id for e in result["auth_chain"]] + ) + # 3. Process any remote auth chain events we haven't seen. + for ev in result["auth_chain"]: + if ev.event_id in seen_remotes.keys(): + continue + + if ev.event_id == event.event_id: + continue + + try: auth_ids = [e_id for e_id, _ in ev.auth_events] auth = { (e.type, e.state_key): e for e in result["auth_chain"] if e.event_id in auth_ids } ev.internal_metadata.outlier = True + + logger.debug( + "do_auth %s different_auth: %s", + event.event_id, e.event_id + ) + yield self._handle_new_event( origin, ev, auth_events=auth ) From 0b1cc7cc0bd81f93ed353a6ac67e06f91b62ab90 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Jan 2015 16:24:40 +0000 Subject: [PATCH 170/180] Return empty list rather than None when there are no emphemeral events for a room --- synapse/handlers/sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index dc69b3cfe7..962686f4bb 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -370,7 +370,7 @@ class SyncHandler(BaseHandler): prev_batch=prev_batch_token, state=state_events_delta, limited=limited, - ephemeral=typing_by_room.get(room_id, None) + ephemeral=typing_by_room.get(room_id, []) ) logging.debug("Room sync: %r", room_sync) From e709d61964834244c9dae673d4e6fefa6075cc5e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Jan 2015 16:56:53 +0000 Subject: [PATCH 171/180] Update documentation to recommend virtual env --- README.rst | 45 +++++++++++++++------------------------------ 1 file changed, 15 insertions(+), 30 deletions(-) diff --git a/README.rst b/README.rst index 768da3df64..ddc5d4f78c 100644 --- a/README.rst +++ b/README.rst @@ -95,7 +95,7 @@ Installing prerequisites on Ubuntu or Debian:: $ sudo apt-get install build-essential python2.7-dev libffi-dev \ python-pip python-setuptools sqlite3 \ - libssl-dev + libssl-dev python-virtualenv libjpeg-dev Installing prerequisites on Mac OS X:: @@ -103,19 +103,12 @@ Installing prerequisites on Mac OS X:: To install the synapse homeserver run:: - $ pip install --user --process-dependency-links https://github.com/matrix-org/synapse/tarball/master + $ virutalenv ~/.synapse + $ source ~/.synapse/bin/activate + $ pip install --process-dependency-links https://github.com/matrix-org/synapse/tarball/master -This installs synapse, along with the libraries it uses, into -``$HOME/.local/lib/`` on Linux or ``$HOME/Library/Python/2.7/lib/`` on OSX. - -Your python may not give priority to locally installed libraries over system -libraries, in which case you must add your local packages to your python path:: - - $ # on Linux: - $ export PYTHONPATH=$HOME/.local/lib/python2.7/site-packages:$PYTHONPATH - - $ # on OSX: - $ export PYTHONPATH=$HOME/Library/Python/2.7/lib/python/site-packages:$PYTHONPATH +This installs synapse, along with the libraries it uses, into a virtual +environment under ``synapse``. For reliable VoIP calls to be routed via this homeserver, you MUST configure a TURN server. See docs/turn-howto.rst for details. @@ -182,23 +175,13 @@ Running Your Homeserver To actually run your new homeserver, pick a working directory for Synapse to run (e.g. ``~/.synapse``), and:: - $ mkdir ~/.synapse $ cd ~/.synapse - - $ # on Linux - $ ~/.local/bin/synctl start - - $ # on OSX - $ ~/Library/Python/2.7/bin/synctl start + $ ./bin/activate + $ synctl start Troubleshooting Running ----------------------- -If ``synctl`` fails with ``pkg_resources.DistributionNotFound`` errors you may -need a newer version of setuptools than that provided by your OS.:: - - $ sudo pip install setuptools --upgrade - If synapse fails with ``missing "sodium.h"`` crypto errors, you may need to manually upgrade PyNaCL, as synapse uses NaCl (http://nacl.cr.yp.to/) for encryption and digital signatures. @@ -225,13 +208,15 @@ directory of your choice:: $ cd synapse The homeserver has a number of external dependencies, that are easiest -to install by making setup.py do so, in --user mode:: +to install using pip and a virtualenv:: - $ python setup.py develop --user + $ virtualenv env + $ env/bin/activate + $ python synapse/dependencies | xargs -i pip install + $ pip install setuptools_trial mock -This will run a process of downloading and installing into your -user's .local/lib directory all of the required dependencies that are -missing. +This will run a process of downloading and installing all the needed +dependencies into a virtual env. Once this is done, you may wish to run the homeserver's unit tests, to check that everything is installed as it should be:: From 33cf48118f3a4fcbd0ad31e52c2ad5b09abb2c60 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Jan 2015 17:00:32 +0000 Subject: [PATCH 172/180] Tell people to "source" the activate script for virtualenv, Remove --user from pip install --- README.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.rst b/README.rst index ddc5d4f78c..285865f5f0 100644 --- a/README.rst +++ b/README.rst @@ -121,19 +121,19 @@ you get errors about ``error: no such option: --process-dependency-links`` you may need to manually upgrade it:: $ sudo pip install --upgrade pip - + If pip crashes mid-installation for reason (e.g. lost terminal), pip may refuse to run until you remove the temporary installation directory it created. To reset the installation:: $ rm -rf /tmp/pip_install_matrix - + pip seems to leak *lots* of memory during installation. For instance, a Linux host with 512MB of RAM may run out of memory whilst installing Twisted. If this happens, you will have to individually install the dependencies which are failing, e.g.:: - $ pip install --user twisted + $ pip install twisted On OSX, if you encounter clang: error: unknown argument: '-mno-fused-madd' you will need to export CFLAGS=-Qunused-arguments. @@ -148,7 +148,7 @@ Synapse can be installed on Cygwin. It requires the following Cygwin packages: - openssl (and openssl-devel, python-openssl) - python - python-setuptools - + The content repository requires additional packages and will be unable to process uploads without them: - libjpeg8 @@ -176,7 +176,7 @@ To actually run your new homeserver, pick a working directory for Synapse to run (e.g. ``~/.synapse``), and:: $ cd ~/.synapse - $ ./bin/activate + $ source ./bin/activate $ synctl start Troubleshooting Running @@ -211,7 +211,7 @@ The homeserver has a number of external dependencies, that are easiest to install using pip and a virtualenv:: $ virtualenv env - $ env/bin/activate + $ source env/bin/activate $ python synapse/dependencies | xargs -i pip install $ pip install setuptools_trial mock @@ -237,7 +237,7 @@ IMPORTANT: Before upgrading an existing homeserver to a new version, please refer to UPGRADE.rst for any additional instructions. Otherwise, simply re-install the new codebase over the current one - e.g. -by ``pip install --user --process-dependency-links +by ``pip install --process-dependency-links https://github.com/matrix-org/synapse/tarball/master`` if using pip, or by ``git pull`` if running off a git working copy. From 166c2cd4f3e3828668f73bb1f8248d850234fc5c Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 30 Jan 2015 17:11:29 +0000 Subject: [PATCH 173/180] add generate config instruction to the HS setup part --- README.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.rst b/README.rst index 285865f5f0..820b16d06c 100644 --- a/README.rst +++ b/README.rst @@ -110,6 +110,15 @@ To install the synapse homeserver run:: This installs synapse, along with the libraries it uses, into a virtual environment under ``synapse``. +To set up your homeserver, run (in your virtualenv, as before):: + + $ python -m synapse.app.homeserver \ + --server-name machine.my.domain.name \ + --config-path homeserver.config \ + --generate-config + +Substituting your host and domain name as appropriate. + For reliable VoIP calls to be routed via this homeserver, you MUST configure a TURN server. See docs/turn-howto.rst for details. From 9ccfdfcd7cc15f5c944670e6b6bcfe74090763f1 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 30 Jan 2015 17:15:39 +0000 Subject: [PATCH 174/180] Add twisted to setup requires so it gets processed before setuptools_trial --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index abccca0f2c..eb1d17c25f 100755 --- a/setup.py +++ b/setup.py @@ -50,6 +50,7 @@ setup( "https://github.com/matrix-org/matrix-angular-sdk/tarball/v0.6.0/#egg=matrix_angular_sdk-0.6.0", ], setup_requires=[ + "Twisted==14.0.2", # Here to override setuptools_trial's dependency on Twisted>=2.4.0 "setuptools_trial", "setuptools>=1.0.0", # Needs setuptools that supports git+ssh. # TODO: Do we need this now? we don't use git+ssh. From 68bd7dfbb72ee37495bf180e277ace85aec6d34b Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 30 Jan 2015 17:37:37 +0000 Subject: [PATCH 175/180] s/homeserver.config/homeserver.yaml/ because that's what synctl looks for. --- README.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index 820b16d06c..cac387292a 100644 --- a/README.rst +++ b/README.rst @@ -114,7 +114,7 @@ To set up your homeserver, run (in your virtualenv, as before):: $ python -m synapse.app.homeserver \ --server-name machine.my.domain.name \ - --config-path homeserver.config \ + --config-path homeserver.yaml \ --generate-config Substituting your host and domain name as appropriate. @@ -273,9 +273,9 @@ For the first form, simply pass the required hostname (of the machine) as the $ python -m synapse.app.homeserver \ --server-name machine.my.domain.name \ - --config-path homeserver.config \ + --config-path homeserver.yaml \ --generate-config - $ python -m synapse.app.homeserver --config-path homeserver.config + $ python -m synapse.app.homeserver --config-path homeserver.yaml Alternatively, you can run ``synctl start`` to guide you through the process. @@ -295,9 +295,9 @@ SRV record, as that is the name other machines will expect it to have:: $ python -m synapse.app.homeserver \ --server-name YOURDOMAIN \ --bind-port 8448 \ - --config-path homeserver.config \ + --config-path homeserver.yaml \ --generate-config - $ python -m synapse.app.homeserver --config-path homeserver.config + $ python -m synapse.app.homeserver --config-path homeserver.yaml You may additionally want to pass one or more "-v" options, in order to From 4c0da49d7ca58774adad870da98a7e168a5be18c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 22:53:13 +0000 Subject: [PATCH 176/180] Resign events when we return them via /query_auth/ --- synapse/handlers/federation.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 623c54d584..8bf5a4cc11 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -741,6 +741,15 @@ class FederationHandler(BaseHandler): local_auth_chain, remote_auth_chain ) + for event in ret["auth_chain"]: + event.signatures.update( + compute_event_signature( + event, + self.hs.hostname, + self.hs.config.signing_key[0] + ) + ) + logger.debug("on_query_auth reutrning: %s", ret) defer.returnValue(ret) From fe10b882b74479b6d139039956932b98629c8165 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 30 Jan 2015 23:05:49 +0000 Subject: [PATCH 177/180] Don't assume all member events have a display nme. --- synapse/push/__init__.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 6a302305fb..75867b3c3c 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -204,10 +204,11 @@ class Pusher(object): event_type='m.room.member', state_key=ev['user_id'] ) - if len(their_member_events_for_room) > 0: - dn = their_member_events_for_room[0].content['displayname'] - if dn is not None: - ctx['sender_display_name'] = dn + for mev in their_member_events_for_room: + if mev.content['membership'] == 'join' and 'displayname' in mev.content: + dn = mev.content['displayname'] + if dn is not None: + ctx['sender_display_name'] = dn defer.returnValue(ctx) From 0b354fcb8452540baf59b714f4d2186615dc5383 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 30 Jan 2015 23:10:35 +0000 Subject: [PATCH 178/180] Again, don't assume all member events have displayname. --- synapse/push/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 75867b3c3c..28e5dae81d 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -101,7 +101,7 @@ class Pusher(object): # This loop does two things: # 1) Find our current display name - if mev.state_key == self.user_name: + if mev.state_key == self.user_name and 'displayname' in mev.content: my_display_name = mev.content['displayname'] # and 2) Get the number of people in that room From e9dfc4cfae3a9abc85ae2882ebfbd0df689a9d2f Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sat, 31 Jan 2015 06:09:59 +0100 Subject: [PATCH 179/180] fix OSX stuff and typos --- README.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index cac387292a..282a53873f 100644 --- a/README.rst +++ b/README.rst @@ -100,15 +100,16 @@ Installing prerequisites on Ubuntu or Debian:: Installing prerequisites on Mac OS X:: $ xcode-select --install + $ sudo pip install virtualenv To install the synapse homeserver run:: - $ virutalenv ~/.synapse + $ virtualenv ~/.synapse $ source ~/.synapse/bin/activate $ pip install --process-dependency-links https://github.com/matrix-org/synapse/tarball/master This installs synapse, along with the libraries it uses, into a virtual -environment under ``synapse``. +environment under ``~/.synapse``. To set up your homeserver, run (in your virtualenv, as before):: From 365e007bee27e230f1280a2174cfc9abfb2deac2 Mon Sep 17 00:00:00 2001 From: David Baker Date: Sat, 31 Jan 2015 12:48:06 +0000 Subject: [PATCH 180/180] Ignore empty strings for display names & room names in notifications --- synapse/push/httppusher.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index d4c5f03b01..7c6953c989 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -92,9 +92,9 @@ class HttpPusher(Pusher): if len(ctx['aliases']): d['notification']['room_alias'] = ctx['aliases'][0] - if 'sender_display_name' in ctx: + if 'sender_display_name' in ctx and len(ctx['sender_display_name']) > 0: d['notification']['sender_display_name'] = ctx['sender_display_name'] - if 'name' in ctx: + if 'name' in ctx and len(ctx['name']) > 0: d['notification']['room_name'] = ctx['name'] defer.returnValue(d)