From 8256a8ece7e228bf69fcd352f1b4adfa2138719a Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 28 Aug 2015 15:31:49 +0100 Subject: [PATCH 01/18] Allow users to redact their own events --- synapse/api/auth.py | 33 +++++++++++++++++++++++++-------- synapse/handlers/_base.py | 19 +++++++++++++++++-- 2 files changed, 42 insertions(+), 10 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 65ee1452ce..f63d2daad8 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -20,7 +20,7 @@ from twisted.internet import defer from synapse.api.constants import EventTypes, Membership, JoinRules from synapse.api.errors import AuthError, Codes, SynapseError from synapse.util.logutils import log_function -from synapse.types import UserID +from synapse.types import UserID, EventID import logging @@ -91,7 +91,7 @@ class Auth(object): self._check_power_levels(event, auth_events) if event.type == EventTypes.Redaction: - self._check_redaction(event, auth_events) + self.check_redaction(event, auth_events) logger.debug("Allowing! %s", event) except AuthError as e: @@ -541,16 +541,33 @@ class Auth(object): return True - def _check_redaction(self, event, auth_events): + def check_redaction(self, event, auth_events): + """Check whether the event sender is allowed to redact the target event. + + Returns: + True if the the sender is allowed to redact the target event if the + target event was created by them. + False if the sender is allowed to redact the target event with no + further checks. + + Raises: + AuthError if the event sender is definitely not allowed to redact + the target event. + """ user_level = self._get_user_power_level(event.user_id, auth_events) redact_level = self._get_named_level(auth_events, "redact", 50) - if user_level < redact_level: - raise AuthError( - 403, - "You don't have permission to redact events" - ) + if user_level > redact_level: + return False + + if EventID.from_string(event.redacts).domain == self.hs.get_config().server_name: + return True + + raise AuthError( + 403, + "You don't have permission to redact events" + ) def _check_power_levels(self, event, auth_events): user_list = event.content.get("users", {}) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index e91f1129db..9d36e3b6d2 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -15,7 +15,7 @@ from twisted.internet import defer -from synapse.api.errors import LimitExceededError, SynapseError +from synapse.api.errors import LimitExceededError, SynapseError, AuthError from synapse.crypto.event_signing import add_hashes_and_signatures from synapse.api.constants import Membership, EventTypes from synapse.types import UserID, RoomAlias @@ -131,7 +131,7 @@ class BaseHandler(object): ) if event.type == EventTypes.CanonicalAlias: - # Check the alias is acually valid (at this time at least) + # Check the alias is actually valid (at this time at least) room_alias_str = event.content.get("alias", None) if room_alias_str: room_alias = RoomAlias.from_string(room_alias_str) @@ -146,6 +146,21 @@ class BaseHandler(object): ) ) + if event.type == EventTypes.Redaction: + if self.auth.check_redaction(event, auth_events=context.current_state): + original_event = yield self.store.get_event( + event.redacts, + check_redacted=False, + get_prev_content=False, + allow_rejected=False, + allow_none=False + ) + if event.user_id != original_event.user_id: + raise AuthError( + 403, + "You don't have permission to redact events" + ) + destinations = set(extra_destinations) for k, s in context.current_state.items(): try: From b854a375b0a859f8c4e16adf9f2fcec75b5a6816 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 1 Sep 2015 11:53:31 +0100 Subject: [PATCH 02/18] Check domain of events properly Federated servers still need to delegate authority to owning servers --- synapse/api/auth.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index f63d2daad8..0c0d678562 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -561,7 +561,9 @@ class Auth(object): if user_level > redact_level: return False - if EventID.from_string(event.redacts).domain == self.hs.get_config().server_name: + redacter_domain = EventID.from_string(event.event_id).domain + redactee_domain = EventID.from_string(event.redacts).domain + if redacter_domain == redactee_domain: return True raise AuthError( From 7ab401d4dc353c43217ef5656347d68fe7d2ffa4 Mon Sep 17 00:00:00 2001 From: pztrn Date: Tue, 1 Sep 2015 19:48:22 +0500 Subject: [PATCH 03/18] Ignore development virtualenv and generated logger configuration as well. Signed-off-by: Stanislav Nikitin --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 4c336b710d..f8c4000134 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,6 @@ build/ localhost-800*/ static/client/register/register_config.js .tox + +env/ +*.config From a9ad647fb27aaa5085a828157f30627edfbe76aa Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 2 Sep 2015 11:11:11 +0100 Subject: [PATCH 04/18] Make port script handle empty sent_transactions table --- scripts/synapse_port_db | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db index e7ed4c309b..c02dff5ba4 100755 --- a/scripts/synapse_port_db +++ b/scripts/synapse_port_db @@ -412,14 +412,17 @@ class Porter(object): self._convert_rows("sent_transactions", headers, rows) inserted_rows = len(rows) - max_inserted_rowid = max(r[0] for r in rows) + if inserted_rows: + max_inserted_rowid = max(r[0] for r in rows) - def insert(txn): - self.postgres_store.insert_many_txn( - txn, "sent_transactions", headers[1:], rows - ) + def insert(txn): + self.postgres_store.insert_many_txn( + txn, "sent_transactions", headers[1:], rows + ) - yield self.postgres_store.execute(insert) + yield self.postgres_store.execute(insert) + else: + max_inserted_rowid = 0 def get_start_id(txn): txn.execute( From aaf319820a8b59936a210c8998ed8feef58257cd Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 2 Sep 2015 15:29:03 +0100 Subject: [PATCH 05/18] Update README to include RAM requirements --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index d8d179135b..5b30ee3563 100644 --- a/README.rst +++ b/README.rst @@ -94,6 +94,7 @@ Synapse is the reference python/twisted Matrix homeserver implementation. System requirements: - POSIX-compliant system (tested on Linux & OS X) - Python 2.7 +- At least 512 MB RAM. Synapse is written in python but some of the libraries is uses are written in C. So before we can install synapse itself we need a working C compiler and the From e90f32646f2f4a51079a1e64405cadce616e54c3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 2 Sep 2015 17:16:14 +0100 Subject: [PATCH 06/18] Bump version and changelog --- CHANGES.rst | 6 ++++++ synapse/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 8b9916c960..a964ffca99 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,9 @@ +Changes in synapse v0.10.0-rc6 (2015-09-02) +=========================================== + +* Remove some of the old database upgrade scripts. +* Fix database port script to work with newly created sqlite databases. + Changes in synapse v0.10.0-rc5 (2015-08-27) =========================================== diff --git a/synapse/__init__.py b/synapse/__init__.py index 57b8304d35..f8cb48ff15 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.10.0-rc5" +__version__ = "0.10.0-rc6" From fd0a919af3fa9cd200b6af8c9f185545ffe331ef Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 2 Sep 2015 17:27:59 +0100 Subject: [PATCH 07/18] Lists use 'append' --- synapse/config/_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/config/_base.py b/synapse/config/_base.py index 1a6784a714..8a75c48733 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -182,7 +182,7 @@ class Config(object): ) % (entry_path, ) continue - files.add(config_path) + files.append(entry_path) config_files.extend(sorted(files)) else: From bdf2e5865a6c47b89dbcb11fb747cfc87c26fdbd Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 3 Sep 2015 09:51:42 +0300 Subject: [PATCH 08/18] update logger to match new ambiguous script name... --- scripts/synapse_port_db | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db index e7ed4c309b..c241fdec5a 100755 --- a/scripts/synapse_port_db +++ b/scripts/synapse_port_db @@ -29,7 +29,7 @@ import traceback import yaml -logger = logging.getLogger("port_from_sqlite_to_postgres") +logger = logging.getLogger("synapse_port_db") BOOLEAN_COLUMNS = { From 9ad38c9807a215925d002869844b479ffc47fa0e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 3 Sep 2015 09:49:54 +0100 Subject: [PATCH 09/18] Bump version and changelog --- CHANGES.rst | 5 +++++ synapse/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index a964ffca99..2ec10516fd 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,8 @@ +Changes in synapse v0.10.0 (2015-09-03) +======================================= + +No change from release candidate. + Changes in synapse v0.10.0-rc6 (2015-09-02) =========================================== diff --git a/synapse/__init__.py b/synapse/__init__.py index f8cb48ff15..d85bb3dce0 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.10.0-rc6" +__version__ = "0.10.0" From 1002bbd7322e48c9ecbb1ea21b3fb8a44e4a4360 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 3 Sep 2015 09:51:01 +0100 Subject: [PATCH 10/18] Change log level to info --- synapse/app/homeserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index ff7807c2e6..fefefffb8f 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -341,7 +341,7 @@ def get_version_string(): ) ).encode("ascii") except Exception as e: - logger.warn("Failed to check for git repository: %s", e) + logger.info("Failed to check for git repository: %s", e) return ("Synapse/%s" % (synapse.__version__,)).encode("ascii") From 227b77409ffc1286ac3c8e44a55d54b72654f5c5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 4 Sep 2015 08:56:23 +0100 Subject: [PATCH 11/18] DEPENDENCY_LINKS was turned to a list --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 16ccc0f1b8..60ab8c7893 100755 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ setup( "setuptools_trial", "mock" ], - dependency_links=dependencies["DEPENDENCY_LINKS"], + dependency_links=dependencies["DEPENDENCY_LINKS"].values(), include_package_data=True, zip_safe=False, long_description=long_description, From 3ead04ceefc0d5a3ab207cabc2b17170229921ae Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 7 Sep 2015 14:57:00 +0100 Subject: [PATCH 12/18] Add instructions for upgrading setuptools for when people encounter a message "mock requires setuptools>=17.1" --- README.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.rst b/README.rst index 5b30ee3563..6c8431aa86 100644 --- a/README.rst +++ b/README.rst @@ -121,6 +121,7 @@ To install the synapse homeserver run:: virtualenv -p python2.7 ~/.synapse source ~/.synapse/bin/activate + pip install --upgrade setuptools pip install --process-dependency-links https://github.com/matrix-org/synapse/tarball/master This installs synapse, along with the libraries it uses, into a virtual @@ -285,6 +286,11 @@ may need to manually upgrade it:: sudo pip install --upgrade pip +Installing may fail with ``mock requires setuptools>=17.1. Aborting installation``. +You can fix this by upgrading setuptools:: + + pip install --upgrade setuptools + If pip crashes mid-installation for reason (e.g. lost terminal), pip may refuse to run until you remove the temporary installation directory it created. To reset the installation:: From 9e4dacd5e7e560bae614ddf390a9f02c2c19a25d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 7 Sep 2015 16:45:48 +0100 Subject: [PATCH 13/18] The maxrss reported by getrusage is in kilobytes, not pages --- synapse/metrics/__init__.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py index d7bcad8a8a..3f7b7158aa 100644 --- a/synapse/metrics/__init__.py +++ b/synapse/metrics/__init__.py @@ -17,7 +17,7 @@ from __future__ import absolute_import import logging -from resource import getrusage, getpagesize, RUSAGE_SELF +from resource import getrusage, RUSAGE_SELF import functools import os import stat @@ -100,7 +100,6 @@ def render_all(): # process resource usage rusage = None -PAGE_SIZE = getpagesize() def update_resource_metrics(): @@ -113,8 +112,8 @@ resource_metrics = get_metrics_for("process.resource") resource_metrics.register_callback("utime", lambda: rusage.ru_utime * 1000) resource_metrics.register_callback("stime", lambda: rusage.ru_stime * 1000) -# pages -resource_metrics.register_callback("maxrss", lambda: rusage.ru_maxrss * PAGE_SIZE) +# kilobytes +resource_metrics.register_callback("maxrss", lambda: rusage.ru_maxrss * 1024) TYPES = { stat.S_IFSOCK: "SOCK", From 709ba99afda371bb19a34e4448dc771a6cdab320 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 7 Sep 2015 16:45:55 +0100 Subject: [PATCH 14/18] Check that /proc/self/fd exists before listing it --- synapse/metrics/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py index 3f7b7158aa..943d637459 100644 --- a/synapse/metrics/__init__.py +++ b/synapse/metrics/__init__.py @@ -130,6 +130,10 @@ def _process_fds(): counts = {(k,): 0 for k in TYPES.values()} counts[("other",)] = 0 + # Not every OS will have a /proc/self/fd directory + if not os.path.exists("/proc/self/fd"): + return counts + for fd in os.listdir("/proc/self/fd"): try: s = os.stat("/proc/self/fd/%s" % (fd)) From c0d1f37baf33aeeab22e635b5fd7905ab07e39e3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 9 Sep 2015 13:47:14 +0100 Subject: [PATCH 15/18] Don't require pdus in check_auth script --- scripts-dev/check_auth.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts-dev/check_auth.py b/scripts-dev/check_auth.py index b889ac7fa7..4fa8792a5f 100644 --- a/scripts-dev/check_auth.py +++ b/scripts-dev/check_auth.py @@ -56,10 +56,9 @@ if __name__ == '__main__': js = json.load(args.json) - auth = Auth(Mock()) check_auth( auth, [FrozenEvent(d) for d in js["auth_chain"]], - [FrozenEvent(d) for d in js["pdus"]], + [FrozenEvent(d) for d in js.get("pdus", [])], ) From dd0867f5ba3da5af191d82ccf1292842b21da84e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 9 Sep 2015 17:02:39 +0100 Subject: [PATCH 16/18] Various bug fixes to crypto.keyring --- synapse/crypto/keyring.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index a692cdbe55..e251ab6af3 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -162,7 +162,9 @@ class Keyring(object): def remove_deferreds(res, server_name, group_id): server_to_gids[server_name].discard(group_id) if not server_to_gids[server_name]: - server_to_deferred.pop(server_name).callback(None) + d = server_to_deferred.pop(server_name, None) + if d: + d.callback(None) return res for g_id, deferred in deferreds.items(): @@ -200,8 +202,15 @@ class Keyring(object): else: break - for server_name, deferred in server_to_deferred: - self.key_downloads[server_name] = ObservableDeferred(deferred) + for server_name, deferred in server_to_deferred.items(): + d = ObservableDeferred(deferred) + self.key_downloads[server_name] = d + + def rm(r, server_name): + self.key_downloads.pop(server_name, None) + return r + + d.addBoth(rm, server_name) def get_server_verify_keys(self, group_id_to_group, group_id_to_deferred): """Takes a dict of KeyGroups and tries to find at least one key for @@ -220,9 +229,8 @@ class Keyring(object): merged_results = {} missing_keys = { - group.server_name: key_id + group.server_name: set(group.key_ids) for group in group_id_to_group.values() - for key_id in group.key_ids } for fn in key_fetch_fns: @@ -279,16 +287,15 @@ class Keyring(object): def get_keys_from_store(self, server_name_and_key_ids): res = yield defer.gatherResults( [ - self.store.get_server_verify_keys(server_name, key_ids) + self.store.get_server_verify_keys( + server_name, key_ids + ).addCallback(lambda ks, server: (server, ks), server_name) for server_name, key_ids in server_name_and_key_ids ], consumeErrors=True, ).addErrback(unwrapFirstError) - defer.returnValue(dict(zip( - [server_name for server_name, _ in server_name_and_key_ids], - res - ))) + defer.returnValue(dict(res)) @defer.inlineCallbacks def get_keys_from_perspectives(self, server_name_and_key_ids): From 30768dcf4082d54101c0a6a9ac3d04462a0395ac Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 10 Sep 2015 10:33:48 +0100 Subject: [PATCH 17/18] Fix adding threepids to an existing account --- synapse/rest/client/v2_alpha/account.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index 522a312c9e..6281e2d029 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -96,6 +96,7 @@ class ThreepidRestServlet(RestServlet): self.hs = hs self.identity_handler = hs.get_handlers().identity_handler self.auth = hs.get_auth() + self.auth_handler = hs.get_handlers().auth_handler @defer.inlineCallbacks def on_GET(self, request): From dffc9c4ae0eea0616cc017c7f858f8a923202075 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 14 Sep 2015 14:41:37 +0100 Subject: [PATCH 18/18] Drop unused index --- .../storage/schema/delta/23/drop_state_index.sql | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 synapse/storage/schema/delta/23/drop_state_index.sql diff --git a/synapse/storage/schema/delta/23/drop_state_index.sql b/synapse/storage/schema/delta/23/drop_state_index.sql new file mode 100644 index 0000000000..07d0ea5cb2 --- /dev/null +++ b/synapse/storage/schema/delta/23/drop_state_index.sql @@ -0,0 +1,16 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +DROP INDEX IF EXISTS state_groups_state_tuple;