Merge remote-tracking branch 'origin/develop' into store_event_actions

This commit is contained in:
David Baker 2015-12-22 17:23:35 +00:00
commit 140a50f641
6 changed files with 210 additions and 91 deletions

View file

@ -6,7 +6,6 @@ export PYTHONDONTWRITEBYTECODE=yep
export TRIAL_FLAGS="--reporter=subunit" export TRIAL_FLAGS="--reporter=subunit"
export TOXSUFFIX="| subunit-1to2 | subunit2junitxml --no-passthrough --output-to=results.xml" export TOXSUFFIX="| subunit-1to2 | subunit2junitxml --no-passthrough --output-to=results.xml"
# Write coverage reports to a separate file for each process # Write coverage reports to a separate file for each process
# Include branch coverage
export COVERAGE_OPTS="-p" export COVERAGE_OPTS="-p"
export DUMP_COVERAGE_COMMAND="coverage help" export DUMP_COVERAGE_COMMAND="coverage help"
@ -16,15 +15,13 @@ export DUMP_COVERAGE_COMMAND="coverage help"
# UNSTABLE or FAILURE this build. # UNSTABLE or FAILURE this build.
export PEP8SUFFIX="--output-file=violations.flake8.log || echo flake8 finished with status code \$?" export PEP8SUFFIX="--output-file=violations.flake8.log || echo flake8 finished with status code \$?"
rm .coverage.* || echo "No files to remove" rm .coverage* || echo "No coverage files to remove"
tox tox
: ${GIT_BRANCH:="origin/$(git rev-parse --abbrev-ref HEAD)"} : ${GIT_BRANCH:="origin/$(git rev-parse --abbrev-ref HEAD)"}
set +u TOX_BIN=$WORKSPACE/.tox/py27/bin
. .tox/py27/bin/activate
set -u
if [[ ! -e .sytest-base ]]; then if [[ ! -e .sytest-base ]]; then
git clone https://github.com/matrix-org/sytest.git .sytest-base --mirror git clone https://github.com/matrix-org/sytest.git .sytest-base --mirror
@ -48,7 +45,8 @@ export PERL5LIB PERL_MB_OPT PERL_MM_OPT
: ${PORT_BASE:=8000} : ${PORT_BASE:=8000}
echo >&2 "Running sytest with SQLite3"; echo >&2 "Running sytest with SQLite3";
./run-tests.pl --coverage -O tap --synapse-directory .. --all --port-base $PORT_BASE > results-sqlite3.tap ./run-tests.pl --coverage -O tap --synapse-directory $WORKSPACE \
--python $TOX_BIN/python --all --port-base $PORT_BASE > results-sqlite3.tap
RUN_POSTGRES="" RUN_POSTGRES=""
@ -66,8 +64,9 @@ done
# Run if both postgresql databases exist # Run if both postgresql databases exist
if test $RUN_POSTGRES = ":$(($PORT_BASE + 1)):$(($PORT_BASE + 2))"; then if test $RUN_POSTGRES = ":$(($PORT_BASE + 1)):$(($PORT_BASE + 2))"; then
echo >&2 "Running sytest with PostgreSQL"; echo >&2 "Running sytest with PostgreSQL";
pip install psycopg2 $TOX_BIN/pip install psycopg2
./run-tests.pl --coverage -O tap --synapse-directory .. --all --port-base $PORT_BASE > results-postgresql.tap ./run-tests.pl --coverage -O tap --synapse-directory $WORKSPACE \
--python $TOX_BIN/python --all --port-base $PORT_BASE > results-postgresql.tap
else else
echo >&2 "Skipping running sytest with PostgreSQL, $RUN_POSTGRES" echo >&2 "Skipping running sytest with PostgreSQL, $RUN_POSTGRES"
fi fi
@ -76,6 +75,7 @@ cd ..
cp sytest/.coverage.* . cp sytest/.coverage.* .
# Combine the coverage reports # Combine the coverage reports
python -m coverage combine echo "Combining:" .coverage.*
$TOX_BIN/python -m coverage combine
# Output coverage to coverage.xml # Output coverage to coverage.xml
coverage xml -o coverage.xml $TOX_BIN/coverage xml -o coverage.xml

View file

@ -120,6 +120,22 @@ class AuthError(SynapseError):
super(AuthError, self).__init__(*args, **kwargs) super(AuthError, self).__init__(*args, **kwargs)
class GuestAccessError(AuthError):
"""An error raised when a there is a problem with a guest user accessing
a room"""
def __init__(self, rooms, *args, **kwargs):
self.rooms = rooms
super(GuestAccessError, self).__init__(*args, **kwargs)
def error_dict(self):
return cs_error(
self.msg,
self.errcode,
rooms=self.rooms,
)
class EventSizeError(SynapseError): class EventSizeError(SynapseError):
"""An error raised when an event is too big.""" """An error raised when an event is too big."""

View file

@ -62,10 +62,29 @@ class Filtering(object):
self._check_definition(user_filter_json[key]) self._check_definition(user_filter_json[key])
if "room" in user_filter_json: if "room" in user_filter_json:
self._check_definition_room_lists(user_filter_json["room"])
for key in room_level_definitions: for key in room_level_definitions:
if key in user_filter_json["room"]: if key in user_filter_json["room"]:
self._check_definition(user_filter_json["room"][key]) self._check_definition(user_filter_json["room"][key])
def _check_definition_room_lists(self, definition):
"""Check that "rooms" and "not_rooms" are lists of room ids if they
are present
Args:
definition(dict): The filter definition
Raises:
SynapseError: If there was a problem with this definition.
"""
# check rooms are valid room IDs
room_id_keys = ["rooms", "not_rooms"]
for key in room_id_keys:
if key in definition:
if type(definition[key]) != list:
raise SynapseError(400, "Expected %s to be a list." % key)
for room_id in definition[key]:
RoomID.from_string(room_id)
def _check_definition(self, definition): def _check_definition(self, definition):
"""Check if the provided definition is valid. """Check if the provided definition is valid.
@ -85,14 +104,7 @@ class Filtering(object):
400, "Expected JSON object, not %s" % (definition,) 400, "Expected JSON object, not %s" % (definition,)
) )
# check rooms are valid room IDs self._check_definition_room_lists(definition)
room_id_keys = ["rooms", "not_rooms"]
for key in room_id_keys:
if key in definition:
if type(definition[key]) != list:
raise SynapseError(400, "Expected %s to be a list." % key)
for room_id in definition[key]:
RoomID.from_string(room_id)
# check senders are valid user IDs # check senders are valid user IDs
user_id_keys = ["senders", "not_senders"] user_id_keys = ["senders", "not_senders"]
@ -119,34 +131,27 @@ class FilterCollection(object):
def __init__(self, filter_json): def __init__(self, filter_json):
self.filter_json = filter_json self.filter_json = filter_json
self.room_timeline_filter = Filter( room_filter_json = self.filter_json.get("room", {})
self.filter_json.get("room", {}).get("timeline", {})
)
self.room_state_filter = Filter( self.room_filter = Filter({
self.filter_json.get("room", {}).get("state", {}) k: v for k, v in room_filter_json.items()
) if k in ("rooms", "not_rooms")
})
self.room_ephemeral_filter = Filter( self.room_timeline_filter = Filter(room_filter_json.get("timeline", {}))
self.filter_json.get("room", {}).get("ephemeral", {}) self.room_state_filter = Filter(room_filter_json.get("state", {}))
) self.room_ephemeral_filter = Filter(room_filter_json.get("ephemeral", {}))
self.room_account_data = Filter(room_filter_json.get("account_data", {}))
self.room_account_data = Filter( self.presence_filter = Filter(self.filter_json.get("presence", {}))
self.filter_json.get("room", {}).get("account_data", {}) self.account_data = Filter(self.filter_json.get("account_data", {}))
)
self.presence_filter = Filter(
self.filter_json.get("presence", {})
)
self.account_data = Filter(
self.filter_json.get("account_data", {})
)
self.include_leave = self.filter_json.get("room", {}).get( self.include_leave = self.filter_json.get("room", {}).get(
"include_leave", False "include_leave", False
) )
def list_rooms(self):
return self.room_filter.list_rooms()
def timeline_limit(self): def timeline_limit(self):
return self.room_timeline_filter.limit() return self.room_timeline_filter.limit()
@ -163,22 +168,31 @@ class FilterCollection(object):
return self.account_data.filter(events) return self.account_data.filter(events)
def filter_room_state(self, events): def filter_room_state(self, events):
return self.room_state_filter.filter(events) return self.room_state_filter.filter(self.room_filter.filter(events))
def filter_room_timeline(self, events): def filter_room_timeline(self, events):
return self.room_timeline_filter.filter(events) return self.room_timeline_filter.filter(self.room_filter.filter(events))
def filter_room_ephemeral(self, events): def filter_room_ephemeral(self, events):
return self.room_ephemeral_filter.filter(events) return self.room_ephemeral_filter.filter(self.room_filter.filter(events))
def filter_room_account_data(self, events): def filter_room_account_data(self, events):
return self.room_account_data.filter(events) return self.room_account_data.filter(self.room_filter.filter(events))
class Filter(object): class Filter(object):
def __init__(self, filter_json): def __init__(self, filter_json):
self.filter_json = filter_json self.filter_json = filter_json
def list_rooms(self):
"""The list of room_id strings this filter restricts the output to
or None if the this filter doesn't list the room ids.
"""
if "rooms" in self.filter_json:
return list(set(self.filter_json["rooms"]))
else:
return None
def check(self, event): def check(self, event):
"""Checks whether the filter matches the given event. """Checks whether the filter matches the given event.

View file

@ -816,7 +816,8 @@ class RoomListHandler(BaseHandler):
@defer.inlineCallbacks @defer.inlineCallbacks
def get_public_room_list(self): def get_public_room_list(self):
chunk = yield self.store.get_rooms(is_public=True) chunk = yield self.store.get_rooms(is_public=True)
results = yield defer.gatherResults(
room_members = yield defer.gatherResults(
[ [
self.store.get_users_in_room(room["room_id"]) self.store.get_users_in_room(room["room_id"])
for room in chunk for room in chunk
@ -824,12 +825,30 @@ class RoomListHandler(BaseHandler):
consumeErrors=True, consumeErrors=True,
).addErrback(unwrapFirstError) ).addErrback(unwrapFirstError)
avatar_urls = yield defer.gatherResults(
[
self.get_room_avatar_url(room["room_id"])
for room in chunk
],
consumeErrors=True,
).addErrback(unwrapFirstError)
for i, room in enumerate(chunk): for i, room in enumerate(chunk):
room["num_joined_members"] = len(results[i]) room["num_joined_members"] = len(room_members[i])
if avatar_urls[i]:
room["avatar_url"] = avatar_urls[i]
# FIXME (erikj): START is no longer a valid value # FIXME (erikj): START is no longer a valid value
defer.returnValue({"start": "START", "end": "END", "chunk": chunk}) defer.returnValue({"start": "START", "end": "END", "chunk": chunk})
@defer.inlineCallbacks
def get_room_avatar_url(self, room_id):
event = yield self.hs.get_state_handler().get_current_state(
room_id, "m.room.avatar"
)
if event and "url" in event.content:
defer.returnValue(event.content["url"])
class RoomContextHandler(BaseHandler): class RoomContextHandler(BaseHandler):
@defer.inlineCallbacks @defer.inlineCallbacks

View file

@ -15,8 +15,8 @@
from ._base import BaseHandler from ._base import BaseHandler
from synapse.streams.config import PaginationConfig
from synapse.api.constants import Membership, EventTypes from synapse.api.constants import Membership, EventTypes
from synapse.api.errors import GuestAccessError
from synapse.util import unwrapFirstError from synapse.util import unwrapFirstError
from twisted.internet import defer from twisted.internet import defer
@ -29,6 +29,7 @@ logger = logging.getLogger(__name__)
SyncConfig = collections.namedtuple("SyncConfig", [ SyncConfig = collections.namedtuple("SyncConfig", [
"user", "user",
"is_guest",
"filter", "filter",
]) ])
@ -120,6 +121,8 @@ class SyncResult(collections.namedtuple("SyncResult", [
self.presence or self.joined or self.invited self.presence or self.joined or self.invited
) )
GuestRoom = collections.namedtuple("GuestRoom", ("room_id", "membership"))
class SyncHandler(BaseHandler): class SyncHandler(BaseHandler):
@ -138,6 +141,18 @@ class SyncHandler(BaseHandler):
A Deferred SyncResult. A Deferred SyncResult.
""" """
if sync_config.is_guest:
bad_rooms = []
for room_id in sync_config.filter.list_rooms():
world_readable = yield self._is_world_readable(room_id)
if not world_readable:
bad_rooms.append(room_id)
if bad_rooms:
raise GuestAccessError(
bad_rooms, 403, "Guest access not allowed"
)
if timeout == 0 or since_token is None or full_state: if timeout == 0 or since_token is None or full_state:
# we are going to return immediately, so don't bother calling # we are going to return immediately, so don't bother calling
# notifier.wait_for_events. # notifier.wait_for_events.
@ -154,6 +169,17 @@ class SyncHandler(BaseHandler):
) )
defer.returnValue(result) defer.returnValue(result)
@defer.inlineCallbacks
def _is_world_readable(self, room_id):
state = yield self.hs.get_state_handler().get_current_state(
room_id,
EventTypes.RoomHistoryVisibility
)
if state and "history_visibility" in state.content:
defer.returnValue(state.content["history_visibility"] == "world_readable")
else:
defer.returnValue(False)
def current_sync_for_user(self, sync_config, since_token=None, def current_sync_for_user(self, sync_config, since_token=None,
full_state=False): full_state=False):
"""Get the sync for client needed to match what the server has now. """Get the sync for client needed to match what the server has now.
@ -189,20 +215,17 @@ class SyncHandler(BaseHandler):
""" """
now_token = yield self.event_sources.get_current_token() now_token = yield self.event_sources.get_current_token()
now_token, ephemeral_by_room = yield self.ephemeral_by_room( if sync_config.is_guest:
sync_config, now_token room_list = [
) GuestRoom(room_id, Membership.JOIN)
for room_id in sync_config.filter.list_rooms()
]
presence_stream = self.event_sources.sources["presence"] account_data = {}
# TODO (mjark): This looks wrong, shouldn't we be getting the presence account_data_by_room = {}
# UP to the present rather than after the present? tags_by_room = {}
pagination_config = PaginationConfig(from_token=now_token)
presence, _ = yield presence_stream.get_pagination_rows(
user=sync_config.user,
pagination_config=pagination_config.get_source_config("presence"),
key=None
)
else:
membership_list = (Membership.INVITE, Membership.JOIN) membership_list = (Membership.INVITE, Membership.JOIN)
if sync_config.filter.include_leave: if sync_config.filter.include_leave:
membership_list += (Membership.LEAVE, Membership.BAN) membership_list += (Membership.LEAVE, Membership.BAN)
@ -222,6 +245,24 @@ class SyncHandler(BaseHandler):
sync_config.user.to_string() sync_config.user.to_string()
) )
presence_stream = self.event_sources.sources["presence"]
joined_room_ids = [
room.room_id for room in room_list
if room.membership == Membership.JOIN
]
presence, _ = yield presence_stream.get_new_events(
from_key=0,
user=sync_config.user,
room_ids=joined_room_ids,
is_guest=sync_config.is_guest,
)
now_token, ephemeral_by_room = yield self.ephemeral_by_room(
sync_config, now_token, joined_room_ids
)
joined = [] joined = []
invited = [] invited = []
archived = [] archived = []
@ -338,11 +379,13 @@ class SyncHandler(BaseHandler):
return account_data_events return account_data_events
@defer.inlineCallbacks @defer.inlineCallbacks
def ephemeral_by_room(self, sync_config, now_token, since_token=None): def ephemeral_by_room(self, sync_config, now_token, room_ids,
since_token=None):
"""Get the ephemeral events for each room the user is in """Get the ephemeral events for each room the user is in
Args: Args:
sync_config (SyncConfig): The flags, filters and user for the sync. sync_config (SyncConfig): The flags, filters and user for the sync.
now_token (StreamToken): Where the server is currently up to. now_token (StreamToken): Where the server is currently up to.
room_ids (list): List of room id strings to get data for.
since_token (StreamToken): Where the server was when the client since_token (StreamToken): Where the server was when the client
last synced. last synced.
Returns: Returns:
@ -353,9 +396,6 @@ class SyncHandler(BaseHandler):
typing_key = since_token.typing_key if since_token else "0" typing_key = since_token.typing_key if since_token else "0"
rooms = yield self.store.get_rooms_for_user(sync_config.user.to_string())
room_ids = [room.room_id for room in rooms]
typing_source = self.event_sources.sources["typing"] typing_source = self.event_sources.sources["typing"]
typing, typing_key = yield typing_source.get_new_events( typing, typing_key = yield typing_source.get_new_events(
user=sync_config.user, user=sync_config.user,
@ -433,17 +473,46 @@ class SyncHandler(BaseHandler):
""" """
now_token = yield self.event_sources.get_current_token() now_token = yield self.event_sources.get_current_token()
rooms = yield self.store.get_rooms_for_user(sync_config.user.to_string()) if sync_config.is_guest:
room_ids = sync_config.filter.list_rooms()
tags_by_room = {}
account_data = {}
account_data_by_room = {}
else:
rooms = yield self.store.get_rooms_for_user(
sync_config.user.to_string()
)
room_ids = [room.room_id for room in rooms] room_ids = [room.room_id for room in rooms]
now_token, ephemeral_by_room = yield self.ephemeral_by_room(
sync_config, now_token, since_token
)
tags_by_room = yield self.store.get_updated_tags(
sync_config.user.to_string(),
since_token.account_data_key,
)
account_data, account_data_by_room = (
yield self.store.get_updated_account_data_for_user(
sync_config.user.to_string(),
since_token.account_data_key,
)
)
now_token, ephemeral_by_room = yield self.ephemeral_by_room(
sync_config, now_token, room_ids, since_token
)
presence_source = self.event_sources.sources["presence"] presence_source = self.event_sources.sources["presence"]
presence, presence_key = yield presence_source.get_new_events( presence, presence_key = yield presence_source.get_new_events(
user=sync_config.user, user=sync_config.user,
from_key=since_token.presence_key, from_key=since_token.presence_key,
limit=sync_config.filter.presence_limit(), limit=sync_config.filter.presence_limit(),
room_ids=room_ids, room_ids=room_ids,
# /sync doesn't support guest access, they can't get to this point in code is_guest=sync_config.is_guest,
is_guest=False,
) )
now_token = now_token.copy_and_replace("presence_key", presence_key) now_token = now_token.copy_and_replace("presence_key", presence_key)
@ -477,18 +546,8 @@ class SyncHandler(BaseHandler):
from_key=since_token.room_key, from_key=since_token.room_key,
to_key=now_token.room_key, to_key=now_token.room_key,
limit=timeline_limit + 1, limit=timeline_limit + 1,
) room_ids=room_ids if sync_config.is_guest else (),
is_guest=sync_config.is_guest,
tags_by_room = yield self.store.get_updated_tags(
sync_config.user.to_string(),
since_token.account_data_key,
)
account_data, account_data_by_room = (
yield self.store.get_updated_account_data_for_user(
sync_config.user.to_string(),
since_token.account_data_key,
)
) )
joined = [] joined = []
@ -628,7 +687,10 @@ class SyncHandler(BaseHandler):
end_key = "s" + room_key.split('-')[-1] end_key = "s" + room_key.split('-')[-1]
loaded_recents = sync_config.filter.filter_room_timeline(events) loaded_recents = sync_config.filter.filter_room_timeline(events)
loaded_recents = yield self._filter_events_for_client( loaded_recents = yield self._filter_events_for_client(
sync_config.user.to_string(), loaded_recents, sync_config.user.to_string(),
loaded_recents,
is_guest=sync_config.is_guest,
require_all_visible_for_guests=False
) )
loaded_recents.extend(recents) loaded_recents.extend(recents)
recents = loaded_recents recents = loaded_recents

View file

@ -85,7 +85,9 @@ class SyncRestServlet(RestServlet):
@defer.inlineCallbacks @defer.inlineCallbacks
def on_GET(self, request): def on_GET(self, request):
user, token_id, _ = yield self.auth.get_user_by_req(request) user, token_id, is_guest = yield self.auth.get_user_by_req(
request, allow_guest=True
)
timeout = parse_integer(request, "timeout", default=0) timeout = parse_integer(request, "timeout", default=0)
since = parse_string(request, "since") since = parse_string(request, "since")
@ -118,8 +120,14 @@ class SyncRestServlet(RestServlet):
except: except:
filter = FilterCollection({}) filter = FilterCollection({})
if is_guest and filter.list_rooms() is None:
raise SynapseError(
400, "Guest users must provide a list of rooms in the filter"
)
sync_config = SyncConfig( sync_config = SyncConfig(
user=user, user=user,
is_guest=is_guest,
filter=filter, filter=filter,
) )