mirror of
https://github.com/element-hq/synapse
synced 2024-08-18 19:10:52 +00:00
Merge remote-tracking branch 'origin/develop' into store_event_actions
This commit is contained in:
commit
140a50f641
6 changed files with 210 additions and 91 deletions
20
jenkins.sh
20
jenkins.sh
|
@ -6,7 +6,6 @@ export PYTHONDONTWRITEBYTECODE=yep
|
|||
export TRIAL_FLAGS="--reporter=subunit"
|
||||
export TOXSUFFIX="| subunit-1to2 | subunit2junitxml --no-passthrough --output-to=results.xml"
|
||||
# Write coverage reports to a separate file for each process
|
||||
# Include branch coverage
|
||||
export COVERAGE_OPTS="-p"
|
||||
export DUMP_COVERAGE_COMMAND="coverage help"
|
||||
|
||||
|
@ -16,15 +15,13 @@ export DUMP_COVERAGE_COMMAND="coverage help"
|
|||
# UNSTABLE or FAILURE this build.
|
||||
export PEP8SUFFIX="--output-file=violations.flake8.log || echo flake8 finished with status code \$?"
|
||||
|
||||
rm .coverage.* || echo "No files to remove"
|
||||
rm .coverage* || echo "No coverage files to remove"
|
||||
|
||||
tox
|
||||
|
||||
: ${GIT_BRANCH:="origin/$(git rev-parse --abbrev-ref HEAD)"}
|
||||
|
||||
set +u
|
||||
. .tox/py27/bin/activate
|
||||
set -u
|
||||
TOX_BIN=$WORKSPACE/.tox/py27/bin
|
||||
|
||||
if [[ ! -e .sytest-base ]]; then
|
||||
git clone https://github.com/matrix-org/sytest.git .sytest-base --mirror
|
||||
|
@ -48,7 +45,8 @@ export PERL5LIB PERL_MB_OPT PERL_MM_OPT
|
|||
: ${PORT_BASE:=8000}
|
||||
|
||||
echo >&2 "Running sytest with SQLite3";
|
||||
./run-tests.pl --coverage -O tap --synapse-directory .. --all --port-base $PORT_BASE > results-sqlite3.tap
|
||||
./run-tests.pl --coverage -O tap --synapse-directory $WORKSPACE \
|
||||
--python $TOX_BIN/python --all --port-base $PORT_BASE > results-sqlite3.tap
|
||||
|
||||
RUN_POSTGRES=""
|
||||
|
||||
|
@ -66,8 +64,9 @@ done
|
|||
# Run if both postgresql databases exist
|
||||
if test $RUN_POSTGRES = ":$(($PORT_BASE + 1)):$(($PORT_BASE + 2))"; then
|
||||
echo >&2 "Running sytest with PostgreSQL";
|
||||
pip install psycopg2
|
||||
./run-tests.pl --coverage -O tap --synapse-directory .. --all --port-base $PORT_BASE > results-postgresql.tap
|
||||
$TOX_BIN/pip install psycopg2
|
||||
./run-tests.pl --coverage -O tap --synapse-directory $WORKSPACE \
|
||||
--python $TOX_BIN/python --all --port-base $PORT_BASE > results-postgresql.tap
|
||||
else
|
||||
echo >&2 "Skipping running sytest with PostgreSQL, $RUN_POSTGRES"
|
||||
fi
|
||||
|
@ -76,6 +75,7 @@ cd ..
|
|||
cp sytest/.coverage.* .
|
||||
|
||||
# Combine the coverage reports
|
||||
python -m coverage combine
|
||||
echo "Combining:" .coverage.*
|
||||
$TOX_BIN/python -m coverage combine
|
||||
# Output coverage to coverage.xml
|
||||
coverage xml -o coverage.xml
|
||||
$TOX_BIN/coverage xml -o coverage.xml
|
||||
|
|
|
@ -120,6 +120,22 @@ class AuthError(SynapseError):
|
|||
super(AuthError, self).__init__(*args, **kwargs)
|
||||
|
||||
|
||||
class GuestAccessError(AuthError):
|
||||
"""An error raised when a there is a problem with a guest user accessing
|
||||
a room"""
|
||||
|
||||
def __init__(self, rooms, *args, **kwargs):
|
||||
self.rooms = rooms
|
||||
super(GuestAccessError, self).__init__(*args, **kwargs)
|
||||
|
||||
def error_dict(self):
|
||||
return cs_error(
|
||||
self.msg,
|
||||
self.errcode,
|
||||
rooms=self.rooms,
|
||||
)
|
||||
|
||||
|
||||
class EventSizeError(SynapseError):
|
||||
"""An error raised when an event is too big."""
|
||||
|
||||
|
|
|
@ -62,10 +62,29 @@ class Filtering(object):
|
|||
self._check_definition(user_filter_json[key])
|
||||
|
||||
if "room" in user_filter_json:
|
||||
self._check_definition_room_lists(user_filter_json["room"])
|
||||
for key in room_level_definitions:
|
||||
if key in user_filter_json["room"]:
|
||||
self._check_definition(user_filter_json["room"][key])
|
||||
|
||||
def _check_definition_room_lists(self, definition):
|
||||
"""Check that "rooms" and "not_rooms" are lists of room ids if they
|
||||
are present
|
||||
|
||||
Args:
|
||||
definition(dict): The filter definition
|
||||
Raises:
|
||||
SynapseError: If there was a problem with this definition.
|
||||
"""
|
||||
# check rooms are valid room IDs
|
||||
room_id_keys = ["rooms", "not_rooms"]
|
||||
for key in room_id_keys:
|
||||
if key in definition:
|
||||
if type(definition[key]) != list:
|
||||
raise SynapseError(400, "Expected %s to be a list." % key)
|
||||
for room_id in definition[key]:
|
||||
RoomID.from_string(room_id)
|
||||
|
||||
def _check_definition(self, definition):
|
||||
"""Check if the provided definition is valid.
|
||||
|
||||
|
@ -85,14 +104,7 @@ class Filtering(object):
|
|||
400, "Expected JSON object, not %s" % (definition,)
|
||||
)
|
||||
|
||||
# check rooms are valid room IDs
|
||||
room_id_keys = ["rooms", "not_rooms"]
|
||||
for key in room_id_keys:
|
||||
if key in definition:
|
||||
if type(definition[key]) != list:
|
||||
raise SynapseError(400, "Expected %s to be a list." % key)
|
||||
for room_id in definition[key]:
|
||||
RoomID.from_string(room_id)
|
||||
self._check_definition_room_lists(definition)
|
||||
|
||||
# check senders are valid user IDs
|
||||
user_id_keys = ["senders", "not_senders"]
|
||||
|
@ -119,34 +131,27 @@ class FilterCollection(object):
|
|||
def __init__(self, filter_json):
|
||||
self.filter_json = filter_json
|
||||
|
||||
self.room_timeline_filter = Filter(
|
||||
self.filter_json.get("room", {}).get("timeline", {})
|
||||
)
|
||||
room_filter_json = self.filter_json.get("room", {})
|
||||
|
||||
self.room_state_filter = Filter(
|
||||
self.filter_json.get("room", {}).get("state", {})
|
||||
)
|
||||
self.room_filter = Filter({
|
||||
k: v for k, v in room_filter_json.items()
|
||||
if k in ("rooms", "not_rooms")
|
||||
})
|
||||
|
||||
self.room_ephemeral_filter = Filter(
|
||||
self.filter_json.get("room", {}).get("ephemeral", {})
|
||||
)
|
||||
|
||||
self.room_account_data = Filter(
|
||||
self.filter_json.get("room", {}).get("account_data", {})
|
||||
)
|
||||
|
||||
self.presence_filter = Filter(
|
||||
self.filter_json.get("presence", {})
|
||||
)
|
||||
|
||||
self.account_data = Filter(
|
||||
self.filter_json.get("account_data", {})
|
||||
)
|
||||
self.room_timeline_filter = Filter(room_filter_json.get("timeline", {}))
|
||||
self.room_state_filter = Filter(room_filter_json.get("state", {}))
|
||||
self.room_ephemeral_filter = Filter(room_filter_json.get("ephemeral", {}))
|
||||
self.room_account_data = Filter(room_filter_json.get("account_data", {}))
|
||||
self.presence_filter = Filter(self.filter_json.get("presence", {}))
|
||||
self.account_data = Filter(self.filter_json.get("account_data", {}))
|
||||
|
||||
self.include_leave = self.filter_json.get("room", {}).get(
|
||||
"include_leave", False
|
||||
)
|
||||
|
||||
def list_rooms(self):
|
||||
return self.room_filter.list_rooms()
|
||||
|
||||
def timeline_limit(self):
|
||||
return self.room_timeline_filter.limit()
|
||||
|
||||
|
@ -163,22 +168,31 @@ class FilterCollection(object):
|
|||
return self.account_data.filter(events)
|
||||
|
||||
def filter_room_state(self, events):
|
||||
return self.room_state_filter.filter(events)
|
||||
return self.room_state_filter.filter(self.room_filter.filter(events))
|
||||
|
||||
def filter_room_timeline(self, events):
|
||||
return self.room_timeline_filter.filter(events)
|
||||
return self.room_timeline_filter.filter(self.room_filter.filter(events))
|
||||
|
||||
def filter_room_ephemeral(self, events):
|
||||
return self.room_ephemeral_filter.filter(events)
|
||||
return self.room_ephemeral_filter.filter(self.room_filter.filter(events))
|
||||
|
||||
def filter_room_account_data(self, events):
|
||||
return self.room_account_data.filter(events)
|
||||
return self.room_account_data.filter(self.room_filter.filter(events))
|
||||
|
||||
|
||||
class Filter(object):
|
||||
def __init__(self, filter_json):
|
||||
self.filter_json = filter_json
|
||||
|
||||
def list_rooms(self):
|
||||
"""The list of room_id strings this filter restricts the output to
|
||||
or None if the this filter doesn't list the room ids.
|
||||
"""
|
||||
if "rooms" in self.filter_json:
|
||||
return list(set(self.filter_json["rooms"]))
|
||||
else:
|
||||
return None
|
||||
|
||||
def check(self, event):
|
||||
"""Checks whether the filter matches the given event.
|
||||
|
||||
|
|
|
@ -816,7 +816,8 @@ class RoomListHandler(BaseHandler):
|
|||
@defer.inlineCallbacks
|
||||
def get_public_room_list(self):
|
||||
chunk = yield self.store.get_rooms(is_public=True)
|
||||
results = yield defer.gatherResults(
|
||||
|
||||
room_members = yield defer.gatherResults(
|
||||
[
|
||||
self.store.get_users_in_room(room["room_id"])
|
||||
for room in chunk
|
||||
|
@ -824,12 +825,30 @@ class RoomListHandler(BaseHandler):
|
|||
consumeErrors=True,
|
||||
).addErrback(unwrapFirstError)
|
||||
|
||||
avatar_urls = yield defer.gatherResults(
|
||||
[
|
||||
self.get_room_avatar_url(room["room_id"])
|
||||
for room in chunk
|
||||
],
|
||||
consumeErrors=True,
|
||||
).addErrback(unwrapFirstError)
|
||||
|
||||
for i, room in enumerate(chunk):
|
||||
room["num_joined_members"] = len(results[i])
|
||||
room["num_joined_members"] = len(room_members[i])
|
||||
if avatar_urls[i]:
|
||||
room["avatar_url"] = avatar_urls[i]
|
||||
|
||||
# FIXME (erikj): START is no longer a valid value
|
||||
defer.returnValue({"start": "START", "end": "END", "chunk": chunk})
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_room_avatar_url(self, room_id):
|
||||
event = yield self.hs.get_state_handler().get_current_state(
|
||||
room_id, "m.room.avatar"
|
||||
)
|
||||
if event and "url" in event.content:
|
||||
defer.returnValue(event.content["url"])
|
||||
|
||||
|
||||
class RoomContextHandler(BaseHandler):
|
||||
@defer.inlineCallbacks
|
||||
|
|
|
@ -15,8 +15,8 @@
|
|||
|
||||
from ._base import BaseHandler
|
||||
|
||||
from synapse.streams.config import PaginationConfig
|
||||
from synapse.api.constants import Membership, EventTypes
|
||||
from synapse.api.errors import GuestAccessError
|
||||
from synapse.util import unwrapFirstError
|
||||
|
||||
from twisted.internet import defer
|
||||
|
@ -29,6 +29,7 @@ logger = logging.getLogger(__name__)
|
|||
|
||||
SyncConfig = collections.namedtuple("SyncConfig", [
|
||||
"user",
|
||||
"is_guest",
|
||||
"filter",
|
||||
])
|
||||
|
||||
|
@ -120,6 +121,8 @@ class SyncResult(collections.namedtuple("SyncResult", [
|
|||
self.presence or self.joined or self.invited
|
||||
)
|
||||
|
||||
GuestRoom = collections.namedtuple("GuestRoom", ("room_id", "membership"))
|
||||
|
||||
|
||||
class SyncHandler(BaseHandler):
|
||||
|
||||
|
@ -138,6 +141,18 @@ class SyncHandler(BaseHandler):
|
|||
A Deferred SyncResult.
|
||||
"""
|
||||
|
||||
if sync_config.is_guest:
|
||||
bad_rooms = []
|
||||
for room_id in sync_config.filter.list_rooms():
|
||||
world_readable = yield self._is_world_readable(room_id)
|
||||
if not world_readable:
|
||||
bad_rooms.append(room_id)
|
||||
|
||||
if bad_rooms:
|
||||
raise GuestAccessError(
|
||||
bad_rooms, 403, "Guest access not allowed"
|
||||
)
|
||||
|
||||
if timeout == 0 or since_token is None or full_state:
|
||||
# we are going to return immediately, so don't bother calling
|
||||
# notifier.wait_for_events.
|
||||
|
@ -154,6 +169,17 @@ class SyncHandler(BaseHandler):
|
|||
)
|
||||
defer.returnValue(result)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _is_world_readable(self, room_id):
|
||||
state = yield self.hs.get_state_handler().get_current_state(
|
||||
room_id,
|
||||
EventTypes.RoomHistoryVisibility
|
||||
)
|
||||
if state and "history_visibility" in state.content:
|
||||
defer.returnValue(state.content["history_visibility"] == "world_readable")
|
||||
else:
|
||||
defer.returnValue(False)
|
||||
|
||||
def current_sync_for_user(self, sync_config, since_token=None,
|
||||
full_state=False):
|
||||
"""Get the sync for client needed to match what the server has now.
|
||||
|
@ -189,37 +215,52 @@ class SyncHandler(BaseHandler):
|
|||
"""
|
||||
now_token = yield self.event_sources.get_current_token()
|
||||
|
||||
now_token, ephemeral_by_room = yield self.ephemeral_by_room(
|
||||
sync_config, now_token
|
||||
)
|
||||
if sync_config.is_guest:
|
||||
room_list = [
|
||||
GuestRoom(room_id, Membership.JOIN)
|
||||
for room_id in sync_config.filter.list_rooms()
|
||||
]
|
||||
|
||||
presence_stream = self.event_sources.sources["presence"]
|
||||
# TODO (mjark): This looks wrong, shouldn't we be getting the presence
|
||||
# UP to the present rather than after the present?
|
||||
pagination_config = PaginationConfig(from_token=now_token)
|
||||
presence, _ = yield presence_stream.get_pagination_rows(
|
||||
user=sync_config.user,
|
||||
pagination_config=pagination_config.get_source_config("presence"),
|
||||
key=None
|
||||
)
|
||||
account_data = {}
|
||||
account_data_by_room = {}
|
||||
tags_by_room = {}
|
||||
|
||||
membership_list = (Membership.INVITE, Membership.JOIN)
|
||||
if sync_config.filter.include_leave:
|
||||
membership_list += (Membership.LEAVE, Membership.BAN)
|
||||
else:
|
||||
membership_list = (Membership.INVITE, Membership.JOIN)
|
||||
if sync_config.filter.include_leave:
|
||||
membership_list += (Membership.LEAVE, Membership.BAN)
|
||||
|
||||
room_list = yield self.store.get_rooms_for_user_where_membership_is(
|
||||
user_id=sync_config.user.to_string(),
|
||||
membership_list=membership_list
|
||||
)
|
||||
room_list = yield self.store.get_rooms_for_user_where_membership_is(
|
||||
user_id=sync_config.user.to_string(),
|
||||
membership_list=membership_list
|
||||
)
|
||||
|
||||
account_data, account_data_by_room = (
|
||||
yield self.store.get_account_data_for_user(
|
||||
account_data, account_data_by_room = (
|
||||
yield self.store.get_account_data_for_user(
|
||||
sync_config.user.to_string()
|
||||
)
|
||||
)
|
||||
|
||||
tags_by_room = yield self.store.get_tags_for_user(
|
||||
sync_config.user.to_string()
|
||||
)
|
||||
|
||||
presence_stream = self.event_sources.sources["presence"]
|
||||
|
||||
joined_room_ids = [
|
||||
room.room_id for room in room_list
|
||||
if room.membership == Membership.JOIN
|
||||
]
|
||||
|
||||
presence, _ = yield presence_stream.get_new_events(
|
||||
from_key=0,
|
||||
user=sync_config.user,
|
||||
room_ids=joined_room_ids,
|
||||
is_guest=sync_config.is_guest,
|
||||
)
|
||||
|
||||
tags_by_room = yield self.store.get_tags_for_user(
|
||||
sync_config.user.to_string()
|
||||
now_token, ephemeral_by_room = yield self.ephemeral_by_room(
|
||||
sync_config, now_token, joined_room_ids
|
||||
)
|
||||
|
||||
joined = []
|
||||
|
@ -338,11 +379,13 @@ class SyncHandler(BaseHandler):
|
|||
return account_data_events
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def ephemeral_by_room(self, sync_config, now_token, since_token=None):
|
||||
def ephemeral_by_room(self, sync_config, now_token, room_ids,
|
||||
since_token=None):
|
||||
"""Get the ephemeral events for each room the user is in
|
||||
Args:
|
||||
sync_config (SyncConfig): The flags, filters and user for the sync.
|
||||
now_token (StreamToken): Where the server is currently up to.
|
||||
room_ids (list): List of room id strings to get data for.
|
||||
since_token (StreamToken): Where the server was when the client
|
||||
last synced.
|
||||
Returns:
|
||||
|
@ -353,9 +396,6 @@ class SyncHandler(BaseHandler):
|
|||
|
||||
typing_key = since_token.typing_key if since_token else "0"
|
||||
|
||||
rooms = yield self.store.get_rooms_for_user(sync_config.user.to_string())
|
||||
room_ids = [room.room_id for room in rooms]
|
||||
|
||||
typing_source = self.event_sources.sources["typing"]
|
||||
typing, typing_key = yield typing_source.get_new_events(
|
||||
user=sync_config.user,
|
||||
|
@ -433,8 +473,38 @@ class SyncHandler(BaseHandler):
|
|||
"""
|
||||
now_token = yield self.event_sources.get_current_token()
|
||||
|
||||
rooms = yield self.store.get_rooms_for_user(sync_config.user.to_string())
|
||||
room_ids = [room.room_id for room in rooms]
|
||||
if sync_config.is_guest:
|
||||
room_ids = sync_config.filter.list_rooms()
|
||||
|
||||
tags_by_room = {}
|
||||
account_data = {}
|
||||
account_data_by_room = {}
|
||||
|
||||
else:
|
||||
rooms = yield self.store.get_rooms_for_user(
|
||||
sync_config.user.to_string()
|
||||
)
|
||||
room_ids = [room.room_id for room in rooms]
|
||||
|
||||
now_token, ephemeral_by_room = yield self.ephemeral_by_room(
|
||||
sync_config, now_token, since_token
|
||||
)
|
||||
|
||||
tags_by_room = yield self.store.get_updated_tags(
|
||||
sync_config.user.to_string(),
|
||||
since_token.account_data_key,
|
||||
)
|
||||
|
||||
account_data, account_data_by_room = (
|
||||
yield self.store.get_updated_account_data_for_user(
|
||||
sync_config.user.to_string(),
|
||||
since_token.account_data_key,
|
||||
)
|
||||
)
|
||||
|
||||
now_token, ephemeral_by_room = yield self.ephemeral_by_room(
|
||||
sync_config, now_token, room_ids, since_token
|
||||
)
|
||||
|
||||
presence_source = self.event_sources.sources["presence"]
|
||||
presence, presence_key = yield presence_source.get_new_events(
|
||||
|
@ -442,8 +512,7 @@ class SyncHandler(BaseHandler):
|
|||
from_key=since_token.presence_key,
|
||||
limit=sync_config.filter.presence_limit(),
|
||||
room_ids=room_ids,
|
||||
# /sync doesn't support guest access, they can't get to this point in code
|
||||
is_guest=False,
|
||||
is_guest=sync_config.is_guest,
|
||||
)
|
||||
now_token = now_token.copy_and_replace("presence_key", presence_key)
|
||||
|
||||
|
@ -477,18 +546,8 @@ class SyncHandler(BaseHandler):
|
|||
from_key=since_token.room_key,
|
||||
to_key=now_token.room_key,
|
||||
limit=timeline_limit + 1,
|
||||
)
|
||||
|
||||
tags_by_room = yield self.store.get_updated_tags(
|
||||
sync_config.user.to_string(),
|
||||
since_token.account_data_key,
|
||||
)
|
||||
|
||||
account_data, account_data_by_room = (
|
||||
yield self.store.get_updated_account_data_for_user(
|
||||
sync_config.user.to_string(),
|
||||
since_token.account_data_key,
|
||||
)
|
||||
room_ids=room_ids if sync_config.is_guest else (),
|
||||
is_guest=sync_config.is_guest,
|
||||
)
|
||||
|
||||
joined = []
|
||||
|
@ -628,7 +687,10 @@ class SyncHandler(BaseHandler):
|
|||
end_key = "s" + room_key.split('-')[-1]
|
||||
loaded_recents = sync_config.filter.filter_room_timeline(events)
|
||||
loaded_recents = yield self._filter_events_for_client(
|
||||
sync_config.user.to_string(), loaded_recents,
|
||||
sync_config.user.to_string(),
|
||||
loaded_recents,
|
||||
is_guest=sync_config.is_guest,
|
||||
require_all_visible_for_guests=False
|
||||
)
|
||||
loaded_recents.extend(recents)
|
||||
recents = loaded_recents
|
||||
|
|
|
@ -85,7 +85,9 @@ class SyncRestServlet(RestServlet):
|
|||
|
||||
@defer.inlineCallbacks
|
||||
def on_GET(self, request):
|
||||
user, token_id, _ = yield self.auth.get_user_by_req(request)
|
||||
user, token_id, is_guest = yield self.auth.get_user_by_req(
|
||||
request, allow_guest=True
|
||||
)
|
||||
|
||||
timeout = parse_integer(request, "timeout", default=0)
|
||||
since = parse_string(request, "since")
|
||||
|
@ -118,8 +120,14 @@ class SyncRestServlet(RestServlet):
|
|||
except:
|
||||
filter = FilterCollection({})
|
||||
|
||||
if is_guest and filter.list_rooms() is None:
|
||||
raise SynapseError(
|
||||
400, "Guest users must provide a list of rooms in the filter"
|
||||
)
|
||||
|
||||
sync_config = SyncConfig(
|
||||
user=user,
|
||||
is_guest=is_guest,
|
||||
filter=filter,
|
||||
)
|
||||
|
||||
|
|
Loading…
Reference in a new issue