From c12ee0d5ba5da8da8bdc0d2318d8a8bdfc7228aa Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 13 Jun 2024 13:56:58 -0500 Subject: [PATCH] Add `is_dm` filtering to Sliding Sync `/sync` (#17277) Based on [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575): Sliding Sync --- changelog.d/17277.feature | 1 + synapse/handlers/sliding_sync.py | 118 +++++++++++++++++++++-- synapse/types/rest/client/__init__.py | 47 ++++++++++ tests/handlers/test_sliding_sync.py | 130 +++++++++++++++++++++++++- tests/rest/client/test_sync.py | 127 +++++++++++++++++++++++++ 5 files changed, 416 insertions(+), 7 deletions(-) create mode 100644 changelog.d/17277.feature diff --git a/changelog.d/17277.feature b/changelog.d/17277.feature new file mode 100644 index 0000000000..5c16342c11 --- /dev/null +++ b/changelog.d/17277.feature @@ -0,0 +1 @@ +Add `is_dm` filtering to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index de4f33abb8..78fb66d6e2 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -22,7 +22,7 @@ from typing import TYPE_CHECKING, AbstractSet, Dict, List, Optional from immutabledict import immutabledict -from synapse.api.constants import Membership +from synapse.api.constants import AccountDataTypes, Membership from synapse.events import EventBase from synapse.types import Requester, RoomStreamToken, StreamToken, UserID from synapse.types.handlers import OperationType, SlidingSyncConfig, SlidingSyncResult @@ -69,9 +69,19 @@ class SlidingSyncHandler: from_token: Optional[StreamToken] = None, timeout_ms: int = 0, ) -> SlidingSyncResult: - """Get the sync for a client if we have new data for it now. Otherwise + """ + Get the sync for a client if we have new data for it now. Otherwise wait for new data to arrive on the server. If the timeout expires, then return an empty sync result. + + Args: + requester: The user making the request + sync_config: Sync configuration + from_token: The point in the stream to sync from. Token of the end of the + previous batch. May be `None` if this is the initial sync request. + timeout_ms: The time in milliseconds to wait for new data to arrive. If 0, + we will immediately but there might not be any new data so we just return an + empty response. """ # If the user is not part of the mau group, then check that limits have # not been exceeded (if not part of the group by this point, almost certain @@ -143,6 +153,14 @@ class SlidingSyncHandler: """ Generates the response body of a Sliding Sync result, represented as a `SlidingSyncResult`. + + We fetch data according to the token range (> `from_token` and <= `to_token`). + + Args: + sync_config: Sync configuration + to_token: The point in the stream to sync up to. + from_token: The point in the stream to sync from. Token of the end of the + previous batch. May be `None` if this is the initial sync request. """ user_id = sync_config.user.to_string() app_service = self.store.get_app_service_by_user_id(user_id) @@ -163,11 +181,12 @@ class SlidingSyncHandler: lists: Dict[str, SlidingSyncResult.SlidingWindowList] = {} if sync_config.lists: for list_key, list_config in sync_config.lists.items(): - # TODO: Apply filters - # - # TODO: Exclude partially stated rooms unless the `required_state` has - # `["m.room.member", "$LAZY"]` + # Apply filters filtered_room_ids = room_id_set + if list_config.filters is not None: + filtered_room_ids = await self.filter_rooms( + sync_config.user, room_id_set, list_config.filters, to_token + ) # TODO: Apply sorts sorted_room_ids = sorted(filtered_room_ids) @@ -217,6 +236,12 @@ class SlidingSyncHandler: `forgotten` flag to the `room_memberships` table in Synapse. There isn't a way to tell when a room was forgotten at the moment so we can't factor it into the from/to range. + + + Args: + user: User to fetch rooms for + to_token: The token to fetch rooms up to. + from_token: The point in the stream to sync from. """ user_id = user.to_string() @@ -439,3 +464,84 @@ class SlidingSyncHandler: sync_room_id_set.add(room_id) return sync_room_id_set + + async def filter_rooms( + self, + user: UserID, + room_id_set: AbstractSet[str], + filters: SlidingSyncConfig.SlidingSyncList.Filters, + to_token: StreamToken, + ) -> AbstractSet[str]: + """ + Filter rooms based on the sync request. + + Args: + user: User to filter rooms for + room_id_set: Set of room IDs to filter down + filters: Filters to apply + to_token: We filter based on the state of the room at this token + """ + user_id = user.to_string() + + # TODO: Apply filters + # + # TODO: Exclude partially stated rooms unless the `required_state` has + # `["m.room.member", "$LAZY"]` + + filtered_room_id_set = set(room_id_set) + + # Filter for Direct-Message (DM) rooms + if filters.is_dm is not None: + # We're using global account data (`m.direct`) instead of checking for + # `is_direct` on membership events because that property only appears for + # the invitee membership event (doesn't show up for the inviter). Account + # data is set by the client so it needs to be scrutinized. + # + # We're unable to take `to_token` into account for global account data since + # we only keep track of the latest account data for the user. + dm_map = await self.store.get_global_account_data_by_type_for_user( + user_id, AccountDataTypes.DIRECT + ) + + # Flatten out the map + dm_room_id_set = set() + if dm_map: + for room_ids in dm_map.values(): + # Account data should be a list of room IDs. Ignore anything else + if isinstance(room_ids, list): + for room_id in room_ids: + if isinstance(room_id, str): + dm_room_id_set.add(room_id) + + if filters.is_dm: + # Only DM rooms please + filtered_room_id_set = filtered_room_id_set.intersection(dm_room_id_set) + else: + # Only non-DM rooms please + filtered_room_id_set = filtered_room_id_set.difference(dm_room_id_set) + + if filters.spaces: + raise NotImplementedError() + + if filters.is_encrypted: + raise NotImplementedError() + + if filters.is_invite: + raise NotImplementedError() + + if filters.room_types: + raise NotImplementedError() + + if filters.not_room_types: + raise NotImplementedError() + + if filters.room_name_like: + raise NotImplementedError() + + if filters.tags: + raise NotImplementedError() + + if filters.not_tags: + raise NotImplementedError() + + return filtered_room_id_set diff --git a/synapse/types/rest/client/__init__.py b/synapse/types/rest/client/__init__.py index ef261518a0..ec83d0daa6 100644 --- a/synapse/types/rest/client/__init__.py +++ b/synapse/types/rest/client/__init__.py @@ -238,6 +238,53 @@ class SlidingSyncBody(RequestBodyModel): """ class Filters(RequestBodyModel): + """ + All fields are applied with AND operators, hence if `is_dm: True` and + `is_encrypted: True` then only Encrypted DM rooms will be returned. The + absence of fields implies no filter on that criteria: it does NOT imply + `False`. These fields may be expanded through use of extensions. + + Attributes: + is_dm: Flag which only returns rooms present (or not) in the DM section + of account data. If unset, both DM rooms and non-DM rooms are returned. + If False, only non-DM rooms are returned. If True, only DM rooms are + returned. + spaces: Filter the room based on the space they belong to according to + `m.space.child` state events. If multiple spaces are present, a room can + be part of any one of the listed spaces (OR'd). The server will inspect + the `m.space.child` state events for the JOINED space room IDs given. + Servers MUST NOT navigate subspaces. It is up to the client to give a + complete list of spaces to navigate. Only rooms directly mentioned as + `m.space.child` events in these spaces will be returned. Unknown spaces + or spaces the user is not joined to will be ignored. + is_encrypted: Flag which only returns rooms which have an + `m.room.encryption` state event. If unset, both encrypted and + unencrypted rooms are returned. If `False`, only unencrypted rooms are + returned. If `True`, only encrypted rooms are returned. + is_invite: Flag which only returns rooms the user is currently invited + to. If unset, both invited and joined rooms are returned. If `False`, no + invited rooms are returned. If `True`, only invited rooms are returned. + room_types: If specified, only rooms where the `m.room.create` event has + a `type` matching one of the strings in this array will be returned. If + this field is unset, all rooms are returned regardless of type. This can + be used to get the initial set of spaces for an account. For rooms which + do not have a room type, use `null`/`None` to include them. + not_room_types: Same as `room_types` but inverted. This can be used to + filter out spaces from the room list. If a type is in both `room_types` + and `not_room_types`, then `not_room_types` wins and they are not included + in the result. + room_name_like: Filter the room name. Case-insensitive partial matching + e.g 'foo' matches 'abFooab'. The term 'like' is inspired by SQL 'LIKE', + and the text here is similar to '%foo%'. + tags: Filter the room based on its room tags. If multiple tags are + present, a room can have any one of the listed tags (OR'd). + not_tags: Filter the room based on its room tags. Takes priority over + `tags`. For example, a room with tags A and B with filters `tags: [A]` + `not_tags: [B]` would NOT be included because `not_tags` takes priority over + `tags`. This filter is useful if your rooms list does NOT include the + list of favourite rooms again. + """ + is_dm: Optional[StrictBool] = None spaces: Optional[List[StrictStr]] = None is_encrypted: Optional[StrictBool] = None diff --git a/tests/handlers/test_sliding_sync.py b/tests/handlers/test_sliding_sync.py index 41ceb517f0..62fe1214fe 100644 --- a/tests/handlers/test_sliding_sync.py +++ b/tests/handlers/test_sliding_sync.py @@ -22,8 +22,9 @@ from unittest.mock import patch from twisted.test.proto_helpers import MemoryReactor -from synapse.api.constants import EventTypes, JoinRules, Membership +from synapse.api.constants import AccountDataTypes, EventTypes, JoinRules, Membership from synapse.api.room_versions import RoomVersions +from synapse.handlers.sliding_sync import SlidingSyncConfig from synapse.rest import admin from synapse.rest.client import knock, login, room from synapse.server import HomeServer @@ -1116,3 +1117,130 @@ class GetSyncRoomIdsForUserEventShardTestCase(BaseMultiWorkerStreamTestCase): room_id3, }, ) + + +class FilterRoomsTestCase(HomeserverTestCase): + """ + Tests Sliding Sync handler `filter_rooms()` to make sure it includes/excludes rooms + correctly. + """ + + servlets = [ + admin.register_servlets, + knock.register_servlets, + login.register_servlets, + room.register_servlets, + ] + + def default_config(self) -> JsonDict: + config = super().default_config() + # Enable sliding sync + config["experimental_features"] = {"msc3575_enabled": True} + return config + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.sliding_sync_handler = self.hs.get_sliding_sync_handler() + self.store = self.hs.get_datastores().main + self.event_sources = hs.get_event_sources() + + def _create_dm_room( + self, + inviter_user_id: str, + inviter_tok: str, + invitee_user_id: str, + invitee_tok: str, + ) -> str: + """ + Helper to create a DM room as the "inviter" and invite the "invitee" user to the room. The + "invitee" user also will join the room. The `m.direct` account data will be set + for both users. + """ + + # Create a room and send an invite the other user + room_id = self.helper.create_room_as( + inviter_user_id, + is_public=False, + tok=inviter_tok, + ) + self.helper.invite( + room_id, + src=inviter_user_id, + targ=invitee_user_id, + tok=inviter_tok, + extra_data={"is_direct": True}, + ) + # Person that was invited joins the room + self.helper.join(room_id, invitee_user_id, tok=invitee_tok) + + # Mimic the client setting the room as a direct message in the global account + # data + self.get_success( + self.store.add_account_data_for_user( + invitee_user_id, + AccountDataTypes.DIRECT, + {inviter_user_id: [room_id]}, + ) + ) + self.get_success( + self.store.add_account_data_for_user( + inviter_user_id, + AccountDataTypes.DIRECT, + {invitee_user_id: [room_id]}, + ) + ) + + return room_id + + def test_filter_dm_rooms(self) -> None: + """ + Test `filter.is_dm` for DM rooms + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + # Create a normal room + room_id = self.helper.create_room_as( + user1_id, + is_public=False, + tok=user1_tok, + ) + + # Create a DM room + dm_room_id = self._create_dm_room( + inviter_user_id=user1_id, + inviter_tok=user1_tok, + invitee_user_id=user2_id, + invitee_tok=user2_tok, + ) + + after_rooms_token = self.event_sources.get_current_token() + + # Try with `is_dm=True` + truthy_filtered_room_ids = self.get_success( + self.sliding_sync_handler.filter_rooms( + UserID.from_string(user1_id), + {room_id, dm_room_id}, + SlidingSyncConfig.SlidingSyncList.Filters( + is_dm=True, + ), + after_rooms_token, + ) + ) + + self.assertEqual(truthy_filtered_room_ids, {dm_room_id}) + + # Try with `is_dm=False` + falsy_filtered_room_ids = self.get_success( + self.sliding_sync_handler.filter_rooms( + UserID.from_string(user1_id), + {room_id, dm_room_id}, + SlidingSyncConfig.SlidingSyncList.Filters( + is_dm=False, + ), + after_rooms_token, + ) + ) + + self.assertEqual(falsy_filtered_room_ids, {room_id}) diff --git a/tests/rest/client/test_sync.py b/tests/rest/client/test_sync.py index a20a3fb40d..40870b2cfe 100644 --- a/tests/rest/client/test_sync.py +++ b/tests/rest/client/test_sync.py @@ -27,6 +27,7 @@ from twisted.test.proto_helpers import MemoryReactor import synapse.rest.admin from synapse.api.constants import ( + AccountDataTypes, EventContentFields, EventTypes, ReceiptTypes, @@ -1226,10 +1227,59 @@ class SlidingSyncTestCase(unittest.HomeserverTestCase): return config def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = hs.get_datastores().main self.sync_endpoint = "/_matrix/client/unstable/org.matrix.msc3575/sync" self.store = hs.get_datastores().main self.event_sources = hs.get_event_sources() + def _create_dm_room( + self, + inviter_user_id: str, + inviter_tok: str, + invitee_user_id: str, + invitee_tok: str, + ) -> str: + """ + Helper to create a DM room as the "inviter" and invite the "invitee" user to the + room. The "invitee" user also will join the room. The `m.direct` account data + will be set for both users. + """ + + # Create a room and send an invite the other user + room_id = self.helper.create_room_as( + inviter_user_id, + is_public=False, + tok=inviter_tok, + ) + self.helper.invite( + room_id, + src=inviter_user_id, + targ=invitee_user_id, + tok=inviter_tok, + extra_data={"is_direct": True}, + ) + # Person that was invited joins the room + self.helper.join(room_id, invitee_user_id, tok=invitee_tok) + + # Mimic the client setting the room as a direct message in the global account + # data + self.get_success( + self.store.add_account_data_for_user( + invitee_user_id, + AccountDataTypes.DIRECT, + {inviter_user_id: [room_id]}, + ) + ) + self.get_success( + self.store.add_account_data_for_user( + inviter_user_id, + AccountDataTypes.DIRECT, + {invitee_user_id: [room_id]}, + ) + ) + + return room_id + def test_sync_list(self) -> None: """ Test that room IDs show up in the Sliding Sync lists @@ -1336,3 +1386,80 @@ class SlidingSyncTestCase(unittest.HomeserverTestCase): self.assertEqual( channel.json_body["next_pos"], future_position_token_serialized ) + + def test_filter_list(self) -> None: + """ + Test that filters apply to lists + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + # Create a DM room + dm_room_id = self._create_dm_room( + inviter_user_id=user1_id, + inviter_tok=user1_tok, + invitee_user_id=user2_id, + invitee_tok=user2_tok, + ) + + # Create a normal room + room_id = self.helper.create_room_as(user1_id, tok=user1_tok, is_public=True) + + # Make the Sliding Sync request + channel = self.make_request( + "POST", + self.sync_endpoint, + { + "lists": { + "dms": { + "ranges": [[0, 99]], + "sort": ["by_recency"], + "required_state": [], + "timeline_limit": 1, + "filters": {"is_dm": True}, + }, + "foo-list": { + "ranges": [[0, 99]], + "sort": ["by_recency"], + "required_state": [], + "timeline_limit": 1, + "filters": {"is_dm": False}, + }, + } + }, + access_token=user1_tok, + ) + self.assertEqual(channel.code, 200, channel.json_body) + + # Make sure it has the foo-list we requested + self.assertListEqual( + list(channel.json_body["lists"].keys()), + ["dms", "foo-list"], + channel.json_body["lists"].keys(), + ) + + # Make sure the list includes the room we are joined to + self.assertListEqual( + list(channel.json_body["lists"]["dms"]["ops"]), + [ + { + "op": "SYNC", + "range": [0, 99], + "room_ids": [dm_room_id], + } + ], + list(channel.json_body["lists"]["dms"]), + ) + self.assertListEqual( + list(channel.json_body["lists"]["foo-list"]["ops"]), + [ + { + "op": "SYNC", + "range": [0, 99], + "room_ids": [room_id], + } + ], + list(channel.json_body["lists"]["foo-list"]), + )