diff --git a/changelog.d/17296.feature b/changelog.d/17296.feature new file mode 100644 index 0000000000..4ea5e7be85 --- /dev/null +++ b/changelog.d/17296.feature @@ -0,0 +1 @@ +Add support for the unstable [MSC4151](https://github.com/matrix-org/matrix-spec-proposals/pull/4151) report room API. diff --git a/changelog.d/17297.misc b/changelog.d/17297.misc new file mode 100644 index 0000000000..7ec351d2c1 --- /dev/null +++ b/changelog.d/17297.misc @@ -0,0 +1 @@ +Bump `mypy` from 1.8.0 to 1.9.0. \ No newline at end of file diff --git a/changelog.d/17300.misc b/changelog.d/17300.misc new file mode 100644 index 0000000000..cdc40bb2e5 --- /dev/null +++ b/changelog.d/17300.misc @@ -0,0 +1 @@ +Expose the worker instance that persisted the event on `event.internal_metadata.instance_name`. diff --git a/changelog.d/17301.bugfix b/changelog.d/17301.bugfix new file mode 100644 index 0000000000..50383cb4a4 --- /dev/null +++ b/changelog.d/17301.bugfix @@ -0,0 +1 @@ +Add initial implementation of an experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. diff --git a/poetry.lock b/poetry.lock index 028a11f527..7b169ceb6e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1384,38 +1384,38 @@ files = [ [[package]] name = "mypy" -version = "1.8.0" +version = "1.9.0" description = "Optional static typing for Python" optional = false python-versions = ">=3.8" files = [ - {file = "mypy-1.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:485a8942f671120f76afffff70f259e1cd0f0cfe08f81c05d8816d958d4577d3"}, - {file = "mypy-1.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:df9824ac11deaf007443e7ed2a4a26bebff98d2bc43c6da21b2b64185da011c4"}, - {file = "mypy-1.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2afecd6354bbfb6e0160f4e4ad9ba6e4e003b767dd80d85516e71f2e955ab50d"}, - {file = "mypy-1.8.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8963b83d53ee733a6e4196954502b33567ad07dfd74851f32be18eb932fb1cb9"}, - {file = "mypy-1.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:e46f44b54ebddbeedbd3d5b289a893219065ef805d95094d16a0af6630f5d410"}, - {file = "mypy-1.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:855fe27b80375e5c5878492f0729540db47b186509c98dae341254c8f45f42ae"}, - {file = "mypy-1.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4c886c6cce2d070bd7df4ec4a05a13ee20c0aa60cb587e8d1265b6c03cf91da3"}, - {file = "mypy-1.8.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d19c413b3c07cbecf1f991e2221746b0d2a9410b59cb3f4fb9557f0365a1a817"}, - {file = "mypy-1.8.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9261ed810972061388918c83c3f5cd46079d875026ba97380f3e3978a72f503d"}, - {file = "mypy-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:51720c776d148bad2372ca21ca29256ed483aa9a4cdefefcef49006dff2a6835"}, - {file = "mypy-1.8.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:52825b01f5c4c1c4eb0db253ec09c7aa17e1a7304d247c48b6f3599ef40db8bd"}, - {file = "mypy-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f5ac9a4eeb1ec0f1ccdc6f326bcdb464de5f80eb07fb38b5ddd7b0de6bc61e55"}, - {file = "mypy-1.8.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afe3fe972c645b4632c563d3f3eff1cdca2fa058f730df2b93a35e3b0c538218"}, - {file = "mypy-1.8.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:42c6680d256ab35637ef88891c6bd02514ccb7e1122133ac96055ff458f93fc3"}, - {file = "mypy-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:720a5ca70e136b675af3af63db533c1c8c9181314d207568bbe79051f122669e"}, - {file = "mypy-1.8.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:028cf9f2cae89e202d7b6593cd98db6759379f17a319b5faf4f9978d7084cdc6"}, - {file = "mypy-1.8.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4e6d97288757e1ddba10dd9549ac27982e3e74a49d8d0179fc14d4365c7add66"}, - {file = "mypy-1.8.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f1478736fcebb90f97e40aff11a5f253af890c845ee0c850fe80aa060a267c6"}, - {file = "mypy-1.8.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:42419861b43e6962a649068a61f4a4839205a3ef525b858377a960b9e2de6e0d"}, - {file = "mypy-1.8.0-cp38-cp38-win_amd64.whl", hash = "sha256:2b5b6c721bd4aabaadead3a5e6fa85c11c6c795e0c81a7215776ef8afc66de02"}, - {file = "mypy-1.8.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5c1538c38584029352878a0466f03a8ee7547d7bd9f641f57a0f3017a7c905b8"}, - {file = "mypy-1.8.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4ef4be7baf08a203170f29e89d79064463b7fc7a0908b9d0d5114e8009c3a259"}, - {file = "mypy-1.8.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7178def594014aa6c35a8ff411cf37d682f428b3b5617ca79029d8ae72f5402b"}, - {file = "mypy-1.8.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ab3c84fa13c04aeeeabb2a7f67a25ef5d77ac9d6486ff33ded762ef353aa5592"}, - {file = "mypy-1.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:99b00bc72855812a60d253420d8a2eae839b0afa4938f09f4d2aa9bb4654263a"}, - {file = "mypy-1.8.0-py3-none-any.whl", hash = "sha256:538fd81bb5e430cc1381a443971c0475582ff9f434c16cd46d2c66763ce85d9d"}, - {file = "mypy-1.8.0.tar.gz", hash = "sha256:6ff8b244d7085a0b425b56d327b480c3b29cafbd2eff27316a004f9a7391ae07"}, + {file = "mypy-1.9.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f8a67616990062232ee4c3952f41c779afac41405806042a8126fe96e098419f"}, + {file = "mypy-1.9.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d357423fa57a489e8c47b7c85dfb96698caba13d66e086b412298a1a0ea3b0ed"}, + {file = "mypy-1.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49c87c15aed320de9b438ae7b00c1ac91cd393c1b854c2ce538e2a72d55df150"}, + {file = "mypy-1.9.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:48533cdd345c3c2e5ef48ba3b0d3880b257b423e7995dada04248725c6f77374"}, + {file = "mypy-1.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:4d3dbd346cfec7cb98e6cbb6e0f3c23618af826316188d587d1c1bc34f0ede03"}, + {file = "mypy-1.9.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:653265f9a2784db65bfca694d1edd23093ce49740b2244cde583aeb134c008f3"}, + {file = "mypy-1.9.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3a3c007ff3ee90f69cf0a15cbcdf0995749569b86b6d2f327af01fd1b8aee9dc"}, + {file = "mypy-1.9.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2418488264eb41f69cc64a69a745fad4a8f86649af4b1041a4c64ee61fc61129"}, + {file = "mypy-1.9.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:68edad3dc7d70f2f17ae4c6c1b9471a56138ca22722487eebacfd1eb5321d612"}, + {file = "mypy-1.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:85ca5fcc24f0b4aeedc1d02f93707bccc04733f21d41c88334c5482219b1ccb3"}, + {file = "mypy-1.9.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aceb1db093b04db5cd390821464504111b8ec3e351eb85afd1433490163d60cd"}, + {file = "mypy-1.9.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0235391f1c6f6ce487b23b9dbd1327b4ec33bb93934aa986efe8a9563d9349e6"}, + {file = "mypy-1.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d4d5ddc13421ba3e2e082a6c2d74c2ddb3979c39b582dacd53dd5d9431237185"}, + {file = "mypy-1.9.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:190da1ee69b427d7efa8aa0d5e5ccd67a4fb04038c380237a0d96829cb157913"}, + {file = "mypy-1.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:fe28657de3bfec596bbeef01cb219833ad9d38dd5393fc649f4b366840baefe6"}, + {file = "mypy-1.9.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e54396d70be04b34f31d2edf3362c1edd023246c82f1730bbf8768c28db5361b"}, + {file = "mypy-1.9.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5e6061f44f2313b94f920e91b204ec600982961e07a17e0f6cd83371cb23f5c2"}, + {file = "mypy-1.9.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81a10926e5473c5fc3da8abb04119a1f5811a236dc3a38d92015cb1e6ba4cb9e"}, + {file = "mypy-1.9.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b685154e22e4e9199fc95f298661deea28aaede5ae16ccc8cbb1045e716b3e04"}, + {file = "mypy-1.9.0-cp38-cp38-win_amd64.whl", hash = "sha256:5d741d3fc7c4da608764073089e5f58ef6352bedc223ff58f2f038c2c4698a89"}, + {file = "mypy-1.9.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:587ce887f75dd9700252a3abbc9c97bbe165a4a630597845c61279cf32dfbf02"}, + {file = "mypy-1.9.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f88566144752999351725ac623471661c9d1cd8caa0134ff98cceeea181789f4"}, + {file = "mypy-1.9.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:61758fabd58ce4b0720ae1e2fea5cfd4431591d6d590b197775329264f86311d"}, + {file = "mypy-1.9.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e49499be624dead83927e70c756970a0bc8240e9f769389cdf5714b0784ca6bf"}, + {file = "mypy-1.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:571741dc4194b4f82d344b15e8837e8c5fcc462d66d076748142327626a1b6e9"}, + {file = "mypy-1.9.0-py3-none-any.whl", hash = "sha256:a260627a570559181a9ea5de61ac6297aa5af202f06fd7ab093ce74e7181e43e"}, + {file = "mypy-1.9.0.tar.gz", hash = "sha256:3cc5da0127e6a478cddd906068496a97a7618a21ce9b54bde5bf7e539c7af974"}, ] [package.dependencies] diff --git a/synapse/rest/client/reporting.py b/synapse/rest/client/reporting.py index a95b83b14d..4eee53e5a8 100644 --- a/synapse/rest/client/reporting.py +++ b/synapse/rest/client/reporting.py @@ -107,7 +107,15 @@ class ReportEventRestServlet(RestServlet): class ReportRoomRestServlet(RestServlet): - # https://github.com/matrix-org/matrix-spec-proposals/pull/4151 + """This endpoint lets clients report a room for abuse. + + Whilst MSC4151 is not yet merged, this unstable endpoint is enabled on matrix.org + for content moderation purposes, and therefore backwards compatibility should be + carefully considered when changing anything on this endpoint. + + More details on the MSC: https://github.com/matrix-org/matrix-spec-proposals/pull/4151 + """ + PATTERNS = client_patterns( "/org.matrix.msc4151/rooms/(?P[^/]*)/report$", releases=[], diff --git a/synapse/storage/databases/main/stream.py b/synapse/storage/databases/main/stream.py index d7cd5ae414..31a387f37d 100644 --- a/synapse/storage/databases/main/stream.py +++ b/synapse/storage/databases/main/stream.py @@ -946,14 +946,14 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): # (sharded event persisters). The first subquery handles the events that # would be within the vector clock and gets all rows between the minimum and # maximum stream ordering in the token which need to be filtered against the - # `instance_map`. The second subquery handles the "before" case and finds a - # row before the token. We then filter out any results past the token's - # vector clock and return the first row that matches. + # `instance_map`. The second subquery handles the "before" case and finds + # the first row before the token. We then filter out any results past the + # token's vector clock and return the first row that matches. min_stream = end_token.stream max_stream = end_token.get_max_stream_pos() # We use `union all` because we don't need any of the deduplication logic - # (`union` is really a union + distinct). `UNION ALL`` does preserve the + # (`union` is really a union + distinct). `UNION ALL` does preserve the # ordering of the operand queries but there is no actual gurantee that it # has this behavior in all scenarios so we need the extra `ORDER BY` at the # bottom. diff --git a/tests/push/test_email.py b/tests/push/test_email.py index c927a73fa6..e0aab1c046 100644 --- a/tests/push/test_email.py +++ b/tests/push/test_email.py @@ -205,8 +205,24 @@ class EmailPusherTests(HomeserverTestCase): # Multipart: plain text, base 64 encoded; html, base 64 encoded multipart_msg = email.message_from_bytes(msg) - txt = multipart_msg.get_payload()[0].get_payload(decode=True).decode() - html = multipart_msg.get_payload()[1].get_payload(decode=True).decode() + + # Extract the text (non-HTML) portion of the multipart Message, + # as a Message. + txt_message = multipart_msg.get_payload(i=0) + assert isinstance(txt_message, email.message.Message) + + # Extract the actual bytes from the Message object, and decode them to a `str`. + txt_bytes = txt_message.get_payload(decode=True) + assert isinstance(txt_bytes, bytes) + txt = txt_bytes.decode() + + # Do the same for the HTML portion of the multipart Message. + html_message = multipart_msg.get_payload(i=1) + assert isinstance(html_message, email.message.Message) + html_bytes = html_message.get_payload(decode=True) + assert isinstance(html_bytes, bytes) + html = html_bytes.decode() + self.assertIn("/_synapse/client/unsubscribe", txt) self.assertIn("/_synapse/client/unsubscribe", html) @@ -347,12 +363,17 @@ class EmailPusherTests(HomeserverTestCase): # That email should contain the room's avatar msg: bytes = args[5] # Multipart: plain text, base 64 encoded; html, base 64 encoded - html = ( - email.message_from_bytes(msg) - .get_payload()[1] - .get_payload(decode=True) - .decode() - ) + + # Extract the html Message object from the Multipart Message. + # We need the asserts to convince mypy that this is OK. + html_message = email.message_from_bytes(msg).get_payload(i=1) + assert isinstance(html_message, email.message.Message) + + # Extract the `bytes` from the html Message object, and decode to a `str`. + html = html_message.get_payload(decode=True) + assert isinstance(html, bytes) + html = html.decode() + self.assertIn("_matrix/media/v1/thumbnail/DUMMY_MEDIA_ID", html) def test_empty_room(self) -> None: diff --git a/tests/replication/storage/test_events.py b/tests/replication/storage/test_events.py index 4e41a1c912..a56f1e2d5d 100644 --- a/tests/replication/storage/test_events.py +++ b/tests/replication/storage/test_events.py @@ -141,6 +141,7 @@ class EventsWorkerStoreTestCase(BaseWorkerStoreTestCase): self.persist(type="m.room.create", key="", creator=USER_ID) self.check("get_invited_rooms_for_local_user", [USER_ID_2], []) event = self.persist(type="m.room.member", key=USER_ID_2, membership="invite") + assert event.internal_metadata.instance_name is not None assert event.internal_metadata.stream_ordering is not None self.replicate() @@ -155,7 +156,7 @@ class EventsWorkerStoreTestCase(BaseWorkerStoreTestCase): "invite", event.event_id, PersistedEventPosition( - self.hs.get_instance_name(), + event.internal_metadata.instance_name, event.internal_metadata.stream_ordering, ), RoomVersions.V1.identifier, @@ -232,11 +233,12 @@ class EventsWorkerStoreTestCase(BaseWorkerStoreTestCase): j2 = self.persist( type="m.room.member", sender=USER_ID_2, key=USER_ID_2, membership="join" ) + assert j2.internal_metadata.instance_name is not None assert j2.internal_metadata.stream_ordering is not None self.replicate() expected_pos = PersistedEventPosition( - "master", j2.internal_metadata.stream_ordering + j2.internal_metadata.instance_name, j2.internal_metadata.stream_ordering ) self.check( "get_rooms_for_user_with_stream_ordering", @@ -288,6 +290,7 @@ class EventsWorkerStoreTestCase(BaseWorkerStoreTestCase): msg, msgctx = self.build_event() self.get_success(self.persistance.persist_events([(j2, j2ctx), (msg, msgctx)])) self.replicate() + assert j2.internal_metadata.instance_name is not None assert j2.internal_metadata.stream_ordering is not None event_source = RoomEventSource(self.hs) @@ -329,7 +332,8 @@ class EventsWorkerStoreTestCase(BaseWorkerStoreTestCase): # joined_rooms list. if membership_changes: expected_pos = PersistedEventPosition( - "master", j2.internal_metadata.stream_ordering + j2.internal_metadata.instance_name, + j2.internal_metadata.stream_ordering, ) self.assertEqual( joined_rooms, diff --git a/tests/rest/client/test_account.py b/tests/rest/client/test_account.py index 992421ffe2..a85ea994de 100644 --- a/tests/rest/client/test_account.py +++ b/tests/rest/client/test_account.py @@ -427,13 +427,23 @@ class PasswordResetTestCase(unittest.HomeserverTestCase): text = None for part in mail.walk(): if part.get_content_type() == "text/plain": - text = part.get_payload(decode=True).decode("UTF-8") + text = part.get_payload(decode=True) + if text is not None: + # According to the logic table in `get_payload`, we know that + # the result of `get_payload` will be `bytes`, but mypy doesn't + # know this and complains. Thus, we assert the type. + assert isinstance(text, bytes) + text = text.decode("UTF-8") + break if not text: self.fail("Could not find text portion of email to parse") - assert text is not None + # `text` must be a `str`, after being decoded and determined just above + # to not be `None` or an empty `str`. + assert isinstance(text, str) + match = re.search(r"https://example.com\S+", text) assert match, "Could not find link in email" @@ -1209,13 +1219,23 @@ class ThreepidEmailRestTestCase(unittest.HomeserverTestCase): text = None for part in mail.walk(): if part.get_content_type() == "text/plain": - text = part.get_payload(decode=True).decode("UTF-8") + text = part.get_payload(decode=True) + if text is not None: + # According to the logic table in `get_payload`, we know that + # the result of `get_payload` will be `bytes`, but mypy doesn't + # know this and complains. Thus, we assert the type. + assert isinstance(text, bytes) + text = text.decode("UTF-8") + break if not text: self.fail("Could not find text portion of email to parse") - assert text is not None + # `text` must be a `str`, after being decoded and determined just above + # to not be `None` or an empty `str`. + assert isinstance(text, str) + match = re.search(r"https://example.com\S+", text) assert match, "Could not find link in email" diff --git a/tests/storage/test_stream.py b/tests/storage/test_stream.py index b8dd796a71..36bfbd8197 100644 --- a/tests/storage/test_stream.py +++ b/tests/storage/test_stream.py @@ -347,7 +347,7 @@ class GetLastEventInRoomBeforeStreamOrderingTestCase(HomeserverTestCase): def test_after_room_created(self) -> None: """ - Test that an event returned if we are using a token after the room was created + Test that an event is returned if we are using a token after the room was created """ user1_id = self.register_user("user1", "pass") user1_tok = self.login(user1_id, "pass") @@ -369,8 +369,8 @@ class GetLastEventInRoomBeforeStreamOrderingTestCase(HomeserverTestCase): def test_activity_in_other_rooms(self) -> None: """ - Test to make sure that the last event in room is returned even if the - `stream_ordering` has advanced past it (and that's from the correct room) + Test to make sure that the last event in the room is returned even if the + `stream_ordering` has advanced from activity in other rooms. """ user1_id = self.register_user("user1", "pass") user1_tok = self.login(user1_id, "pass") @@ -488,8 +488,8 @@ class GetLastEventInRoomBeforeStreamOrderingTestCase(HomeserverTestCase): def test_last_event_before_sharded_token(self) -> None: """ - Test to make sure we can find the last event that that is *before* the sharded - token (a token that has an `instance_map` and looks like + Test to make sure we can find the last event that is *before* the sharded token + (a token that has an `instance_map` and looks like `m{min_pos}~{writer1}.{pos1}~{writer2}.{pos2}`). """ user1_id = self.register_user("user1", "pass") @@ -504,7 +504,6 @@ class GetLastEventInRoomBeforeStreamOrderingTestCase(HomeserverTestCase): ) # Create another room to advance the `stream_ordering` on the same worker - # so we can sandwich event3 in the middle of the token room_id2 = self.helper.create_room_as(user1_id, tok=user1_tok, is_public=True) event_response3, event_pos3 = self._send_event_on_instance( "worker1", room_id2, user1_tok