diff --git a/.gitignore b/.gitignore index 9bb5bdd647..2bccf19997 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ _trial_temp*/ # stuff that is likely to exist when you run a server locally /*.db /*.log +/*.log.* /*.log.config /*.pid /.python-version diff --git a/CHANGES.md b/CHANGES.md index d6fa92d81c..2aebe92cac 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,5 @@ -Synapse 1.25.0 (2020-xx-xx) -=========================== +Synapse 1.25.0rc1 (2021-01-06) +============================== Removal warning --------------- @@ -12,6 +12,92 @@ are deprecated and will be removed in a future release. They will be replaced by `POST /_synapse/admin/v1/rooms//delete` replaces `POST /_synapse/admin/v1/purge_room` and `POST /_synapse/admin/v1/shutdown_room/`. +Features +-------- + +- Add an admin API that lets server admins get power in rooms in which local users have power. ([\#8756](https://github.com/matrix-org/synapse/issues/8756)) +- Add optional HTTP authentication to replication endpoints. ([\#8853](https://github.com/matrix-org/synapse/issues/8853)) +- Improve the error messages printed as a result of configuration problems for extension modules. ([\#8874](https://github.com/matrix-org/synapse/issues/8874)) +- Add the number of local devices to Room Details Admin API. Contributed by @dklimpel. ([\#8886](https://github.com/matrix-org/synapse/issues/8886)) +- Add `X-Robots-Tag` header to stop web crawlers from indexing media. Contributed by Aaron Raimist. ([\#8887](https://github.com/matrix-org/synapse/issues/8887)) +- Spam-checkers may now define their methods as `async`. ([\#8890](https://github.com/matrix-org/synapse/issues/8890)) +- Add support for allowing users to pick their own user ID during a single-sign-on login. ([\#8897](https://github.com/matrix-org/synapse/issues/8897), [\#8900](https://github.com/matrix-org/synapse/issues/8900), [\#8911](https://github.com/matrix-org/synapse/issues/8911), [\#8938](https://github.com/matrix-org/synapse/issues/8938), [\#8941](https://github.com/matrix-org/synapse/issues/8941), [\#8942](https://github.com/matrix-org/synapse/issues/8942), [\#8951](https://github.com/matrix-org/synapse/issues/8951)) +- Add an `email.invite_client_location` configuration option to send a web client location to the invite endpoint on the identity server which allows customisation of the email template. ([\#8930](https://github.com/matrix-org/synapse/issues/8930)) +- The search term in the list room and list user Admin APIs is now treated as case-insensitive. ([\#8931](https://github.com/matrix-org/synapse/issues/8931)) +- Apply an IP range blacklist to push and key revocation requests. ([\#8821](https://github.com/matrix-org/synapse/issues/8821), [\#8870](https://github.com/matrix-org/synapse/issues/8870), [\#8954](https://github.com/matrix-org/synapse/issues/8954)) +- Add an option to allow re-use of user-interactive authentication sessions for a period of time. ([\#8970](https://github.com/matrix-org/synapse/issues/8970)) +- Allow running the redact endpoint on workers. ([\#8994](https://github.com/matrix-org/synapse/issues/8994)) + + +Bugfixes +-------- + +- Fix bug where we might not correctly calculate the current state for rooms with multiple extremities. ([\#8827](https://github.com/matrix-org/synapse/issues/8827)) +- Fix a long-standing bug in the register admin endpoint (`/_synapse/admin/v1/register`) when the `mac` field was not provided. The endpoint now properly returns a 400 error. Contributed by @edwargix. ([\#8837](https://github.com/matrix-org/synapse/issues/8837)) +- Fix a long-standing bug on Synapse instances supporting Single-Sign-On, where users would be prompted to enter their password to confirm certain actions, even though they have not set a password. ([\#8858](https://github.com/matrix-org/synapse/issues/8858)) +- Fix a longstanding bug where a 500 error would be returned if the `Content-Length` header was not provided to the upload media resource. ([\#8862](https://github.com/matrix-org/synapse/issues/8862)) +- Add additional validation to pusher URLs to be compliant with the specification. ([\#8865](https://github.com/matrix-org/synapse/issues/8865)) +- Fix the error code that is returned when a user tries to register on a homeserver on which new-user registration has been disabled. ([\#8867](https://github.com/matrix-org/synapse/issues/8867)) +- Fix a bug where `PUT /_synapse/admin/v2/users/` failed to create a new user when `avatar_url` is specified. Bug introduced in Synapse v1.9.0. ([\#8872](https://github.com/matrix-org/synapse/issues/8872)) +- Fix a 500 error when attempting to preview an empty HTML file. ([\#8883](https://github.com/matrix-org/synapse/issues/8883)) +- Fix occasional deadlock when handling SIGHUP. ([\#8918](https://github.com/matrix-org/synapse/issues/8918)) +- Fix login API to not ratelimit application services that have ratelimiting disabled. ([\#8920](https://github.com/matrix-org/synapse/issues/8920)) +- Fix bug where we ratelimited auto joining of rooms on registration (using `auto_join_rooms` config). ([\#8921](https://github.com/matrix-org/synapse/issues/8921)) +- Fix a bug where deactivated users appeared in the user directory when their profile information was updated. ([\#8933](https://github.com/matrix-org/synapse/issues/8933), [\#8964](https://github.com/matrix-org/synapse/issues/8964)) +- Fix bug introduced in Synapse v1.24.0 which would cause an exception on startup if both `enabled` and `localdb_enabled` were set to `False` in the `password_config` setting of the configuration file. ([\#8937](https://github.com/matrix-org/synapse/issues/8937)) +- Fix a bug where 500 errors would be returned if the `m.room_history_visibility` event had invalid content. ([\#8945](https://github.com/matrix-org/synapse/issues/8945)) +- Fix a bug causing common English words to not be considered for a user directory search. ([\#8959](https://github.com/matrix-org/synapse/issues/8959)) +- Fix bug where application services couldn't register new ghost users if the server had reached its MAU limit. ([\#8962](https://github.com/matrix-org/synapse/issues/8962)) +- Fix a long-standing bug where a `m.image` event without a `url` would cause errors on push. ([\#8965](https://github.com/matrix-org/synapse/issues/8965)) +- Fix a small bug in v2 state resolution algorithm, which could also cause performance issues for rooms with large numbers of power levels. ([\#8971](https://github.com/matrix-org/synapse/issues/8971)) +- Add validation to the `sendToDevice` API to raise a missing parameters error instead of a 500 error. ([\#8975](https://github.com/matrix-org/synapse/issues/8975)) +- Add validation of group IDs to raise a 400 error instead of a 500 eror. ([\#8977](https://github.com/matrix-org/synapse/issues/8977)) + + +Improved Documentation +---------------------- + +- Fix the "Event persist rate" section of the included grafana dashboard by adding missing prometheus rules. ([\#8802](https://github.com/matrix-org/synapse/issues/8802)) +- Combine related media admin API docs. ([\#8839](https://github.com/matrix-org/synapse/issues/8839)) +- Fix an error in the documentation for the SAML username mapping provider. ([\#8873](https://github.com/matrix-org/synapse/issues/8873)) +- Clarify comments around template directories in `sample_config.yaml`. ([\#8891](https://github.com/matrix-org/synapse/issues/8891)) +- Moved instructions for database setup, adjusted heading levels and improved syntax highlighting in [INSTALL.md](../INSTALL.md). Contributed by fossterer. ([\#8987](https://github.com/matrix-org/synapse/issues/8987)) +- Update the example value of `group_creation_prefix` in the sample configuration. ([\#8992](https://github.com/matrix-org/synapse/issues/8992)) +- Link the Synapse developer room to the development section in the docs. ([\#9002](https://github.com/matrix-org/synapse/issues/9002)) + + +Deprecations and Removals +------------------------- + +- Deprecate Shutdown Room and Purge Room Admin APIs. ([\#8829](https://github.com/matrix-org/synapse/issues/8829)) + + +Internal Changes +---------------- + +- Properly store the mapping of external ID to Matrix ID for CAS users. ([\#8856](https://github.com/matrix-org/synapse/issues/8856), [\#8958](https://github.com/matrix-org/synapse/issues/8958)) +- Remove some unnecessary stubbing from unit tests. ([\#8861](https://github.com/matrix-org/synapse/issues/8861)) +- Remove unused `FakeResponse` class from unit tests. ([\#8864](https://github.com/matrix-org/synapse/issues/8864)) +- Pass `room_id` to `get_auth_chain_difference`. ([\#8879](https://github.com/matrix-org/synapse/issues/8879)) +- Add type hints to push module. ([\#8880](https://github.com/matrix-org/synapse/issues/8880), [\#8882](https://github.com/matrix-org/synapse/issues/8882), [\#8901](https://github.com/matrix-org/synapse/issues/8901), [\#8940](https://github.com/matrix-org/synapse/issues/8940), [\#8943](https://github.com/matrix-org/synapse/issues/8943), [\#9020](https://github.com/matrix-org/synapse/issues/9020)) +- Simplify logic for handling user-interactive-auth via single-sign-on servers. ([\#8881](https://github.com/matrix-org/synapse/issues/8881)) +- Skip the SAML tests if the requirements (`pysaml2` and `xmlsec1`) aren't available. ([\#8905](https://github.com/matrix-org/synapse/issues/8905)) +- Fix multiarch docker image builds. ([\#8906](https://github.com/matrix-org/synapse/issues/8906)) +- Don't publish `latest` docker image until all archs are built. ([\#8909](https://github.com/matrix-org/synapse/issues/8909)) +- Various clean-ups to the structured logging and logging context code. ([\#8916](https://github.com/matrix-org/synapse/issues/8916), [\#8935](https://github.com/matrix-org/synapse/issues/8935)) +- Automatically drop stale forward-extremities under some specific conditions. ([\#8929](https://github.com/matrix-org/synapse/issues/8929)) +- Refactor test utilities for injecting HTTP requests. ([\#8946](https://github.com/matrix-org/synapse/issues/8946)) +- Add a maximum size of 50 kilobytes to .well-known lookups. ([\#8950](https://github.com/matrix-org/synapse/issues/8950)) +- Fix bug in `generate_log_config` script which made it write empty files. ([\#8952](https://github.com/matrix-org/synapse/issues/8952)) +- Clean up tox.ini file; disable coverage checking for non-test runs. ([\#8963](https://github.com/matrix-org/synapse/issues/8963)) +- Add type hints to the admin and room list handlers. ([\#8973](https://github.com/matrix-org/synapse/issues/8973)) +- Add type hints to the receipts and user directory handlers. ([\#8976](https://github.com/matrix-org/synapse/issues/8976)) +- Drop the unused `local_invites` table. ([\#8979](https://github.com/matrix-org/synapse/issues/8979)) +- Add type hints to the base storage code. ([\#8980](https://github.com/matrix-org/synapse/issues/8980)) +- Support using PyJWT v2.0.0 in the test suite. ([\#8986](https://github.com/matrix-org/synapse/issues/8986)) +- Fix `tests.federation.transport.RoomDirectoryFederationTests` and ensure it runs in CI. ([\#8998](https://github.com/matrix-org/synapse/issues/8998)) +- Add type hints to the crypto module. ([\#8999](https://github.com/matrix-org/synapse/issues/8999)) + Synapse 1.24.0 (2020-12-09) =========================== diff --git a/changelog.d/8756.feature b/changelog.d/8756.feature deleted file mode 100644 index 03eb79fb0a..0000000000 --- a/changelog.d/8756.feature +++ /dev/null @@ -1 +0,0 @@ -Add admin API that lets server admins get power in rooms in which local users have power. diff --git a/changelog.d/8802.doc b/changelog.d/8802.doc deleted file mode 100644 index 580c4281f8..0000000000 --- a/changelog.d/8802.doc +++ /dev/null @@ -1 +0,0 @@ -Fix the "Event persist rate" section of the included grafana dashboard by adding missing prometheus rules. diff --git a/changelog.d/8821.bugfix b/changelog.d/8821.bugfix deleted file mode 100644 index 39f53174ad..0000000000 --- a/changelog.d/8821.bugfix +++ /dev/null @@ -1 +0,0 @@ -Apply an IP range blacklist to push and key revocation requests. diff --git a/changelog.d/8827.bugfix b/changelog.d/8827.bugfix deleted file mode 100644 index 18195680d3..0000000000 --- a/changelog.d/8827.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug where we might not correctly calculate the current state for rooms with multiple extremities. diff --git a/changelog.d/8829.removal b/changelog.d/8829.removal deleted file mode 100644 index 2f3708218b..0000000000 --- a/changelog.d/8829.removal +++ /dev/null @@ -1 +0,0 @@ -Deprecate Shutdown Room and Purge Room Admin APIs. diff --git a/changelog.d/8837.bugfix b/changelog.d/8837.bugfix deleted file mode 100644 index b2977d0c31..0000000000 --- a/changelog.d/8837.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a long standing bug in the register admin endpoint (`/_synapse/admin/v1/register`) when the `mac` field was not provided. The endpoint now properly returns a 400 error. Contributed by @edwargix. diff --git a/changelog.d/8839.doc b/changelog.d/8839.doc deleted file mode 100644 index c35c59a763..0000000000 --- a/changelog.d/8839.doc +++ /dev/null @@ -1 +0,0 @@ -Combine related media admin API docs. \ No newline at end of file diff --git a/changelog.d/8853.feature b/changelog.d/8853.feature deleted file mode 100644 index 63c59f4ff2..0000000000 --- a/changelog.d/8853.feature +++ /dev/null @@ -1 +0,0 @@ -Add optional HTTP authentication to replication endpoints. diff --git a/changelog.d/8856.misc b/changelog.d/8856.misc deleted file mode 100644 index 1507073e4f..0000000000 --- a/changelog.d/8856.misc +++ /dev/null @@ -1 +0,0 @@ -Properly store the mapping of external ID to Matrix ID for CAS users. diff --git a/changelog.d/8858.bugfix b/changelog.d/8858.bugfix deleted file mode 100644 index 0d58cb9abc..0000000000 --- a/changelog.d/8858.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a long-standing bug on Synapse instances supporting Single-Sign-On, where users would be prompted to enter their password to confirm certain actions, even though they have not set a password. diff --git a/changelog.d/8861.misc b/changelog.d/8861.misc deleted file mode 100644 index 9821f804cf..0000000000 --- a/changelog.d/8861.misc +++ /dev/null @@ -1 +0,0 @@ -Remove some unnecessary stubbing from unit tests. diff --git a/changelog.d/8862.bugfix b/changelog.d/8862.bugfix deleted file mode 100644 index bdbd633f72..0000000000 --- a/changelog.d/8862.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a longstanding bug where a 500 error would be returned if the `Content-Length` header was not provided to the upload media resource. diff --git a/changelog.d/8864.misc b/changelog.d/8864.misc deleted file mode 100644 index a780883495..0000000000 --- a/changelog.d/8864.misc +++ /dev/null @@ -1 +0,0 @@ -Remove unused `FakeResponse` class from unit tests. diff --git a/changelog.d/8865.bugfix b/changelog.d/8865.bugfix deleted file mode 100644 index a1e625f552..0000000000 --- a/changelog.d/8865.bugfix +++ /dev/null @@ -1 +0,0 @@ -Add additional validation to pusher URLs to be compliant with the specification. diff --git a/changelog.d/8867.bugfix b/changelog.d/8867.bugfix deleted file mode 100644 index f2414ff111..0000000000 --- a/changelog.d/8867.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix the error code that is returned when a user tries to register on a homeserver on which new-user registration has been disabled. diff --git a/changelog.d/8870.bugfix b/changelog.d/8870.bugfix deleted file mode 100644 index 39f53174ad..0000000000 --- a/changelog.d/8870.bugfix +++ /dev/null @@ -1 +0,0 @@ -Apply an IP range blacklist to push and key revocation requests. diff --git a/changelog.d/8872.bugfix b/changelog.d/8872.bugfix deleted file mode 100644 index ed00b70a0f..0000000000 --- a/changelog.d/8872.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug where `PUT /_synapse/admin/v2/users/` failed to create a new user when `avatar_url` is specified. Bug introduced in Synapse v1.9.0. diff --git a/changelog.d/8873.doc b/changelog.d/8873.doc deleted file mode 100644 index 0c2a043bd1..0000000000 --- a/changelog.d/8873.doc +++ /dev/null @@ -1 +0,0 @@ -Fix an error in the documentation for the SAML username mapping provider. diff --git a/changelog.d/8874.feature b/changelog.d/8874.feature deleted file mode 100644 index 720665ecac..0000000000 --- a/changelog.d/8874.feature +++ /dev/null @@ -1 +0,0 @@ -Improve the error messages printed as a result of configuration problems for extension modules. diff --git a/changelog.d/8879.misc b/changelog.d/8879.misc deleted file mode 100644 index 6f9516b314..0000000000 --- a/changelog.d/8879.misc +++ /dev/null @@ -1 +0,0 @@ -Pass `room_id` to `get_auth_chain_difference`. diff --git a/changelog.d/8880.misc b/changelog.d/8880.misc deleted file mode 100644 index 4ff0b94b94..0000000000 --- a/changelog.d/8880.misc +++ /dev/null @@ -1 +0,0 @@ -Add type hints to push module. diff --git a/changelog.d/8881.misc b/changelog.d/8881.misc deleted file mode 100644 index 07d3f30fb2..0000000000 --- a/changelog.d/8881.misc +++ /dev/null @@ -1 +0,0 @@ -Simplify logic for handling user-interactive-auth via single-sign-on servers. diff --git a/changelog.d/8882.misc b/changelog.d/8882.misc deleted file mode 100644 index 4ff0b94b94..0000000000 --- a/changelog.d/8882.misc +++ /dev/null @@ -1 +0,0 @@ -Add type hints to push module. diff --git a/changelog.d/8883.bugfix b/changelog.d/8883.bugfix deleted file mode 100644 index 6137fc5b2b..0000000000 --- a/changelog.d/8883.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a 500 error when attempting to preview an empty HTML file. diff --git a/changelog.d/8886.feature b/changelog.d/8886.feature deleted file mode 100644 index 9e446f28bd..0000000000 --- a/changelog.d/8886.feature +++ /dev/null @@ -1 +0,0 @@ -Add number of local devices to Room Details Admin API. Contributed by @dklimpel. \ No newline at end of file diff --git a/changelog.d/8887.feature b/changelog.d/8887.feature deleted file mode 100644 index 729eb1f1ea..0000000000 --- a/changelog.d/8887.feature +++ /dev/null @@ -1 +0,0 @@ -Add `X-Robots-Tag` header to stop web crawlers from indexing media. diff --git a/changelog.d/8890.feature b/changelog.d/8890.feature deleted file mode 100644 index 97aa72a76e..0000000000 --- a/changelog.d/8890.feature +++ /dev/null @@ -1 +0,0 @@ -Spam-checkers may now define their methods as `async`. diff --git a/changelog.d/8891.doc b/changelog.d/8891.doc deleted file mode 100644 index c3947fe7c2..0000000000 --- a/changelog.d/8891.doc +++ /dev/null @@ -1 +0,0 @@ -Clarify comments around template directories in `sample_config.yaml`. diff --git a/changelog.d/8897.feature b/changelog.d/8897.feature deleted file mode 100644 index d450ef4998..0000000000 --- a/changelog.d/8897.feature +++ /dev/null @@ -1 +0,0 @@ -Add support for allowing users to pick their own user ID during a single-sign-on login. diff --git a/changelog.d/8900.feature b/changelog.d/8900.feature deleted file mode 100644 index d450ef4998..0000000000 --- a/changelog.d/8900.feature +++ /dev/null @@ -1 +0,0 @@ -Add support for allowing users to pick their own user ID during a single-sign-on login. diff --git a/changelog.d/8901.misc b/changelog.d/8901.misc deleted file mode 100644 index 4ff0b94b94..0000000000 --- a/changelog.d/8901.misc +++ /dev/null @@ -1 +0,0 @@ -Add type hints to push module. diff --git a/changelog.d/8905.misc b/changelog.d/8905.misc deleted file mode 100644 index a9a11a2303..0000000000 --- a/changelog.d/8905.misc +++ /dev/null @@ -1 +0,0 @@ -Skip the SAML tests if the requirements (`pysaml2` and `xmlsec1`) aren't available. diff --git a/changelog.d/8906.misc b/changelog.d/8906.misc deleted file mode 100644 index 8b95e4c553..0000000000 --- a/changelog.d/8906.misc +++ /dev/null @@ -1 +0,0 @@ -Fix multiarch docker image builds. diff --git a/changelog.d/8909.misc b/changelog.d/8909.misc deleted file mode 100644 index b45972f0fa..0000000000 --- a/changelog.d/8909.misc +++ /dev/null @@ -1 +0,0 @@ -Don't publish `latest` docker image until all archs are built. diff --git a/changelog.d/8911.feature b/changelog.d/8911.feature deleted file mode 100644 index d450ef4998..0000000000 --- a/changelog.d/8911.feature +++ /dev/null @@ -1 +0,0 @@ -Add support for allowing users to pick their own user ID during a single-sign-on login. diff --git a/changelog.d/8918.bugfix b/changelog.d/8918.bugfix deleted file mode 100644 index ae0f6745d7..0000000000 --- a/changelog.d/8918.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix occasional deadlock when handling SIGHUP. diff --git a/changelog.d/8920.bugfix b/changelog.d/8920.bugfix deleted file mode 100644 index abcf186bda..0000000000 --- a/changelog.d/8920.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix login API to not ratelimit application services that have ratelimiting disabled. diff --git a/changelog.d/8921.bugfix b/changelog.d/8921.bugfix deleted file mode 100644 index 7f6f0b8a76..0000000000 --- a/changelog.d/8921.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug where we ratelimited auto joining of rooms on registration (using `auto_join_rooms` config). diff --git a/changelog.d/8929.misc b/changelog.d/8929.misc deleted file mode 100644 index 157018b6a6..0000000000 --- a/changelog.d/8929.misc +++ /dev/null @@ -1 +0,0 @@ -Automatically drop stale forward-extremities under some specific conditions. diff --git a/changelog.d/8930.feature b/changelog.d/8930.feature deleted file mode 100644 index cb305b5266..0000000000 --- a/changelog.d/8930.feature +++ /dev/null @@ -1 +0,0 @@ -Add an `email.invite_client_location` configuration option to send a web client location to the invite endpoint on the identity server which allows customisation of the email template. diff --git a/changelog.d/8931.feature b/changelog.d/8931.feature deleted file mode 100644 index 35c720eb8c..0000000000 --- a/changelog.d/8931.feature +++ /dev/null @@ -1 +0,0 @@ -Make search statement in List Room and List User Admin API case-insensitive. \ No newline at end of file diff --git a/changelog.d/8933.bugfix b/changelog.d/8933.bugfix deleted file mode 100644 index 295933d6cd..0000000000 --- a/changelog.d/8933.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug where deactivated users appeared in the user directory when their profile information was updated. diff --git a/changelog.d/8935.misc b/changelog.d/8935.misc deleted file mode 100644 index bf94135fd5..0000000000 --- a/changelog.d/8935.misc +++ /dev/null @@ -1 +0,0 @@ -Various clean-ups to the structured logging and logging context code. diff --git a/changelog.d/8937.bugfix b/changelog.d/8937.bugfix deleted file mode 100644 index 01e1848448..0000000000 --- a/changelog.d/8937.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug introduced in Synapse v1.24.0 which would cause an exception on startup if both `enabled` and `localdb_enabled` were set to `False` in the `password_config` setting of the configuration file. diff --git a/changelog.d/8938.feature b/changelog.d/8938.feature deleted file mode 100644 index d450ef4998..0000000000 --- a/changelog.d/8938.feature +++ /dev/null @@ -1 +0,0 @@ -Add support for allowing users to pick their own user ID during a single-sign-on login. diff --git a/changelog.d/8916.misc b/changelog.d/8939.misc similarity index 100% rename from changelog.d/8916.misc rename to changelog.d/8939.misc diff --git a/changelog.d/8940.misc b/changelog.d/8940.misc deleted file mode 100644 index 4ff0b94b94..0000000000 --- a/changelog.d/8940.misc +++ /dev/null @@ -1 +0,0 @@ -Add type hints to push module. diff --git a/changelog.d/8941.feature b/changelog.d/8941.feature deleted file mode 100644 index d450ef4998..0000000000 --- a/changelog.d/8941.feature +++ /dev/null @@ -1 +0,0 @@ -Add support for allowing users to pick their own user ID during a single-sign-on login. diff --git a/changelog.d/8942.feature b/changelog.d/8942.feature deleted file mode 100644 index d450ef4998..0000000000 --- a/changelog.d/8942.feature +++ /dev/null @@ -1 +0,0 @@ -Add support for allowing users to pick their own user ID during a single-sign-on login. diff --git a/changelog.d/8943.misc b/changelog.d/8943.misc deleted file mode 100644 index 4ff0b94b94..0000000000 --- a/changelog.d/8943.misc +++ /dev/null @@ -1 +0,0 @@ -Add type hints to push module. diff --git a/changelog.d/8945.bugfix b/changelog.d/8945.bugfix deleted file mode 100644 index f9e6dbba56..0000000000 --- a/changelog.d/8945.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug where 500 errors would be returned if the `m.room_history_visibility` event had invalid content. diff --git a/changelog.d/8946.misc b/changelog.d/8946.misc deleted file mode 100644 index 54502e9b90..0000000000 --- a/changelog.d/8946.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor test utilities for injecting HTTP requests. diff --git a/changelog.d/8950.misc b/changelog.d/8950.misc deleted file mode 100644 index 42e0335afc..0000000000 --- a/changelog.d/8950.misc +++ /dev/null @@ -1 +0,0 @@ -Add a maximum size of 50 kilobytes to .well-known lookups. diff --git a/changelog.d/8951.feature b/changelog.d/8951.feature deleted file mode 100644 index d450ef4998..0000000000 --- a/changelog.d/8951.feature +++ /dev/null @@ -1 +0,0 @@ -Add support for allowing users to pick their own user ID during a single-sign-on login. diff --git a/changelog.d/8952.misc b/changelog.d/8952.misc deleted file mode 100644 index 4c4a874649..0000000000 --- a/changelog.d/8952.misc +++ /dev/null @@ -1 +0,0 @@ -Fix bug in `generate_log_config` script which made it write empty files. diff --git a/changelog.d/8954.feature b/changelog.d/8954.feature deleted file mode 100644 index 39f53174ad..0000000000 --- a/changelog.d/8954.feature +++ /dev/null @@ -1 +0,0 @@ -Apply an IP range blacklist to push and key revocation requests. diff --git a/changelog.d/8958.misc b/changelog.d/8958.misc deleted file mode 100644 index 1507073e4f..0000000000 --- a/changelog.d/8958.misc +++ /dev/null @@ -1 +0,0 @@ -Properly store the mapping of external ID to Matrix ID for CAS users. diff --git a/changelog.d/8959.bugfix b/changelog.d/8959.bugfix deleted file mode 100644 index 772818bae9..0000000000 --- a/changelog.d/8959.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug causing common English words to not be considered for a user directory search. diff --git a/changelog.d/8962.bugfix b/changelog.d/8962.bugfix deleted file mode 100644 index af1a5e4c3a..0000000000 --- a/changelog.d/8962.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug where application services couldn't register new ghost users if the server had reached its MAU limit. diff --git a/changelog.d/8963.misc b/changelog.d/8963.misc deleted file mode 100644 index 495d89e8ee..0000000000 --- a/changelog.d/8963.misc +++ /dev/null @@ -1 +0,0 @@ -Clean up tox.ini file; disable coverage checking for non-test runs. diff --git a/changelog.d/8964.bugfix b/changelog.d/8964.bugfix deleted file mode 100644 index 295933d6cd..0000000000 --- a/changelog.d/8964.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug where deactivated users appeared in the user directory when their profile information was updated. diff --git a/changelog.d/8965.bugfix b/changelog.d/8965.bugfix deleted file mode 100644 index cbccebddb5..0000000000 --- a/changelog.d/8965.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a longstanding bug where a `m.image` event without a `url` would cause errors on push. diff --git a/changelog.d/8970.feature b/changelog.d/8970.feature deleted file mode 100644 index 6d5b3303a6..0000000000 --- a/changelog.d/8970.feature +++ /dev/null @@ -1 +0,0 @@ -Allow re-using an user-interactive authentication session for a period of time. diff --git a/changelog.d/8971.bugfix b/changelog.d/8971.bugfix deleted file mode 100644 index c3e44b8c0b..0000000000 --- a/changelog.d/8971.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix small bug in v2 state resolution algorithm, which could also cause performance issues for rooms with large numbers of power levels. diff --git a/changelog.d/8973.misc b/changelog.d/8973.misc deleted file mode 100644 index be744f5dc7..0000000000 --- a/changelog.d/8973.misc +++ /dev/null @@ -1 +0,0 @@ -Add type hints to the admin and room list handlers. diff --git a/changelog.d/8975.bugfix b/changelog.d/8975.bugfix deleted file mode 100644 index 75049b8e18..0000000000 --- a/changelog.d/8975.bugfix +++ /dev/null @@ -1 +0,0 @@ -Add validation to the `sendToDevice` API to raise a missing parameters error instead of a 500 error. diff --git a/changelog.d/8976.misc b/changelog.d/8976.misc deleted file mode 100644 index 4f4b083b1e..0000000000 --- a/changelog.d/8976.misc +++ /dev/null @@ -1 +0,0 @@ -Add type hints to the receipts and user directory handlers. diff --git a/changelog.d/8977.bugfix b/changelog.d/8977.bugfix deleted file mode 100644 index ae0b6bec14..0000000000 --- a/changelog.d/8977.bugfix +++ /dev/null @@ -1 +0,0 @@ -Properly return 400 errors on invalid group IDs. diff --git a/changelog.d/8979.misc b/changelog.d/8979.misc deleted file mode 100644 index 670821cf90..0000000000 --- a/changelog.d/8979.misc +++ /dev/null @@ -1 +0,0 @@ -Drop the unused `local_invites` table. diff --git a/changelog.d/8980.misc b/changelog.d/8980.misc deleted file mode 100644 index 83ef3c5def..0000000000 --- a/changelog.d/8980.misc +++ /dev/null @@ -1 +0,0 @@ -Add type hints to the base storage code. diff --git a/changelog.d/8984.feature b/changelog.d/8984.feature new file mode 100644 index 0000000000..4db629746e --- /dev/null +++ b/changelog.d/8984.feature @@ -0,0 +1 @@ +Implement [MSC2176](https://github.com/matrix-org/matrix-doc/pull/2176) in an experimental room version. diff --git a/changelog.d/8986.misc b/changelog.d/8986.misc deleted file mode 100644 index 6aefc78784..0000000000 --- a/changelog.d/8986.misc +++ /dev/null @@ -1 +0,0 @@ -Support using PyJWT v2.0.0 in the test suite. diff --git a/changelog.d/8987.doc b/changelog.d/8987.doc deleted file mode 100644 index c6e4932729..0000000000 --- a/changelog.d/8987.doc +++ /dev/null @@ -1 +0,0 @@ -Moved instructions for database setup, adjusted heading levels and improved syntax highlighting in [INSTALL.md](../INSTALL.md). Contributed by fossterer. diff --git a/changelog.d/8992.doc b/changelog.d/8992.doc deleted file mode 100644 index 6a47bda26b..0000000000 --- a/changelog.d/8992.doc +++ /dev/null @@ -1 +0,0 @@ -Update the example value of `group_creation_prefix` in the sample configuration. diff --git a/changelog.d/8994.feature b/changelog.d/8994.feature deleted file mode 100644 index 76aeb185cb..0000000000 --- a/changelog.d/8994.feature +++ /dev/null @@ -1 +0,0 @@ -Allow running the redact endpoint on workers. diff --git a/changelog.d/8998.misc b/changelog.d/8998.misc deleted file mode 100644 index 81346694bd..0000000000 --- a/changelog.d/8998.misc +++ /dev/null @@ -1 +0,0 @@ -Fix `tests.federation.transport.RoomDirectoryFederationTests` and ensure it runs in CI. \ No newline at end of file diff --git a/changelog.d/8999.misc b/changelog.d/8999.misc deleted file mode 100644 index 3987204f06..0000000000 --- a/changelog.d/8999.misc +++ /dev/null @@ -1 +0,0 @@ -Add type hints to the crypto module. diff --git a/changelog.d/9002.doc b/changelog.d/9002.doc deleted file mode 100644 index 26928c9a93..0000000000 --- a/changelog.d/9002.doc +++ /dev/null @@ -1 +0,0 @@ -Link the Synapse developer room to the development section in the docs. diff --git a/changelog.d/9015.feature b/changelog.d/9015.feature new file mode 100644 index 0000000000..01a24dcf49 --- /dev/null +++ b/changelog.d/9015.feature @@ -0,0 +1 @@ +Add support for multiple SSO Identity Providers. diff --git a/changelog.d/9017.feature b/changelog.d/9017.feature new file mode 100644 index 0000000000..01a24dcf49 --- /dev/null +++ b/changelog.d/9017.feature @@ -0,0 +1 @@ +Add support for multiple SSO Identity Providers. diff --git a/changelog.d/9018.misc b/changelog.d/9018.misc new file mode 100644 index 0000000000..bb31eb4a46 --- /dev/null +++ b/changelog.d/9018.misc @@ -0,0 +1 @@ +Ignore date-rotated homeserver logs saved to disk. diff --git a/changelog.d/9020.misc b/changelog.d/9020.misc deleted file mode 100644 index 4ff0b94b94..0000000000 --- a/changelog.d/9020.misc +++ /dev/null @@ -1 +0,0 @@ -Add type hints to push module. diff --git a/changelog.d/9023.bugfix b/changelog.d/9023.bugfix new file mode 100644 index 0000000000..deae64d933 --- /dev/null +++ b/changelog.d/9023.bugfix @@ -0,0 +1 @@ +Fix a longstanding issue where an internal server error would occur when requesting a profile over federation that did not include a display name / avatar URL. diff --git a/changelog.d/9028.bugfix b/changelog.d/9028.bugfix new file mode 100644 index 0000000000..66666886a4 --- /dev/null +++ b/changelog.d/9028.bugfix @@ -0,0 +1 @@ +Fix a long-standing bug where some caches could grow larger than configured. diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index dd981609ac..c8ae46d1b3 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -1909,6 +1909,31 @@ sso: # # Synapse will look for the following templates in this directory: # + # * HTML page to prompt the user to choose an Identity Provider during + # login: 'sso_login_idp_picker.html'. + # + # This is only used if multiple SSO Identity Providers are configured. + # + # When rendering, this template is given the following variables: + # * redirect_url: the URL that the user will be redirected to after + # login. Needs manual escaping (see + # https://jinja.palletsprojects.com/en/2.11.x/templates/#html-escaping). + # + # * server_name: the homeserver's name. + # + # * providers: a list of available Identity Providers. Each element is + # an object with the following attributes: + # * idp_id: unique identifier for the IdP + # * idp_name: user-facing name for the IdP + # + # The rendered HTML page should contain a form which submits its results + # back as a GET request, with the following query parameters: + # + # * redirectUrl: the client redirect URI (ie, the `redirect_url` passed + # to the template) + # + # * idp: the 'idp_id' of the chosen IDP. + # # * HTML page for a confirmation step before redirecting back to the client # with the login token: 'sso_redirect_confirm.html'. # diff --git a/synapse/__init__.py b/synapse/__init__.py index f2d3ac68eb..99fb675748 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -48,7 +48,7 @@ try: except ImportError: pass -__version__ = "1.24.0" +__version__ = "1.25.0rc1" if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)): # We import here so that we don't have to install a bunch of deps when diff --git a/synapse/api/room_versions.py b/synapse/api/room_versions.py index f3ecbf36b6..de2cc15d33 100644 --- a/synapse/api/room_versions.py +++ b/synapse/api/room_versions.py @@ -51,11 +51,11 @@ class RoomDisposition: class RoomVersion: """An object which describes the unique attributes of a room version.""" - identifier = attr.ib() # str; the identifier for this version - disposition = attr.ib() # str; one of the RoomDispositions - event_format = attr.ib() # int; one of the EventFormatVersions - state_res = attr.ib() # int; one of the StateResolutionVersions - enforce_key_validity = attr.ib() # bool + identifier = attr.ib(type=str) # the identifier for this version + disposition = attr.ib(type=str) # one of the RoomDispositions + event_format = attr.ib(type=int) # one of the EventFormatVersions + state_res = attr.ib(type=int) # one of the StateResolutionVersions + enforce_key_validity = attr.ib(type=bool) # bool: before MSC2261/MSC2432, m.room.aliases had special auth rules and redaction rules special_case_aliases_auth = attr.ib(type=bool) @@ -64,9 +64,11 @@ class RoomVersion: # * Floats # * NaN, Infinity, -Infinity strict_canonicaljson = attr.ib(type=bool) - # bool: MSC2209: Check 'notifications' key while verifying + # MSC2209: Check 'notifications' key while verifying # m.room.power_levels auth rules. limit_notifications_power_levels = attr.ib(type=bool) + # MSC2174/MSC2176: Apply updated redaction rules algorithm. + msc2176_redaction_rules = attr.ib(type=bool) class RoomVersions: @@ -79,6 +81,7 @@ class RoomVersions: special_case_aliases_auth=True, strict_canonicaljson=False, limit_notifications_power_levels=False, + msc2176_redaction_rules=False, ) V2 = RoomVersion( "2", @@ -89,6 +92,7 @@ class RoomVersions: special_case_aliases_auth=True, strict_canonicaljson=False, limit_notifications_power_levels=False, + msc2176_redaction_rules=False, ) V3 = RoomVersion( "3", @@ -99,6 +103,7 @@ class RoomVersions: special_case_aliases_auth=True, strict_canonicaljson=False, limit_notifications_power_levels=False, + msc2176_redaction_rules=False, ) V4 = RoomVersion( "4", @@ -109,6 +114,7 @@ class RoomVersions: special_case_aliases_auth=True, strict_canonicaljson=False, limit_notifications_power_levels=False, + msc2176_redaction_rules=False, ) V5 = RoomVersion( "5", @@ -119,6 +125,7 @@ class RoomVersions: special_case_aliases_auth=True, strict_canonicaljson=False, limit_notifications_power_levels=False, + msc2176_redaction_rules=False, ) V6 = RoomVersion( "6", @@ -129,6 +136,18 @@ class RoomVersions: special_case_aliases_auth=False, strict_canonicaljson=True, limit_notifications_power_levels=True, + msc2176_redaction_rules=False, + ) + MSC2176 = RoomVersion( + "org.matrix.msc2176", + RoomDisposition.UNSTABLE, + EventFormatVersions.V3, + StateResolutionVersions.V2, + enforce_key_validity=True, + special_case_aliases_auth=False, + strict_canonicaljson=True, + limit_notifications_power_levels=True, + msc2176_redaction_rules=True, ) @@ -141,5 +160,6 @@ KNOWN_ROOM_VERSIONS = { RoomVersions.V4, RoomVersions.V5, RoomVersions.V6, + RoomVersions.MSC2176, ) } # type: Dict[str, RoomVersion] diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 8d9b53be53..b1d9817a6a 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -63,6 +63,7 @@ from synapse.rest import ClientRestResource from synapse.rest.admin import AdminRestResource from synapse.rest.health import HealthResource from synapse.rest.key.v2 import KeyApiV2Resource +from synapse.rest.synapse.client.pick_idp import PickIdpResource from synapse.rest.synapse.client.pick_username import pick_username_resource from synapse.rest.well_known import WellKnownResource from synapse.server import HomeServer @@ -194,6 +195,7 @@ class SynapseHomeServer(HomeServer): "/.well-known/matrix/client": WellKnownResource(self), "/_synapse/admin": AdminRestResource(self), "/_synapse/client/pick_username": pick_username_resource(self), + "/_synapse/client/pick_idp": PickIdpResource(self), } ) diff --git a/synapse/config/sso.py b/synapse/config/sso.py index 93bbd40937..1aeb1c5c92 100644 --- a/synapse/config/sso.py +++ b/synapse/config/sso.py @@ -31,6 +31,7 @@ class SSOConfig(Config): # Read templates from disk ( + self.sso_login_idp_picker_template, self.sso_redirect_confirm_template, self.sso_auth_confirm_template, self.sso_error_template, @@ -38,6 +39,7 @@ class SSOConfig(Config): sso_auth_success_template, ) = self.read_templates( [ + "sso_login_idp_picker.html", "sso_redirect_confirm.html", "sso_auth_confirm.html", "sso_error.html", @@ -98,6 +100,31 @@ class SSOConfig(Config): # # Synapse will look for the following templates in this directory: # + # * HTML page to prompt the user to choose an Identity Provider during + # login: 'sso_login_idp_picker.html'. + # + # This is only used if multiple SSO Identity Providers are configured. + # + # When rendering, this template is given the following variables: + # * redirect_url: the URL that the user will be redirected to after + # login. Needs manual escaping (see + # https://jinja.palletsprojects.com/en/2.11.x/templates/#html-escaping). + # + # * server_name: the homeserver's name. + # + # * providers: a list of available Identity Providers. Each element is + # an object with the following attributes: + # * idp_id: unique identifier for the IdP + # * idp_name: user-facing name for the IdP + # + # The rendered HTML page should contain a form which submits its results + # back as a GET request, with the following query parameters: + # + # * redirectUrl: the client redirect URI (ie, the `redirect_url` passed + # to the template) + # + # * idp: the 'idp_id' of the chosen IDP. + # # * HTML page for a confirmation step before redirecting back to the client # with the login token: 'sso_redirect_confirm.html'. # diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 14f7f1156f..9c22e33813 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -79,13 +79,15 @@ def prune_event_dict(room_version: RoomVersion, event_dict: dict) -> dict: "state_key", "depth", "prev_events", - "prev_state", "auth_events", "origin", "origin_server_ts", - "membership", ] + # Room versions from before MSC2176 had additional allowed keys. + if not room_version.msc2176_redaction_rules: + allowed_keys.extend(["prev_state", "membership"]) + event_type = event_dict["type"] new_content = {} @@ -98,6 +100,10 @@ def prune_event_dict(room_version: RoomVersion, event_dict: dict) -> dict: if event_type == EventTypes.Member: add_fields("membership") elif event_type == EventTypes.Create: + # MSC2176 rules state that create events cannot be redacted. + if room_version.msc2176_redaction_rules: + return event_dict + add_fields("creator") elif event_type == EventTypes.JoinRules: add_fields("join_rule") @@ -112,10 +118,16 @@ def prune_event_dict(room_version: RoomVersion, event_dict: dict) -> dict: "kick", "redact", ) + + if room_version.msc2176_redaction_rules: + add_fields("invite") + elif event_type == EventTypes.Aliases and room_version.special_case_aliases_auth: add_fields("aliases") elif event_type == EventTypes.RoomHistoryVisibility: add_fields("history_visibility") + elif event_type == EventTypes.Redaction and room_version.msc2176_redaction_rules: + add_fields("redacts") allowed_fields = {k: v for k, v in event_dict.items() if k in allowed_keys} diff --git a/synapse/handlers/cas_handler.py b/synapse/handlers/cas_handler.py index fca210a5a6..f3430c6713 100644 --- a/synapse/handlers/cas_handler.py +++ b/synapse/handlers/cas_handler.py @@ -75,10 +75,15 @@ class CasHandler: self._http_client = hs.get_proxied_http_client() # identifier for the external_ids table - self._auth_provider_id = "cas" + self.idp_id = "cas" + + # user-facing name of this auth provider + self.idp_name = "CAS" self._sso_handler = hs.get_sso_handler() + self._sso_handler.register_identity_provider(self) + def _build_service_param(self, args: Dict[str, str]) -> str: """ Generates a value to use as the "service" parameter when redirecting or @@ -105,7 +110,7 @@ class CasHandler: Args: ticket: The CAS ticket from the client. service_args: Additional arguments to include in the service URL. - Should be the same as those passed to `get_redirect_url`. + Should be the same as those passed to `handle_redirect_request`. Raises: CasError: If there's an error parsing the CAS response. @@ -184,16 +189,31 @@ class CasHandler: return CasResponse(user, attributes) - def get_redirect_url(self, service_args: Dict[str, str]) -> str: - """ - Generates a URL for the CAS server where the client should be redirected. + async def handle_redirect_request( + self, + request: SynapseRequest, + client_redirect_url: Optional[bytes], + ui_auth_session_id: Optional[str] = None, + ) -> str: + """Generates a URL for the CAS server where the client should be redirected. Args: - service_args: Additional arguments to include in the final redirect URL. + request: the incoming HTTP request + client_redirect_url: the URL that we should redirect the + client to after login (or None for UI Auth). + ui_auth_session_id: The session ID of the ongoing UI Auth (or + None if this is a login). Returns: - The URL to redirect the client to. + URL to redirect to """ + + if ui_auth_session_id: + service_args = {"session": ui_auth_session_id} + else: + assert client_redirect_url + service_args = {"redirectUrl": client_redirect_url.decode("utf8")} + args = urllib.parse.urlencode( {"service": self._build_service_param(service_args)} ) @@ -275,7 +295,7 @@ class CasHandler: # first check if we're doing a UIA if session: return await self._sso_handler.complete_sso_ui_auth_request( - self._auth_provider_id, cas_response.username, session, request, + self.idp_id, cas_response.username, session, request, ) # otherwise, we're handling a login request. @@ -375,7 +395,7 @@ class CasHandler: return None await self._sso_handler.complete_sso_login_request( - self._auth_provider_id, + self.idp_id, cas_response.username, request, client_redirect_url, diff --git a/synapse/handlers/oidc_handler.py b/synapse/handlers/oidc_handler.py index 709f8dfc13..6835c6c462 100644 --- a/synapse/handlers/oidc_handler.py +++ b/synapse/handlers/oidc_handler.py @@ -119,10 +119,15 @@ class OidcHandler(BaseHandler): self._macaroon_secret_key = hs.config.macaroon_secret_key # identifier for the external_ids table - self._auth_provider_id = "oidc" + self.idp_id = "oidc" + + # user-facing name of this auth provider + self.idp_name = "OIDC" self._sso_handler = hs.get_sso_handler() + self._sso_handler.register_identity_provider(self) + def _validate_metadata(self): """Verifies the provider metadata. @@ -475,7 +480,7 @@ class OidcHandler(BaseHandler): async def handle_redirect_request( self, request: SynapseRequest, - client_redirect_url: bytes, + client_redirect_url: Optional[bytes], ui_auth_session_id: Optional[str] = None, ) -> str: """Handle an incoming request to /login/sso/redirect @@ -499,7 +504,7 @@ class OidcHandler(BaseHandler): request: the incoming request from the browser. We'll respond to it with a redirect and a cookie. client_redirect_url: the URL that we should redirect the client to - when everything is done + when everything is done (or None for UI Auth) ui_auth_session_id: The session ID of the ongoing UI Auth (or None if this is a login). @@ -511,6 +516,9 @@ class OidcHandler(BaseHandler): state = generate_token() nonce = generate_token() + if not client_redirect_url: + client_redirect_url = b"" + cookie = self._generate_oidc_session_token( state=state, nonce=nonce, @@ -682,7 +690,7 @@ class OidcHandler(BaseHandler): return return await self._sso_handler.complete_sso_ui_auth_request( - self._auth_provider_id, remote_user_id, ui_auth_session_id, request + self.idp_id, remote_user_id, ui_auth_session_id, request ) # otherwise, it's a login @@ -923,7 +931,7 @@ class OidcHandler(BaseHandler): extra_attributes = await get_extra_attributes(userinfo, token) await self._sso_handler.complete_sso_login_request( - self._auth_provider_id, + self.idp_id, remote_user_id, request, client_redirect_url, diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index dee0ef45e7..36f9ee4b71 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -156,7 +156,7 @@ class ProfileHandler(BaseHandler): except HttpResponseException as e: raise e.to_synapse_error() - return result["displayname"] + return result.get("displayname") async def set_displayname( self, @@ -246,7 +246,7 @@ class ProfileHandler(BaseHandler): except HttpResponseException as e: raise e.to_synapse_error() - return result["avatar_url"] + return result.get("avatar_url") async def set_avatar_url( self, diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 1f809fa161..3bece6d668 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -365,7 +365,7 @@ class RoomCreationHandler(BaseHandler): creation_content = { "room_version": new_room_version.identifier, "predecessor": {"room_id": old_room_id, "event_id": tombstone_event_id}, - } + } # type: JsonDict # Check if old room was non-federatable diff --git a/synapse/handlers/saml_handler.py b/synapse/handlers/saml_handler.py index 5fa7ab3f8b..a8376543c9 100644 --- a/synapse/handlers/saml_handler.py +++ b/synapse/handlers/saml_handler.py @@ -73,27 +73,41 @@ class SamlHandler(BaseHandler): ) # identifier for the external_ids table - self._auth_provider_id = "saml" + self.idp_id = "saml" + + # user-facing name of this auth provider + self.idp_name = "SAML" # a map from saml session id to Saml2SessionData object self._outstanding_requests_dict = {} # type: Dict[str, Saml2SessionData] self._sso_handler = hs.get_sso_handler() + self._sso_handler.register_identity_provider(self) - def handle_redirect_request( - self, client_redirect_url: bytes, ui_auth_session_id: Optional[str] = None - ) -> bytes: + async def handle_redirect_request( + self, + request: SynapseRequest, + client_redirect_url: Optional[bytes], + ui_auth_session_id: Optional[str] = None, + ) -> str: """Handle an incoming request to /login/sso/redirect Args: + request: the incoming HTTP request client_redirect_url: the URL that we should redirect the - client to when everything is done + client to after login (or None for UI Auth). ui_auth_session_id: The session ID of the ongoing UI Auth (or None if this is a login). Returns: URL to redirect to """ + if not client_redirect_url: + # Some SAML identity providers (e.g. Google) require a + # RelayState parameter on requests, so pass in a dummy redirect URL + # (which will never get used). + client_redirect_url = b"unused" + reqid, info = self._saml_client.prepare_for_authenticate( entityid=self._saml_idp_entityid, relay_state=client_redirect_url ) @@ -210,7 +224,7 @@ class SamlHandler(BaseHandler): return return await self._sso_handler.complete_sso_ui_auth_request( - self._auth_provider_id, + self.idp_id, remote_user_id, current_session.ui_auth_session_id, request, @@ -306,7 +320,7 @@ class SamlHandler(BaseHandler): return None await self._sso_handler.complete_sso_login_request( - self._auth_provider_id, + self.idp_id, remote_user_id, request, client_redirect_url, diff --git a/synapse/handlers/sso.py b/synapse/handlers/sso.py index 33cd6bc178..2da1ea2223 100644 --- a/synapse/handlers/sso.py +++ b/synapse/handlers/sso.py @@ -12,15 +12,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import abc import logging -from typing import TYPE_CHECKING, Awaitable, Callable, Dict, List, Optional +from typing import TYPE_CHECKING, Awaitable, Callable, Dict, List, Mapping, Optional +from urllib.parse import urlencode import attr -from typing_extensions import NoReturn +from typing_extensions import NoReturn, Protocol from twisted.web.http import Request -from synapse.api.errors import RedirectException, SynapseError +from synapse.api.errors import Codes, RedirectException, SynapseError from synapse.http.server import respond_with_html from synapse.http.site import SynapseRequest from synapse.types import JsonDict, UserID, contains_invalid_mxid_characters @@ -40,6 +42,58 @@ class MappingException(Exception): """ +class SsoIdentityProvider(Protocol): + """Abstract base class to be implemented by SSO Identity Providers + + An Identity Provider, or IdP, is an external HTTP service which authenticates a user + to say whether they should be allowed to log in, or perform a given action. + + Synapse supports various implementations of IdPs, including OpenID Connect, SAML, + and CAS. + + The main entry point is `handle_redirect_request`, which should return a URI to + redirect the user's browser to the IdP's authentication page. + + Each IdP should be registered with the SsoHandler via + `hs.get_sso_handler().register_identity_provider()`, so that requests to + `/_matrix/client/r0/login/sso/redirect` can be correctly dispatched. + """ + + @property + @abc.abstractmethod + def idp_id(self) -> str: + """A unique identifier for this SSO provider + + Eg, "saml", "cas", "github" + """ + + @property + @abc.abstractmethod + def idp_name(self) -> str: + """User-facing name for this provider""" + + @abc.abstractmethod + async def handle_redirect_request( + self, + request: SynapseRequest, + client_redirect_url: Optional[bytes], + ui_auth_session_id: Optional[str] = None, + ) -> str: + """Handle an incoming request to /login/sso/redirect + + Args: + request: the incoming HTTP request + client_redirect_url: the URL that we should redirect the + client to after login (or None for UI Auth). + ui_auth_session_id: The session ID of the ongoing UI Auth (or + None if this is a login). + + Returns: + URL to redirect to + """ + raise NotImplementedError() + + @attr.s class UserAttributes: # the localpart of the mxid that the mapper has assigned to the user. @@ -100,6 +154,18 @@ class SsoHandler: # a map from session id to session data self._username_mapping_sessions = {} # type: Dict[str, UsernameMappingSession] + # map from idp_id to SsoIdentityProvider + self._identity_providers = {} # type: Dict[str, SsoIdentityProvider] + + def register_identity_provider(self, p: SsoIdentityProvider): + p_id = p.idp_id + assert p_id not in self._identity_providers + self._identity_providers[p_id] = p + + def get_identity_providers(self) -> Mapping[str, SsoIdentityProvider]: + """Get the configured identity providers""" + return self._identity_providers + def render_error( self, request: Request, @@ -124,6 +190,34 @@ class SsoHandler: ) respond_with_html(request, code, html) + async def handle_redirect_request( + self, request: SynapseRequest, client_redirect_url: bytes, + ) -> str: + """Handle a request to /login/sso/redirect + + Args: + request: incoming HTTP request + client_redirect_url: the URL that we should redirect the + client to after login. + + Returns: + the URI to redirect to + """ + if not self._identity_providers: + raise SynapseError( + 400, "Homeserver not configured for SSO.", errcode=Codes.UNRECOGNIZED + ) + + # if we only have one auth provider, redirect to it directly + if len(self._identity_providers) == 1: + ap = next(iter(self._identity_providers.values())) + return await ap.handle_redirect_request(request, client_redirect_url) + + # otherwise, redirect to the IDP picker + return "/_synapse/client/pick_idp?" + urlencode( + (("redirectUrl", client_redirect_url),) + ) + async def get_sso_user_by_remote_user_id( self, auth_provider_id: str, remote_user_id: str ) -> Optional[str]: diff --git a/synapse/logging/context.py b/synapse/logging/context.py index a507a83e93..c2db8b45f3 100644 --- a/synapse/logging/context.py +++ b/synapse/logging/context.py @@ -252,7 +252,12 @@ class LoggingContext: "scope", ] - def __init__(self, name=None, parent_context=None, request=None) -> None: + def __init__( + self, + name: Optional[str] = None, + parent_context: "Optional[LoggingContext]" = None, + request: Optional[str] = None, + ) -> None: self.previous_context = current_context() self.name = name @@ -536,20 +541,20 @@ class LoggingContextFilter(logging.Filter): def __init__(self, request: str = ""): self._default_request = request - def filter(self, record) -> Literal[True]: + def filter(self, record: logging.LogRecord) -> Literal[True]: """Add each fields from the logging contexts to the record. Returns: True to include the record in the log output. """ context = current_context() - record.request = self._default_request + record.request = self._default_request # type: ignore # context should never be None, but if it somehow ends up being, then # we end up in a death spiral of infinite loops, so let's check, for # robustness' sake. if context is not None: # Logging is interested in the request. - record.request = context.request + record.request = context.request # type: ignore return True @@ -616,9 +621,7 @@ def set_current_context(context: LoggingContextOrSentinel) -> LoggingContextOrSe return current -def nested_logging_context( - suffix: str, parent_context: Optional[LoggingContext] = None -) -> LoggingContext: +def nested_logging_context(suffix: str) -> LoggingContext: """Creates a new logging context as a child of another. The nested logging context will have a 'request' made up of the parent context's @@ -632,20 +635,23 @@ def nested_logging_context( # ... do stuff Args: - suffix (str): suffix to add to the parent context's 'request'. - parent_context (LoggingContext|None): parent context. Will use the current context - if None. + suffix: suffix to add to the parent context's 'request'. Returns: LoggingContext: new logging context. """ - if parent_context is not None: - context = parent_context # type: LoggingContextOrSentinel + curr_context = current_context() + if not curr_context: + logger.warning( + "Starting nested logging context from sentinel context: metrics will be lost" + ) + parent_context = None + prefix = "" else: - context = current_context() - return LoggingContext( - parent_context=context, request=str(context.request) + "-" + suffix - ) + assert isinstance(curr_context, LoggingContext) + parent_context = curr_context + prefix = str(parent_context.request) + return LoggingContext(parent_context=parent_context, request=prefix + "-" + suffix) def preserve_fn(f): @@ -822,10 +828,18 @@ def defer_to_threadpool(reactor, threadpool, f, *args, **kwargs): Deferred: A Deferred which fires a callback with the result of `f`, or an errback if `f` throws an exception. """ - logcontext = current_context() + curr_context = current_context() + if not curr_context: + logger.warning( + "Calling defer_to_threadpool from sentinel context: metrics will be lost" + ) + parent_context = None + else: + assert isinstance(curr_context, LoggingContext) + parent_context = curr_context def g(): - with LoggingContext(parent_context=logcontext): + with LoggingContext(parent_context=parent_context): return f(*args, **kwargs) return make_deferred_yieldable(threads.deferToThreadPool(reactor, threadpool, g)) diff --git a/synapse/res/templates/sso_login_idp_picker.html b/synapse/res/templates/sso_login_idp_picker.html new file mode 100644 index 0000000000..f53c9cd679 --- /dev/null +++ b/synapse/res/templates/sso_login_idp_picker.html @@ -0,0 +1,28 @@ + + + + + + {{server_name | e}} Login + + +
+

{{server_name | e}} Login

+ +
+ + diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index 5f4c6703db..ebc346105b 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -311,48 +311,31 @@ class LoginRestServlet(RestServlet): return result -class BaseSSORedirectServlet(RestServlet): - """Common base class for /login/sso/redirect impls""" - +class SsoRedirectServlet(RestServlet): PATTERNS = client_patterns("/login/(cas|sso)/redirect", v1=True) + def __init__(self, hs: "HomeServer"): + # make sure that the relevant handlers are instantiated, so that they + # register themselves with the main SSOHandler. + if hs.config.cas_enabled: + hs.get_cas_handler() + elif hs.config.saml2_enabled: + hs.get_saml_handler() + elif hs.config.oidc_enabled: + hs.get_oidc_handler() + self._sso_handler = hs.get_sso_handler() + async def on_GET(self, request: SynapseRequest): - args = request.args - if b"redirectUrl" not in args: - return 400, "Redirect URL not specified for SSO auth" - client_redirect_url = args[b"redirectUrl"][0] - sso_url = await self.get_sso_url(request, client_redirect_url) + client_redirect_url = parse_string( + request, "redirectUrl", required=True, encoding=None + ) + sso_url = await self._sso_handler.handle_redirect_request( + request, client_redirect_url + ) + logger.info("Redirecting to %s", sso_url) request.redirect(sso_url) finish_request(request) - async def get_sso_url( - self, request: SynapseRequest, client_redirect_url: bytes - ) -> bytes: - """Get the URL to redirect to, to perform SSO auth - - Args: - request: The client request to redirect. - client_redirect_url: the URL that we should redirect the - client to when everything is done - - Returns: - URL to redirect to - """ - # to be implemented by subclasses - raise NotImplementedError() - - -class CasRedirectServlet(BaseSSORedirectServlet): - def __init__(self, hs): - self._cas_handler = hs.get_cas_handler() - - async def get_sso_url( - self, request: SynapseRequest, client_redirect_url: bytes - ) -> bytes: - return self._cas_handler.get_redirect_url( - {"redirectUrl": client_redirect_url} - ).encode("ascii") - class CasTicketServlet(RestServlet): PATTERNS = client_patterns("/login/cas/ticket", v1=True) @@ -379,40 +362,8 @@ class CasTicketServlet(RestServlet): ) -class SAMLRedirectServlet(BaseSSORedirectServlet): - PATTERNS = client_patterns("/login/sso/redirect", v1=True) - - def __init__(self, hs): - self._saml_handler = hs.get_saml_handler() - - async def get_sso_url( - self, request: SynapseRequest, client_redirect_url: bytes - ) -> bytes: - return self._saml_handler.handle_redirect_request(client_redirect_url) - - -class OIDCRedirectServlet(BaseSSORedirectServlet): - """Implementation for /login/sso/redirect for the OIDC login flow.""" - - PATTERNS = client_patterns("/login/sso/redirect", v1=True) - - def __init__(self, hs): - self._oidc_handler = hs.get_oidc_handler() - - async def get_sso_url( - self, request: SynapseRequest, client_redirect_url: bytes - ) -> bytes: - return await self._oidc_handler.handle_redirect_request( - request, client_redirect_url - ) - - def register_servlets(hs, http_server): LoginRestServlet(hs).register(http_server) + SsoRedirectServlet(hs).register(http_server) if hs.config.cas_enabled: - CasRedirectServlet(hs).register(http_server) CasTicketServlet(hs).register(http_server) - elif hs.config.saml2_enabled: - SAMLRedirectServlet(hs).register(http_server) - elif hs.config.oidc_enabled: - OIDCRedirectServlet(hs).register(http_server) diff --git a/synapse/rest/client/v2_alpha/auth.py b/synapse/rest/client/v2_alpha/auth.py index fab077747f..9b9514632f 100644 --- a/synapse/rest/client/v2_alpha/auth.py +++ b/synapse/rest/client/v2_alpha/auth.py @@ -14,15 +14,20 @@ # limitations under the License. import logging +from typing import TYPE_CHECKING from synapse.api.constants import LoginType from synapse.api.errors import SynapseError from synapse.api.urls import CLIENT_API_PREFIX +from synapse.handlers.sso import SsoIdentityProvider from synapse.http.server import respond_with_html from synapse.http.servlet import RestServlet, parse_string from ._base import client_patterns +if TYPE_CHECKING: + from synapse.server import HomeServer + logger = logging.getLogger(__name__) @@ -35,7 +40,7 @@ class AuthRestServlet(RestServlet): PATTERNS = client_patterns(r"/auth/(?P[\w\.]*)/fallback/web") - def __init__(self, hs): + def __init__(self, hs: "HomeServer"): super().__init__() self.hs = hs self.auth = hs.get_auth() @@ -85,31 +90,20 @@ class AuthRestServlet(RestServlet): elif stagetype == LoginType.SSO: # Display a confirmation page which prompts the user to # re-authenticate with their SSO provider. + if self._cas_enabled: - # Generate a request to CAS that redirects back to an endpoint - # to verify the successful authentication. - sso_redirect_url = self._cas_handler.get_redirect_url( - {"session": session}, - ) - + sso_auth_provider = self._cas_handler # type: SsoIdentityProvider elif self._saml_enabled: - # Some SAML identity providers (e.g. Google) require a - # RelayState parameter on requests. It is not necessary here, so - # pass in a dummy redirect URL (which will never get used). - client_redirect_url = b"unused" - sso_redirect_url = self._saml_handler.handle_redirect_request( - client_redirect_url, session - ) - + sso_auth_provider = self._saml_handler elif self._oidc_enabled: - client_redirect_url = b"" - sso_redirect_url = await self._oidc_handler.handle_redirect_request( - request, client_redirect_url, session - ) - + sso_auth_provider = self._oidc_handler else: raise SynapseError(400, "Homeserver not configured for SSO.") + sso_redirect_url = await sso_auth_provider.handle_redirect_request( + request, None, session + ) + html = await self.auth_handler.start_sso_ui_auth(sso_redirect_url, session) else: diff --git a/synapse/rest/synapse/client/pick_idp.py b/synapse/rest/synapse/client/pick_idp.py new file mode 100644 index 0000000000..e5b720bbca --- /dev/null +++ b/synapse/rest/synapse/client/pick_idp.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +# Copyright 2021 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +from typing import TYPE_CHECKING + +from synapse.http.server import ( + DirectServeHtmlResource, + finish_request, + respond_with_html, +) +from synapse.http.servlet import parse_string +from synapse.http.site import SynapseRequest + +if TYPE_CHECKING: + from synapse.server import HomeServer + +logger = logging.getLogger(__name__) + + +class PickIdpResource(DirectServeHtmlResource): + """IdP picker resource. + + This resource gets mounted under /_synapse/client/pick_idp. It serves an HTML page + which prompts the user to choose an Identity Provider from the list. + """ + + def __init__(self, hs: "HomeServer"): + super().__init__() + self._sso_handler = hs.get_sso_handler() + self._sso_login_idp_picker_template = ( + hs.config.sso.sso_login_idp_picker_template + ) + self._server_name = hs.hostname + + async def _async_render_GET(self, request: SynapseRequest) -> None: + client_redirect_url = parse_string(request, "redirectUrl", required=True) + idp = parse_string(request, "idp", required=False) + + # if we need to pick an IdP, do so + if not idp: + return await self._serve_id_picker(request, client_redirect_url) + + # otherwise, redirect to the IdP's redirect URI + providers = self._sso_handler.get_identity_providers() + auth_provider = providers.get(idp) + if not auth_provider: + logger.info("Unknown idp %r", idp) + self._sso_handler.render_error( + request, "unknown_idp", "Unknown identity provider ID" + ) + return + + sso_url = await auth_provider.handle_redirect_request( + request, client_redirect_url.encode("utf8") + ) + logger.info("Redirecting to %s", sso_url) + request.redirect(sso_url) + finish_request(request) + + async def _serve_id_picker( + self, request: SynapseRequest, client_redirect_url: str + ) -> None: + # otherwise, serve up the IdP picker + providers = self._sso_handler.get_identity_providers() + html = self._sso_login_idp_picker_template.render( + redirect_url=client_redirect_url, + server_name=self._server_name, + providers=providers.values(), + ) + respond_with_html(request, 200, html) diff --git a/synapse/static/client/login/style.css b/synapse/static/client/login/style.css index 83e4f6abc8..dd76714a92 100644 --- a/synapse/static/client/login/style.css +++ b/synapse/static/client/login/style.css @@ -31,6 +31,11 @@ form { margin: 10px 0 0 0; } +ul.radiobuttons { + text-align: left; + list-style: none; +} + /* * Add some padding to the viewport. */ diff --git a/synapse/storage/database.py b/synapse/storage/database.py index d1b5760c2c..b70ca3087b 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -42,7 +42,6 @@ from synapse.api.errors import StoreError from synapse.config.database import DatabaseConnectionConfig from synapse.logging.context import ( LoggingContext, - LoggingContextOrSentinel, current_context, make_deferred_yieldable, ) @@ -671,12 +670,15 @@ class DatabasePool: Returns: The result of func """ - parent_context = current_context() # type: Optional[LoggingContextOrSentinel] - if not parent_context: + curr_context = current_context() + if not curr_context: logger.warning( "Starting db connection from sentinel context: metrics will be lost" ) parent_context = None + else: + assert isinstance(curr_context, LoggingContext) + parent_context = curr_context start_time = monotonic_time() diff --git a/synapse/util/caches/deferred_cache.py b/synapse/util/caches/deferred_cache.py index 601305487c..1adc92eb90 100644 --- a/synapse/util/caches/deferred_cache.py +++ b/synapse/util/caches/deferred_cache.py @@ -105,7 +105,7 @@ class DeferredCache(Generic[KT, VT]): keylen=keylen, cache_name=name, cache_type=cache_type, - size_callback=(lambda d: len(d)) if iterable else None, + size_callback=(lambda d: len(d) or 1) if iterable else None, metrics_collection_callback=metrics_cb, apply_cache_factor_from_config=apply_cache_factor_from_config, ) # type: LruCache[KT, VT] diff --git a/synapse/util/metrics.py b/synapse/util/metrics.py index ffdea0de8d..24123d5cc4 100644 --- a/synapse/util/metrics.py +++ b/synapse/util/metrics.py @@ -108,7 +108,15 @@ class Measure: def __init__(self, clock, name): self.clock = clock self.name = name - parent_context = current_context() + curr_context = current_context() + if not curr_context: + logger.warning( + "Starting metrics collection from sentinel context: metrics will be lost" + ) + parent_context = None + else: + assert isinstance(curr_context, LoggingContext) + parent_context = curr_context self._logging_context = LoggingContext( "Measure[%s]" % (self.name,), parent_context ) diff --git a/tests/events/test_utils.py b/tests/events/test_utils.py index c1274c14af..8ba36c6074 100644 --- a/tests/events/test_utils.py +++ b/tests/events/test_utils.py @@ -34,11 +34,17 @@ def MockEvent(**kwargs): class PruneEventTestCase(unittest.TestCase): - """ Asserts that a new event constructed with `evdict` will look like - `matchdict` when it is redacted. """ - def run_test(self, evdict, matchdict, **kwargs): - self.assertEquals( + """ + Asserts that a new event constructed with `evdict` will look like + `matchdict` when it is redacted. + + Args: + evdict: The dictionary to build the event from. + matchdict: The expected resulting dictionary. + kwargs: Additional keyword arguments used to create the event. + """ + self.assertEqual( prune_event(make_event_from_dict(evdict, **kwargs)).get_dict(), matchdict ) @@ -55,54 +61,80 @@ class PruneEventTestCase(unittest.TestCase): ) def test_basic_keys(self): + """Ensure that the keys that should be untouched are kept.""" + # Note that some of the values below don't really make sense, but the + # pruning of events doesn't worry about the values of any fields (with + # the exception of the content field). self.run_test( { + "event_id": "$3:domain", "type": "A", "room_id": "!1:domain", "sender": "@2:domain", - "event_id": "$3:domain", + "state_key": "B", + "content": {"other_key": "foo"}, + "hashes": "hashes", + "signatures": {"domain": {"algo:1": "sigs"}}, + "depth": 4, + "prev_events": "prev_events", + "prev_state": "prev_state", + "auth_events": "auth_events", "origin": "domain", + "origin_server_ts": 1234, + "membership": "join", + # Also include a key that should be removed. + "other_key": "foo", }, { + "event_id": "$3:domain", "type": "A", "room_id": "!1:domain", "sender": "@2:domain", - "event_id": "$3:domain", + "state_key": "B", + "hashes": "hashes", + "depth": 4, + "prev_events": "prev_events", + "prev_state": "prev_state", + "auth_events": "auth_events", "origin": "domain", + "origin_server_ts": 1234, + "membership": "join", "content": {}, - "signatures": {}, + "signatures": {"domain": {"algo:1": "sigs"}}, "unsigned": {}, }, ) - def test_unsigned_age_ts(self): + # As of MSC2176 we now redact the membership and prev_states keys. self.run_test( - {"type": "B", "event_id": "$test:domain", "unsigned": {"age_ts": 20}}, - { - "type": "B", - "event_id": "$test:domain", - "content": {}, - "signatures": {}, - "unsigned": {"age_ts": 20}, - }, + {"type": "A", "prev_state": "prev_state", "membership": "join"}, + {"type": "A", "content": {}, "signatures": {}, "unsigned": {}}, + room_version=RoomVersions.MSC2176, ) + def test_unsigned(self): + """Ensure that unsigned properties get stripped (except age_ts and replaces_state).""" self.run_test( { "type": "B", "event_id": "$test:domain", - "unsigned": {"other_key": "here"}, + "unsigned": { + "age_ts": 20, + "replaces_state": "$test2:domain", + "other_key": "foo", + }, }, { "type": "B", "event_id": "$test:domain", "content": {}, "signatures": {}, - "unsigned": {}, + "unsigned": {"age_ts": 20, "replaces_state": "$test2:domain"}, }, ) def test_content(self): + """The content dictionary should be stripped in most cases.""" self.run_test( {"type": "C", "event_id": "$test:domain", "content": {"things": "here"}}, { @@ -114,11 +146,35 @@ class PruneEventTestCase(unittest.TestCase): }, ) + # Some events keep a single content key/value. + EVENT_KEEP_CONTENT_KEYS = [ + ("member", "membership", "join"), + ("join_rules", "join_rule", "invite"), + ("history_visibility", "history_visibility", "shared"), + ] + for event_type, key, value in EVENT_KEEP_CONTENT_KEYS: + self.run_test( + { + "type": "m.room." + event_type, + "event_id": "$test:domain", + "content": {key: value, "other_key": "foo"}, + }, + { + "type": "m.room." + event_type, + "event_id": "$test:domain", + "content": {key: value}, + "signatures": {}, + "unsigned": {}, + }, + ) + + def test_create(self): + """Create events are partially redacted until MSC2176.""" self.run_test( { "type": "m.room.create", "event_id": "$test:domain", - "content": {"creator": "@2:domain", "other_field": "here"}, + "content": {"creator": "@2:domain", "other_key": "foo"}, }, { "type": "m.room.create", @@ -129,6 +185,68 @@ class PruneEventTestCase(unittest.TestCase): }, ) + # After MSC2176, create events get nothing redacted. + self.run_test( + {"type": "m.room.create", "content": {"not_a_real_key": True}}, + { + "type": "m.room.create", + "content": {"not_a_real_key": True}, + "signatures": {}, + "unsigned": {}, + }, + room_version=RoomVersions.MSC2176, + ) + + def test_power_levels(self): + """Power level events keep a variety of content keys.""" + self.run_test( + { + "type": "m.room.power_levels", + "event_id": "$test:domain", + "content": { + "ban": 1, + "events": {"m.room.name": 100}, + "events_default": 2, + "invite": 3, + "kick": 4, + "redact": 5, + "state_default": 6, + "users": {"@admin:domain": 100}, + "users_default": 7, + "other_key": 8, + }, + }, + { + "type": "m.room.power_levels", + "event_id": "$test:domain", + "content": { + "ban": 1, + "events": {"m.room.name": 100}, + "events_default": 2, + # Note that invite is not here. + "kick": 4, + "redact": 5, + "state_default": 6, + "users": {"@admin:domain": 100}, + "users_default": 7, + }, + "signatures": {}, + "unsigned": {}, + }, + ) + + # After MSC2176, power levels events keep the invite key. + self.run_test( + {"type": "m.room.power_levels", "content": {"invite": 75}}, + { + "type": "m.room.power_levels", + "content": {"invite": 75}, + "signatures": {}, + "unsigned": {}, + }, + room_version=RoomVersions.MSC2176, + ) + def test_alias_event(self): """Alias events have special behavior up through room version 6.""" self.run_test( @@ -146,8 +264,7 @@ class PruneEventTestCase(unittest.TestCase): }, ) - def test_msc2432_alias_event(self): - """After MSC2432, alias events have no special behavior.""" + # After MSC2432, alias events have no special behavior. self.run_test( {"type": "m.room.aliases", "content": {"aliases": ["test"]}}, { @@ -159,6 +276,32 @@ class PruneEventTestCase(unittest.TestCase): room_version=RoomVersions.V6, ) + def test_redacts(self): + """Redaction events have no special behaviour until MSC2174/MSC2176.""" + + self.run_test( + {"type": "m.room.redaction", "content": {"redacts": "$test2:domain"}}, + { + "type": "m.room.redaction", + "content": {}, + "signatures": {}, + "unsigned": {}, + }, + room_version=RoomVersions.V6, + ) + + # After MSC2174, redaction events keep the redacts content key. + self.run_test( + {"type": "m.room.redaction", "content": {"redacts": "$test2:domain"}}, + { + "type": "m.room.redaction", + "content": {"redacts": "$test2:domain"}, + "signatures": {}, + "unsigned": {}, + }, + room_version=RoomVersions.MSC2176, + ) + class SerializeEventTestCase(unittest.TestCase): def serialize(self, ev, fields): diff --git a/tests/rest/client/v1/test_login.py b/tests/rest/client/v1/test_login.py index 18932d7518..999d628315 100644 --- a/tests/rest/client/v1/test_login.py +++ b/tests/rest/client/v1/test_login.py @@ -385,7 +385,7 @@ class CASTestCase(unittest.HomeserverTestCase): channel = self.make_request("GET", cas_ticket_url) # Test that the response is HTML. - self.assertEqual(channel.code, 200) + self.assertEqual(channel.code, 200, channel.result) content_type_header_value = "" for header in channel.result.get("headers", []): if header[0] == b"Content-Type": diff --git a/tests/util/caches/test_deferred_cache.py b/tests/util/caches/test_deferred_cache.py index dadfabd46d..ecd9efc4df 100644 --- a/tests/util/caches/test_deferred_cache.py +++ b/tests/util/caches/test_deferred_cache.py @@ -25,13 +25,8 @@ from tests.unittest import TestCase class DeferredCacheTestCase(TestCase): def test_empty(self): cache = DeferredCache("test") - failed = False - try: + with self.assertRaises(KeyError): cache.get("foo") - except KeyError: - failed = True - - self.assertTrue(failed) def test_hit(self): cache = DeferredCache("test") @@ -155,13 +150,8 @@ class DeferredCacheTestCase(TestCase): cache.prefill(("foo",), 123) cache.invalidate(("foo",)) - failed = False - try: + with self.assertRaises(KeyError): cache.get(("foo",)) - except KeyError: - failed = True - - self.assertTrue(failed) def test_invalidate_all(self): cache = DeferredCache("testcache") @@ -215,13 +205,8 @@ class DeferredCacheTestCase(TestCase): cache.prefill(2, "two") cache.prefill(3, "three") # 1 will be evicted - failed = False - try: + with self.assertRaises(KeyError): cache.get(1) - except KeyError: - failed = True - - self.assertTrue(failed) cache.get(2) cache.get(3) @@ -239,13 +224,55 @@ class DeferredCacheTestCase(TestCase): cache.prefill(3, "three") - failed = False - try: + with self.assertRaises(KeyError): cache.get(2) - except KeyError: - failed = True - - self.assertTrue(failed) cache.get(1) cache.get(3) + + def test_eviction_iterable(self): + cache = DeferredCache( + "test", max_entries=3, apply_cache_factor_from_config=False, iterable=True, + ) + + cache.prefill(1, ["one", "two"]) + cache.prefill(2, ["three"]) + + # Now access 1 again, thus causing 2 to be least-recently used + cache.get(1) + + # Now add an item to the cache, which evicts 2. + cache.prefill(3, ["four"]) + with self.assertRaises(KeyError): + cache.get(2) + + # Ensure 1 & 3 are in the cache. + cache.get(1) + cache.get(3) + + # Now access 1 again, thus causing 3 to be least-recently used + cache.get(1) + + # Now add an item with multiple elements to the cache + cache.prefill(4, ["five", "six"]) + + # Both 1 and 3 are evicted since there's too many elements. + with self.assertRaises(KeyError): + cache.get(1) + with self.assertRaises(KeyError): + cache.get(3) + + # Now add another item to fill the cache again. + cache.prefill(5, ["seven"]) + + # Now access 4, thus causing 5 to be least-recently used + cache.get(4) + + # Add an empty item. + cache.prefill(6, []) + + # 5 gets evicted and replaced since an empty element counts as an item. + with self.assertRaises(KeyError): + cache.get(5) + cache.get(4) + cache.get(6)