mirror of
https://github.com/element-hq/synapse
synced 2024-09-17 20:05:10 +00:00
Merge branch 'develop' into madlittlemods/11850-migrate-to-opentelemetry
Conflicts: synapse/storage/schema/__init__.py
This commit is contained in:
commit
19c6f6ecc9
70 changed files with 290 additions and 94 deletions
4
.github/workflows/tests.yml
vendored
4
.github/workflows/tests.yml
vendored
|
@ -468,10 +468,6 @@ jobs:
|
||||||
tests-done:
|
tests-done:
|
||||||
if: ${{ always() }}
|
if: ${{ always() }}
|
||||||
needs:
|
needs:
|
||||||
- check-sampleconfig
|
|
||||||
- lint
|
|
||||||
- lint-crlf
|
|
||||||
- lint-newsfile
|
|
||||||
- trial
|
- trial
|
||||||
- trial-olddeps
|
- trial-olddeps
|
||||||
- sytest
|
- sytest
|
||||||
|
|
108
CHANGES.md
108
CHANGES.md
|
@ -1,3 +1,111 @@
|
||||||
|
Synapse 1.67.0 (2022-09-13)
|
||||||
|
===========================
|
||||||
|
|
||||||
|
This release removes using the deprecated direct TCP replication configuration
|
||||||
|
for workers. Server admins should use Redis instead. See the [upgrade
|
||||||
|
notes](https://matrix-org.github.io/synapse/v1.67/upgrade.html#upgrading-to-v1670).
|
||||||
|
|
||||||
|
The minimum version of `poetry` supported for managing source checkouts is now
|
||||||
|
1.2.0.
|
||||||
|
|
||||||
|
**Notice:** from the next major release (1.68.0) installing Synapse from a source
|
||||||
|
checkout will require a recent Rust compiler. Those using packages or
|
||||||
|
`pip install matrix-synapse` will not be affected. See the [upgrade
|
||||||
|
notes](https://matrix-org.github.io/synapse/v1.67/upgrade.html#upgrading-to-v1670).
|
||||||
|
|
||||||
|
**Notice:** from the next major release (1.68.0), running Synapse with a SQLite
|
||||||
|
database will require SQLite version 3.27.0 or higher. (The [current minimum
|
||||||
|
version is SQLite 3.22.0](https://github.com/matrix-org/synapse/blob/release-v1.67/synapse/storage/engines/sqlite.py#L69-L78).)
|
||||||
|
See [#12983](https://github.com/matrix-org/synapse/issues/12983) and the [upgrade notes](https://matrix-org.github.io/synapse/v1.67/upgrade.html#upgrading-to-v1670) for more details.
|
||||||
|
|
||||||
|
|
||||||
|
No significant changes since 1.67.0rc1.
|
||||||
|
|
||||||
|
|
||||||
|
Synapse 1.67.0rc1 (2022-09-06)
|
||||||
|
==============================
|
||||||
|
|
||||||
|
Features
|
||||||
|
--------
|
||||||
|
|
||||||
|
- Support setting the registration shared secret in a file, via a new `registration_shared_secret_path` configuration option. ([\#13614](https://github.com/matrix-org/synapse/issues/13614))
|
||||||
|
- Change the default startup behaviour so that any missing "additional" configuration files (signing key, etc) are generated automatically. ([\#13615](https://github.com/matrix-org/synapse/issues/13615))
|
||||||
|
- Improve performance of sending messages in rooms with thousands of local users. ([\#13634](https://github.com/matrix-org/synapse/issues/13634))
|
||||||
|
|
||||||
|
|
||||||
|
Bugfixes
|
||||||
|
--------
|
||||||
|
|
||||||
|
- Fix a bug introduced in Synapse 1.13 where the [List Rooms admin API](https://matrix-org.github.io/synapse/develop/admin_api/rooms.html#list-room-api) would return integers instead of booleans for the `federatable` and `public` fields when using a Sqlite database. ([\#13509](https://github.com/matrix-org/synapse/issues/13509))
|
||||||
|
- Fix bug that user cannot `/forget` rooms after the last member has left the room. ([\#13546](https://github.com/matrix-org/synapse/issues/13546))
|
||||||
|
- Faster Room Joins: fix `/make_knock` blocking indefinitely when the room in question is a partial-stated room. ([\#13583](https://github.com/matrix-org/synapse/issues/13583))
|
||||||
|
- Fix loading the current stream position behind the actual position. ([\#13585](https://github.com/matrix-org/synapse/issues/13585))
|
||||||
|
- Fix a longstanding bug in `register_new_matrix_user` which meant it was always necessary to explicitly give a server URL. ([\#13616](https://github.com/matrix-org/synapse/issues/13616))
|
||||||
|
- Fix the running of [MSC1763](https://github.com/matrix-org/matrix-spec-proposals/pull/1763) retention purge_jobs in deployments with background jobs running on a worker by forcing them back onto the main worker. Contributed by Brad @ Beeper. ([\#13632](https://github.com/matrix-org/synapse/issues/13632))
|
||||||
|
- Fix a long-standing bug that downloaded media for URL previews was not deleted while database background updates were running. ([\#13657](https://github.com/matrix-org/synapse/issues/13657))
|
||||||
|
- Fix [MSC3030](https://github.com/matrix-org/matrix-spec-proposals/pull/3030) `/timestamp_to_event` endpoint to return the correct next event when the events have the same timestamp. ([\#13658](https://github.com/matrix-org/synapse/issues/13658))
|
||||||
|
- Fix bug where we wedge media plugins if clients disconnect early. Introduced in v1.22.0. ([\#13660](https://github.com/matrix-org/synapse/issues/13660))
|
||||||
|
- Fix a long-standing bug which meant that keys for unwhitelisted servers were not returned by `/_matrix/key/v2/query`. ([\#13683](https://github.com/matrix-org/synapse/issues/13683))
|
||||||
|
- Fix a bug introduced in Synapse v1.20.0 that would cause the unstable unread counts from [MSC2654](https://github.com/matrix-org/matrix-spec-proposals/pull/2654) to be calculated even if the feature is disabled. ([\#13694](https://github.com/matrix-org/synapse/issues/13694))
|
||||||
|
|
||||||
|
|
||||||
|
Updates to the Docker image
|
||||||
|
---------------------------
|
||||||
|
|
||||||
|
- Update docker image to use a stable version of poetry. ([\#13688](https://github.com/matrix-org/synapse/issues/13688))
|
||||||
|
|
||||||
|
|
||||||
|
Improved Documentation
|
||||||
|
----------------------
|
||||||
|
|
||||||
|
- Improve the description of the ["chain cover index"](https://matrix-org.github.io/synapse/latest/auth_chain_difference_algorithm.html) used internally by Synapse. ([\#13602](https://github.com/matrix-org/synapse/issues/13602))
|
||||||
|
- Document how ["monthly active users"](https://matrix-org.github.io/synapse/latest/usage/administration/monthly_active_users.html) is calculated and used. ([\#13617](https://github.com/matrix-org/synapse/issues/13617))
|
||||||
|
- Improve documentation around user registration. ([\#13640](https://github.com/matrix-org/synapse/issues/13640))
|
||||||
|
- Remove documentation of legacy `frontend_proxy` worker app. ([\#13645](https://github.com/matrix-org/synapse/issues/13645))
|
||||||
|
- Clarify documentation that HTTP replication traffic can be protected with a shared secret. ([\#13656](https://github.com/matrix-org/synapse/issues/13656))
|
||||||
|
- Remove unintentional colons from [config manual](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html) headers. ([\#13665](https://github.com/matrix-org/synapse/issues/13665))
|
||||||
|
- Update docs to make enabling metrics more clear. ([\#13678](https://github.com/matrix-org/synapse/issues/13678))
|
||||||
|
- Clarify `(room_id, event_id)` global uniqueness and how we should scope our database schemas. ([\#13701](https://github.com/matrix-org/synapse/issues/13701))
|
||||||
|
|
||||||
|
|
||||||
|
Deprecations and Removals
|
||||||
|
-------------------------
|
||||||
|
|
||||||
|
- Drop support for calling `/_matrix/client/v3/rooms/{roomId}/invite` without an `id_access_token`, which was not permitted by the spec. Contributed by @Vetchu. ([\#13241](https://github.com/matrix-org/synapse/issues/13241))
|
||||||
|
- Remove redundant `_get_joined_users_from_context` cache. Contributed by Nick @ Beeper (@fizzadar). ([\#13569](https://github.com/matrix-org/synapse/issues/13569))
|
||||||
|
- Remove the ability to use direct TCP replication with workers. Direct TCP replication was deprecated in Synapse v1.18.0. Workers now require using Redis. ([\#13647](https://github.com/matrix-org/synapse/issues/13647))
|
||||||
|
- Remove support for unstable [private read receipts](https://github.com/matrix-org/matrix-spec-proposals/pull/2285). ([\#13653](https://github.com/matrix-org/synapse/issues/13653), [\#13692](https://github.com/matrix-org/synapse/issues/13692))
|
||||||
|
|
||||||
|
|
||||||
|
Internal Changes
|
||||||
|
----------------
|
||||||
|
|
||||||
|
- Extend the release script to wait for GitHub Actions to finish and to be usable as a guide for the whole process. ([\#13483](https://github.com/matrix-org/synapse/issues/13483))
|
||||||
|
- Add experimental configuration option to allow disabling legacy Prometheus metric names. ([\#13540](https://github.com/matrix-org/synapse/issues/13540))
|
||||||
|
- Cache user IDs instead of profiles to reduce cache memory usage. Contributed by Nick @ Beeper (@fizzadar). ([\#13573](https://github.com/matrix-org/synapse/issues/13573), [\#13600](https://github.com/matrix-org/synapse/issues/13600))
|
||||||
|
- Optimize how Synapse calculates domains to fetch from during backfill. ([\#13575](https://github.com/matrix-org/synapse/issues/13575))
|
||||||
|
- Comment about a better future where we can get the state diff between two events. ([\#13586](https://github.com/matrix-org/synapse/issues/13586))
|
||||||
|
- Instrument `_check_sigs_and_hash_and_fetch` to trace time spent in child concurrent calls for understandable traces in Jaeger. ([\#13588](https://github.com/matrix-org/synapse/issues/13588))
|
||||||
|
- Improve performance of `@cachedList`. ([\#13591](https://github.com/matrix-org/synapse/issues/13591))
|
||||||
|
- Minor speed up of fetching large numbers of push rules. ([\#13592](https://github.com/matrix-org/synapse/issues/13592))
|
||||||
|
- Optimise push action fetching queries. Contributed by Nick @ Beeper (@fizzadar). ([\#13597](https://github.com/matrix-org/synapse/issues/13597))
|
||||||
|
- Rename `event_map` to `unpersisted_events` when computing the auth differences. ([\#13603](https://github.com/matrix-org/synapse/issues/13603))
|
||||||
|
- Refactor `get_users_in_room(room_id)` mis-use with dedicated `get_current_hosts_in_room(room_id)` function. ([\#13605](https://github.com/matrix-org/synapse/issues/13605))
|
||||||
|
- Use dedicated `get_local_users_in_room(room_id)` function to find local users when calculating `join_authorised_via_users_server` of a `/make_join` request. ([\#13606](https://github.com/matrix-org/synapse/issues/13606))
|
||||||
|
- Refactor `get_users_in_room(room_id)` mis-use to lookup single local user with dedicated `check_local_user_in_room(...)` function. ([\#13608](https://github.com/matrix-org/synapse/issues/13608))
|
||||||
|
- Drop unused column `application_services_state.last_txn`. ([\#13627](https://github.com/matrix-org/synapse/issues/13627))
|
||||||
|
- Improve readability of Complement CI logs by printing failure results last. ([\#13639](https://github.com/matrix-org/synapse/issues/13639))
|
||||||
|
- Generalise the `@cancellable` annotation so it can be used on functions other than just servlet methods. ([\#13662](https://github.com/matrix-org/synapse/issues/13662))
|
||||||
|
- Introduce a `CommonUsageMetrics` class to share some usage metrics between the Prometheus exporter and the phone home stats. ([\#13671](https://github.com/matrix-org/synapse/issues/13671))
|
||||||
|
- Add some logging to help track down #13444. ([\#13679](https://github.com/matrix-org/synapse/issues/13679))
|
||||||
|
- Update poetry lock file for v1.2.0. ([\#13689](https://github.com/matrix-org/synapse/issues/13689))
|
||||||
|
- Add cache to `is_partial_state_room`. ([\#13693](https://github.com/matrix-org/synapse/issues/13693))
|
||||||
|
- Update the Grafana dashboard that is included with Synapse in the `contrib` directory. ([\#13697](https://github.com/matrix-org/synapse/issues/13697))
|
||||||
|
- Only run trial CI on all python versions on non-PRs. ([\#13698](https://github.com/matrix-org/synapse/issues/13698))
|
||||||
|
- Fix typechecking with latest types-jsonschema. ([\#13712](https://github.com/matrix-org/synapse/issues/13712))
|
||||||
|
- Reduce number of CI checks we run for PRs. ([\#13713](https://github.com/matrix-org/synapse/issues/13713))
|
||||||
|
|
||||||
|
|
||||||
Synapse 1.66.0 (2022-08-31)
|
Synapse 1.66.0 (2022-08-31)
|
||||||
===========================
|
===========================
|
||||||
|
|
||||||
|
|
|
@ -1 +0,0 @@
|
||||||
Drop support for calling `/_matrix/client/v3/rooms/{roomId}/invite` without an `id_access_token`, which was not permitted by the spec. Contributed by @Vetchu.
|
|
1
changelog.d/13480.doc
Normal file
1
changelog.d/13480.doc
Normal file
|
@ -0,0 +1 @@
|
||||||
|
Note that `libpq` is required on ARM-based Macs.
|
|
@ -1 +0,0 @@
|
||||||
Extend the release script to wait for GitHub Actions to finish and to be usable as a guide for the whole process.
|
|
|
@ -1 +0,0 @@
|
||||||
Fix a bug introduced in Synapse 1.13 where the [List Rooms admin API](https://matrix-org.github.io/synapse/develop/admin_api/rooms.html#list-room-api) would return integers instead of booleans for the `federatable` and `public` fields when using a Sqlite database.
|
|
|
@ -1 +0,0 @@
|
||||||
Add experimental configuration option to allow disabling legacy Prometheus metric names.
|
|
|
@ -1 +0,0 @@
|
||||||
Fix bug that user cannot `/forget` rooms after the last member has left the room.
|
|
|
@ -1 +0,0 @@
|
||||||
Remove redundant `_get_joined_users_from_context` cache. Contributed by Nick @ Beeper (@fizzadar).
|
|
|
@ -1 +0,0 @@
|
||||||
Cache user IDs instead of profiles to reduce cache memory usage. Contributed by Nick @ Beeper (@fizzadar).
|
|
|
@ -1 +0,0 @@
|
||||||
Optimize how Synapse calculates domains to fetch from during backfill.
|
|
|
@ -1 +0,0 @@
|
||||||
Faster Room Joins: fix `/make_knock` blocking indefinitely when the room in question is a partial-stated room.
|
|
|
@ -1 +0,0 @@
|
||||||
Fix loading the current stream position behind the actual position.
|
|
|
@ -1 +0,0 @@
|
||||||
Comment about a better future where we can get the state diff between two events.
|
|
|
@ -1 +0,0 @@
|
||||||
Instrument `_check_sigs_and_hash_and_fetch` to trace time spent in child concurrent calls for understandable traces in Jaeger.
|
|
|
@ -1 +0,0 @@
|
||||||
Improve performance of `@cachedList`.
|
|
|
@ -1 +0,0 @@
|
||||||
Minor speed up of fetching large numbers of push rules.
|
|
|
@ -1 +0,0 @@
|
||||||
Optimise push action fetching queries. Contributed by Nick @ Beeper (@fizzadar).
|
|
|
@ -1 +0,0 @@
|
||||||
Cache user IDs instead of profiles to reduce cache memory usage. Contributed by Nick @ Beeper (@fizzadar).
|
|
|
@ -1 +0,0 @@
|
||||||
Improve the description of the ["chain cover index"](https://matrix-org.github.io/synapse/latest/auth_chain_difference_algorithm.html) used internally by Synapse.
|
|
|
@ -1 +0,0 @@
|
||||||
Rename `event_map` to `unpersisted_events` when computing the auth differences.
|
|
|
@ -1 +0,0 @@
|
||||||
Refactor `get_users_in_room(room_id)` mis-use with dedicated `get_current_hosts_in_room(room_id)` function.
|
|
|
@ -1 +0,0 @@
|
||||||
Use dedicated `get_local_users_in_room(room_id)` function to find local users when calculating `join_authorised_via_users_server` of a `/make_join` request.
|
|
|
@ -1 +0,0 @@
|
||||||
Refactor `get_users_in_room(room_id)` mis-use to lookup single local user with dedicated `check_local_user_in_room(...)` function.
|
|
|
@ -1 +0,0 @@
|
||||||
Support setting the registration shared secret in a file, via a new `registration_shared_secret_path` configuration option.
|
|
|
@ -1 +0,0 @@
|
||||||
Change the default startup behaviour so that any missing "additional" configuration files (signing key, etc) are generated automatically.
|
|
|
@ -1 +0,0 @@
|
||||||
Fix a longstanding bug in `register_new_matrix_user` which meant it was always necessary to explicitly give a server URL.
|
|
|
@ -1 +0,0 @@
|
||||||
Document how ["monthly active users"](https://matrix-org.github.io/synapse/latest/usage/administration/monthly_active_users.html) is calculated and used.
|
|
|
@ -1 +0,0 @@
|
||||||
Drop unused column `application_services_state.last_txn`.
|
|
|
@ -1 +0,0 @@
|
||||||
Fix the running of MSC1763 retention purge_jobs in deployments with background jobs running on a worker by forcing them back onto the main worker. Contributed by Brad @ Beeper.
|
|
|
@ -1 +0,0 @@
|
||||||
Improve performance of sending messages in rooms with thousands of local users.
|
|
|
@ -1 +0,0 @@
|
||||||
Improve readability of Complement CI logs by printing failure results last.
|
|
|
@ -1 +0,0 @@
|
||||||
Improve documentation around user registration.
|
|
|
@ -1 +0,0 @@
|
||||||
Remove documentation of legacy `frontend_proxy` worker app.
|
|
|
@ -1 +0,0 @@
|
||||||
Remove the ability to use direct TCP replication with workers. Direct TCP replication was deprecated in Synapse v1.18.0. Workers now require using Redis.
|
|
|
@ -1 +0,0 @@
|
||||||
Remove support for unstable [private read receipts](https://github.com/matrix-org/matrix-spec-proposals/pull/2285).
|
|
|
@ -1 +0,0 @@
|
||||||
Clarify documentation that HTTP replication traffic can be protected with a shared secret.
|
|
|
@ -1 +0,0 @@
|
||||||
Fix a long-standing bug that downloaded media for URL previews was not deleted while database background updates were running.
|
|
|
@ -1 +0,0 @@
|
||||||
Fix MSC3030 `/timestamp_to_event` endpoint to return the correct next event when the events have the same timestamp.
|
|
|
@ -1 +0,0 @@
|
||||||
Fix bug where we wedge media plugins if clients disconnect early. Introduced in v1.22.0.
|
|
|
@ -1 +0,0 @@
|
||||||
Generalise the `@cancellable` annotation so it can be used on functions other than just servlet methods.
|
|
|
@ -1 +0,0 @@
|
||||||
Remove unintentional colons from [config manual](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html) headers.
|
|
|
@ -1 +0,0 @@
|
||||||
Introduce a `CommonUsageMetrics` class to share some usage metrics between the Prometheus exporter and the phone home stats.
|
|
|
@ -1 +0,0 @@
|
||||||
Update docs to make enabling metrics more clear.
|
|
|
@ -1 +0,0 @@
|
||||||
Add some logging to help track down #13444.
|
|
|
@ -1 +0,0 @@
|
||||||
Fix a long-standing bug which meant that keys for unwhitelisted servers were not returned by `/_matrix/key/v2/query`.
|
|
|
@ -1 +0,0 @@
|
||||||
Update docker image to use a stable version of poetry.
|
|
|
@ -1 +0,0 @@
|
||||||
Update poetry lock file for v1.2.0.
|
|
|
@ -1 +0,0 @@
|
||||||
Remove support for unstable [private read receipts](https://github.com/matrix-org/matrix-spec-proposals/pull/2285).
|
|
|
@ -1 +0,0 @@
|
||||||
Add cache to `is_partial_state_room`.
|
|
|
@ -1 +0,0 @@
|
||||||
Fix a bug introduced in Synapse v1.20.0 that would cause the unstable unread counts from [MSC2654](https://github.com/matrix-org/matrix-spec-proposals/pull/2654) to be calculated even if the feature is disabled.
|
|
|
@ -1 +0,0 @@
|
||||||
Update the Grafana dashboard that is included with Synapse in the `contrib` directory.
|
|
|
@ -1 +0,0 @@
|
||||||
Only run trial CI on all python versions on non-PRs.
|
|
|
@ -1 +0,0 @@
|
||||||
Clarify `(room_id, event_id)` global uniqueness and how we should scope our database schemas.
|
|
1
changelog.d/13703.misc
Normal file
1
changelog.d/13703.misc
Normal file
|
@ -0,0 +1 @@
|
||||||
|
Add & populate `event_stream_ordering` column on receipts table for future optimisation of push action processing. Contributed by Nick @ Beeper (@fizzadar).
|
|
@ -1 +0,0 @@
|
||||||
Fix typechecking with latest types-jsonschema.
|
|
|
@ -1 +0,0 @@
|
||||||
Reduce number of CI checks we run for PRs.
|
|
1
changelog.d/13766.bugfix
Normal file
1
changelog.d/13766.bugfix
Normal file
|
@ -0,0 +1 @@
|
||||||
|
Fix a long-standing bug where the `cache_invalidation_stream_seq` sequence would begin at 1 instead of 2.
|
1
changelog.d/13770.misc
Normal file
1
changelog.d/13770.misc
Normal file
|
@ -0,0 +1 @@
|
||||||
|
Update the script which makes full schema dumps.
|
1
changelog.d/13784.misc
Normal file
1
changelog.d/13784.misc
Normal file
|
@ -0,0 +1 @@
|
||||||
|
Simplify the dependency DAG in the tests workflow.
|
14
debian/changelog
vendored
14
debian/changelog
vendored
|
@ -1,8 +1,18 @@
|
||||||
matrix-synapse-py3 (1.66.0ubuntu1) UNRELEASED; urgency=medium
|
matrix-synapse-py3 (1.67.0) stable; urgency=medium
|
||||||
|
|
||||||
|
* New Synapse release 1.67.0.
|
||||||
|
|
||||||
|
-- Synapse Packaging team <packages@matrix.org> Tue, 13 Sep 2022 09:19:56 +0100
|
||||||
|
|
||||||
|
matrix-synapse-py3 (1.67.0~rc1) stable; urgency=medium
|
||||||
|
|
||||||
|
[ Erik Johnston ]
|
||||||
* Use stable poetry 1.2.0 version, rather than a prerelease.
|
* Use stable poetry 1.2.0 version, rather than a prerelease.
|
||||||
|
|
||||||
-- Erik Johnston <erik@matrix.org> Thu, 01 Sep 2022 13:48:31 +0100
|
[ Synapse Packaging team ]
|
||||||
|
* New Synapse release 1.67.0rc1.
|
||||||
|
|
||||||
|
-- Synapse Packaging team <packages@matrix.org> Tue, 06 Sep 2022 09:01:06 +0100
|
||||||
|
|
||||||
matrix-synapse-py3 (1.66.0) stable; urgency=medium
|
matrix-synapse-py3 (1.66.0) stable; urgency=medium
|
||||||
|
|
||||||
|
|
|
@ -303,9 +303,10 @@ You may need to install the latest Xcode developer tools:
|
||||||
xcode-select --install
|
xcode-select --install
|
||||||
```
|
```
|
||||||
|
|
||||||
On ARM-based Macs you may need to explicitly install libjpeg which is a pillow dependency. You can use Homebrew (https://brew.sh):
|
On ARM-based Macs you may need to install libjpeg and libpq.
|
||||||
|
You can use Homebrew (https://brew.sh):
|
||||||
```sh
|
```sh
|
||||||
brew install jpeg
|
brew install jpeg libpq
|
||||||
```
|
```
|
||||||
|
|
||||||
On macOS Catalina (10.15) you may need to explicitly install OpenSSL
|
On macOS Catalina (10.15) you may need to explicitly install OpenSSL
|
||||||
|
|
|
@ -111,6 +111,30 @@ and remove the TCP `replication` listener from config of the master and
|
||||||
The minimum supported version of poetry is now 1.2. This should only affect
|
The minimum supported version of poetry is now 1.2. This should only affect
|
||||||
those installing from a source checkout.
|
those installing from a source checkout.
|
||||||
|
|
||||||
|
## Rust requirement in the next release
|
||||||
|
|
||||||
|
From the next major release (v1.68.0) installing Synapse from a source checkout
|
||||||
|
will require a recent Rust compiler. Those using packages or
|
||||||
|
`pip install matrix-synapse` will not be affected.
|
||||||
|
|
||||||
|
The simplest way of installing Rust is via [rustup.rs](https://rustup.rs/)
|
||||||
|
|
||||||
|
## SQLite version requirement in the next release
|
||||||
|
|
||||||
|
From the next major release (v1.68.0) Synapse will require SQLite 3.27.0 or
|
||||||
|
higher. Synapse v1.67.0 will be the last major release supporting SQLite
|
||||||
|
versions 3.22 to 3.26.
|
||||||
|
|
||||||
|
Those using docker images or Debian packages from Matrix.org will not be
|
||||||
|
affected. If you have installed from source, you should check the version of
|
||||||
|
SQLite used by Python with:
|
||||||
|
|
||||||
|
```shell
|
||||||
|
python -c "import sqlite3; print(sqlite3.sqlite_version)"
|
||||||
|
```
|
||||||
|
|
||||||
|
If this is too old, refer to your distribution for advice on upgrading.
|
||||||
|
|
||||||
# Upgrading to v1.66.0
|
# Upgrading to v1.66.0
|
||||||
|
|
||||||
## Delegation of email validation no longer supported
|
## Delegation of email validation no longer supported
|
||||||
|
|
|
@ -9,8 +9,10 @@
|
||||||
export PGHOST="localhost"
|
export PGHOST="localhost"
|
||||||
POSTGRES_DB_NAME="synapse_full_schema.$$"
|
POSTGRES_DB_NAME="synapse_full_schema.$$"
|
||||||
|
|
||||||
SQLITE_FULL_SCHEMA_OUTPUT_FILE="full.sql.sqlite"
|
SQLITE_SCHEMA_FILE="schema.sql.sqlite"
|
||||||
POSTGRES_FULL_SCHEMA_OUTPUT_FILE="full.sql.postgres"
|
SQLITE_ROWS_FILE="rows.sql.sqlite"
|
||||||
|
POSTGRES_SCHEMA_FILE="full.sql.postgres"
|
||||||
|
POSTGRES_ROWS_FILE="rows.sql.postgres"
|
||||||
|
|
||||||
REQUIRED_DEPS=("matrix-synapse" "psycopg2")
|
REQUIRED_DEPS=("matrix-synapse" "psycopg2")
|
||||||
|
|
||||||
|
@ -22,7 +24,7 @@ usage() {
|
||||||
echo " Username to connect to local postgres instance. The password will be requested"
|
echo " Username to connect to local postgres instance. The password will be requested"
|
||||||
echo " during script execution."
|
echo " during script execution."
|
||||||
echo "-c"
|
echo "-c"
|
||||||
echo " CI mode. Enables coverage tracking and prints every command that the script runs."
|
echo " CI mode. Prints every command that the script runs."
|
||||||
echo "-o <path>"
|
echo "-o <path>"
|
||||||
echo " Directory to output full schema files to."
|
echo " Directory to output full schema files to."
|
||||||
echo "-h"
|
echo "-h"
|
||||||
|
@ -37,11 +39,6 @@ while getopts "p:co:h" opt; do
|
||||||
c)
|
c)
|
||||||
# Print all commands that are being executed
|
# Print all commands that are being executed
|
||||||
set -x
|
set -x
|
||||||
|
|
||||||
# Modify required dependencies for coverage
|
|
||||||
REQUIRED_DEPS+=("coverage" "coverage-enable-subprocess")
|
|
||||||
|
|
||||||
COVERAGE=1
|
|
||||||
;;
|
;;
|
||||||
o)
|
o)
|
||||||
command -v realpath > /dev/null || (echo "The -o flag requires the 'realpath' binary to be installed" && exit 1)
|
command -v realpath > /dev/null || (echo "The -o flag requires the 'realpath' binary to be installed" && exit 1)
|
||||||
|
@ -102,6 +99,7 @@ SQLITE_DB=$TMPDIR/homeserver.db
|
||||||
POSTGRES_CONFIG=$TMPDIR/postgres.conf
|
POSTGRES_CONFIG=$TMPDIR/postgres.conf
|
||||||
|
|
||||||
# Ensure these files are delete on script exit
|
# Ensure these files are delete on script exit
|
||||||
|
# TODO: the trap should also drop the temp postgres DB
|
||||||
trap 'rm -rf $TMPDIR' EXIT
|
trap 'rm -rf $TMPDIR' EXIT
|
||||||
|
|
||||||
cat > "$SQLITE_CONFIG" <<EOF
|
cat > "$SQLITE_CONFIG" <<EOF
|
||||||
|
@ -147,48 +145,34 @@ python -m synapse.app.homeserver --generate-keys -c "$SQLITE_CONFIG"
|
||||||
|
|
||||||
# Make sure the SQLite3 database is using the latest schema and has no pending background update.
|
# Make sure the SQLite3 database is using the latest schema and has no pending background update.
|
||||||
echo "Running db background jobs..."
|
echo "Running db background jobs..."
|
||||||
synapse/_scripts/update_synapse_database.py --database-config --run-background-updates "$SQLITE_CONFIG"
|
synapse/_scripts/update_synapse_database.py --database-config "$SQLITE_CONFIG" --run-background-updates
|
||||||
|
|
||||||
# Create the PostgreSQL database.
|
# Create the PostgreSQL database.
|
||||||
echo "Creating postgres database..."
|
echo "Creating postgres database..."
|
||||||
createdb --lc-collate=C --lc-ctype=C --template=template0 "$POSTGRES_DB_NAME"
|
createdb --lc-collate=C --lc-ctype=C --template=template0 "$POSTGRES_DB_NAME"
|
||||||
|
|
||||||
echo "Copying data from SQLite3 to Postgres with synapse_port_db..."
|
echo "Running db background jobs..."
|
||||||
if [ -z "$COVERAGE" ]; then
|
synapse/_scripts/update_synapse_database.py --database-config "$POSTGRES_CONFIG" --run-background-updates
|
||||||
# No coverage needed
|
|
||||||
synapse/_scripts/synapse_port_db.py --sqlite-database "$SQLITE_DB" --postgres-config "$POSTGRES_CONFIG"
|
|
||||||
else
|
|
||||||
# Coverage desired
|
|
||||||
coverage run synapse/_scripts/synapse_port_db.py --sqlite-database "$SQLITE_DB" --postgres-config "$POSTGRES_CONFIG"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Delete schema_version, applied_schema_deltas and applied_module_schemas tables
|
# Delete schema_version, applied_schema_deltas and applied_module_schemas tables
|
||||||
# Also delete any shadow tables from fts4
|
# Also delete any shadow tables from fts4
|
||||||
# This needs to be done after synapse_port_db is run
|
|
||||||
echo "Dropping unwanted db tables..."
|
echo "Dropping unwanted db tables..."
|
||||||
SQL="
|
SQL="
|
||||||
DROP TABLE schema_version;
|
DROP TABLE schema_version;
|
||||||
DROP TABLE applied_schema_deltas;
|
DROP TABLE applied_schema_deltas;
|
||||||
DROP TABLE applied_module_schemas;
|
DROP TABLE applied_module_schemas;
|
||||||
DROP TABLE event_search_content;
|
|
||||||
DROP TABLE event_search_segments;
|
|
||||||
DROP TABLE event_search_segdir;
|
|
||||||
DROP TABLE event_search_docsize;
|
|
||||||
DROP TABLE event_search_stat;
|
|
||||||
DROP TABLE user_directory_search_content;
|
|
||||||
DROP TABLE user_directory_search_segments;
|
|
||||||
DROP TABLE user_directory_search_segdir;
|
|
||||||
DROP TABLE user_directory_search_docsize;
|
|
||||||
DROP TABLE user_directory_search_stat;
|
|
||||||
"
|
"
|
||||||
sqlite3 "$SQLITE_DB" <<< "$SQL"
|
sqlite3 "$SQLITE_DB" <<< "$SQL"
|
||||||
psql "$POSTGRES_DB_NAME" -w <<< "$SQL"
|
psql "$POSTGRES_DB_NAME" -w <<< "$SQL"
|
||||||
|
|
||||||
echo "Dumping SQLite3 schema to '$OUTPUT_DIR/$SQLITE_FULL_SCHEMA_OUTPUT_FILE'..."
|
echo "Dumping SQLite3 schema to '$OUTPUT_DIR/$SQLITE_SCHEMA_FILE' and '$OUTPUT_DIR/$SQLITE_ROWS_FILE'..."
|
||||||
sqlite3 "$SQLITE_DB" ".dump" > "$OUTPUT_DIR/$SQLITE_FULL_SCHEMA_OUTPUT_FILE"
|
sqlite3 "$SQLITE_DB" ".schema --indent" > "$OUTPUT_DIR/$SQLITE_SCHEMA_FILE"
|
||||||
|
sqlite3 "$SQLITE_DB" ".dump --data-only --nosys" > "$OUTPUT_DIR/$SQLITE_ROWS_FILE"
|
||||||
|
|
||||||
echo "Dumping Postgres schema to '$OUTPUT_DIR/$POSTGRES_FULL_SCHEMA_OUTPUT_FILE'..."
|
echo "Dumping Postgres schema to '$OUTPUT_DIR/$POSTGRES_SCHEMA_FILE' and '$OUTPUT_DIR/$POSTGRES_ROWS_FILE'..."
|
||||||
pg_dump --format=plain --no-tablespaces --no-acl --no-owner $POSTGRES_DB_NAME | sed -e '/^--/d' -e 's/public\.//g' -e '/^SET /d' -e '/^SELECT /d' > "$OUTPUT_DIR/$POSTGRES_FULL_SCHEMA_OUTPUT_FILE"
|
pg_dump --format=plain --schema-only --no-tablespaces --no-acl --no-owner "$POSTGRES_DB_NAME" | sed -e '/^$/d' -e '/^--/d' -e 's/public\.//g' -e '/^SET /d' -e '/^SELECT /d' > "$OUTPUT_DIR/$POSTGRES_SCHEMA_FILE"
|
||||||
|
pg_dump --format=plain --data-only --inserts --no-tablespaces --no-acl --no-owner "$POSTGRES_DB_NAME" | sed -e '/^$/d' -e '/^--/d' -e 's/public\.//g' -e '/^SET /d' -e '/^SELECT /d' > "$OUTPUT_DIR/$POSTGRES_ROWS_FILE"
|
||||||
|
|
||||||
echo "Cleaning up temporary Postgres database..."
|
echo "Cleaning up temporary Postgres database..."
|
||||||
dropdb $POSTGRES_DB_NAME
|
dropdb $POSTGRES_DB_NAME
|
||||||
|
|
|
@ -67,6 +67,7 @@ from synapse.storage.databases.main.media_repository import (
|
||||||
)
|
)
|
||||||
from synapse.storage.databases.main.presence import PresenceBackgroundUpdateStore
|
from synapse.storage.databases.main.presence import PresenceBackgroundUpdateStore
|
||||||
from synapse.storage.databases.main.pusher import PusherWorkerStore
|
from synapse.storage.databases.main.pusher import PusherWorkerStore
|
||||||
|
from synapse.storage.databases.main.receipts import ReceiptsBackgroundUpdateStore
|
||||||
from synapse.storage.databases.main.registration import (
|
from synapse.storage.databases.main.registration import (
|
||||||
RegistrationBackgroundUpdateStore,
|
RegistrationBackgroundUpdateStore,
|
||||||
find_max_generated_user_id_localpart,
|
find_max_generated_user_id_localpart,
|
||||||
|
@ -203,6 +204,7 @@ class Store(
|
||||||
PushRuleStore,
|
PushRuleStore,
|
||||||
PusherWorkerStore,
|
PusherWorkerStore,
|
||||||
PresenceBackgroundUpdateStore,
|
PresenceBackgroundUpdateStore,
|
||||||
|
ReceiptsBackgroundUpdateStore,
|
||||||
):
|
):
|
||||||
def execute(self, f: Callable[..., R], *args: Any, **kwargs: Any) -> Awaitable[R]:
|
def execute(self, f: Callable[..., R], *args: Any, **kwargs: Any) -> Awaitable[R]:
|
||||||
return self.db_pool.runInteraction(f.__name__, f, *args, **kwargs)
|
return self.db_pool.runInteraction(f.__name__, f, *args, **kwargs)
|
||||||
|
|
|
@ -675,6 +675,7 @@ class ReceiptsWorkerStore(SQLBaseStore):
|
||||||
values={
|
values={
|
||||||
"stream_id": stream_id,
|
"stream_id": stream_id,
|
||||||
"event_id": event_id,
|
"event_id": event_id,
|
||||||
|
"event_stream_ordering": stream_ordering,
|
||||||
"data": json_encoder.encode(data),
|
"data": json_encoder.encode(data),
|
||||||
},
|
},
|
||||||
# receipts_linearized has a unique constraint on
|
# receipts_linearized has a unique constraint on
|
||||||
|
@ -830,5 +831,76 @@ class ReceiptsWorkerStore(SQLBaseStore):
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class ReceiptsStore(ReceiptsWorkerStore):
|
class ReceiptsBackgroundUpdateStore(SQLBaseStore):
|
||||||
|
POPULATE_RECEIPT_EVENT_STREAM_ORDERING = "populate_event_stream_ordering"
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
database: DatabasePool,
|
||||||
|
db_conn: LoggingDatabaseConnection,
|
||||||
|
hs: "HomeServer",
|
||||||
|
):
|
||||||
|
super().__init__(database, db_conn, hs)
|
||||||
|
|
||||||
|
self.db_pool.updates.register_background_update_handler(
|
||||||
|
self.POPULATE_RECEIPT_EVENT_STREAM_ORDERING,
|
||||||
|
self._populate_receipt_event_stream_ordering,
|
||||||
|
)
|
||||||
|
|
||||||
|
async def _populate_receipt_event_stream_ordering(
|
||||||
|
self, progress: JsonDict, batch_size: int
|
||||||
|
) -> int:
|
||||||
|
def _populate_receipt_event_stream_ordering_txn(
|
||||||
|
txn: LoggingTransaction,
|
||||||
|
) -> bool:
|
||||||
|
|
||||||
|
if "max_stream_id" in progress:
|
||||||
|
max_stream_id = progress["max_stream_id"]
|
||||||
|
else:
|
||||||
|
txn.execute("SELECT max(stream_id) FROM receipts_linearized")
|
||||||
|
res = txn.fetchone()
|
||||||
|
if res is None or res[0] is None:
|
||||||
|
return True
|
||||||
|
else:
|
||||||
|
max_stream_id = res[0]
|
||||||
|
|
||||||
|
start = progress.get("stream_id", 0)
|
||||||
|
stop = start + batch_size
|
||||||
|
|
||||||
|
sql = """
|
||||||
|
UPDATE receipts_linearized
|
||||||
|
SET event_stream_ordering = (
|
||||||
|
SELECT stream_ordering
|
||||||
|
FROM events
|
||||||
|
WHERE event_id = receipts_linearized.event_id
|
||||||
|
)
|
||||||
|
WHERE stream_id >= ? AND stream_id < ?
|
||||||
|
"""
|
||||||
|
txn.execute(sql, (start, stop))
|
||||||
|
|
||||||
|
self.db_pool.updates._background_update_progress_txn(
|
||||||
|
txn,
|
||||||
|
self.POPULATE_RECEIPT_EVENT_STREAM_ORDERING,
|
||||||
|
{
|
||||||
|
"stream_id": stop,
|
||||||
|
"max_stream_id": max_stream_id,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
return stop > max_stream_id
|
||||||
|
|
||||||
|
finished = await self.db_pool.runInteraction(
|
||||||
|
"_remove_devices_from_device_inbox_txn",
|
||||||
|
_populate_receipt_event_stream_ordering_txn,
|
||||||
|
)
|
||||||
|
|
||||||
|
if finished:
|
||||||
|
await self.db_pool.updates._end_background_update(
|
||||||
|
self.POPULATE_RECEIPT_EVENT_STREAM_ORDERING
|
||||||
|
)
|
||||||
|
|
||||||
|
return batch_size
|
||||||
|
|
||||||
|
|
||||||
|
class ReceiptsStore(ReceiptsWorkerStore, ReceiptsBackgroundUpdateStore):
|
||||||
pass
|
pass
|
||||||
|
|
|
@ -76,6 +76,7 @@ Changes in SCHEMA_VERSION = 72:
|
||||||
- event_edges.(room_id, is_state) are no longer written to.
|
- event_edges.(room_id, is_state) are no longer written to.
|
||||||
- Tables related to groups are dropped.
|
- Tables related to groups are dropped.
|
||||||
- Unused column application_services_state.last_txn is dropped
|
- Unused column application_services_state.last_txn is dropped
|
||||||
|
- Cache invalidation stream id sequence now begins at 2 to match code expectation.
|
||||||
- Rename column in `device_lists_outbound_pokes` and `device_lists_changes_in_room`
|
- Rename column in `device_lists_outbound_pokes` and `device_lists_changes_in_room`
|
||||||
from `opentracing_context` to generalized `tracing_context`.
|
from `opentracing_context` to generalized `tracing_context`.
|
||||||
"""
|
"""
|
||||||
|
|
|
@ -0,0 +1,19 @@
|
||||||
|
/* Copyright 2022 Beeper
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
ALTER TABLE receipts_linearized ADD COLUMN event_stream_ordering BIGINT;
|
||||||
|
|
||||||
|
INSERT INTO background_updates (update_name, progress_json) VALUES
|
||||||
|
('populate_event_stream_ordering', '{}');
|
|
@ -0,0 +1,23 @@
|
||||||
|
/* Copyright 2022 The Matrix.org Foundation C.I.C
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
|
||||||
|
-- The sequence needs to begin at 2 because a bunch of code assumes that
|
||||||
|
-- get_next_id_txn will return values >= 2, cf this comment:
|
||||||
|
-- https://github.com/matrix-org/synapse/blob/b93bd95e8ab64d27ae26841020f62ee61272a5f2/synapse/storage/util/id_generators.py#L344
|
||||||
|
|
||||||
|
SELECT setval('cache_invalidation_stream_seq', (
|
||||||
|
SELECT COALESCE(MAX(last_value), 1) FROM cache_invalidation_stream_seq
|
||||||
|
));
|
|
@ -26,6 +26,10 @@
|
||||||
* (event, state) pair, we can use that stream_ordering to identify when
|
* (event, state) pair, we can use that stream_ordering to identify when
|
||||||
* the new state was assigned for the event.
|
* the new state was assigned for the event.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
/* NB: This table belongs to the `main` logical database; it should not be present
|
||||||
|
* in `state`.
|
||||||
|
*/
|
||||||
CREATE TABLE IF NOT EXISTS ex_outlier_stream(
|
CREATE TABLE IF NOT EXISTS ex_outlier_stream(
|
||||||
event_stream_ordering BIGINT PRIMARY KEY NOT NULL,
|
event_stream_ordering BIGINT PRIMARY KEY NOT NULL,
|
||||||
event_id TEXT NOT NULL,
|
event_id TEXT NOT NULL,
|
||||||
|
|
Loading…
Reference in a new issue