mirror of
https://github.com/element-hq/synapse
synced 2024-09-12 09:25:11 +00:00
Merge branch 'develop' into madlittlemods/11850-migrate-to-opentelemetry
Conflicts: synapse/storage/schema/__init__.py
This commit is contained in:
Eric Eastwood
commit
19c6f6ecc9
70 changed files with 290 additions and 94 deletions
4
.github/workflows/tests.yml
vendored
4
.github/workflows/tests.yml
vendored
|
|
@ -468,10 +468,6 @@ jobs:
|
|||
tests-done:
|
||||
if: ${{ always() }}
|
||||
needs:
|
||||
- check-sampleconfig
|
||||
- lint
|
||||
- lint-crlf
|
||||
- lint-newsfile
|
||||
- trial
|
||||
- trial-olddeps
|
||||
- sytest
|
||||
|
|
|
|||
108
CHANGES.md
108
CHANGES.md
|
|
@ -1,3 +1,111 @@
|
|||
Synapse 1.67.0 (2022-09-13)
|
||||
===========================
|
||||
|
||||
This release removes using the deprecated direct TCP replication configuration
|
||||
for workers. Server admins should use Redis instead. See the [upgrade
|
||||
notes](https://matrix-org.github.io/synapse/v1.67/upgrade.html#upgrading-to-v1670).
|
||||
|
||||
The minimum version of `poetry` supported for managing source checkouts is now
|
||||
1.2.0.
|
||||
|
||||
**Notice:** from the next major release (1.68.0) installing Synapse from a source
|
||||
checkout will require a recent Rust compiler. Those using packages or
|
||||
`pip install matrix-synapse` will not be affected. See the [upgrade
|
||||
notes](https://matrix-org.github.io/synapse/v1.67/upgrade.html#upgrading-to-v1670).
|
||||
|
||||
**Notice:** from the next major release (1.68.0), running Synapse with a SQLite
|
||||
database will require SQLite version 3.27.0 or higher. (The [current minimum
|
||||
version is SQLite 3.22.0](https://github.com/matrix-org/synapse/blob/release-v1.67/synapse/storage/engines/sqlite.py#L69-L78).)
|
||||
See [#12983](https://github.com/matrix-org/synapse/issues/12983) and the [upgrade notes](https://matrix-org.github.io/synapse/v1.67/upgrade.html#upgrading-to-v1670) for more details.
|
||||
|
||||
|
||||
No significant changes since 1.67.0rc1.
|
||||
|
||||
|
||||
Synapse 1.67.0rc1 (2022-09-06)
|
||||
==============================
|
||||
|
||||
Features
|
||||
--------
|
||||
|
||||
- Support setting the registration shared secret in a file, via a new `registration_shared_secret_path` configuration option. ([\#13614](https://github.com/matrix-org/synapse/issues/13614))
|
||||
- Change the default startup behaviour so that any missing "additional" configuration files (signing key, etc) are generated automatically. ([\#13615](https://github.com/matrix-org/synapse/issues/13615))
|
||||
- Improve performance of sending messages in rooms with thousands of local users. ([\#13634](https://github.com/matrix-org/synapse/issues/13634))
|
||||
|
||||
|
||||
Bugfixes
|
||||
--------
|
||||
|
||||
- Fix a bug introduced in Synapse 1.13 where the [List Rooms admin API](https://matrix-org.github.io/synapse/develop/admin_api/rooms.html#list-room-api) would return integers instead of booleans for the `federatable` and `public` fields when using a Sqlite database. ([\#13509](https://github.com/matrix-org/synapse/issues/13509))
|
||||
- Fix bug that user cannot `/forget` rooms after the last member has left the room. ([\#13546](https://github.com/matrix-org/synapse/issues/13546))
|
||||
- Faster Room Joins: fix `/make_knock` blocking indefinitely when the room in question is a partial-stated room. ([\#13583](https://github.com/matrix-org/synapse/issues/13583))
|
||||
- Fix loading the current stream position behind the actual position. ([\#13585](https://github.com/matrix-org/synapse/issues/13585))
|
||||
- Fix a longstanding bug in `register_new_matrix_user` which meant it was always necessary to explicitly give a server URL. ([\#13616](https://github.com/matrix-org/synapse/issues/13616))
|
||||
- Fix the running of [MSC1763](https://github.com/matrix-org/matrix-spec-proposals/pull/1763) retention purge_jobs in deployments with background jobs running on a worker by forcing them back onto the main worker. Contributed by Brad @ Beeper. ([\#13632](https://github.com/matrix-org/synapse/issues/13632))
|
||||
- Fix a long-standing bug that downloaded media for URL previews was not deleted while database background updates were running. ([\#13657](https://github.com/matrix-org/synapse/issues/13657))
|
||||
- Fix [MSC3030](https://github.com/matrix-org/matrix-spec-proposals/pull/3030) `/timestamp_to_event` endpoint to return the correct next event when the events have the same timestamp. ([\#13658](https://github.com/matrix-org/synapse/issues/13658))
|
||||
- Fix bug where we wedge media plugins if clients disconnect early. Introduced in v1.22.0. ([\#13660](https://github.com/matrix-org/synapse/issues/13660))
|
||||
- Fix a long-standing bug which meant that keys for unwhitelisted servers were not returned by `/_matrix/key/v2/query`. ([\#13683](https://github.com/matrix-org/synapse/issues/13683))
|
||||
- Fix a bug introduced in Synapse v1.20.0 that would cause the unstable unread counts from [MSC2654](https://github.com/matrix-org/matrix-spec-proposals/pull/2654) to be calculated even if the feature is disabled. ([\#13694](https://github.com/matrix-org/synapse/issues/13694))
|
||||
|
||||
|
||||
Updates to the Docker image
|
||||
---------------------------
|
||||
|
||||
- Update docker image to use a stable version of poetry. ([\#13688](https://github.com/matrix-org/synapse/issues/13688))
|
||||
|
||||
|
||||
Improved Documentation
|
||||
----------------------
|
||||
|
||||
- Improve the description of the ["chain cover index"](https://matrix-org.github.io/synapse/latest/auth_chain_difference_algorithm.html) used internally by Synapse. ([\#13602](https://github.com/matrix-org/synapse/issues/13602))
|
||||
- Document how ["monthly active users"](https://matrix-org.github.io/synapse/latest/usage/administration/monthly_active_users.html) is calculated and used. ([\#13617](https://github.com/matrix-org/synapse/issues/13617))
|
||||
- Improve documentation around user registration. ([\#13640](https://github.com/matrix-org/synapse/issues/13640))
|
||||
- Remove documentation of legacy `frontend_proxy` worker app. ([\#13645](https://github.com/matrix-org/synapse/issues/13645))
|
||||
- Clarify documentation that HTTP replication traffic can be protected with a shared secret. ([\#13656](https://github.com/matrix-org/synapse/issues/13656))
|
||||
- Remove unintentional colons from [config manual](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html) headers. ([\#13665](https://github.com/matrix-org/synapse/issues/13665))
|
||||
- Update docs to make enabling metrics more clear. ([\#13678](https://github.com/matrix-org/synapse/issues/13678))
|
||||
- Clarify `(room_id, event_id)` global uniqueness and how we should scope our database schemas. ([\#13701](https://github.com/matrix-org/synapse/issues/13701))
|
||||
|
||||
|
||||
Deprecations and Removals
|
||||
-------------------------
|
||||
|
||||
- Drop support for calling `/_matrix/client/v3/rooms/{roomId}/invite` without an `id_access_token`, which was not permitted by the spec. Contributed by @Vetchu. ([\#13241](https://github.com/matrix-org/synapse/issues/13241))
|
||||
- Remove redundant `_get_joined_users_from_context` cache. Contributed by Nick @ Beeper (@fizzadar). ([\#13569](https://github.com/matrix-org/synapse/issues/13569))
|
||||
- Remove the ability to use direct TCP replication with workers. Direct TCP replication was deprecated in Synapse v1.18.0. Workers now require using Redis. ([\#13647](https://github.com/matrix-org/synapse/issues/13647))
|
||||
- Remove support for unstable [private read receipts](https://github.com/matrix-org/matrix-spec-proposals/pull/2285). ([\#13653](https://github.com/matrix-org/synapse/issues/13653), [\#13692](https://github.com/matrix-org/synapse/issues/13692))
|
||||
|
||||
|
||||
Internal Changes
|
||||
----------------
|
||||
|
||||
- Extend the release script to wait for GitHub Actions to finish and to be usable as a guide for the whole process. ([\#13483](https://github.com/matrix-org/synapse/issues/13483))
|
||||
- Add experimental configuration option to allow disabling legacy Prometheus metric names. ([\#13540](https://github.com/matrix-org/synapse/issues/13540))
|
||||
- Cache user IDs instead of profiles to reduce cache memory usage. Contributed by Nick @ Beeper (@fizzadar). ([\#13573](https://github.com/matrix-org/synapse/issues/13573), [\#13600](https://github.com/matrix-org/synapse/issues/13600))
|
||||
- Optimize how Synapse calculates domains to fetch from during backfill. ([\#13575](https://github.com/matrix-org/synapse/issues/13575))
|
||||
- Comment about a better future where we can get the state diff between two events. ([\#13586](https://github.com/matrix-org/synapse/issues/13586))
|
||||
- Instrument `_check_sigs_and_hash_and_fetch` to trace time spent in child concurrent calls for understandable traces in Jaeger. ([\#13588](https://github.com/matrix-org/synapse/issues/13588))
|
||||
- Improve performance of `@cachedList`. ([\#13591](https://github.com/matrix-org/synapse/issues/13591))
|
||||
- Minor speed up of fetching large numbers of push rules. ([\#13592](https://github.com/matrix-org/synapse/issues/13592))
|
||||
- Optimise push action fetching queries. Contributed by Nick @ Beeper (@fizzadar). ([\#13597](https://github.com/matrix-org/synapse/issues/13597))
|
||||
- Rename `event_map` to `unpersisted_events` when computing the auth differences. ([\#13603](https://github.com/matrix-org/synapse/issues/13603))
|
||||
- Refactor `get_users_in_room(room_id)` mis-use with dedicated `get_current_hosts_in_room(room_id)` function. ([\#13605](https://github.com/matrix-org/synapse/issues/13605))
|
||||
- Use dedicated `get_local_users_in_room(room_id)` function to find local users when calculating `join_authorised_via_users_server` of a `/make_join` request. ([\#13606](https://github.com/matrix-org/synapse/issues/13606))
|
||||
- Refactor `get_users_in_room(room_id)` mis-use to lookup single local user with dedicated `check_local_user_in_room(...)` function. ([\#13608](https://github.com/matrix-org/synapse/issues/13608))
|
||||
- Drop unused column `application_services_state.last_txn`. ([\#13627](https://github.com/matrix-org/synapse/issues/13627))
|
||||
- Improve readability of Complement CI logs by printing failure results last. ([\#13639](https://github.com/matrix-org/synapse/issues/13639))
|
||||
- Generalise the `@cancellable` annotation so it can be used on functions other than just servlet methods. ([\#13662](https://github.com/matrix-org/synapse/issues/13662))
|
||||
- Introduce a `CommonUsageMetrics` class to share some usage metrics between the Prometheus exporter and the phone home stats. ([\#13671](https://github.com/matrix-org/synapse/issues/13671))
|
||||
- Add some logging to help track down #13444. ([\#13679](https://github.com/matrix-org/synapse/issues/13679))
|
||||
- Update poetry lock file for v1.2.0. ([\#13689](https://github.com/matrix-org/synapse/issues/13689))
|
||||
- Add cache to `is_partial_state_room`. ([\#13693](https://github.com/matrix-org/synapse/issues/13693))
|
||||
- Update the Grafana dashboard that is included with Synapse in the `contrib` directory. ([\#13697](https://github.com/matrix-org/synapse/issues/13697))
|
||||
- Only run trial CI on all python versions on non-PRs. ([\#13698](https://github.com/matrix-org/synapse/issues/13698))
|
||||
- Fix typechecking with latest types-jsonschema. ([\#13712](https://github.com/matrix-org/synapse/issues/13712))
|
||||
- Reduce number of CI checks we run for PRs. ([\#13713](https://github.com/matrix-org/synapse/issues/13713))
|
||||
|
||||
|
||||
Synapse 1.66.0 (2022-08-31)
|
||||
===========================
|
||||
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
Drop support for calling `/_matrix/client/v3/rooms/{roomId}/invite` without an `id_access_token`, which was not permitted by the spec. Contributed by @Vetchu.
|
||||
1
changelog.d/13480.doc
Normal file
1
changelog.d/13480.doc
Normal file
|
|
@ -0,0 +1 @@
|
|||
Note that `libpq` is required on ARM-based Macs.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Extend the release script to wait for GitHub Actions to finish and to be usable as a guide for the whole process.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Fix a bug introduced in Synapse 1.13 where the [List Rooms admin API](https://matrix-org.github.io/synapse/develop/admin_api/rooms.html#list-room-api) would return integers instead of booleans for the `federatable` and `public` fields when using a Sqlite database.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Add experimental configuration option to allow disabling legacy Prometheus metric names.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Fix bug that user cannot `/forget` rooms after the last member has left the room.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Remove redundant `_get_joined_users_from_context` cache. Contributed by Nick @ Beeper (@fizzadar).
|
||||
|
|
@ -1 +0,0 @@
|
|||
Cache user IDs instead of profiles to reduce cache memory usage. Contributed by Nick @ Beeper (@fizzadar).
|
||||
|
|
@ -1 +0,0 @@
|
|||
Optimize how Synapse calculates domains to fetch from during backfill.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Faster Room Joins: fix `/make_knock` blocking indefinitely when the room in question is a partial-stated room.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Fix loading the current stream position behind the actual position.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Comment about a better future where we can get the state diff between two events.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Instrument `_check_sigs_and_hash_and_fetch` to trace time spent in child concurrent calls for understandable traces in Jaeger.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Improve performance of `@cachedList`.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Minor speed up of fetching large numbers of push rules.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Optimise push action fetching queries. Contributed by Nick @ Beeper (@fizzadar).
|
||||
|
|
@ -1 +0,0 @@
|
|||
Cache user IDs instead of profiles to reduce cache memory usage. Contributed by Nick @ Beeper (@fizzadar).
|
||||
|
|
@ -1 +0,0 @@
|
|||
Improve the description of the ["chain cover index"](https://matrix-org.github.io/synapse/latest/auth_chain_difference_algorithm.html) used internally by Synapse.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Rename `event_map` to `unpersisted_events` when computing the auth differences.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Refactor `get_users_in_room(room_id)` mis-use with dedicated `get_current_hosts_in_room(room_id)` function.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Use dedicated `get_local_users_in_room(room_id)` function to find local users when calculating `join_authorised_via_users_server` of a `/make_join` request.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Refactor `get_users_in_room(room_id)` mis-use to lookup single local user with dedicated `check_local_user_in_room(...)` function.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Support setting the registration shared secret in a file, via a new `registration_shared_secret_path` configuration option.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Change the default startup behaviour so that any missing "additional" configuration files (signing key, etc) are generated automatically.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Fix a longstanding bug in `register_new_matrix_user` which meant it was always necessary to explicitly give a server URL.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Document how ["monthly active users"](https://matrix-org.github.io/synapse/latest/usage/administration/monthly_active_users.html) is calculated and used.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Drop unused column `application_services_state.last_txn`.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Fix the running of MSC1763 retention purge_jobs in deployments with background jobs running on a worker by forcing them back onto the main worker. Contributed by Brad @ Beeper.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Improve performance of sending messages in rooms with thousands of local users.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Improve readability of Complement CI logs by printing failure results last.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Improve documentation around user registration.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Remove documentation of legacy `frontend_proxy` worker app.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Remove the ability to use direct TCP replication with workers. Direct TCP replication was deprecated in Synapse v1.18.0. Workers now require using Redis.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Remove support for unstable [private read receipts](https://github.com/matrix-org/matrix-spec-proposals/pull/2285).
|
||||
|
|
@ -1 +0,0 @@
|
|||
Clarify documentation that HTTP replication traffic can be protected with a shared secret.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Fix a long-standing bug that downloaded media for URL previews was not deleted while database background updates were running.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Fix MSC3030 `/timestamp_to_event` endpoint to return the correct next event when the events have the same timestamp.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Fix bug where we wedge media plugins if clients disconnect early. Introduced in v1.22.0.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Generalise the `@cancellable` annotation so it can be used on functions other than just servlet methods.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Remove unintentional colons from [config manual](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html) headers.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Introduce a `CommonUsageMetrics` class to share some usage metrics between the Prometheus exporter and the phone home stats.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Update docs to make enabling metrics more clear.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Add some logging to help track down #13444.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Fix a long-standing bug which meant that keys for unwhitelisted servers were not returned by `/_matrix/key/v2/query`.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Update docker image to use a stable version of poetry.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Update poetry lock file for v1.2.0.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Remove support for unstable [private read receipts](https://github.com/matrix-org/matrix-spec-proposals/pull/2285).
|
||||
|
|
@ -1 +0,0 @@
|
|||
Add cache to `is_partial_state_room`.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Fix a bug introduced in Synapse v1.20.0 that would cause the unstable unread counts from [MSC2654](https://github.com/matrix-org/matrix-spec-proposals/pull/2654) to be calculated even if the feature is disabled.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Update the Grafana dashboard that is included with Synapse in the `contrib` directory.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Only run trial CI on all python versions on non-PRs.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Clarify `(room_id, event_id)` global uniqueness and how we should scope our database schemas.
|
||||
1
changelog.d/13703.misc
Normal file
1
changelog.d/13703.misc
Normal file
|
|
@ -0,0 +1 @@
|
|||
Add & populate `event_stream_ordering` column on receipts table for future optimisation of push action processing. Contributed by Nick @ Beeper (@fizzadar).
|
||||
|
|
@ -1 +0,0 @@
|
|||
Fix typechecking with latest types-jsonschema.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Reduce number of CI checks we run for PRs.
|
||||
1
changelog.d/13766.bugfix
Normal file
1
changelog.d/13766.bugfix
Normal file
|
|
@ -0,0 +1 @@
|
|||
Fix a long-standing bug where the `cache_invalidation_stream_seq` sequence would begin at 1 instead of 2.
|
||||
1
changelog.d/13770.misc
Normal file
1
changelog.d/13770.misc
Normal file
|
|
@ -0,0 +1 @@
|
|||
Update the script which makes full schema dumps.
|
||||
1
changelog.d/13784.misc
Normal file
1
changelog.d/13784.misc
Normal file
|
|
@ -0,0 +1 @@
|
|||
Simplify the dependency DAG in the tests workflow.
|
||||
14
debian/changelog
vendored
14
debian/changelog
vendored
|
|
@ -1,8 +1,18 @@
|
|||
matrix-synapse-py3 (1.66.0ubuntu1) UNRELEASED; urgency=medium
|
||||
matrix-synapse-py3 (1.67.0) stable; urgency=medium
|
||||
|
||||
* New Synapse release 1.67.0.
|
||||
|
||||
-- Synapse Packaging team <packages@matrix.org> Tue, 13 Sep 2022 09:19:56 +0100
|
||||
|
||||
matrix-synapse-py3 (1.67.0~rc1) stable; urgency=medium
|
||||
|
||||
[ Erik Johnston ]
|
||||
* Use stable poetry 1.2.0 version, rather than a prerelease.
|
||||
|
||||
-- Erik Johnston <erik@matrix.org> Thu, 01 Sep 2022 13:48:31 +0100
|
||||
[ Synapse Packaging team ]
|
||||
* New Synapse release 1.67.0rc1.
|
||||
|
||||
-- Synapse Packaging team <packages@matrix.org> Tue, 06 Sep 2022 09:01:06 +0100
|
||||
|
||||
matrix-synapse-py3 (1.66.0) stable; urgency=medium
|
||||
|
||||
|
|
|
|||
|
|
@ -303,9 +303,10 @@ You may need to install the latest Xcode developer tools:
|
|||
xcode-select --install
|
||||
```
|
||||
|
||||
On ARM-based Macs you may need to explicitly install libjpeg which is a pillow dependency. You can use Homebrew (https://brew.sh):
|
||||
On ARM-based Macs you may need to install libjpeg and libpq.
|
||||
You can use Homebrew (https://brew.sh):
|
||||
```sh
|
||||
brew install jpeg
|
||||
brew install jpeg libpq
|
||||
```
|
||||
|
||||
On macOS Catalina (10.15) you may need to explicitly install OpenSSL
|
||||
|
|
|
|||
|
|
@ -111,6 +111,30 @@ and remove the TCP `replication` listener from config of the master and
|
|||
The minimum supported version of poetry is now 1.2. This should only affect
|
||||
those installing from a source checkout.
|
||||
|
||||
## Rust requirement in the next release
|
||||
|
||||
From the next major release (v1.68.0) installing Synapse from a source checkout
|
||||
will require a recent Rust compiler. Those using packages or
|
||||
`pip install matrix-synapse` will not be affected.
|
||||
|
||||
The simplest way of installing Rust is via [rustup.rs](https://rustup.rs/)
|
||||
|
||||
## SQLite version requirement in the next release
|
||||
|
||||
From the next major release (v1.68.0) Synapse will require SQLite 3.27.0 or
|
||||
higher. Synapse v1.67.0 will be the last major release supporting SQLite
|
||||
versions 3.22 to 3.26.
|
||||
|
||||
Those using docker images or Debian packages from Matrix.org will not be
|
||||
affected. If you have installed from source, you should check the version of
|
||||
SQLite used by Python with:
|
||||
|
||||
```shell
|
||||
python -c "import sqlite3; print(sqlite3.sqlite_version)"
|
||||
```
|
||||
|
||||
If this is too old, refer to your distribution for advice on upgrading.
|
||||
|
||||
# Upgrading to v1.66.0
|
||||
|
||||
## Delegation of email validation no longer supported
|
||||
|
|
|
|||
|
|
@ -9,8 +9,10 @@
|
|||
export PGHOST="localhost"
|
||||
POSTGRES_DB_NAME="synapse_full_schema.$$"
|
||||
|
||||
SQLITE_FULL_SCHEMA_OUTPUT_FILE="full.sql.sqlite"
|
||||
POSTGRES_FULL_SCHEMA_OUTPUT_FILE="full.sql.postgres"
|
||||
SQLITE_SCHEMA_FILE="schema.sql.sqlite"
|
||||
SQLITE_ROWS_FILE="rows.sql.sqlite"
|
||||
POSTGRES_SCHEMA_FILE="full.sql.postgres"
|
||||
POSTGRES_ROWS_FILE="rows.sql.postgres"
|
||||
|
||||
REQUIRED_DEPS=("matrix-synapse" "psycopg2")
|
||||
|
||||
|
|
@ -22,7 +24,7 @@ usage() {
|
|||
echo " Username to connect to local postgres instance. The password will be requested"
|
||||
echo " during script execution."
|
||||
echo "-c"
|
||||
echo " CI mode. Enables coverage tracking and prints every command that the script runs."
|
||||
echo " CI mode. Prints every command that the script runs."
|
||||
echo "-o <path>"
|
||||
echo " Directory to output full schema files to."
|
||||
echo "-h"
|
||||
|
|
@ -37,11 +39,6 @@ while getopts "p:co:h" opt; do
|
|||
c)
|
||||
# Print all commands that are being executed
|
||||
set -x
|
||||
|
||||
# Modify required dependencies for coverage
|
||||
REQUIRED_DEPS+=("coverage" "coverage-enable-subprocess")
|
||||
|
||||
COVERAGE=1
|
||||
;;
|
||||
o)
|
||||
command -v realpath > /dev/null || (echo "The -o flag requires the 'realpath' binary to be installed" && exit 1)
|
||||
|
|
@ -102,6 +99,7 @@ SQLITE_DB=$TMPDIR/homeserver.db
|
|||
POSTGRES_CONFIG=$TMPDIR/postgres.conf
|
||||
|
||||
# Ensure these files are delete on script exit
|
||||
# TODO: the trap should also drop the temp postgres DB
|
||||
trap 'rm -rf $TMPDIR' EXIT
|
||||
|
||||
cat > "$SQLITE_CONFIG" <<EOF
|
||||
|
|
@ -147,48 +145,34 @@ python -m synapse.app.homeserver --generate-keys -c "$SQLITE_CONFIG"
|
|||
|
||||
# Make sure the SQLite3 database is using the latest schema and has no pending background update.
|
||||
echo "Running db background jobs..."
|
||||
synapse/_scripts/update_synapse_database.py --database-config --run-background-updates "$SQLITE_CONFIG"
|
||||
synapse/_scripts/update_synapse_database.py --database-config "$SQLITE_CONFIG" --run-background-updates
|
||||
|
||||
# Create the PostgreSQL database.
|
||||
echo "Creating postgres database..."
|
||||
createdb --lc-collate=C --lc-ctype=C --template=template0 "$POSTGRES_DB_NAME"
|
||||
|
||||
echo "Copying data from SQLite3 to Postgres with synapse_port_db..."
|
||||
if [ -z "$COVERAGE" ]; then
|
||||
# No coverage needed
|
||||
synapse/_scripts/synapse_port_db.py --sqlite-database "$SQLITE_DB" --postgres-config "$POSTGRES_CONFIG"
|
||||
else
|
||||
# Coverage desired
|
||||
coverage run synapse/_scripts/synapse_port_db.py --sqlite-database "$SQLITE_DB" --postgres-config "$POSTGRES_CONFIG"
|
||||
fi
|
||||
echo "Running db background jobs..."
|
||||
synapse/_scripts/update_synapse_database.py --database-config "$POSTGRES_CONFIG" --run-background-updates
|
||||
|
||||
|
||||
# Delete schema_version, applied_schema_deltas and applied_module_schemas tables
|
||||
# Also delete any shadow tables from fts4
|
||||
# This needs to be done after synapse_port_db is run
|
||||
echo "Dropping unwanted db tables..."
|
||||
SQL="
|
||||
DROP TABLE schema_version;
|
||||
DROP TABLE applied_schema_deltas;
|
||||
DROP TABLE applied_module_schemas;
|
||||
DROP TABLE event_search_content;
|
||||
DROP TABLE event_search_segments;
|
||||
DROP TABLE event_search_segdir;
|
||||
DROP TABLE event_search_docsize;
|
||||
DROP TABLE event_search_stat;
|
||||
DROP TABLE user_directory_search_content;
|
||||
DROP TABLE user_directory_search_segments;
|
||||
DROP TABLE user_directory_search_segdir;
|
||||
DROP TABLE user_directory_search_docsize;
|
||||
DROP TABLE user_directory_search_stat;
|
||||
"
|
||||
sqlite3 "$SQLITE_DB" <<< "$SQL"
|
||||
psql "$POSTGRES_DB_NAME" -w <<< "$SQL"
|
||||
|
||||
echo "Dumping SQLite3 schema to '$OUTPUT_DIR/$SQLITE_FULL_SCHEMA_OUTPUT_FILE'..."
|
||||
sqlite3 "$SQLITE_DB" ".dump" > "$OUTPUT_DIR/$SQLITE_FULL_SCHEMA_OUTPUT_FILE"
|
||||
echo "Dumping SQLite3 schema to '$OUTPUT_DIR/$SQLITE_SCHEMA_FILE' and '$OUTPUT_DIR/$SQLITE_ROWS_FILE'..."
|
||||
sqlite3 "$SQLITE_DB" ".schema --indent" > "$OUTPUT_DIR/$SQLITE_SCHEMA_FILE"
|
||||
sqlite3 "$SQLITE_DB" ".dump --data-only --nosys" > "$OUTPUT_DIR/$SQLITE_ROWS_FILE"
|
||||
|
||||
echo "Dumping Postgres schema to '$OUTPUT_DIR/$POSTGRES_FULL_SCHEMA_OUTPUT_FILE'..."
|
||||
pg_dump --format=plain --no-tablespaces --no-acl --no-owner $POSTGRES_DB_NAME | sed -e '/^--/d' -e 's/public\.//g' -e '/^SET /d' -e '/^SELECT /d' > "$OUTPUT_DIR/$POSTGRES_FULL_SCHEMA_OUTPUT_FILE"
|
||||
echo "Dumping Postgres schema to '$OUTPUT_DIR/$POSTGRES_SCHEMA_FILE' and '$OUTPUT_DIR/$POSTGRES_ROWS_FILE'..."
|
||||
pg_dump --format=plain --schema-only --no-tablespaces --no-acl --no-owner "$POSTGRES_DB_NAME" | sed -e '/^$/d' -e '/^--/d' -e 's/public\.//g' -e '/^SET /d' -e '/^SELECT /d' > "$OUTPUT_DIR/$POSTGRES_SCHEMA_FILE"
|
||||
pg_dump --format=plain --data-only --inserts --no-tablespaces --no-acl --no-owner "$POSTGRES_DB_NAME" | sed -e '/^$/d' -e '/^--/d' -e 's/public\.//g' -e '/^SET /d' -e '/^SELECT /d' > "$OUTPUT_DIR/$POSTGRES_ROWS_FILE"
|
||||
|
||||
echo "Cleaning up temporary Postgres database..."
|
||||
dropdb $POSTGRES_DB_NAME
|
||||
|
|
|
|||
|
|
@ -67,6 +67,7 @@ from synapse.storage.databases.main.media_repository import (
|
|||
)
|
||||
from synapse.storage.databases.main.presence import PresenceBackgroundUpdateStore
|
||||
from synapse.storage.databases.main.pusher import PusherWorkerStore
|
||||
from synapse.storage.databases.main.receipts import ReceiptsBackgroundUpdateStore
|
||||
from synapse.storage.databases.main.registration import (
|
||||
RegistrationBackgroundUpdateStore,
|
||||
find_max_generated_user_id_localpart,
|
||||
|
|
@ -203,6 +204,7 @@ class Store(
|
|||
PushRuleStore,
|
||||
PusherWorkerStore,
|
||||
PresenceBackgroundUpdateStore,
|
||||
ReceiptsBackgroundUpdateStore,
|
||||
):
|
||||
def execute(self, f: Callable[..., R], *args: Any, **kwargs: Any) -> Awaitable[R]:
|
||||
return self.db_pool.runInteraction(f.__name__, f, *args, **kwargs)
|
||||
|
|
|
|||
|
|
@ -675,6 +675,7 @@ class ReceiptsWorkerStore(SQLBaseStore):
|
|||
values={
|
||||
"stream_id": stream_id,
|
||||
"event_id": event_id,
|
||||
"event_stream_ordering": stream_ordering,
|
||||
"data": json_encoder.encode(data),
|
||||
},
|
||||
# receipts_linearized has a unique constraint on
|
||||
|
|
@ -830,5 +831,76 @@ class ReceiptsWorkerStore(SQLBaseStore):
|
|||
)
|
||||
|
||||
|
||||
class ReceiptsStore(ReceiptsWorkerStore):
|
||||
class ReceiptsBackgroundUpdateStore(SQLBaseStore):
|
||||
POPULATE_RECEIPT_EVENT_STREAM_ORDERING = "populate_event_stream_ordering"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
database: DatabasePool,
|
||||
db_conn: LoggingDatabaseConnection,
|
||||
hs: "HomeServer",
|
||||
):
|
||||
super().__init__(database, db_conn, hs)
|
||||
|
||||
self.db_pool.updates.register_background_update_handler(
|
||||
self.POPULATE_RECEIPT_EVENT_STREAM_ORDERING,
|
||||
self._populate_receipt_event_stream_ordering,
|
||||
)
|
||||
|
||||
async def _populate_receipt_event_stream_ordering(
|
||||
self, progress: JsonDict, batch_size: int
|
||||
) -> int:
|
||||
def _populate_receipt_event_stream_ordering_txn(
|
||||
txn: LoggingTransaction,
|
||||
) -> bool:
|
||||
|
||||
if "max_stream_id" in progress:
|
||||
max_stream_id = progress["max_stream_id"]
|
||||
else:
|
||||
txn.execute("SELECT max(stream_id) FROM receipts_linearized")
|
||||
res = txn.fetchone()
|
||||
if res is None or res[0] is None:
|
||||
return True
|
||||
else:
|
||||
max_stream_id = res[0]
|
||||
|
||||
start = progress.get("stream_id", 0)
|
||||
stop = start + batch_size
|
||||
|
||||
sql = """
|
||||
UPDATE receipts_linearized
|
||||
SET event_stream_ordering = (
|
||||
SELECT stream_ordering
|
||||
FROM events
|
||||
WHERE event_id = receipts_linearized.event_id
|
||||
)
|
||||
WHERE stream_id >= ? AND stream_id < ?
|
||||
"""
|
||||
txn.execute(sql, (start, stop))
|
||||
|
||||
self.db_pool.updates._background_update_progress_txn(
|
||||
txn,
|
||||
self.POPULATE_RECEIPT_EVENT_STREAM_ORDERING,
|
||||
{
|
||||
"stream_id": stop,
|
||||
"max_stream_id": max_stream_id,
|
||||
},
|
||||
)
|
||||
|
||||
return stop > max_stream_id
|
||||
|
||||
finished = await self.db_pool.runInteraction(
|
||||
"_remove_devices_from_device_inbox_txn",
|
||||
_populate_receipt_event_stream_ordering_txn,
|
||||
)
|
||||
|
||||
if finished:
|
||||
await self.db_pool.updates._end_background_update(
|
||||
self.POPULATE_RECEIPT_EVENT_STREAM_ORDERING
|
||||
)
|
||||
|
||||
return batch_size
|
||||
|
||||
|
||||
class ReceiptsStore(ReceiptsWorkerStore, ReceiptsBackgroundUpdateStore):
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -76,6 +76,7 @@ Changes in SCHEMA_VERSION = 72:
|
|||
- event_edges.(room_id, is_state) are no longer written to.
|
||||
- Tables related to groups are dropped.
|
||||
- Unused column application_services_state.last_txn is dropped
|
||||
- Cache invalidation stream id sequence now begins at 2 to match code expectation.
|
||||
- Rename column in `device_lists_outbound_pokes` and `device_lists_changes_in_room`
|
||||
from `opentracing_context` to generalized `tracing_context`.
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -0,0 +1,19 @@
|
|||
/* Copyright 2022 Beeper
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
ALTER TABLE receipts_linearized ADD COLUMN event_stream_ordering BIGINT;
|
||||
|
||||
INSERT INTO background_updates (update_name, progress_json) VALUES
|
||||
('populate_event_stream_ordering', '{}');
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
/* Copyright 2022 The Matrix.org Foundation C.I.C
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
|
||||
-- The sequence needs to begin at 2 because a bunch of code assumes that
|
||||
-- get_next_id_txn will return values >= 2, cf this comment:
|
||||
-- https://github.com/matrix-org/synapse/blob/b93bd95e8ab64d27ae26841020f62ee61272a5f2/synapse/storage/util/id_generators.py#L344
|
||||
|
||||
SELECT setval('cache_invalidation_stream_seq', (
|
||||
SELECT COALESCE(MAX(last_value), 1) FROM cache_invalidation_stream_seq
|
||||
));
|
||||
|
|
@ -26,6 +26,10 @@
|
|||
* (event, state) pair, we can use that stream_ordering to identify when
|
||||
* the new state was assigned for the event.
|
||||
*/
|
||||
|
||||
/* NB: This table belongs to the `main` logical database; it should not be present
|
||||
* in `state`.
|
||||
*/
|
||||
CREATE TABLE IF NOT EXISTS ex_outlier_stream(
|
||||
event_stream_ordering BIGINT PRIMARY KEY NOT NULL,
|
||||
event_id TEXT NOT NULL,
|
||||
|
|
|
|||
Loading…
Reference in a new issue