Automatically apply SQL for inconsistent sequence (#17305)

Rather than forcing the server operator to apply the SQL manually.

This should be safe, as there should be only one writer for these
sequences.
This commit is contained in:
Erik Johnston 2024-06-14 16:40:29 +01:00 committed by GitHub
parent e6816babf6
commit a3cb244755
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 25 additions and 37 deletions

1
changelog.d/17305.misc Normal file
View file

@ -0,0 +1 @@
When rolling back to a previous Synapse version and then forwards again to this release, don't require server operators to manually run SQL.

View file

@ -255,13 +255,3 @@ however extreme care must be taken to avoid database corruption.
Note that the above may fail with an error about duplicate rows if corruption Note that the above may fail with an error about duplicate rows if corruption
has already occurred, and such duplicate rows will need to be manually removed. has already occurred, and such duplicate rows will need to be manually removed.
### Fixing inconsistent sequences error
Synapse uses Postgres sequences to generate IDs for various tables. A sequence
and associated table can get out of sync if, for example, Synapse has been
downgraded and then upgraded again.
To fix the issue shut down Synapse (including any and all workers) and run the
SQL command included in the error message. Once done Synapse should start
successfully.

View file

@ -36,21 +36,6 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
_INCONSISTENT_SEQUENCE_ERROR = """
Postgres sequence '%(seq)s' is inconsistent with associated
table '%(table)s'. This can happen if Synapse has been downgraded and
then upgraded again, or due to a bad migration.
To fix this error, shut down Synapse (including any and all workers)
and run the following SQL:
SELECT setval('%(seq)s', (
%(max_id_sql)s
));
See docs/postgres.md for more information.
"""
_INCONSISTENT_STREAM_ERROR = """ _INCONSISTENT_STREAM_ERROR = """
Postgres sequence '%(seq)s' is inconsistent with associated stream position Postgres sequence '%(seq)s' is inconsistent with associated stream position
of '%(stream_name)s' in the 'stream_positions' table. of '%(stream_name)s' in the 'stream_positions' table.
@ -169,25 +154,33 @@ class PostgresSequenceGenerator(SequenceGenerator):
if row: if row:
max_in_stream_positions = row[0] max_in_stream_positions = row[0]
txn.close()
# If `is_called` is False then `last_value` is actually the value that # If `is_called` is False then `last_value` is actually the value that
# will be generated next, so we decrement to get the true "last value". # will be generated next, so we decrement to get the true "last value".
if not is_called: if not is_called:
last_value -= 1 last_value -= 1
if max_stream_id > last_value: if max_stream_id > last_value:
# The sequence is lagging behind the tables. This is probably due to
# rolling back to a version before the sequence was used and then
# forwards again. We resolve this by setting the sequence to the
# right value.
logger.warning( logger.warning(
"Postgres sequence %s is behind table %s: %d < %d", "Postgres sequence %s is behind table %s: %d < %d. Updating sequence.",
self._sequence_name, self._sequence_name,
table, table,
last_value, last_value,
max_stream_id, max_stream_id,
) )
raise IncorrectDatabaseSetup(
_INCONSISTENT_SEQUENCE_ERROR sql = f"""
% {"seq": self._sequence_name, "table": table, "max_id_sql": table_sql} SELECT setval('{self._sequence_name}', GREATEST(
) (SELECT last_value FROM {self._sequence_name}),
({table_sql})
));
"""
txn.execute(sql)
txn.close()
# If we have values in the stream positions table then they have to be # If we have values in the stream positions table then they have to be
# less than or equal to `last_value` # less than or equal to `last_value`

View file

@ -28,7 +28,6 @@ from synapse.storage.database import (
LoggingDatabaseConnection, LoggingDatabaseConnection,
LoggingTransaction, LoggingTransaction,
) )
from synapse.storage.engines import IncorrectDatabaseSetup
from synapse.storage.types import Cursor from synapse.storage.types import Cursor
from synapse.storage.util.id_generators import MultiWriterIdGenerator from synapse.storage.util.id_generators import MultiWriterIdGenerator
from synapse.storage.util.sequence import ( from synapse.storage.util.sequence import (
@ -525,7 +524,7 @@ class WorkerMultiWriterIdGeneratorTestCase(MultiWriterIdGeneratorBase):
self.assertEqual(id_gen_5.get_current_token_for_writer("third"), 6) self.assertEqual(id_gen_5.get_current_token_for_writer("third"), 6)
def test_sequence_consistency(self) -> None: def test_sequence_consistency(self) -> None:
"""Test that we error out if the table and sequence diverges.""" """Test that we correct the sequence if the table and sequence diverges."""
# Prefill with some rows # Prefill with some rows
self._insert_row_with_id("master", 3) self._insert_row_with_id("master", 3)
@ -536,9 +535,14 @@ class WorkerMultiWriterIdGeneratorTestCase(MultiWriterIdGeneratorBase):
self.get_success(self.db_pool.runInteraction("_insert", _insert)) self.get_success(self.db_pool.runInteraction("_insert", _insert))
# Creating the ID gen should error # Creating the ID gen should now fix the inconsistency
with self.assertRaises(IncorrectDatabaseSetup): id_gen = self._create_id_generator()
self._create_id_generator("first")
async def _get_next_async() -> None:
async with id_gen.get_next() as stream_id:
self.assertEqual(stream_id, 27)
self.get_success(_get_next_async())
def test_minimal_local_token(self) -> None: def test_minimal_local_token(self) -> None:
self._insert_rows("first", 3) self._insert_rows("first", 3)