diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index f5952d1fc0..874d41447a 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -300,6 +300,9 @@ class SQLBaseStore(object): def inner_func(conn, *args, **kwargs): with LoggingContext("runInteraction") as context: + if self.database_engine.is_connection_closed(conn): + conn.reconnect() + current_context.copy_to(context) start = time.time() * 1000 txn_id = self._TXN_ID @@ -322,12 +325,35 @@ class SQLBaseStore(object): LoggingTransaction(txn, name, self.database_engine), *args, **kwargs ) + except self.database_engine.module.OperationalError as e: + # This can happen if the database disappears mid + # transaction. + logger.warn( + "[TXN OPERROR] {%s} %s %d/%d", + name, e, i, N + ) + if i < N: + i += 1 + try: + conn.rollback() + except self.database_engine.module.Error as e1: + logger.warn( + "[TXN EROLL] {%s} %s", + name, e1, + ) + continue except self.database_engine.module.DatabaseError as e: if self.database_engine.is_deadlock(e): logger.warn("[TXN DEADLOCK] {%s} %d/%d", name, i, N) if i < N: i += 1 - conn.rollback() + try: + conn.rollback() + except self.database_engine.module.Error as e1: + logger.warn( + "[TXN EROLL] {%s} %s", + name, e1, + ) continue raise except Exception as e: diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index 6f75245fa7..00dbae7b60 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -39,3 +39,6 @@ class PostgresEngine(object): if isinstance(error, self.module.DatabaseError): return error.pgcode in ["40001", "40P01"] return False + + def is_connection_closed(self, conn): + return bool(conn) diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py index 72c11df461..39828a597c 100644 --- a/synapse/storage/engines/sqlite3.py +++ b/synapse/storage/engines/sqlite3.py @@ -35,3 +35,6 @@ class Sqlite3Engine(object): def is_deadlock(self, error): return False + + def is_connection_closed(self, conn): + return False