mirror of
https://github.com/element-hq/synapse
synced 2024-08-18 23:50:52 +00:00
Minor perf fixes to get_auth_chain_ids
.
This commit is contained in:
parent
fc87d2ffb3
commit
7b7c3cedf2
3 changed files with 6 additions and 7 deletions
1
changelog.d/6954.misc
Normal file
1
changelog.d/6954.misc
Normal file
|
@ -0,0 +1 @@
|
||||||
|
Minor perf fixes to `get_auth_chain_ids`.
|
|
@ -16,7 +16,6 @@ import itertools
|
||||||
import logging
|
import logging
|
||||||
from typing import List, Optional, Set
|
from typing import List, Optional, Set
|
||||||
|
|
||||||
from six.moves import range
|
|
||||||
from six.moves.queue import Empty, PriorityQueue
|
from six.moves.queue import Empty, PriorityQueue
|
||||||
|
|
||||||
from twisted.internet import defer
|
from twisted.internet import defer
|
||||||
|
@ -28,6 +27,7 @@ from synapse.storage.data_stores.main.events_worker import EventsWorkerStore
|
||||||
from synapse.storage.data_stores.main.signatures import SignatureWorkerStore
|
from synapse.storage.data_stores.main.signatures import SignatureWorkerStore
|
||||||
from synapse.storage.database import Database
|
from synapse.storage.database import Database
|
||||||
from synapse.util.caches.descriptors import cached
|
from synapse.util.caches.descriptors import cached
|
||||||
|
from synapse.util.iterutils import batch_iter
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
@ -88,14 +88,12 @@ class EventFederationWorkerStore(EventsWorkerStore, SignatureWorkerStore, SQLBas
|
||||||
front = set(event_ids)
|
front = set(event_ids)
|
||||||
while front:
|
while front:
|
||||||
new_front = set()
|
new_front = set()
|
||||||
front_list = list(front)
|
for chunk in batch_iter(front, 100):
|
||||||
chunks = [front_list[x : x + 100] for x in range(0, len(front), 100)]
|
|
||||||
for chunk in chunks:
|
|
||||||
clause, args = make_in_list_sql_clause(
|
clause, args = make_in_list_sql_clause(
|
||||||
txn.database_engine, "event_id", chunk
|
txn.database_engine, "event_id", chunk
|
||||||
)
|
)
|
||||||
txn.execute(base_sql + clause, list(args))
|
txn.execute(base_sql + clause, args)
|
||||||
new_front.update([r[0] for r in txn])
|
new_front.update(r[0] for r in txn)
|
||||||
|
|
||||||
new_front -= ignore_events
|
new_front -= ignore_events
|
||||||
new_front -= results
|
new_front -= results
|
||||||
|
|
|
@ -1504,7 +1504,7 @@ class Database(object):
|
||||||
|
|
||||||
def make_in_list_sql_clause(
|
def make_in_list_sql_clause(
|
||||||
database_engine, column: str, iterable: Iterable
|
database_engine, column: str, iterable: Iterable
|
||||||
) -> Tuple[str, Iterable]:
|
) -> Tuple[str, list]:
|
||||||
"""Returns an SQL clause that checks the given column is in the iterable.
|
"""Returns an SQL clause that checks the given column is in the iterable.
|
||||||
|
|
||||||
On SQLite this expands to `column IN (?, ?, ...)`, whereas on Postgres
|
On SQLite this expands to `column IN (?, ?, ...)`, whereas on Postgres
|
||||||
|
|
Loading…
Reference in a new issue