mirror of
https://github.com/element-hq/synapse
synced 2024-08-19 06:50:52 +00:00
Fix batching for fetching cross-signing keys
There's no point carefully dividing a list into batches, and then completely ignoring the batches.
This commit is contained in:
parent
5b8023dc7f
commit
db5f9031b7
1 changed files with 15 additions and 23 deletions
|
@ -25,7 +25,9 @@ from twisted.internet import defer
|
|||
|
||||
from synapse.logging.opentracing import log_kv, set_tag, trace
|
||||
from synapse.storage._base import SQLBaseStore, db_to_json
|
||||
from synapse.storage.database import make_in_list_sql_clause
|
||||
from synapse.util.caches.descriptors import cached, cachedList
|
||||
from synapse.util.iterutils import batch_iter
|
||||
|
||||
|
||||
class EndToEndKeyWorkerStore(SQLBaseStore):
|
||||
|
@ -391,26 +393,24 @@ class EndToEndKeyWorkerStore(SQLBaseStore):
|
|||
"""
|
||||
result = {}
|
||||
|
||||
batch_size = 100
|
||||
chunks = [
|
||||
user_ids[i : i + batch_size] for i in range(0, len(user_ids), batch_size)
|
||||
]
|
||||
for user_chunk in chunks:
|
||||
sql = """
|
||||
for user_chunk in batch_iter(user_ids, 100):
|
||||
clause, params = make_in_list_sql_clause(
|
||||
txn.database_engine, "k.user_id", user_chunk
|
||||
)
|
||||
sql = (
|
||||
"""
|
||||
SELECT k.user_id, k.keytype, k.keydata, k.stream_id
|
||||
FROM e2e_cross_signing_keys k
|
||||
INNER JOIN (SELECT user_id, keytype, MAX(stream_id) AS stream_id
|
||||
FROM e2e_cross_signing_keys
|
||||
GROUP BY user_id, keytype) s
|
||||
USING (user_id, stream_id, keytype)
|
||||
WHERE k.user_id IN (%s)
|
||||
""" % (
|
||||
",".join("?" for u in user_chunk),
|
||||
WHERE
|
||||
"""
|
||||
+ clause
|
||||
)
|
||||
query_params = []
|
||||
query_params.extend(user_chunk)
|
||||
|
||||
txn.execute(sql, query_params)
|
||||
txn.execute(sql, params)
|
||||
rows = self.db.cursor_to_dict(txn)
|
||||
|
||||
for row in rows:
|
||||
|
@ -453,15 +453,7 @@ class EndToEndKeyWorkerStore(SQLBaseStore):
|
|||
device_id = k
|
||||
devices[(user_id, device_id)] = key_type
|
||||
|
||||
device_list = list(devices)
|
||||
|
||||
# split into batches
|
||||
batch_size = 100
|
||||
chunks = [
|
||||
device_list[i : i + batch_size]
|
||||
for i in range(0, len(device_list), batch_size)
|
||||
]
|
||||
for user_chunk in chunks:
|
||||
for batch in batch_iter(devices.keys(), size=100):
|
||||
sql = """
|
||||
SELECT target_user_id, target_device_id, key_id, signature
|
||||
FROM e2e_cross_signing_signatures
|
||||
|
@ -469,11 +461,11 @@ class EndToEndKeyWorkerStore(SQLBaseStore):
|
|||
AND (%s)
|
||||
""" % (
|
||||
" OR ".join(
|
||||
"(target_user_id = ? AND target_device_id = ?)" for d in devices
|
||||
"(target_user_id = ? AND target_device_id = ?)" for _ in batch
|
||||
)
|
||||
)
|
||||
query_params = [from_user_id]
|
||||
for item in devices:
|
||||
for item in batch:
|
||||
# item is a (user_id, device_id) tuple
|
||||
query_params.extend(item)
|
||||
|
||||
|
|
Loading…
Reference in a new issue