mirror of
https://github.com/element-hq/synapse
synced 2024-08-20 06:10:21 +00:00
Merge pull request #2256 from matrix-org/erikj/faster_device_updates
Split up device_lists_outbound_pokes table for faster updates.
This commit is contained in:
commit
671f0afa1d
2 changed files with 65 additions and 13 deletions
|
@ -368,7 +368,7 @@ class DeviceStore(SQLBaseStore):
|
||||||
|
|
||||||
prev_sent_id_sql = """
|
prev_sent_id_sql = """
|
||||||
SELECT coalesce(max(stream_id), 0) as stream_id
|
SELECT coalesce(max(stream_id), 0) as stream_id
|
||||||
FROM device_lists_outbound_pokes
|
FROM device_lists_outbound_last_success
|
||||||
WHERE destination = ? AND user_id = ? AND stream_id <= ?
|
WHERE destination = ? AND user_id = ? AND stream_id <= ?
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
@ -510,32 +510,43 @@ class DeviceStore(SQLBaseStore):
|
||||||
)
|
)
|
||||||
|
|
||||||
def _mark_as_sent_devices_by_remote_txn(self, txn, destination, stream_id):
|
def _mark_as_sent_devices_by_remote_txn(self, txn, destination, stream_id):
|
||||||
# First we DELETE all rows such that only the latest row for each
|
# We update the device_lists_outbound_last_success with the successfully
|
||||||
# (destination, user_id is left. We do this by selecting first and
|
# poked users. We do the join to see which users need to be inserted and
|
||||||
# deleting.
|
# which updated.
|
||||||
sql = """
|
sql = """
|
||||||
SELECT user_id, coalesce(max(stream_id), 0) FROM device_lists_outbound_pokes
|
SELECT user_id, coalesce(max(o.stream_id), 0), (max(s.stream_id) IS NOT NULL)
|
||||||
WHERE destination = ? AND stream_id <= ?
|
FROM device_lists_outbound_pokes as o
|
||||||
|
LEFT JOIN device_lists_outbound_last_success as s
|
||||||
|
USING (destination, user_id)
|
||||||
|
WHERE destination = ? AND o.stream_id <= ?
|
||||||
GROUP BY user_id
|
GROUP BY user_id
|
||||||
HAVING count(*) > 1
|
|
||||||
"""
|
"""
|
||||||
txn.execute(sql, (destination, stream_id,))
|
txn.execute(sql, (destination, stream_id,))
|
||||||
rows = txn.fetchall()
|
rows = txn.fetchall()
|
||||||
|
|
||||||
sql = """
|
sql = """
|
||||||
DELETE FROM device_lists_outbound_pokes
|
UPDATE device_lists_outbound_last_success
|
||||||
WHERE destination = ? AND user_id = ? AND stream_id < ?
|
SET stream_id = ?
|
||||||
|
WHERE destination = ? AND user_id = ?
|
||||||
"""
|
"""
|
||||||
txn.executemany(
|
txn.executemany(
|
||||||
sql, ((destination, row[0], row[1],) for row in rows)
|
sql, ((row[1], destination, row[0],) for row in rows if row[2])
|
||||||
)
|
)
|
||||||
|
|
||||||
# Mark everything that is left as sent
|
|
||||||
sql = """
|
sql = """
|
||||||
UPDATE device_lists_outbound_pokes SET sent = ?
|
INSERT INTO device_lists_outbound_last_success
|
||||||
|
(destination, user_id, stream_id) VALUES (?, ?, ?)
|
||||||
|
"""
|
||||||
|
txn.executemany(
|
||||||
|
sql, ((destination, row[0], row[1],) for row in rows if not row[2])
|
||||||
|
)
|
||||||
|
|
||||||
|
# Delete all sent outbound pokes
|
||||||
|
sql = """
|
||||||
|
DELETE FROM device_lists_outbound_pokes
|
||||||
WHERE destination = ? AND stream_id <= ?
|
WHERE destination = ? AND stream_id <= ?
|
||||||
"""
|
"""
|
||||||
txn.execute(sql, (True, destination, stream_id,))
|
txn.execute(sql, (destination, stream_id,))
|
||||||
|
|
||||||
@defer.inlineCallbacks
|
@defer.inlineCallbacks
|
||||||
def get_user_whose_devices_changed(self, from_key):
|
def get_user_whose_devices_changed(self, from_key):
|
||||||
|
@ -670,6 +681,14 @@ class DeviceStore(SQLBaseStore):
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Since we've deleted unsent deltas, we need to remove the entry
|
||||||
|
# of last successful sent so that the prev_ids are correctly set.
|
||||||
|
sql = """
|
||||||
|
DELETE FROM device_lists_outbound_last_success
|
||||||
|
WHERE destination = ? AND user_id = ?
|
||||||
|
"""
|
||||||
|
txn.executemany(sql, ((row[0], row[1]) for row in rows))
|
||||||
|
|
||||||
logger.info("Pruned %d device list outbound pokes", txn.rowcount)
|
logger.info("Pruned %d device list outbound pokes", txn.rowcount)
|
||||||
|
|
||||||
return self.runInteraction(
|
return self.runInteraction(
|
||||||
|
|
33
synapse/storage/schema/delta/42/device_list_last_id.sql
Normal file
33
synapse/storage/schema/delta/42/device_list_last_id.sql
Normal file
|
@ -0,0 +1,33 @@
|
||||||
|
/* Copyright 2017 Vector Creations Ltd
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
|
||||||
|
-- Table of last stream_id that we sent to destination for user_id. This is
|
||||||
|
-- used to fill out the `prev_id` fields of outbound device list updates.
|
||||||
|
CREATE TABLE device_lists_outbound_last_success (
|
||||||
|
destination TEXT NOT NULL,
|
||||||
|
user_id TEXT NOT NULL,
|
||||||
|
stream_id BIGINT NOT NULL
|
||||||
|
);
|
||||||
|
|
||||||
|
INSERT INTO device_lists_outbound_last_success
|
||||||
|
SELECT destination, user_id, coalesce(max(stream_id), 0) as stream_id
|
||||||
|
FROM device_lists_outbound_pokes
|
||||||
|
WHERE sent = (1 = 1) -- sqlite doesn't have inbuilt boolean values
|
||||||
|
GROUP BY destination, user_id;
|
||||||
|
|
||||||
|
CREATE INDEX device_lists_outbound_last_success_idx ON device_lists_outbound_last_success(
|
||||||
|
destination, user_id, stream_id
|
||||||
|
);
|
Loading…
Reference in a new issue