mirror of
https://github.com/element-hq/synapse
synced 2024-08-20 03:50:20 +00:00
Merge pull request #2256 from matrix-org/erikj/faster_device_updates
Split up device_lists_outbound_pokes table for faster updates.
This commit is contained in:
commit
671f0afa1d
2 changed files with 65 additions and 13 deletions
|
@ -368,7 +368,7 @@ class DeviceStore(SQLBaseStore):
|
|||
|
||||
prev_sent_id_sql = """
|
||||
SELECT coalesce(max(stream_id), 0) as stream_id
|
||||
FROM device_lists_outbound_pokes
|
||||
FROM device_lists_outbound_last_success
|
||||
WHERE destination = ? AND user_id = ? AND stream_id <= ?
|
||||
"""
|
||||
|
||||
|
@ -510,32 +510,43 @@ class DeviceStore(SQLBaseStore):
|
|||
)
|
||||
|
||||
def _mark_as_sent_devices_by_remote_txn(self, txn, destination, stream_id):
|
||||
# First we DELETE all rows such that only the latest row for each
|
||||
# (destination, user_id is left. We do this by selecting first and
|
||||
# deleting.
|
||||
# We update the device_lists_outbound_last_success with the successfully
|
||||
# poked users. We do the join to see which users need to be inserted and
|
||||
# which updated.
|
||||
sql = """
|
||||
SELECT user_id, coalesce(max(stream_id), 0) FROM device_lists_outbound_pokes
|
||||
WHERE destination = ? AND stream_id <= ?
|
||||
SELECT user_id, coalesce(max(o.stream_id), 0), (max(s.stream_id) IS NOT NULL)
|
||||
FROM device_lists_outbound_pokes as o
|
||||
LEFT JOIN device_lists_outbound_last_success as s
|
||||
USING (destination, user_id)
|
||||
WHERE destination = ? AND o.stream_id <= ?
|
||||
GROUP BY user_id
|
||||
HAVING count(*) > 1
|
||||
"""
|
||||
txn.execute(sql, (destination, stream_id,))
|
||||
rows = txn.fetchall()
|
||||
|
||||
sql = """
|
||||
DELETE FROM device_lists_outbound_pokes
|
||||
WHERE destination = ? AND user_id = ? AND stream_id < ?
|
||||
UPDATE device_lists_outbound_last_success
|
||||
SET stream_id = ?
|
||||
WHERE destination = ? AND user_id = ?
|
||||
"""
|
||||
txn.executemany(
|
||||
sql, ((destination, row[0], row[1],) for row in rows)
|
||||
sql, ((row[1], destination, row[0],) for row in rows if row[2])
|
||||
)
|
||||
|
||||
# Mark everything that is left as sent
|
||||
sql = """
|
||||
UPDATE device_lists_outbound_pokes SET sent = ?
|
||||
INSERT INTO device_lists_outbound_last_success
|
||||
(destination, user_id, stream_id) VALUES (?, ?, ?)
|
||||
"""
|
||||
txn.executemany(
|
||||
sql, ((destination, row[0], row[1],) for row in rows if not row[2])
|
||||
)
|
||||
|
||||
# Delete all sent outbound pokes
|
||||
sql = """
|
||||
DELETE FROM device_lists_outbound_pokes
|
||||
WHERE destination = ? AND stream_id <= ?
|
||||
"""
|
||||
txn.execute(sql, (True, destination, stream_id,))
|
||||
txn.execute(sql, (destination, stream_id,))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_user_whose_devices_changed(self, from_key):
|
||||
|
@ -670,6 +681,14 @@ class DeviceStore(SQLBaseStore):
|
|||
)
|
||||
)
|
||||
|
||||
# Since we've deleted unsent deltas, we need to remove the entry
|
||||
# of last successful sent so that the prev_ids are correctly set.
|
||||
sql = """
|
||||
DELETE FROM device_lists_outbound_last_success
|
||||
WHERE destination = ? AND user_id = ?
|
||||
"""
|
||||
txn.executemany(sql, ((row[0], row[1]) for row in rows))
|
||||
|
||||
logger.info("Pruned %d device list outbound pokes", txn.rowcount)
|
||||
|
||||
return self.runInteraction(
|
||||
|
|
33
synapse/storage/schema/delta/42/device_list_last_id.sql
Normal file
33
synapse/storage/schema/delta/42/device_list_last_id.sql
Normal file
|
@ -0,0 +1,33 @@
|
|||
/* Copyright 2017 Vector Creations Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
|
||||
-- Table of last stream_id that we sent to destination for user_id. This is
|
||||
-- used to fill out the `prev_id` fields of outbound device list updates.
|
||||
CREATE TABLE device_lists_outbound_last_success (
|
||||
destination TEXT NOT NULL,
|
||||
user_id TEXT NOT NULL,
|
||||
stream_id BIGINT NOT NULL
|
||||
);
|
||||
|
||||
INSERT INTO device_lists_outbound_last_success
|
||||
SELECT destination, user_id, coalesce(max(stream_id), 0) as stream_id
|
||||
FROM device_lists_outbound_pokes
|
||||
WHERE sent = (1 = 1) -- sqlite doesn't have inbuilt boolean values
|
||||
GROUP BY destination, user_id;
|
||||
|
||||
CREATE INDEX device_lists_outbound_last_success_idx ON device_lists_outbound_last_success(
|
||||
destination, user_id, stream_id
|
||||
);
|
Loading…
Reference in a new issue