Merge pull request #2256 from matrix-org/erikj/faster_device_updates

Split up device_lists_outbound_pokes table for faster updates.
This commit is contained in:
Erik Johnston 2017-06-07 11:48:00 +01:00 committed by GitHub
commit 671f0afa1d
2 changed files with 65 additions and 13 deletions

View file

@ -368,7 +368,7 @@ class DeviceStore(SQLBaseStore):
prev_sent_id_sql = """ prev_sent_id_sql = """
SELECT coalesce(max(stream_id), 0) as stream_id SELECT coalesce(max(stream_id), 0) as stream_id
FROM device_lists_outbound_pokes FROM device_lists_outbound_last_success
WHERE destination = ? AND user_id = ? AND stream_id <= ? WHERE destination = ? AND user_id = ? AND stream_id <= ?
""" """
@ -510,32 +510,43 @@ class DeviceStore(SQLBaseStore):
) )
def _mark_as_sent_devices_by_remote_txn(self, txn, destination, stream_id): def _mark_as_sent_devices_by_remote_txn(self, txn, destination, stream_id):
# First we DELETE all rows such that only the latest row for each # We update the device_lists_outbound_last_success with the successfully
# (destination, user_id is left. We do this by selecting first and # poked users. We do the join to see which users need to be inserted and
# deleting. # which updated.
sql = """ sql = """
SELECT user_id, coalesce(max(stream_id), 0) FROM device_lists_outbound_pokes SELECT user_id, coalesce(max(o.stream_id), 0), (max(s.stream_id) IS NOT NULL)
WHERE destination = ? AND stream_id <= ? FROM device_lists_outbound_pokes as o
LEFT JOIN device_lists_outbound_last_success as s
USING (destination, user_id)
WHERE destination = ? AND o.stream_id <= ?
GROUP BY user_id GROUP BY user_id
HAVING count(*) > 1
""" """
txn.execute(sql, (destination, stream_id,)) txn.execute(sql, (destination, stream_id,))
rows = txn.fetchall() rows = txn.fetchall()
sql = """ sql = """
DELETE FROM device_lists_outbound_pokes UPDATE device_lists_outbound_last_success
WHERE destination = ? AND user_id = ? AND stream_id < ? SET stream_id = ?
WHERE destination = ? AND user_id = ?
""" """
txn.executemany( txn.executemany(
sql, ((destination, row[0], row[1],) for row in rows) sql, ((row[1], destination, row[0],) for row in rows if row[2])
) )
# Mark everything that is left as sent
sql = """ sql = """
UPDATE device_lists_outbound_pokes SET sent = ? INSERT INTO device_lists_outbound_last_success
(destination, user_id, stream_id) VALUES (?, ?, ?)
"""
txn.executemany(
sql, ((destination, row[0], row[1],) for row in rows if not row[2])
)
# Delete all sent outbound pokes
sql = """
DELETE FROM device_lists_outbound_pokes
WHERE destination = ? AND stream_id <= ? WHERE destination = ? AND stream_id <= ?
""" """
txn.execute(sql, (True, destination, stream_id,)) txn.execute(sql, (destination, stream_id,))
@defer.inlineCallbacks @defer.inlineCallbacks
def get_user_whose_devices_changed(self, from_key): def get_user_whose_devices_changed(self, from_key):
@ -670,6 +681,14 @@ class DeviceStore(SQLBaseStore):
) )
) )
# Since we've deleted unsent deltas, we need to remove the entry
# of last successful sent so that the prev_ids are correctly set.
sql = """
DELETE FROM device_lists_outbound_last_success
WHERE destination = ? AND user_id = ?
"""
txn.executemany(sql, ((row[0], row[1]) for row in rows))
logger.info("Pruned %d device list outbound pokes", txn.rowcount) logger.info("Pruned %d device list outbound pokes", txn.rowcount)
return self.runInteraction( return self.runInteraction(

View file

@ -0,0 +1,33 @@
/* Copyright 2017 Vector Creations Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-- Table of last stream_id that we sent to destination for user_id. This is
-- used to fill out the `prev_id` fields of outbound device list updates.
CREATE TABLE device_lists_outbound_last_success (
destination TEXT NOT NULL,
user_id TEXT NOT NULL,
stream_id BIGINT NOT NULL
);
INSERT INTO device_lists_outbound_last_success
SELECT destination, user_id, coalesce(max(stream_id), 0) as stream_id
FROM device_lists_outbound_pokes
WHERE sent = (1 = 1) -- sqlite doesn't have inbuilt boolean values
GROUP BY destination, user_id;
CREATE INDEX device_lists_outbound_last_success_idx ON device_lists_outbound_last_success(
destination, user_id, stream_id
);