mirror of
https://github.com/element-hq/synapse
synced 2024-09-02 02:10:20 +00:00
Merge branch 'develop' of github.com:matrix-org/synapse into use-simplejson
This commit is contained in:
commit
42bc56dad3
10 changed files with 260 additions and 24 deletions
|
@ -27,6 +27,16 @@ class Config(object):
|
||||||
def __init__(self, args):
|
def __init__(self, args):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def parse_size(string):
|
||||||
|
sizes = {"K": 1024, "M": 1024 * 1024}
|
||||||
|
size = 1
|
||||||
|
suffix = string[-1]
|
||||||
|
if suffix in sizes:
|
||||||
|
string = string[:-1]
|
||||||
|
size = sizes[suffix]
|
||||||
|
return int(string) * size
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def abspath(file_path):
|
def abspath(file_path):
|
||||||
return os.path.abspath(file_path) if file_path else file_path
|
return os.path.abspath(file_path) if file_path else file_path
|
||||||
|
|
|
@ -24,6 +24,7 @@ class DatabaseConfig(Config):
|
||||||
self.database_path = ":memory:"
|
self.database_path = ":memory:"
|
||||||
else:
|
else:
|
||||||
self.database_path = self.abspath(args.database_path)
|
self.database_path = self.abspath(args.database_path)
|
||||||
|
self.event_cache_size = self.parse_size(args.event_cache_size)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def add_arguments(cls, parser):
|
def add_arguments(cls, parser):
|
||||||
|
@ -33,6 +34,10 @@ class DatabaseConfig(Config):
|
||||||
"-d", "--database-path", default="homeserver.db",
|
"-d", "--database-path", default="homeserver.db",
|
||||||
help="The database name."
|
help="The database name."
|
||||||
)
|
)
|
||||||
|
db_group.add_argument(
|
||||||
|
"--event-cache-size", default="100K",
|
||||||
|
help="Number of events to cache in memory."
|
||||||
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def generate_config(cls, args, config_dir_path):
|
def generate_config(cls, args, config_dir_path):
|
||||||
|
|
|
@ -4,7 +4,7 @@ from distutils.version import LooseVersion
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
REQUIREMENTS = {
|
REQUIREMENTS = {
|
||||||
"syutil==0.0.2": ["syutil"],
|
"syutil>=0.0.2": ["syutil"],
|
||||||
"matrix_angular_sdk>=0.6.1": ["syweb>=0.6.1"],
|
"matrix_angular_sdk>=0.6.1": ["syweb>=0.6.1"],
|
||||||
"Twisted==14.0.2": ["twisted==14.0.2"],
|
"Twisted==14.0.2": ["twisted==14.0.2"],
|
||||||
"service_identity>=1.0.0": ["service_identity>=1.0.0"],
|
"service_identity>=1.0.0": ["service_identity>=1.0.0"],
|
||||||
|
|
|
@ -163,6 +163,9 @@ class DataStore(RoomMemberStore, RoomStore,
|
||||||
stream_ordering=None, is_new_state=True,
|
stream_ordering=None, is_new_state=True,
|
||||||
current_state=None):
|
current_state=None):
|
||||||
|
|
||||||
|
# Remove the any existing cache entries for the event_id
|
||||||
|
self._get_event_cache.pop(event.event_id)
|
||||||
|
|
||||||
# We purposefully do this first since if we include a `current_state`
|
# We purposefully do this first since if we include a `current_state`
|
||||||
# key, we *want* to update the `current_state_events` table
|
# key, we *want* to update the `current_state_events` table
|
||||||
if current_state:
|
if current_state:
|
||||||
|
@ -426,6 +429,8 @@ class DataStore(RoomMemberStore, RoomStore,
|
||||||
)
|
)
|
||||||
|
|
||||||
def _store_redaction(self, txn, event):
|
def _store_redaction(self, txn, event):
|
||||||
|
# invalidate the cache for the redacted event
|
||||||
|
self._get_event_cache.pop(event.redacts)
|
||||||
txn.execute(
|
txn.execute(
|
||||||
"INSERT OR IGNORE INTO redactions "
|
"INSERT OR IGNORE INTO redactions "
|
||||||
"(event_id, redacts) VALUES (?,?)",
|
"(event_id, redacts) VALUES (?,?)",
|
||||||
|
|
|
@ -19,6 +19,7 @@ from synapse.events import FrozenEvent
|
||||||
from synapse.events.utils import prune_event
|
from synapse.events.utils import prune_event
|
||||||
from synapse.util.logutils import log_function
|
from synapse.util.logutils import log_function
|
||||||
from synapse.util.logcontext import PreserveLoggingContext, LoggingContext
|
from synapse.util.logcontext import PreserveLoggingContext, LoggingContext
|
||||||
|
from synapse.util.lrucache import LruCache
|
||||||
|
|
||||||
from twisted.internet import defer
|
from twisted.internet import defer
|
||||||
|
|
||||||
|
@ -128,6 +129,8 @@ class SQLBaseStore(object):
|
||||||
self._txn_perf_counters = PerformanceCounters()
|
self._txn_perf_counters = PerformanceCounters()
|
||||||
self._get_event_counters = PerformanceCounters()
|
self._get_event_counters = PerformanceCounters()
|
||||||
|
|
||||||
|
self._get_event_cache = LruCache(hs.config.event_cache_size)
|
||||||
|
|
||||||
def start_profiling(self):
|
def start_profiling(self):
|
||||||
self._previous_loop_ts = self._clock.time_msec()
|
self._previous_loop_ts = self._clock.time_msec()
|
||||||
|
|
||||||
|
@ -579,6 +582,19 @@ class SQLBaseStore(object):
|
||||||
|
|
||||||
def _get_event_txn(self, txn, event_id, check_redacted=True,
|
def _get_event_txn(self, txn, event_id, check_redacted=True,
|
||||||
get_prev_content=False, allow_rejected=False):
|
get_prev_content=False, allow_rejected=False):
|
||||||
|
|
||||||
|
start_time = time.time() * 1000
|
||||||
|
update_counter = self._get_event_counters.update
|
||||||
|
|
||||||
|
try:
|
||||||
|
cache = self._get_event_cache.setdefault(event_id, {})
|
||||||
|
# Separate cache entries for each way to invoke _get_event_txn
|
||||||
|
return cache[(check_redacted, get_prev_content, allow_rejected)]
|
||||||
|
except KeyError:
|
||||||
|
pass
|
||||||
|
finally:
|
||||||
|
start_time = update_counter("event_cache", start_time)
|
||||||
|
|
||||||
sql = (
|
sql = (
|
||||||
"SELECT e.internal_metadata, e.json, r.event_id, rej.reason "
|
"SELECT e.internal_metadata, e.json, r.event_id, rej.reason "
|
||||||
"FROM event_json as e "
|
"FROM event_json as e "
|
||||||
|
@ -588,8 +604,6 @@ class SQLBaseStore(object):
|
||||||
"LIMIT 1 "
|
"LIMIT 1 "
|
||||||
)
|
)
|
||||||
|
|
||||||
start_time = time.time() * 1000
|
|
||||||
|
|
||||||
txn.execute(sql, (event_id,))
|
txn.execute(sql, (event_id,))
|
||||||
|
|
||||||
res = txn.fetchone()
|
res = txn.fetchone()
|
||||||
|
@ -599,14 +613,16 @@ class SQLBaseStore(object):
|
||||||
|
|
||||||
internal_metadata, js, redacted, rejected_reason = res
|
internal_metadata, js, redacted, rejected_reason = res
|
||||||
|
|
||||||
self._get_event_counters.update("select_event", start_time)
|
start_time = update_counter("select_event", start_time)
|
||||||
|
|
||||||
if allow_rejected or not rejected_reason:
|
if allow_rejected or not rejected_reason:
|
||||||
return self._get_event_from_row_txn(
|
result = self._get_event_from_row_txn(
|
||||||
txn, internal_metadata, js, redacted,
|
txn, internal_metadata, js, redacted,
|
||||||
check_redacted=check_redacted,
|
check_redacted=check_redacted,
|
||||||
get_prev_content=get_prev_content,
|
get_prev_content=get_prev_content,
|
||||||
)
|
)
|
||||||
|
cache[(check_redacted, get_prev_content, allow_rejected)] = result
|
||||||
|
return result
|
||||||
else:
|
else:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
|
@ -35,6 +35,11 @@ RoomsForUser = namedtuple(
|
||||||
|
|
||||||
class RoomMemberStore(SQLBaseStore):
|
class RoomMemberStore(SQLBaseStore):
|
||||||
|
|
||||||
|
def __init__(self, *args, **kw):
|
||||||
|
super(RoomMemberStore, self).__init__(*args, **kw)
|
||||||
|
|
||||||
|
self._user_rooms_cache = {}
|
||||||
|
|
||||||
def _store_room_member_txn(self, txn, event):
|
def _store_room_member_txn(self, txn, event):
|
||||||
"""Store a room member in the database.
|
"""Store a room member in the database.
|
||||||
"""
|
"""
|
||||||
|
@ -98,6 +103,8 @@ class RoomMemberStore(SQLBaseStore):
|
||||||
|
|
||||||
txn.execute(sql, (event.room_id, domain))
|
txn.execute(sql, (event.room_id, domain))
|
||||||
|
|
||||||
|
self.invalidate_rooms_for_user(target_user_id)
|
||||||
|
|
||||||
@defer.inlineCallbacks
|
@defer.inlineCallbacks
|
||||||
def get_room_member(self, user_id, room_id):
|
def get_room_member(self, user_id, room_id):
|
||||||
"""Retrieve the current state of a room member.
|
"""Retrieve the current state of a room member.
|
||||||
|
@ -240,28 +247,53 @@ class RoomMemberStore(SQLBaseStore):
|
||||||
results = self._parse_events_txn(txn, rows)
|
results = self._parse_events_txn(txn, rows)
|
||||||
return results
|
return results
|
||||||
|
|
||||||
|
# TODO(paul): Create a nice @cached decorator to do this
|
||||||
|
# @cached
|
||||||
|
# def get_foo(...)
|
||||||
|
# ...
|
||||||
|
# invalidate_foo = get_foo.invalidator
|
||||||
|
|
||||||
|
@defer.inlineCallbacks
|
||||||
|
def get_rooms_for_user(self, user_id):
|
||||||
|
# TODO(paul): put some performance counters in here so we can easily
|
||||||
|
# track what impact this cache is having
|
||||||
|
if user_id in self._user_rooms_cache:
|
||||||
|
defer.returnValue(self._user_rooms_cache[user_id])
|
||||||
|
|
||||||
|
rooms = yield self.get_rooms_for_user_where_membership_is(
|
||||||
|
user_id, membership_list=[Membership.JOIN],
|
||||||
|
)
|
||||||
|
|
||||||
|
# TODO(paul): Consider applying a maximum size; just evict things at
|
||||||
|
# random, or consider LRU?
|
||||||
|
|
||||||
|
self._user_rooms_cache[user_id] = rooms
|
||||||
|
defer.returnValue(rooms)
|
||||||
|
|
||||||
|
def invalidate_rooms_for_user(self, user_id):
|
||||||
|
if user_id in self._user_rooms_cache:
|
||||||
|
del self._user_rooms_cache[user_id]
|
||||||
|
|
||||||
|
@defer.inlineCallbacks
|
||||||
def user_rooms_intersect(self, user_id_list):
|
def user_rooms_intersect(self, user_id_list):
|
||||||
""" Checks whether all the users whose IDs are given in a list share a
|
""" Checks whether all the users whose IDs are given in a list share a
|
||||||
room.
|
room.
|
||||||
|
|
||||||
|
This is a "hot path" function that's called a lot, e.g. by presence for
|
||||||
|
generating the event stream. As such, it is implemented locally by
|
||||||
|
wrapping logic around heavily-cached database queries.
|
||||||
"""
|
"""
|
||||||
def interaction(txn):
|
if len(user_id_list) < 2:
|
||||||
user_list_clause = " OR ".join(["m.user_id = ?"] * len(user_id_list))
|
defer.returnValue(True)
|
||||||
sql = (
|
|
||||||
"SELECT m.room_id FROM room_memberships as m "
|
|
||||||
"INNER JOIN current_state_events as c "
|
|
||||||
"ON m.event_id = c.event_id "
|
|
||||||
"WHERE m.membership = 'join' "
|
|
||||||
"AND (%(clause)s) "
|
|
||||||
# TODO(paul): We've got duplicate rows in the database somewhere
|
|
||||||
# so we have to DISTINCT m.user_id here
|
|
||||||
"GROUP BY m.room_id HAVING COUNT(DISTINCT m.user_id) = ?"
|
|
||||||
) % {"clause": user_list_clause}
|
|
||||||
|
|
||||||
args = list(user_id_list)
|
deferreds = [self.get_rooms_for_user(u) for u in user_id_list]
|
||||||
args.append(len(user_id_list))
|
|
||||||
|
|
||||||
txn.execute(sql, args)
|
results = yield defer.DeferredList(deferreds)
|
||||||
|
|
||||||
return len(txn.fetchall()) > 0
|
# A list of sets of strings giving room IDs for each user
|
||||||
|
room_id_lists = [set([r.room_id for r in result[1]]) for result in results]
|
||||||
|
|
||||||
return self.runInteraction("user_rooms_intersect", interaction)
|
# There isn't a setintersection(*list_of_sets)
|
||||||
|
ret = len(room_id_lists.pop(0).intersection(*room_id_lists)) > 0
|
||||||
|
|
||||||
|
defer.returnValue(ret)
|
||||||
|
|
110
synapse/util/lrucache.py
Normal file
110
synapse/util/lrucache.py
Normal file
|
@ -0,0 +1,110 @@
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
# Copyright 2015 OpenMarket Ltd
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
|
||||||
|
class LruCache(object):
|
||||||
|
"""Least-recently-used cache."""
|
||||||
|
# TODO(mjark) Add hit/miss counters
|
||||||
|
# TODO(mjark) Add mutex for linked list for thread safety.
|
||||||
|
def __init__(self, max_size):
|
||||||
|
cache = {}
|
||||||
|
list_root = []
|
||||||
|
list_root[:] = [list_root, list_root, None, None]
|
||||||
|
|
||||||
|
PREV, NEXT, KEY, VALUE = 0, 1, 2, 3
|
||||||
|
|
||||||
|
def add_node(key, value):
|
||||||
|
prev_node = list_root
|
||||||
|
next_node = prev_node[NEXT]
|
||||||
|
node = [prev_node, next_node, key, value]
|
||||||
|
prev_node[NEXT] = node
|
||||||
|
next_node[PREV] = node
|
||||||
|
cache[key] = node
|
||||||
|
|
||||||
|
def move_node_to_front(node):
|
||||||
|
prev_node = node[PREV]
|
||||||
|
next_node = node[NEXT]
|
||||||
|
prev_node[NEXT] = next_node
|
||||||
|
next_node[PREV] = prev_node
|
||||||
|
prev_node = list_root
|
||||||
|
next_node = prev_node[NEXT]
|
||||||
|
node[PREV] = prev_node
|
||||||
|
node[NEXT] = next_node
|
||||||
|
prev_node[NEXT] = node
|
||||||
|
next_node[PREV] = node
|
||||||
|
|
||||||
|
def delete_node(node):
|
||||||
|
prev_node = node[PREV]
|
||||||
|
next_node = node[NEXT]
|
||||||
|
prev_node[NEXT] = next_node
|
||||||
|
next_node[PREV] = prev_node
|
||||||
|
cache.pop(node[KEY], None)
|
||||||
|
|
||||||
|
def cache_get(key, default=None):
|
||||||
|
node = cache.get(key, None)
|
||||||
|
if node is not None:
|
||||||
|
move_node_to_front(node)
|
||||||
|
return node[VALUE]
|
||||||
|
else:
|
||||||
|
return default
|
||||||
|
|
||||||
|
def cache_set(key, value):
|
||||||
|
node = cache.get(key, None)
|
||||||
|
if node is not None:
|
||||||
|
move_node_to_front(node)
|
||||||
|
node[VALUE] = value
|
||||||
|
else:
|
||||||
|
add_node(key, value)
|
||||||
|
if len(cache) > max_size:
|
||||||
|
delete_node(list_root[PREV])
|
||||||
|
|
||||||
|
def cache_set_default(key, value):
|
||||||
|
node = cache.get(key, None)
|
||||||
|
if node is not None:
|
||||||
|
return node[VALUE]
|
||||||
|
else:
|
||||||
|
add_node(key, value)
|
||||||
|
if len(cache) > max_size:
|
||||||
|
delete_node(list_root[PREV])
|
||||||
|
return value
|
||||||
|
|
||||||
|
def cache_pop(key, default=None):
|
||||||
|
node = cache.get(key, None)
|
||||||
|
if node:
|
||||||
|
delete_node(node)
|
||||||
|
return node[VALUE]
|
||||||
|
else:
|
||||||
|
return default
|
||||||
|
|
||||||
|
self.sentinel = object()
|
||||||
|
self.get = cache_get
|
||||||
|
self.set = cache_set
|
||||||
|
self.setdefault = cache_set_default
|
||||||
|
self.pop = cache_pop
|
||||||
|
|
||||||
|
def __getitem__(self, key):
|
||||||
|
result = self.get(key, self.sentinel)
|
||||||
|
if result is self.sentinel:
|
||||||
|
raise KeyError()
|
||||||
|
else:
|
||||||
|
return result
|
||||||
|
|
||||||
|
def __setitem__(self, key, value):
|
||||||
|
self.set(key, value)
|
||||||
|
|
||||||
|
def __delitem__(self, key, value):
|
||||||
|
result = self.pop(key, self.sentinel)
|
||||||
|
if result is self.sentinel:
|
||||||
|
raise KeyError()
|
|
@ -38,8 +38,9 @@ class SQLBaseStoreTestCase(unittest.TestCase):
|
||||||
return defer.succeed(func(self.mock_txn, *args, **kwargs))
|
return defer.succeed(func(self.mock_txn, *args, **kwargs))
|
||||||
self.db_pool.runInteraction = runInteraction
|
self.db_pool.runInteraction = runInteraction
|
||||||
|
|
||||||
hs = HomeServer("test",
|
config = Mock()
|
||||||
db_pool=self.db_pool)
|
config.event_cache_size = 1
|
||||||
|
hs = HomeServer("test", db_pool=self.db_pool, config=config)
|
||||||
|
|
||||||
self.datastore = SQLBaseStore(hs)
|
self.datastore = SQLBaseStore(hs)
|
||||||
|
|
||||||
|
|
56
tests/util/test_lrucache.py
Normal file
56
tests/util/test_lrucache.py
Normal file
|
@ -0,0 +1,56 @@
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
# Copyright 2015 OpenMarket Ltd
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
|
||||||
|
from .. import unittest
|
||||||
|
|
||||||
|
from synapse.util.lrucache import LruCache
|
||||||
|
|
||||||
|
class LruCacheTestCase(unittest.TestCase):
|
||||||
|
|
||||||
|
def test_get_set(self):
|
||||||
|
cache = LruCache(1)
|
||||||
|
cache["key"] = "value"
|
||||||
|
self.assertEquals(cache.get("key"), "value")
|
||||||
|
self.assertEquals(cache["key"], "value")
|
||||||
|
|
||||||
|
def test_eviction(self):
|
||||||
|
cache = LruCache(2)
|
||||||
|
cache[1] = 1
|
||||||
|
cache[2] = 2
|
||||||
|
|
||||||
|
self.assertEquals(cache.get(1), 1)
|
||||||
|
self.assertEquals(cache.get(2), 2)
|
||||||
|
|
||||||
|
cache[3] = 3
|
||||||
|
|
||||||
|
self.assertEquals(cache.get(1), None)
|
||||||
|
self.assertEquals(cache.get(2), 2)
|
||||||
|
self.assertEquals(cache.get(3), 3)
|
||||||
|
|
||||||
|
def test_setdefault(self):
|
||||||
|
cache = LruCache(1)
|
||||||
|
self.assertEquals(cache.setdefault("key", 1), 1)
|
||||||
|
self.assertEquals(cache.get("key"), 1)
|
||||||
|
self.assertEquals(cache.setdefault("key", 2), 1)
|
||||||
|
self.assertEquals(cache.get("key"), 1)
|
||||||
|
|
||||||
|
def test_pop(self):
|
||||||
|
cache = LruCache(1)
|
||||||
|
cache["key"] = 1
|
||||||
|
self.assertEquals(cache.pop("key"), 1)
|
||||||
|
self.assertEquals(cache.pop("key"), None)
|
||||||
|
|
||||||
|
|
|
@ -41,6 +41,7 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs):
|
||||||
if config is None:
|
if config is None:
|
||||||
config = Mock()
|
config = Mock()
|
||||||
config.signing_key = [MockKey()]
|
config.signing_key = [MockKey()]
|
||||||
|
config.event_cache_size = 1
|
||||||
|
|
||||||
if datastore is None:
|
if datastore is None:
|
||||||
db_pool = SQLiteMemoryDbPool()
|
db_pool = SQLiteMemoryDbPool()
|
||||||
|
|
Loading…
Reference in a new issue