Add a prometheus metric for active cache lookups. (#5750)

* Add a prometheus metric for active cache lookups.

* changelog
This commit is contained in:
Richard van der Hoff 2019-07-24 11:33:13 +01:00 committed by GitHub
parent 3641784e8c
commit 418635e68a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 34 additions and 2 deletions

1
changelog.d/5750.misc Normal file
View file

@ -0,0 +1 @@
Add a prometheus metric for pending cache lookups.

View file

@ -1,5 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright 2015, 2016 OpenMarket Ltd # Copyright 2015, 2016 OpenMarket Ltd
# Copyright 2019 The Matrix.org Foundation C.I.C.
# #
# Licensed under the Apache License, Version 2.0 (the "License"); # Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License. # you may not use this file except in compliance with the License.
@ -51,7 +52,19 @@ response_cache_evicted = Gauge(
response_cache_total = Gauge("synapse_util_caches_response_cache:total", "", ["name"]) response_cache_total = Gauge("synapse_util_caches_response_cache:total", "", ["name"])
def register_cache(cache_type, cache_name, cache): def register_cache(cache_type, cache_name, cache, collect_callback=None):
"""Register a cache object for metric collection.
Args:
cache_type (str):
cache_name (str): name of the cache
cache (object): cache itself
collect_callback (callable|None): if not None, a function which is called during
metric collection to update additional metrics.
Returns:
CacheMetric: an object which provides inc_{hits,misses,evictions} methods
"""
# Check if the metric is already registered. Unregister it, if so. # Check if the metric is already registered. Unregister it, if so.
# This usually happens during tests, as at runtime these caches are # This usually happens during tests, as at runtime these caches are
@ -90,6 +103,8 @@ def register_cache(cache_type, cache_name, cache):
cache_hits.labels(cache_name).set(self.hits) cache_hits.labels(cache_name).set(self.hits)
cache_evicted.labels(cache_name).set(self.evicted_size) cache_evicted.labels(cache_name).set(self.evicted_size)
cache_total.labels(cache_name).set(self.hits + self.misses) cache_total.labels(cache_name).set(self.hits + self.misses)
if collect_callback:
collect_callback()
except Exception as e: except Exception as e:
logger.warn("Error calculating metrics for %s: %s", cache_name, e) logger.warn("Error calculating metrics for %s: %s", cache_name, e)
raise raise

View file

@ -22,6 +22,8 @@ from collections import namedtuple
import six import six
from six import itervalues, string_types from six import itervalues, string_types
from prometheus_client import Gauge
from twisted.internet import defer from twisted.internet import defer
from synapse.logging.context import make_deferred_yieldable, preserve_fn from synapse.logging.context import make_deferred_yieldable, preserve_fn
@ -37,6 +39,12 @@ from . import register_cache
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
cache_pending_metric = Gauge(
"synapse_util_caches_cache_pending",
"Number of lookups currently pending for this cache",
["name"],
)
_CacheSentinel = object() _CacheSentinel = object()
@ -82,11 +90,19 @@ class Cache(object):
self.name = name self.name = name
self.keylen = keylen self.keylen = keylen
self.thread = None self.thread = None
self.metrics = register_cache("cache", name, self.cache) self.metrics = register_cache(
"cache",
name,
self.cache,
collect_callback=self._metrics_collection_callback,
)
def _on_evicted(self, evicted_count): def _on_evicted(self, evicted_count):
self.metrics.inc_evictions(evicted_count) self.metrics.inc_evictions(evicted_count)
def _metrics_collection_callback(self):
cache_pending_metric.labels(self.name).set(len(self._pending_deferred_cache))
def check_thread(self): def check_thread(self):
expected_thread = self.thread expected_thread = self.thread
if expected_thread is None: if expected_thread is None: