Make the in flight requests metrics thread safe

This commit is contained in:
Erik Johnston 2018-08-20 17:27:52 +01:00
parent 1058d14127
commit b01a755498

View file

@ -15,6 +15,7 @@
# limitations under the License. # limitations under the License.
import logging import logging
import threading
from prometheus_client.core import Counter, Histogram from prometheus_client.core import Counter, Histogram
@ -111,6 +112,9 @@ in_flight_requests_db_sched_duration = Counter(
# The set of all in flight requests, set[RequestMetrics] # The set of all in flight requests, set[RequestMetrics]
_in_flight_requests = set() _in_flight_requests = set()
# Protects the _in_flight_requests set from concurrent accesss
_in_flight_reqeuests_lock = threading.Lock()
def _get_in_flight_counts(): def _get_in_flight_counts():
"""Returns a count of all in flight requests by (method, server_name) """Returns a count of all in flight requests by (method, server_name)
@ -120,7 +124,8 @@ def _get_in_flight_counts():
""" """
# Cast to a list to prevent it changing while the Prometheus # Cast to a list to prevent it changing while the Prometheus
# thread is collecting metrics # thread is collecting metrics
reqs = list(_in_flight_requests) with _in_flight_reqeuests_lock:
reqs = list(_in_flight_requests)
for rm in reqs: for rm in reqs:
rm.update_metrics() rm.update_metrics()
@ -154,10 +159,12 @@ class RequestMetrics(object):
# to the "in flight" metrics. # to the "in flight" metrics.
self._request_stats = self.start_context.get_resource_usage() self._request_stats = self.start_context.get_resource_usage()
_in_flight_requests.add(self) with _in_flight_reqeuests_lock:
_in_flight_requests.add(self)
def stop(self, time_sec, request): def stop(self, time_sec, request):
_in_flight_requests.discard(self) with _in_flight_reqeuests_lock:
_in_flight_requests.discard(self)
context = LoggingContext.current_context() context = LoggingContext.current_context()