mirror of
https://github.com/element-hq/synapse
synced 2024-08-19 01:00:54 +00:00
The existing implementation of the `python_twisted_reactor_tick_time` metric is pretty useless, because it *only* measures the time taken to execute timed calls and callbacks from threads. That neglects everything that happens off the back of I/O, which is obviously quite a lot for us. To improve this, I've hooked into a different place in the reactor - in particular, where it calls `epoll`. That call is the only place it should wait for something to happen - the rest of the loop *should* be quick. I've also removed `python_twisted_reactor_pending_calls`, because I don't believe anyone ever looks at it, and it's a nuisance to populate.
378 lines
11 KiB
HTML
378 lines
11 KiB
HTML
{{ template "head" . }}
|
|
|
|
{{ template "prom_content_head" . }}
|
|
<h1>System Resources</h1>
|
|
|
|
<h3>CPU</h3>
|
|
<div id="process_resource_utime"></div>
|
|
<script>
|
|
new PromConsole.Graph({
|
|
node: document.querySelector("#process_resource_utime"),
|
|
expr: "rate(process_cpu_seconds_total[2m]) * 100",
|
|
name: "[[job]]-[[index]]",
|
|
min: 0,
|
|
max: 100,
|
|
renderer: "line",
|
|
height: 150,
|
|
yAxisFormatter: PromConsole.NumberFormatter.humanize,
|
|
yHoverFormatter: PromConsole.NumberFormatter.humanize,
|
|
yUnits: "%",
|
|
yTitle: "CPU Usage"
|
|
})
|
|
</script>
|
|
|
|
<h3>Memory</h3>
|
|
<div id="process_resident_memory_bytes"></div>
|
|
<script>
|
|
new PromConsole.Graph({
|
|
node: document.querySelector("#process_resident_memory_bytes"),
|
|
expr: "process_resident_memory_bytes",
|
|
name: "[[job]]-[[index]]",
|
|
min: 0,
|
|
renderer: "line",
|
|
height: 150,
|
|
yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
|
|
yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
|
|
yUnits: "bytes",
|
|
yTitle: "Usage"
|
|
})
|
|
</script>
|
|
|
|
<h3>File descriptors</h3>
|
|
<div id="process_fds"></div>
|
|
<script>
|
|
new PromConsole.Graph({
|
|
node: document.querySelector("#process_fds"),
|
|
expr: "process_open_fds",
|
|
name: "[[job]]-[[index]]",
|
|
min: 0,
|
|
renderer: "line",
|
|
height: 150,
|
|
yAxisFormatter: PromConsole.NumberFormatter.humanize,
|
|
yHoverFormatter: PromConsole.NumberFormatter.humanize,
|
|
yUnits: "",
|
|
yTitle: "Descriptors"
|
|
})
|
|
</script>
|
|
|
|
<h1>Reactor</h1>
|
|
|
|
<h3>Total reactor time</h3>
|
|
<div id="reactor_total_time"></div>
|
|
<script>
|
|
new PromConsole.Graph({
|
|
node: document.querySelector("#reactor_total_time"),
|
|
expr: "rate(python_twisted_reactor_tick_time_sum[2m])",
|
|
name: "[[job]]-[[index]]",
|
|
max: 1,
|
|
min: 0,
|
|
renderer: "area",
|
|
height: 150,
|
|
yAxisFormatter: PromConsole.NumberFormatter.humanize,
|
|
yHoverFormatter: PromConsole.NumberFormatter.humanize,
|
|
yUnits: "s/s",
|
|
yTitle: "Usage"
|
|
})
|
|
</script>
|
|
|
|
<h3>Average reactor tick time</h3>
|
|
<div id="reactor_average_time"></div>
|
|
<script>
|
|
new PromConsole.Graph({
|
|
node: document.querySelector("#reactor_average_time"),
|
|
expr: "rate(python_twisted_reactor_tick_time_sum[2m]) / rate(python_twisted_reactor_tick_time_count[2m])",
|
|
name: "[[job]]-[[index]]",
|
|
min: 0,
|
|
renderer: "line",
|
|
height: 150,
|
|
yAxisFormatter: PromConsole.NumberFormatter.humanize,
|
|
yHoverFormatter: PromConsole.NumberFormatter.humanize,
|
|
yUnits: "s",
|
|
yTitle: "Time"
|
|
})
|
|
</script>
|
|
|
|
<h1>Storage</h1>
|
|
|
|
<h3>Queries</h3>
|
|
<div id="synapse_storage_query_time"></div>
|
|
<script>
|
|
new PromConsole.Graph({
|
|
node: document.querySelector("#synapse_storage_query_time"),
|
|
expr: "sum(rate(synapse_storage_query_time_count[2m])) by (verb)",
|
|
name: "[[verb]]",
|
|
yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
|
|
yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
|
|
yUnits: "queries/s",
|
|
yTitle: "Queries"
|
|
})
|
|
</script>
|
|
|
|
<h3>Transactions</h3>
|
|
<div id="synapse_storage_transaction_time"></div>
|
|
<script>
|
|
new PromConsole.Graph({
|
|
node: document.querySelector("#synapse_storage_transaction_time"),
|
|
expr: "topk(10, rate(synapse_storage_transaction_time_count[2m]))",
|
|
name: "[[job]]-[[index]] [[desc]]",
|
|
min: 0,
|
|
yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
|
|
yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
|
|
yUnits: "txn/s",
|
|
yTitle: "Transactions"
|
|
})
|
|
</script>
|
|
|
|
<h3>Transaction execution time</h3>
|
|
<div id="synapse_storage_transactions_time_sec"></div>
|
|
<script>
|
|
new PromConsole.Graph({
|
|
node: document.querySelector("#synapse_storage_transactions_time_sec"),
|
|
expr: "rate(synapse_storage_transaction_time_sum[2m])",
|
|
name: "[[job]]-[[index]] [[desc]]",
|
|
min: 0,
|
|
yAxisFormatter: PromConsole.NumberFormatter.humanize,
|
|
yHoverFormatter: PromConsole.NumberFormatter.humanize,
|
|
yUnits: "s/s",
|
|
yTitle: "Usage"
|
|
})
|
|
</script>
|
|
|
|
<h3>Average time waiting for database connection</h3>
|
|
<div id="synapse_storage_avg_waiting_time"></div>
|
|
<script>
|
|
new PromConsole.Graph({
|
|
node: document.querySelector("#synapse_storage_avg_waiting_time"),
|
|
expr: "rate(synapse_storage_schedule_time_sum[2m]) / rate(synapse_storage_schedule_time_count[2m])",
|
|
name: "[[job]]-[[index]]",
|
|
min: 0,
|
|
yAxisFormatter: PromConsole.NumberFormatter.humanize,
|
|
yHoverFormatter: PromConsole.NumberFormatter.humanize,
|
|
yUnits: "s",
|
|
yTitle: "Time"
|
|
})
|
|
</script>
|
|
|
|
<h3>Cache request rate</h3>
|
|
<div id="synapse_cache_request_rate"></div>
|
|
<script>
|
|
new PromConsole.Graph({
|
|
node: document.querySelector("#synapse_cache_request_rate"),
|
|
expr: "rate(synapse_util_caches_cache:total[2m])",
|
|
name: "[[job]]-[[index]] [[name]]",
|
|
min: 0,
|
|
yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
|
|
yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
|
|
yUnits: "rps",
|
|
yTitle: "Cache request rate"
|
|
})
|
|
</script>
|
|
|
|
<h3>Cache size</h3>
|
|
<div id="synapse_cache_size"></div>
|
|
<script>
|
|
new PromConsole.Graph({
|
|
node: document.querySelector("#synapse_cache_size"),
|
|
expr: "synapse_util_caches_cache:size",
|
|
name: "[[job]]-[[index]] [[name]]",
|
|
yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
|
|
yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
|
|
yUnits: "",
|
|
yTitle: "Items"
|
|
})
|
|
</script>
|
|
|
|
<h1>Requests</h1>
|
|
|
|
<h3>Requests by Servlet</h3>
|
|
<div id="synapse_http_server_request_count_servlet"></div>
|
|
<script>
|
|
new PromConsole.Graph({
|
|
node: document.querySelector("#synapse_http_server_request_count_servlet"),
|
|
expr: "rate(synapse_http_server_in_flight_requests_count[2m])",
|
|
name: "[[job]]-[[index]] [[method]] [[servlet]]",
|
|
yAxisFormatter: PromConsole.NumberFormatter.humanize,
|
|
yHoverFormatter: PromConsole.NumberFormatter.humanize,
|
|
yUnits: "req/s",
|
|
yTitle: "Requests"
|
|
})
|
|
</script>
|
|
<h4> (without <tt>EventStreamRestServlet</tt> or <tt>SyncRestServlet</tt>)</h4>
|
|
<div id="synapse_http_server_request_count_servlet_minus_events"></div>
|
|
<script>
|
|
new PromConsole.Graph({
|
|
node: document.querySelector("#synapse_http_server_request_count_servlet_minus_events"),
|
|
expr: "rate(synapse_http_server_in_flight_requests_count{servlet!=\"EventStreamRestServlet\", servlet!=\"SyncRestServlet\"}[2m])",
|
|
name: "[[job]]-[[index]] [[method]] [[servlet]]",
|
|
yAxisFormatter: PromConsole.NumberFormatter.humanize,
|
|
yHoverFormatter: PromConsole.NumberFormatter.humanize,
|
|
yUnits: "req/s",
|
|
yTitle: "Requests"
|
|
})
|
|
</script>
|
|
|
|
<h3>Average response times</h3>
|
|
<div id="synapse_http_server_response_time_avg"></div>
|
|
<script>
|
|
new PromConsole.Graph({
|
|
node: document.querySelector("#synapse_http_server_response_time_avg"),
|
|
expr: "rate(synapse_http_server_response_time_seconds_sum[2m]) / rate(synapse_http_server_response_count[2m])",
|
|
name: "[[job]]-[[index]] [[servlet]]",
|
|
yAxisFormatter: PromConsole.NumberFormatter.humanize,
|
|
yHoverFormatter: PromConsole.NumberFormatter.humanize,
|
|
yUnits: "s/req",
|
|
yTitle: "Response time"
|
|
})
|
|
</script>
|
|
|
|
<h3>All responses by code</h3>
|
|
<div id="synapse_http_server_responses"></div>
|
|
<script>
|
|
new PromConsole.Graph({
|
|
node: document.querySelector("#synapse_http_server_responses"),
|
|
expr: "rate(synapse_http_server_responses[2m])",
|
|
name: "[[method]] / [[code]]",
|
|
yAxisFormatter: PromConsole.NumberFormatter.humanize,
|
|
yHoverFormatter: PromConsole.NumberFormatter.humanize,
|
|
yUnits: "req/s",
|
|
yTitle: "Requests"
|
|
})
|
|
</script>
|
|
|
|
<h3>Error responses by code</h3>
|
|
<div id="synapse_http_server_responses_err"></div>
|
|
<script>
|
|
new PromConsole.Graph({
|
|
node: document.querySelector("#synapse_http_server_responses_err"),
|
|
expr: "rate(synapse_http_server_responses{code=~\"[45]..\"}[2m])",
|
|
name: "[[method]] / [[code]]",
|
|
yAxisFormatter: PromConsole.NumberFormatter.humanize,
|
|
yHoverFormatter: PromConsole.NumberFormatter.humanize,
|
|
yUnits: "req/s",
|
|
yTitle: "Requests"
|
|
})
|
|
</script>
|
|
|
|
|
|
<h3>CPU Usage</h3>
|
|
<div id="synapse_http_server_response_ru_utime"></div>
|
|
<script>
|
|
new PromConsole.Graph({
|
|
node: document.querySelector("#synapse_http_server_response_ru_utime"),
|
|
expr: "rate(synapse_http_server_response_ru_utime_seconds[2m])",
|
|
name: "[[job]]-[[index]] [[servlet]]",
|
|
yAxisFormatter: PromConsole.NumberFormatter.humanize,
|
|
yHoverFormatter: PromConsole.NumberFormatter.humanize,
|
|
yUnits: "s/s",
|
|
yTitle: "CPU Usage"
|
|
})
|
|
</script>
|
|
|
|
|
|
<h3>DB Usage</h3>
|
|
<div id="synapse_http_server_response_db_txn_duration"></div>
|
|
<script>
|
|
new PromConsole.Graph({
|
|
node: document.querySelector("#synapse_http_server_response_db_txn_duration"),
|
|
expr: "rate(synapse_http_server_response_db_txn_duration_seconds[2m])",
|
|
name: "[[job]]-[[index]] [[servlet]]",
|
|
yAxisFormatter: PromConsole.NumberFormatter.humanize,
|
|
yHoverFormatter: PromConsole.NumberFormatter.humanize,
|
|
yUnits: "s/s",
|
|
yTitle: "DB Usage"
|
|
})
|
|
</script>
|
|
|
|
|
|
<h3>Average event send times</h3>
|
|
<div id="synapse_http_server_send_time_avg"></div>
|
|
<script>
|
|
new PromConsole.Graph({
|
|
node: document.querySelector("#synapse_http_server_send_time_avg"),
|
|
expr: "rate(synapse_http_server_response_time_seconds_sum{servlet='RoomSendEventRestServlet'}[2m]) / rate(synapse_http_server_response_count{servlet='RoomSendEventRestServlet'}[2m])",
|
|
name: "[[job]]-[[index]] [[servlet]]",
|
|
yAxisFormatter: PromConsole.NumberFormatter.humanize,
|
|
yHoverFormatter: PromConsole.NumberFormatter.humanize,
|
|
yUnits: "s/req",
|
|
yTitle: "Response time"
|
|
})
|
|
</script>
|
|
|
|
<h1>Federation</h1>
|
|
|
|
<h3>Sent Messages</h3>
|
|
<div id="synapse_federation_client_sent"></div>
|
|
<script>
|
|
new PromConsole.Graph({
|
|
node: document.querySelector("#synapse_federation_client_sent"),
|
|
expr: "rate(synapse_federation_client_sent[2m])",
|
|
name: "[[job]]-[[index]] [[type]]",
|
|
yAxisFormatter: PromConsole.NumberFormatter.humanize,
|
|
yHoverFormatter: PromConsole.NumberFormatter.humanize,
|
|
yUnits: "req/s",
|
|
yTitle: "Requests"
|
|
})
|
|
</script>
|
|
|
|
<h3>Received Messages</h3>
|
|
<div id="synapse_federation_server_received"></div>
|
|
<script>
|
|
new PromConsole.Graph({
|
|
node: document.querySelector("#synapse_federation_server_received"),
|
|
expr: "rate(synapse_federation_server_received[2m])",
|
|
name: "[[job]]-[[index]] [[type]]",
|
|
yAxisFormatter: PromConsole.NumberFormatter.humanize,
|
|
yHoverFormatter: PromConsole.NumberFormatter.humanize,
|
|
yUnits: "req/s",
|
|
yTitle: "Requests"
|
|
})
|
|
</script>
|
|
|
|
<h3>Pending</h3>
|
|
<div id="synapse_federation_transaction_queue_pending"></div>
|
|
<script>
|
|
new PromConsole.Graph({
|
|
node: document.querySelector("#synapse_federation_transaction_queue_pending"),
|
|
expr: "synapse_federation_transaction_queue_pending",
|
|
name: "[[type]]",
|
|
yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
|
|
yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
|
|
yUnits: "",
|
|
yTitle: "Units"
|
|
})
|
|
</script>
|
|
|
|
<h1>Clients</h1>
|
|
|
|
<h3>Notifiers</h3>
|
|
<div id="synapse_notifier_listeners"></div>
|
|
<script>
|
|
new PromConsole.Graph({
|
|
node: document.querySelector("#synapse_notifier_listeners"),
|
|
expr: "synapse_notifier_listeners",
|
|
name: "[[job]]-[[index]]",
|
|
min: 0,
|
|
yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
|
|
yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
|
|
yUnits: "",
|
|
yTitle: "Listeners"
|
|
})
|
|
</script>
|
|
|
|
<h3>Notified Events</h3>
|
|
<div id="synapse_notifier_notified_events"></div>
|
|
<script>
|
|
new PromConsole.Graph({
|
|
node: document.querySelector("#synapse_notifier_notified_events"),
|
|
expr: "rate(synapse_notifier_notified_events[2m])",
|
|
name: "[[job]]-[[index]]",
|
|
yAxisFormatter: PromConsole.NumberFormatter.humanize,
|
|
yHoverFormatter: PromConsole.NumberFormatter.humanize,
|
|
yUnits: "events/s",
|
|
yTitle: "Event rate"
|
|
})
|
|
</script>
|
|
|
|
{{ template "prom_content_tail" . }}
|
|
|
|
{{ template "tail" }}
|