Rephl3x/Magent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add private Prometheus API metrics and Grafana performance dashboard
Magent CI/CD / verify (push) Successful in 10m24s
Details
Magent CI/CD / deploy-prod (push) Skipped
Details
Magent CI/CD / deploy-beta (push) Skipped
Details

Browse Source
This commit is contained in:
Assclaw
2026-09-07 19:23:35 +12:00
parent bd1f2cb1cb
commit 131b5fc5c7
7 changed files with 553 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/clients/base.py
+6
View File
@@ -5,6 +5,7 @@ import httpx
from ..logging_config import sanitize_headers, sanitize_value
from ..services.operation_progress import finish_remote_call, start_remote_call
from ..metrics import record_remote
_SERVICE_NAMES = {
@@ -309,6 +310,7 @@ class ApiClient:
service_name = _SERVICE_NAMES.get(self.__class__.__name__, self.__class__.__name__.removesuffix("Client"))
active_message, _ = _operation_messages(service_name, method, path)
operation_event_id = start_remote_call(service_name, active_message)
metric_status = 'error'
self.logger.debug(
"outbound request started method=%s url=%s params=%s payload=%s headers=%s",
method,
@@ -327,6 +329,7 @@ class ApiClient:
params=params,
payload=payload,
)
metric_status = str(response.status_code)
response.raise_for_status()
duration_ms = round((time.perf_counter() - started_at) * 1000, 2)
self.logger.debug(
@@ -389,6 +392,9 @@ class ApiClient:
)
raise
finally:
record_remote(service_name, method, metric_status, time.perf_counter() - started_at)
async def get(
self,
path: str,
backend/app/main.py
+4
View File
@@ -44,6 +44,7 @@ from .logging_config import (
summarize_http_body,
)
from .runtime import get_runtime_settings
from .metrics import record_api, start_metrics
logger = logging.getLogger(__name__)
_background_tasks: list[asyncio.Task[None]] = []
@@ -112,6 +113,7 @@ async def log_requests_and_add_security_headers(request: Request, call_next):
response = await call_next(request)
except Exception:
duration_ms = round((time.perf_counter() - started_at) * 1000, 2)
record_api(request, 500, time.perf_counter() - started_at)
logger.exception(
"request failed method=%s path=%s duration_ms=%s",
request.method,
@@ -125,6 +127,7 @@ async def log_requests_and_add_security_headers(request: Request, call_next):
raise
duration_ms = round((time.perf_counter() - started_at) * 1000, 2)
record_api(request, response.status_code, time.perf_counter() - started_at)
response.headers.setdefault("X-Request-ID", request_id)
response.headers.setdefault("X-Content-Type-Options", "nosniff")
response.headers.setdefault("X-Frame-Options", "DENY")
@@ -221,6 +224,7 @@ def _enforce_secure_startup_configuration() -> None:
@app.on_event("startup")
async def startup() -> None:
start_metrics()
configure_logging(
settings.log_level,
settings.log_file,
backend/app/metrics.py
+27
View File
@@ -0,0 +1,27 @@
"""Low-cardinality operational metrics; no URLs, query values or user data."""
import os
from prometheus_client import Counter, Histogram, start_http_server
BUCKETS = (.01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10, 30, 60)
API_CALLS = Counter('magent_api_requests_total', 'API responses by route template', ['method', 'route', 'status'])
API_TIME = Histogram('magent_api_response_seconds', 'Time until response headers (not stream lifetime)', ['method', 'route'], buckets=BUCKETS)
REMOTE_CALLS = Counter('magent_remote_requests_total', 'Logical service client calls', ['service', 'method', 'status'])
REMOTE_TIME = Histogram('magent_remote_response_seconds', 'Logical service client call duration', ['service', 'method'], buckets=BUCKETS)
_server = None
def start_metrics():
global _server
if _server is None and os.getenv('MAGENT_METRICS_ENABLED', '').lower() == 'true':
_server = start_http_server(int(os.getenv('MAGENT_METRICS_PORT', '9108')), addr=os.getenv('MAGENT_METRICS_BIND', '127.0.0.1'))
def record_api(request, status, seconds):
route = getattr(request.scope.get('route'), 'path', 'unmatched')
method = request.method if request.method in {'GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS'} else 'OTHER'
API_CALLS.labels(method, route, str(status)).inc()
API_TIME.labels(method, route).observe(max(0, seconds))
def record_remote(service, method, status, seconds):
service = service if service in {'Seerr', 'Jellyfin', 'Sonarr', 'Radarr', 'Bazarr', 'Prowlarr', 'qBittorrent'} else 'Other'
method = method.upper() if method.upper() in {'GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS'} else 'OTHER'
REMOTE_CALLS.labels(service, method, str(status)).inc()
REMOTE_TIME.labels(service, method).observe(max(0, seconds))
backend/requirements.txt
+1
View File
@@ -7,3 +7,4 @@ PyJWT==2.13.0
passlib==1.7.4
python-multipart==0.0.31
Pillow==12.3.0
prometheus-client==0.22.1
backend/tests/test_metrics.py
+20
View File
@@ -0,0 +1,20 @@
import unittest
from types import SimpleNamespace
from prometheus_client import REGISTRY, generate_latest
from backend.app.metrics import record_api, record_remote
class MetricsTests(unittest.TestCase):
def test_route_template_not_private_path(self):
request = SimpleNamespace(method='GET', scope={'route': SimpleNamespace(path='/requests/{request_id}')})
before = REGISTRY.get_sample_value('magent_api_requests_total', {'method': 'GET', 'route': '/requests/{request_id}', 'status': '200'}) or 0
record_api(request, 200, .1)
self.assertEqual(REGISTRY.get_sample_value('magent_api_requests_total', {'method': 'GET', 'route': '/requests/{request_id}', 'status': '200'}), before + 1)
def test_unknown_route_and_service_are_bounded(self):
record_api(SimpleNamespace(method='SECRET-USER-METHOD', scope={}), 404, .01)
record_remote('secret-service-name', 'GET', 'error', .1)
data = generate_latest().decode()
self.assertNotIn('secret-service-name', data)
self.assertNotIn('SECRET-USER-METHOD', data)
self.assertIn('route="unmatched"', data)
monitoring/README.md
+21
View File
@@ -0,0 +1,21 @@
# Magent monitoring
Grafana dashboard: `grafana/magent-api-performance.json` (Prometheus UID `prometheus`).
Set `MAGENT_METRICS_ENABLED=true`, `MAGENT_METRICS_BIND=0.0.0.0` and
`MAGENT_METRICS_PORT=9108` inside the container. Publish port 9108 **only on a
private interface**; do not proxy it through the public website. By default the
listener is disabled and its bind address is loopback.
Production publishes `100.114.113.88:9108:9108` on GRZ-DKR01's Tailscale interface.
Prometheus on ANA-DKR01 scrapes it every 15 seconds with job name `magent`.
Grafana's existing file provider loads the dashboard from its Magent folder.
API labels contain method, matched route template and HTTP status, never raw
paths, query values, usernames or credentials. API latency measures time to
response headers, not long-lived event-stream duration. Service metrics cover
the shared ApiClient, including background calls; custom client paths and CSRF
subrequests are not separate calls. CPU/memory refer to the Python backend only.
Metrics start at deployment, with no historical backfill. Rate/percentile panels
need multiple scrapes; unused services have no series until called. Prometheus
retains history across Magent restarts, while process counters reset normally.
monitoring/grafana/magent-api-performance.json
+474
View File
@@ -0,0 +1,474 @@
{
"uid": "magent-api-performance",
"title": "Magent — API & Performance",
"tags": [
"magent",
"production"
],
"schemaVersion": 40,
"version": 1,
"refresh": "15s",
"time": {
"from": "now-1h",
"to": "now"
},
"timezone": "browser",
"editable": true,
"description": "Metrics begin when instrumentation is deployed. No historical backfill. API timings are time-to-headers. Outbound metrics cover shared ApiClient calls; CPU/memory cover the Python backend. No user IDs, usernames, tokens, search terms or raw URLs are labels.",
"panels": [
{
"id": 1,
"title": "Magent metrics reachable",
"description": "1 = scrape healthy; 0 = unavailable.",
"type": "stat",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 0,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "up{job=\"magent\"}",
"legendFormat": "Magent"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
]
},
"colorMode": "value"
}
},
{
"id": 2,
"title": "API calls / second",
"description": "",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 0,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "sum(rate(magent_api_requests_total{job=\"magent\"}[$__rate_interval]))",
"legendFormat": "Calls / sec"
}
],
"fieldConfig": {
"defaults": {
"unit": "reqps"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"id": 3,
"title": "API response time — p95 by route",
"description": "Time to response headers; streaming session lifetime is excluded.",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 8,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.95, sum by (le, route) (rate(magent_api_response_seconds_bucket{job=\"magent\"}[$__rate_interval])))",
"legendFormat": "{{route}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"id": 4,
"title": "API responses by status",
"description": "",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 8,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "sum by (status) (rate(magent_api_requests_total{job=\"magent\"}[$__rate_interval]))",
"legendFormat": "HTTP {{status}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "reqps"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"id": 5,
"title": "API server error percentage",
"description": "",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 16,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "100 * (sum(rate(magent_api_requests_total{job=\"magent\",status=~\"5..\"}[$__rate_interval])) or vector(0)) / clamp_min(sum(rate(magent_api_requests_total{job=\"magent\"}[$__rate_interval])), 0.000001)",
"legendFormat": "5xx"
}
],
"fieldConfig": {
"defaults": {
"unit": "percent"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"id": 6,
"title": "Busiest API routes",
"description": "",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 16,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "topk(10,sum by (route) (rate(magent_api_requests_total{job=\"magent\"}[$__rate_interval])))",
"legendFormat": "{{route}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "reqps"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"id": 7,
"title": "Connected service calls / second",
"description": "",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 24,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "sum by (service) (rate(magent_remote_requests_total{job=\"magent\"}[$__rate_interval]))",
"legendFormat": "{{service}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "reqps"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"id": 8,
"title": "Connected services — p95 response time",
"description": "Instrumented shared API-client calls, including background work. Does not count every low-level HTTP exchange.",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 24,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.95,sum by (le,service) (rate(magent_remote_response_seconds_bucket{job=\"magent\"}[$__rate_interval])))",
"legendFormat": "{{service}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"id": 9,
"title": "Service redirects and errors",
"description": "error = connection/transport failure. Redirects are shown because they can prevent API operations.",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 32,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "sum by (service,status) (rate(magent_remote_requests_total{job=\"magent\",status=~\"3..|4..|5..|error\"}[$__rate_interval]))",
"legendFormat": "{{service}} · {{status}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "reqps"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"id": 10,
"title": "Backend memory",
"description": "",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 32,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "process_resident_memory_bytes{job=\"magent\"}",
"legendFormat": "Python backend"
}
],
"fieldConfig": {
"defaults": {
"unit": "bytes"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"id": 11,
"title": "Backend CPU — cores used",
"description": "Backend process only, not the frontend or whole host.",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 40,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "rate(process_cpu_seconds_total{job=\"magent\"}[$__rate_interval])",
"legendFormat": "CPU cores"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"id": 12,
"title": "Backend uptime",
"description": "",
"type": "stat",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 40,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "time() - process_start_time_seconds{job=\"magent\"}",
"legendFormat": "Uptime"
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
]
},
"colorMode": "value"
}
}
]
}