From 131b5fc5c7984d0973f8451565ab1d247c247d00 Mon Sep 17 00:00:00 2001 From: Zak Bearman Date: Mon, 7 Sep 2026 19:23:35 +1200 Subject: [PATCH] Add private Prometheus API metrics and Grafana performance dashboard --- backend/app/clients/base.py | 6 + backend/app/main.py | 4 + backend/app/metrics.py | 27 + backend/requirements.txt | 1 + backend/tests/test_metrics.py | 20 + monitoring/README.md | 21 + .../grafana/magent-api-performance.json | 474 ++++++++++++++++++ 7 files changed, 553 insertions(+) create mode 100644 backend/app/metrics.py create mode 100644 backend/tests/test_metrics.py create mode 100644 monitoring/README.md create mode 100644 monitoring/grafana/magent-api-performance.json diff --git a/backend/app/clients/base.py b/backend/app/clients/base.py index 5d94c68..c8d5b8e 100644 --- a/backend/app/clients/base.py +++ b/backend/app/clients/base.py @@ -5,6 +5,7 @@ import httpx from ..logging_config import sanitize_headers, sanitize_value from ..services.operation_progress import finish_remote_call, start_remote_call +from ..metrics import record_remote _SERVICE_NAMES = { @@ -309,6 +310,7 @@ class ApiClient: service_name = _SERVICE_NAMES.get(self.__class__.__name__, self.__class__.__name__.removesuffix("Client")) active_message, _ = _operation_messages(service_name, method, path) operation_event_id = start_remote_call(service_name, active_message) + metric_status = 'error' self.logger.debug( "outbound request started method=%s url=%s params=%s payload=%s headers=%s", method, @@ -327,6 +329,7 @@ class ApiClient: params=params, payload=payload, ) + metric_status = str(response.status_code) response.raise_for_status() duration_ms = round((time.perf_counter() - started_at) * 1000, 2) self.logger.debug( @@ -389,6 +392,9 @@ class ApiClient: ) raise + finally: + record_remote(service_name, method, metric_status, time.perf_counter() - started_at) + async def get( self, path: str, diff --git a/backend/app/main.py b/backend/app/main.py index 2aa0d8e..8cccdd6 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -44,6 +44,7 @@ from .logging_config import ( summarize_http_body, ) from .runtime import get_runtime_settings +from .metrics import record_api, start_metrics logger = logging.getLogger(__name__) _background_tasks: list[asyncio.Task[None]] = [] @@ -112,6 +113,7 @@ async def log_requests_and_add_security_headers(request: Request, call_next): response = await call_next(request) except Exception: duration_ms = round((time.perf_counter() - started_at) * 1000, 2) + record_api(request, 500, time.perf_counter() - started_at) logger.exception( "request failed method=%s path=%s duration_ms=%s", request.method, @@ -125,6 +127,7 @@ async def log_requests_and_add_security_headers(request: Request, call_next): raise duration_ms = round((time.perf_counter() - started_at) * 1000, 2) + record_api(request, response.status_code, time.perf_counter() - started_at) response.headers.setdefault("X-Request-ID", request_id) response.headers.setdefault("X-Content-Type-Options", "nosniff") response.headers.setdefault("X-Frame-Options", "DENY") @@ -221,6 +224,7 @@ def _enforce_secure_startup_configuration() -> None: @app.on_event("startup") async def startup() -> None: + start_metrics() configure_logging( settings.log_level, settings.log_file, diff --git a/backend/app/metrics.py b/backend/app/metrics.py new file mode 100644 index 0000000..b7e1595 --- /dev/null +++ b/backend/app/metrics.py @@ -0,0 +1,27 @@ +"""Low-cardinality operational metrics; no URLs, query values or user data.""" +import os +from prometheus_client import Counter, Histogram, start_http_server + +BUCKETS = (.01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10, 30, 60) +API_CALLS = Counter('magent_api_requests_total', 'API responses by route template', ['method', 'route', 'status']) +API_TIME = Histogram('magent_api_response_seconds', 'Time until response headers (not stream lifetime)', ['method', 'route'], buckets=BUCKETS) +REMOTE_CALLS = Counter('magent_remote_requests_total', 'Logical service client calls', ['service', 'method', 'status']) +REMOTE_TIME = Histogram('magent_remote_response_seconds', 'Logical service client call duration', ['service', 'method'], buckets=BUCKETS) +_server = None + +def start_metrics(): + global _server + if _server is None and os.getenv('MAGENT_METRICS_ENABLED', '').lower() == 'true': + _server = start_http_server(int(os.getenv('MAGENT_METRICS_PORT', '9108')), addr=os.getenv('MAGENT_METRICS_BIND', '127.0.0.1')) + +def record_api(request, status, seconds): + route = getattr(request.scope.get('route'), 'path', 'unmatched') + method = request.method if request.method in {'GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS'} else 'OTHER' + API_CALLS.labels(method, route, str(status)).inc() + API_TIME.labels(method, route).observe(max(0, seconds)) + +def record_remote(service, method, status, seconds): + service = service if service in {'Seerr', 'Jellyfin', 'Sonarr', 'Radarr', 'Bazarr', 'Prowlarr', 'qBittorrent'} else 'Other' + method = method.upper() if method.upper() in {'GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS'} else 'OTHER' + REMOTE_CALLS.labels(service, method, str(status)).inc() + REMOTE_TIME.labels(service, method).observe(max(0, seconds)) diff --git a/backend/requirements.txt b/backend/requirements.txt index e9b011d..4914997 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -7,3 +7,4 @@ PyJWT==2.13.0 passlib==1.7.4 python-multipart==0.0.31 Pillow==12.3.0 +prometheus-client==0.22.1 diff --git a/backend/tests/test_metrics.py b/backend/tests/test_metrics.py new file mode 100644 index 0000000..f5b37ce --- /dev/null +++ b/backend/tests/test_metrics.py @@ -0,0 +1,20 @@ +import unittest +from types import SimpleNamespace +from prometheus_client import REGISTRY, generate_latest +from backend.app.metrics import record_api, record_remote + + +class MetricsTests(unittest.TestCase): + def test_route_template_not_private_path(self): + request = SimpleNamespace(method='GET', scope={'route': SimpleNamespace(path='/requests/{request_id}')}) + before = REGISTRY.get_sample_value('magent_api_requests_total', {'method': 'GET', 'route': '/requests/{request_id}', 'status': '200'}) or 0 + record_api(request, 200, .1) + self.assertEqual(REGISTRY.get_sample_value('magent_api_requests_total', {'method': 'GET', 'route': '/requests/{request_id}', 'status': '200'}), before + 1) + + def test_unknown_route_and_service_are_bounded(self): + record_api(SimpleNamespace(method='SECRET-USER-METHOD', scope={}), 404, .01) + record_remote('secret-service-name', 'GET', 'error', .1) + data = generate_latest().decode() + self.assertNotIn('secret-service-name', data) + self.assertNotIn('SECRET-USER-METHOD', data) + self.assertIn('route="unmatched"', data) diff --git a/monitoring/README.md b/monitoring/README.md new file mode 100644 index 0000000..00b999f --- /dev/null +++ b/monitoring/README.md @@ -0,0 +1,21 @@ +# Magent monitoring + +Grafana dashboard: `grafana/magent-api-performance.json` (Prometheus UID `prometheus`). +Set `MAGENT_METRICS_ENABLED=true`, `MAGENT_METRICS_BIND=0.0.0.0` and +`MAGENT_METRICS_PORT=9108` inside the container. Publish port 9108 **only on a +private interface**; do not proxy it through the public website. By default the +listener is disabled and its bind address is loopback. + +Production publishes `100.114.113.88:9108:9108` on GRZ-DKR01's Tailscale interface. +Prometheus on ANA-DKR01 scrapes it every 15 seconds with job name `magent`. +Grafana's existing file provider loads the dashboard from its Magent folder. + +API labels contain method, matched route template and HTTP status, never raw +paths, query values, usernames or credentials. API latency measures time to +response headers, not long-lived event-stream duration. Service metrics cover +the shared ApiClient, including background calls; custom client paths and CSRF +subrequests are not separate calls. CPU/memory refer to the Python backend only. + +Metrics start at deployment, with no historical backfill. Rate/percentile panels +need multiple scrapes; unused services have no series until called. Prometheus +retains history across Magent restarts, while process counters reset normally. diff --git a/monitoring/grafana/magent-api-performance.json b/monitoring/grafana/magent-api-performance.json new file mode 100644 index 0000000..c132a2e --- /dev/null +++ b/monitoring/grafana/magent-api-performance.json @@ -0,0 +1,474 @@ +{ + "uid": "magent-api-performance", + "title": "Magent — API & Performance", + "tags": [ + "magent", + "production" + ], + "schemaVersion": 40, + "version": 1, + "refresh": "15s", + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "browser", + "editable": true, + "description": "Metrics begin when instrumentation is deployed. No historical backfill. API timings are time-to-headers. Outbound metrics cover shared ApiClient calls; CPU/memory cover the Python backend. No user IDs, usernames, tokens, search terms or raw URLs are labels.", + "panels": [ + { + "id": 1, + "title": "Magent metrics reachable", + "description": "1 = scrape healthy; 0 = unavailable.", + "type": "stat", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 0, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "up{job=\"magent\"}", + "legendFormat": "Magent" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value" + } + }, + { + "id": 2, + "title": "API calls / second", + "description": "", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 0, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(magent_api_requests_total{job=\"magent\"}[$__rate_interval]))", + "legendFormat": "Calls / sec" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 3, + "title": "API response time — p95 by route", + "description": "Time to response headers; streaming session lifetime is excluded.", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 8, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95, sum by (le, route) (rate(magent_api_response_seconds_bucket{job=\"magent\"}[$__rate_interval])))", + "legendFormat": "{{route}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 4, + "title": "API responses by status", + "description": "", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 8, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (status) (rate(magent_api_requests_total{job=\"magent\"}[$__rate_interval]))", + "legendFormat": "HTTP {{status}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 5, + "title": "API server error percentage", + "description": "", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 16, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "100 * (sum(rate(magent_api_requests_total{job=\"magent\",status=~\"5..\"}[$__rate_interval])) or vector(0)) / clamp_min(sum(rate(magent_api_requests_total{job=\"magent\"}[$__rate_interval])), 0.000001)", + "legendFormat": "5xx" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 6, + "title": "Busiest API routes", + "description": "", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 16, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "topk(10,sum by (route) (rate(magent_api_requests_total{job=\"magent\"}[$__rate_interval])))", + "legendFormat": "{{route}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 7, + "title": "Connected service calls / second", + "description": "", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 24, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (service) (rate(magent_remote_requests_total{job=\"magent\"}[$__rate_interval]))", + "legendFormat": "{{service}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 8, + "title": "Connected services — p95 response time", + "description": "Instrumented shared API-client calls, including background work. Does not count every low-level HTTP exchange.", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 24, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95,sum by (le,service) (rate(magent_remote_response_seconds_bucket{job=\"magent\"}[$__rate_interval])))", + "legendFormat": "{{service}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 9, + "title": "Service redirects and errors", + "description": "error = connection/transport failure. Redirects are shown because they can prevent API operations.", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 32, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (service,status) (rate(magent_remote_requests_total{job=\"magent\",status=~\"3..|4..|5..|error\"}[$__rate_interval]))", + "legendFormat": "{{service}} · {{status}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 10, + "title": "Backend memory", + "description": "", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 32, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "process_resident_memory_bytes{job=\"magent\"}", + "legendFormat": "Python backend" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 11, + "title": "Backend CPU — cores used", + "description": "Backend process only, not the frontend or whole host.", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 40, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "rate(process_cpu_seconds_total{job=\"magent\"}[$__rate_interval])", + "legendFormat": "CPU cores" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 12, + "title": "Backend uptime", + "description": "", + "type": "stat", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 40, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "time() - process_start_time_seconds{job=\"magent\"}", + "legendFormat": "Uptime" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value" + } + } + ] +}