Add private Prometheus API metrics and Grafana performance dashboard

backend/app/clients/base.py

@@ -5,6 +5,7 @@ import httpx
 
 from ..logging_config import sanitize_headers, sanitize_value
 from ..services.operation_progress import finish_remote_call, start_remote_call
+from ..metrics import record_remote
 
 
 _SERVICE_NAMES = {
@@ -309,6 +310,7 @@ class ApiClient:
         service_name = _SERVICE_NAMES.get(self.__class__.__name__, self.__class__.__name__.removesuffix("Client"))
         active_message, _ = _operation_messages(service_name, method, path)
         operation_event_id = start_remote_call(service_name, active_message)
+        metric_status = 'error'
         self.logger.debug(
             "outbound request started method=%s url=%s params=%s payload=%s headers=%s",
             method,
@@ -327,6 +329,7 @@ class ApiClient:
                     params=params,
                     payload=payload,
                 )
+                metric_status = str(response.status_code)
                 response.raise_for_status()
             duration_ms = round((time.perf_counter() - started_at) * 1000, 2)
             self.logger.debug(
@@ -389,6 +392,9 @@ class ApiClient:
             )
             raise
 
+        finally:
+            record_remote(service_name, method, metric_status, time.perf_counter() - started_at)
+
     async def get(
         self,
         path: str,

backend/app/main.py

@@ -44,6 +44,7 @@ from .logging_config import (
     summarize_http_body,
 )
 from .runtime import get_runtime_settings
+from .metrics import record_api, start_metrics
 
 logger = logging.getLogger(__name__)
 _background_tasks: list[asyncio.Task[None]] = []
@@ -112,6 +113,7 @@ async def log_requests_and_add_security_headers(request: Request, call_next):
         response = await call_next(request)
     except Exception:
         duration_ms = round((time.perf_counter() - started_at) * 1000, 2)
+        record_api(request, 500, time.perf_counter() - started_at)
         logger.exception(
             "request failed method=%s path=%s duration_ms=%s",
             request.method,
@@ -125,6 +127,7 @@ async def log_requests_and_add_security_headers(request: Request, call_next):
         raise
 
     duration_ms = round((time.perf_counter() - started_at) * 1000, 2)
+    record_api(request, response.status_code, time.perf_counter() - started_at)
     response.headers.setdefault("X-Request-ID", request_id)
     response.headers.setdefault("X-Content-Type-Options", "nosniff")
     response.headers.setdefault("X-Frame-Options", "DENY")
@@ -221,6 +224,7 @@ def _enforce_secure_startup_configuration() -> None:
 
 @app.on_event("startup")
 async def startup() -> None:
+    start_metrics()
     configure_logging(
         settings.log_level,
         settings.log_file,

backend/app/metrics.py

@@ -0,0 +1,27 @@
+"""Low-cardinality operational metrics; no URLs, query values or user data."""
+import os
+from prometheus_client import Counter, Histogram, start_http_server
+
+BUCKETS = (.01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10, 30, 60)
+API_CALLS = Counter('magent_api_requests_total', 'API responses by route template', ['method', 'route', 'status'])
+API_TIME = Histogram('magent_api_response_seconds', 'Time until response headers (not stream lifetime)', ['method', 'route'], buckets=BUCKETS)
+REMOTE_CALLS = Counter('magent_remote_requests_total', 'Logical service client calls', ['service', 'method', 'status'])
+REMOTE_TIME = Histogram('magent_remote_response_seconds', 'Logical service client call duration', ['service', 'method'], buckets=BUCKETS)
+_server = None
+
+def start_metrics():
+    global _server
+    if _server is None and os.getenv('MAGENT_METRICS_ENABLED', '').lower() == 'true':
+        _server = start_http_server(int(os.getenv('MAGENT_METRICS_PORT', '9108')), addr=os.getenv('MAGENT_METRICS_BIND', '127.0.0.1'))
+
+def record_api(request, status, seconds):
+    route = getattr(request.scope.get('route'), 'path', 'unmatched')
+    method = request.method if request.method in {'GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS'} else 'OTHER'
+    API_CALLS.labels(method, route, str(status)).inc()
+    API_TIME.labels(method, route).observe(max(0, seconds))
+
+def record_remote(service, method, status, seconds):
+    service = service if service in {'Seerr', 'Jellyfin', 'Sonarr', 'Radarr', 'Bazarr', 'Prowlarr', 'qBittorrent'} else 'Other'
+    method = method.upper() if method.upper() in {'GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS'} else 'OTHER'
+    REMOTE_CALLS.labels(service, method, str(status)).inc()
+    REMOTE_TIME.labels(service, method).observe(max(0, seconds))

backend/requirements.txt

@@ -7,3 +7,4 @@ PyJWT==2.13.0
 passlib==1.7.4
 python-multipart==0.0.31
 Pillow==12.3.0
+prometheus-client==0.22.1

backend/tests/test_metrics.py

@@ -0,0 +1,20 @@
+import unittest
+from types import SimpleNamespace
+from prometheus_client import REGISTRY, generate_latest
+from backend.app.metrics import record_api, record_remote
+
+
+class MetricsTests(unittest.TestCase):
+    def test_route_template_not_private_path(self):
+        request = SimpleNamespace(method='GET', scope={'route': SimpleNamespace(path='/requests/{request_id}')})
+        before = REGISTRY.get_sample_value('magent_api_requests_total', {'method': 'GET', 'route': '/requests/{request_id}', 'status': '200'}) or 0
+        record_api(request, 200, .1)
+        self.assertEqual(REGISTRY.get_sample_value('magent_api_requests_total', {'method': 'GET', 'route': '/requests/{request_id}', 'status': '200'}), before + 1)
+
+    def test_unknown_route_and_service_are_bounded(self):
+        record_api(SimpleNamespace(method='SECRET-USER-METHOD', scope={}), 404, .01)
+        record_remote('secret-service-name', 'GET', 'error', .1)
+        data = generate_latest().decode()
+        self.assertNotIn('secret-service-name', data)
+        self.assertNotIn('SECRET-USER-METHOD', data)
+        self.assertIn('route="unmatched"', data)