Add private Prometheus API metrics and Grafana performance dashboard

backend/app/clients/base.py

@@ -5,6 +5,7 @@ import httpx
 
 from ..logging_config import sanitize_headers, sanitize_value
 from ..services.operation_progress import finish_remote_call, start_remote_call
+from ..metrics import record_remote
 
 
 _SERVICE_NAMES = {
@@ -309,6 +310,7 @@ class ApiClient:
         service_name = _SERVICE_NAMES.get(self.__class__.__name__, self.__class__.__name__.removesuffix("Client"))
         active_message, _ = _operation_messages(service_name, method, path)
         operation_event_id = start_remote_call(service_name, active_message)
+        metric_status = 'error'
         self.logger.debug(
             "outbound request started method=%s url=%s params=%s payload=%s headers=%s",
             method,
@@ -327,6 +329,7 @@ class ApiClient:
                     params=params,
                     payload=payload,
                 )
+                metric_status = str(response.status_code)
                 response.raise_for_status()
             duration_ms = round((time.perf_counter() - started_at) * 1000, 2)
             self.logger.debug(
@@ -389,6 +392,9 @@ class ApiClient:
             )
             raise
 
+        finally:
+            record_remote(service_name, method, metric_status, time.perf_counter() - started_at)
+
     async def get(
         self,
         path: str,

backend/app/main.py

@@ -44,6 +44,7 @@ from .logging_config import (
     summarize_http_body,
 )
 from .runtime import get_runtime_settings
+from .metrics import record_api, start_metrics
 
 logger = logging.getLogger(__name__)
 _background_tasks: list[asyncio.Task[None]] = []
@@ -112,6 +113,7 @@ async def log_requests_and_add_security_headers(request: Request, call_next):
         response = await call_next(request)
     except Exception:
         duration_ms = round((time.perf_counter() - started_at) * 1000, 2)
+        record_api(request, 500, time.perf_counter() - started_at)
         logger.exception(
             "request failed method=%s path=%s duration_ms=%s",
             request.method,
@@ -125,6 +127,7 @@ async def log_requests_and_add_security_headers(request: Request, call_next):
         raise
 
     duration_ms = round((time.perf_counter() - started_at) * 1000, 2)
+    record_api(request, response.status_code, time.perf_counter() - started_at)
     response.headers.setdefault("X-Request-ID", request_id)
     response.headers.setdefault("X-Content-Type-Options", "nosniff")
     response.headers.setdefault("X-Frame-Options", "DENY")
@@ -221,6 +224,7 @@ def _enforce_secure_startup_configuration() -> None:
 
 @app.on_event("startup")
 async def startup() -> None:
+    start_metrics()
     configure_logging(
         settings.log_level,
         settings.log_file,

backend/app/metrics.py

@@ -0,0 +1,27 @@
+"""Low-cardinality operational metrics; no URLs, query values or user data."""
+import os
+from prometheus_client import Counter, Histogram, start_http_server
+
+BUCKETS = (.01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10, 30, 60)
+API_CALLS = Counter('magent_api_requests_total', 'API responses by route template', ['method', 'route', 'status'])
+API_TIME = Histogram('magent_api_response_seconds', 'Time until response headers (not stream lifetime)', ['method', 'route'], buckets=BUCKETS)
+REMOTE_CALLS = Counter('magent_remote_requests_total', 'Logical service client calls', ['service', 'method', 'status'])
+REMOTE_TIME = Histogram('magent_remote_response_seconds', 'Logical service client call duration', ['service', 'method'], buckets=BUCKETS)
+_server = None
+
+def start_metrics():
+    global _server
+    if _server is None and os.getenv('MAGENT_METRICS_ENABLED', '').lower() == 'true':
+        _server = start_http_server(int(os.getenv('MAGENT_METRICS_PORT', '9108')), addr=os.getenv('MAGENT_METRICS_BIND', '127.0.0.1'))
+
+def record_api(request, status, seconds):
+    route = getattr(request.scope.get('route'), 'path', 'unmatched')
+    method = request.method if request.method in {'GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS'} else 'OTHER'
+    API_CALLS.labels(method, route, str(status)).inc()
+    API_TIME.labels(method, route).observe(max(0, seconds))
+
+def record_remote(service, method, status, seconds):
+    service = service if service in {'Seerr', 'Jellyfin', 'Sonarr', 'Radarr', 'Bazarr', 'Prowlarr', 'qBittorrent'} else 'Other'
+    method = method.upper() if method.upper() in {'GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS'} else 'OTHER'
+    REMOTE_CALLS.labels(service, method, str(status)).inc()
+    REMOTE_TIME.labels(service, method).observe(max(0, seconds))

backend/requirements.txt

@@ -7,3 +7,4 @@ PyJWT==2.13.0
 passlib==1.7.4
 python-multipart==0.0.31
 Pillow==12.3.0
+prometheus-client==0.22.1

backend/tests/test_metrics.py

@@ -0,0 +1,20 @@
+import unittest
+from types import SimpleNamespace
+from prometheus_client import REGISTRY, generate_latest
+from backend.app.metrics import record_api, record_remote
+
+
+class MetricsTests(unittest.TestCase):
+    def test_route_template_not_private_path(self):
+        request = SimpleNamespace(method='GET', scope={'route': SimpleNamespace(path='/requests/{request_id}')})
+        before = REGISTRY.get_sample_value('magent_api_requests_total', {'method': 'GET', 'route': '/requests/{request_id}', 'status': '200'}) or 0
+        record_api(request, 200, .1)
+        self.assertEqual(REGISTRY.get_sample_value('magent_api_requests_total', {'method': 'GET', 'route': '/requests/{request_id}', 'status': '200'}), before + 1)
+
+    def test_unknown_route_and_service_are_bounded(self):
+        record_api(SimpleNamespace(method='SECRET-USER-METHOD', scope={}), 404, .01)
+        record_remote('secret-service-name', 'GET', 'error', .1)
+        data = generate_latest().decode()
+        self.assertNotIn('secret-service-name', data)
+        self.assertNotIn('SECRET-USER-METHOD', data)
+        self.assertIn('route="unmatched"', data)

monitoring/README.md

@@ -0,0 +1,21 @@
+# Magent monitoring
+
+Grafana dashboard: `grafana/magent-api-performance.json` (Prometheus UID `prometheus`).
+Set `MAGENT_METRICS_ENABLED=true`, `MAGENT_METRICS_BIND=0.0.0.0` and
+`MAGENT_METRICS_PORT=9108` inside the container. Publish port 9108 **only on a
+private interface**; do not proxy it through the public website. By default the
+listener is disabled and its bind address is loopback.
+
+Production publishes `100.114.113.88:9108:9108` on GRZ-DKR01's Tailscale interface.
+Prometheus on ANA-DKR01 scrapes it every 15 seconds with job name `magent`.
+Grafana's existing file provider loads the dashboard from its Magent folder.
+
+API labels contain method, matched route template and HTTP status, never raw
+paths, query values, usernames or credentials. API latency measures time to
+response headers, not long-lived event-stream duration. Service metrics cover
+the shared ApiClient, including background calls; custom client paths and CSRF
+subrequests are not separate calls. CPU/memory refer to the Python backend only.
+
+Metrics start at deployment, with no historical backfill. Rate/percentile panels
+need multiple scrapes; unused services have no series until called. Prometheus
+retains history across Magent restarts, while process counters reset normally.

monitoring/grafana/magent-api-performance.json

@@ -0,0 +1,474 @@
+{
+  "uid": "magent-api-performance",
+  "title": "Magent — API & Performance",
+  "tags": [
+    "magent",
+    "production"
+  ],
+  "schemaVersion": 40,
+  "version": 1,
+  "refresh": "15s",
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timezone": "browser",
+  "editable": true,
+  "description": "Metrics begin when instrumentation is deployed. No historical backfill. API timings are time-to-headers. Outbound metrics cover shared ApiClient calls; CPU/memory cover the Python backend. No user IDs, usernames, tokens, search terms or raw URLs are labels.",
+  "panels": [
+    {
+      "id": 1,
+      "title": "Magent metrics reachable",
+      "description": "1 = scrape healthy; 0 = unavailable.",
+      "type": "stat",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 0,
+        "w": 12,
+        "h": 8
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "up{job=\"magent\"}",
+          "legendFormat": "Magent"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "colorMode": "value"
+      }
+    },
+    {
+      "id": 2,
+      "title": "API calls / second",
+      "description": "",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 0,
+        "w": 12,
+        "h": 8
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "sum(rate(magent_api_requests_total{job=\"magent\"}[$__rate_interval]))",
+          "legendFormat": "Calls / sec"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqps"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "id": 3,
+      "title": "API response time — p95 by route",
+      "description": "Time to response headers; streaming session lifetime is excluded.",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 8,
+        "w": 12,
+        "h": 8
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "histogram_quantile(0.95, sum by (le, route) (rate(magent_api_response_seconds_bucket{job=\"magent\"}[$__rate_interval])))",
+          "legendFormat": "{{route}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "id": 4,
+      "title": "API responses by status",
+      "description": "",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 8,
+        "w": 12,
+        "h": 8
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "sum by (status) (rate(magent_api_requests_total{job=\"magent\"}[$__rate_interval]))",
+          "legendFormat": "HTTP {{status}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqps"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "id": 5,
+      "title": "API server error percentage",
+      "description": "",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 16,
+        "w": 12,
+        "h": 8
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "100 * (sum(rate(magent_api_requests_total{job=\"magent\",status=~\"5..\"}[$__rate_interval])) or vector(0)) / clamp_min(sum(rate(magent_api_requests_total{job=\"magent\"}[$__rate_interval])), 0.000001)",
+          "legendFormat": "5xx"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "id": 6,
+      "title": "Busiest API routes",
+      "description": "",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 16,
+        "w": 12,
+        "h": 8
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "topk(10,sum by (route) (rate(magent_api_requests_total{job=\"magent\"}[$__rate_interval])))",
+          "legendFormat": "{{route}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqps"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "id": 7,
+      "title": "Connected service calls / second",
+      "description": "",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 24,
+        "w": 12,
+        "h": 8
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "sum by (service) (rate(magent_remote_requests_total{job=\"magent\"}[$__rate_interval]))",
+          "legendFormat": "{{service}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqps"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "id": 8,
+      "title": "Connected services — p95 response time",
+      "description": "Instrumented shared API-client calls, including background work. Does not count every low-level HTTP exchange.",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 24,
+        "w": 12,
+        "h": 8
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "histogram_quantile(0.95,sum by (le,service) (rate(magent_remote_response_seconds_bucket{job=\"magent\"}[$__rate_interval])))",
+          "legendFormat": "{{service}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "id": 9,
+      "title": "Service redirects and errors",
+      "description": "error = connection/transport failure. Redirects are shown because they can prevent API operations.",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 32,
+        "w": 12,
+        "h": 8
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "sum by (service,status) (rate(magent_remote_requests_total{job=\"magent\",status=~\"3..|4..|5..|error\"}[$__rate_interval]))",
+          "legendFormat": "{{service}} · {{status}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqps"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "id": 10,
+      "title": "Backend memory",
+      "description": "",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 32,
+        "w": 12,
+        "h": 8
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "process_resident_memory_bytes{job=\"magent\"}",
+          "legendFormat": "Python backend"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "id": 11,
+      "title": "Backend CPU — cores used",
+      "description": "Backend process only, not the frontend or whole host.",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 40,
+        "w": 12,
+        "h": 8
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "rate(process_cpu_seconds_total{job=\"magent\"}[$__rate_interval])",
+          "legendFormat": "CPU cores"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "id": 12,
+      "title": "Backend uptime",
+      "description": "",
+      "type": "stat",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 40,
+        "w": 12,
+        "h": 8
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "time() - process_start_time_seconds{job=\"magent\"}",
+          "legendFormat": "Uptime"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "colorMode": "value"
+      }
+    }
+  ]
+}