Add private Prometheus API metrics and Grafana performance dashboard

backend/app/metrics.py

@@ -0,0 +1,27 @@
+"""Low-cardinality operational metrics; no URLs, query values or user data."""
+import os
+from prometheus_client import Counter, Histogram, start_http_server
+
+BUCKETS = (.01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10, 30, 60)
+API_CALLS = Counter('magent_api_requests_total', 'API responses by route template', ['method', 'route', 'status'])
+API_TIME = Histogram('magent_api_response_seconds', 'Time until response headers (not stream lifetime)', ['method', 'route'], buckets=BUCKETS)
+REMOTE_CALLS = Counter('magent_remote_requests_total', 'Logical service client calls', ['service', 'method', 'status'])
+REMOTE_TIME = Histogram('magent_remote_response_seconds', 'Logical service client call duration', ['service', 'method'], buckets=BUCKETS)
+_server = None
+
+def start_metrics():
+    global _server
+    if _server is None and os.getenv('MAGENT_METRICS_ENABLED', '').lower() == 'true':
+        _server = start_http_server(int(os.getenv('MAGENT_METRICS_PORT', '9108')), addr=os.getenv('MAGENT_METRICS_BIND', '127.0.0.1'))
+
+def record_api(request, status, seconds):
+    route = getattr(request.scope.get('route'), 'path', 'unmatched')
+    method = request.method if request.method in {'GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS'} else 'OTHER'
+    API_CALLS.labels(method, route, str(status)).inc()
+    API_TIME.labels(method, route).observe(max(0, seconds))
+
+def record_remote(service, method, status, seconds):
+    service = service if service in {'Seerr', 'Jellyfin', 'Sonarr', 'Radarr', 'Bazarr', 'Prowlarr', 'qBittorrent'} else 'Other'
+    method = method.upper() if method.upper() in {'GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS'} else 'OTHER'
+    REMOTE_CALLS.labels(service, method, str(status)).inc()
+    REMOTE_TIME.labels(service, method).observe(max(0, seconds))