Add private Prometheus API metrics and Grafana performance dashboard

backend/app/clients/base.py

@@ -5,6 +5,7 @@ import httpx
 
 from ..logging_config import sanitize_headers, sanitize_value
 from ..services.operation_progress import finish_remote_call, start_remote_call
+from ..metrics import record_remote
 
 
 _SERVICE_NAMES = {
@@ -309,6 +310,7 @@ class ApiClient:
         service_name = _SERVICE_NAMES.get(self.__class__.__name__, self.__class__.__name__.removesuffix("Client"))
         active_message, _ = _operation_messages(service_name, method, path)
         operation_event_id = start_remote_call(service_name, active_message)
+        metric_status = 'error'
         self.logger.debug(
             "outbound request started method=%s url=%s params=%s payload=%s headers=%s",
             method,
@@ -327,6 +329,7 @@ class ApiClient:
                     params=params,
                     payload=payload,
                 )
+                metric_status = str(response.status_code)
                 response.raise_for_status()
             duration_ms = round((time.perf_counter() - started_at) * 1000, 2)
             self.logger.debug(
@@ -389,6 +392,9 @@ class ApiClient:
             )
             raise
 
+        finally:
+            record_remote(service_name, method, metric_status, time.perf_counter() - started_at)
+
     async def get(
         self,
         path: str,