Files
Magent/backend/app/services/issue_resolution.py
T
Assclaw 0b59289a2e
Magent CI/CD / verify (push) Canceled after 4m31s
Magent CI/CD / deploy-prod (push) Canceled after 0s
Magent CI/CD / deploy-beta (push) Canceled after 0s
Automate repair completion confirmation
2026-09-01 22:37:23 +12:00

586 lines
23 KiB
Python

from __future__ import annotations
import asyncio
from datetime import datetime, timedelta, timezone
from html import escape
import json
import logging
from typing import Any, Dict, Optional
from ..config import settings as env_settings
from ..db import (
add_portal_item_activity,
get_portal_item,
get_user_by_username,
list_portal_item_activity,
list_portal_items,
update_portal_item,
)
from ..clients.jellyfin import JellyfinClient
from ..clients.sonarr import SonarrClient
from ..runtime import get_runtime_settings
from .invite_email import resolve_user_delivery_email, send_generic_email
from .snapshot import build_snapshot
logger = logging.getLogger(__name__)
_SYSTEM_USER = "Magent"
_MEDIA_REPAIR_STARTED_EVENTS = {"replacement_started", "missing_search_started"}
def _now() -> datetime:
return datetime.now(timezone.utc)
def _parse_datetime(value: Any) -> Optional[datetime]:
if not isinstance(value, str) or not value.strip():
return None
try:
parsed = datetime.fromisoformat(value.strip().replace("Z", "+00:00"))
except ValueError:
return None
return parsed.replace(tzinfo=parsed.tzinfo or timezone.utc).astimezone(timezone.utc)
def _metadata(item: Dict[str, Any]) -> Dict[str, Any]:
raw = item.get("metadata_json")
if not isinstance(raw, str) or not raw.strip():
return {}
try:
parsed = json.loads(raw)
except (TypeError, ValueError):
return {}
return parsed if isinstance(parsed, dict) else {}
def issue_resolution_state(item: Dict[str, Any]) -> Dict[str, Any]:
state = _metadata(item).get("resolutionConfirmation")
return dict(state) if isinstance(state, dict) else {}
def _metadata_with_resolution(item: Dict[str, Any], state: Dict[str, Any]) -> str:
metadata = _metadata(item)
metadata["resolutionConfirmation"] = state
return json.dumps(metadata, separators=(",", ":"), sort_keys=True)
def _interval_delta(value: int, unit: str) -> timedelta:
safe_value = max(1, min(int(value), 365))
normalized_unit = str(unit or "days").strip().lower()
if normalized_unit == "weeks":
return timedelta(weeks=safe_value)
if normalized_unit == "months":
return timedelta(days=30 * safe_value)
return timedelta(days=safe_value)
def _workflow_settings() -> tuple[int, int, str]:
runtime = get_runtime_settings()
attempts = max(0, min(int(runtime.issue_confirmation_contact_attempts or 0), 10))
interval_value = max(1, min(int(runtime.issue_confirmation_interval_value or 1), 365))
interval_unit = str(runtime.issue_confirmation_interval_unit or "days").strip().lower()
if interval_unit not in {"days", "weeks", "months"}:
interval_unit = "days"
return attempts, interval_value, interval_unit
def _app_url() -> str:
runtime = get_runtime_settings()
for value in (runtime.magent_application_url, runtime.magent_proxy_base_url, env_settings.cors_allow_origin):
candidate = str(value or "").strip()
if candidate:
return candidate.rstrip("/")
return f"http://localhost:{int(runtime.magent_application_port or 3000)}"
def _issue_url(item_id: int) -> str:
return f"{_app_url()}/portal/issues?item={item_id}"
def _activity(
item_id: int,
event_type: str,
message: str,
*,
actor_username: str = _SYSTEM_USER,
actor_role: str = "system",
metadata: Optional[Dict[str, Any]] = None,
) -> None:
add_portal_item_activity(
item_id,
event_type=event_type,
actor_username=actor_username,
actor_role=actor_role,
message=message,
metadata_json=json.dumps(metadata, separators=(",", ":"), sort_keys=True) if metadata else None,
)
def _activity_metadata(entry: Dict[str, Any]) -> Dict[str, Any]:
raw = entry.get("metadata_json")
if not isinstance(raw, str) or not raw.strip():
return {}
try:
parsed = json.loads(raw)
except (TypeError, ValueError):
return {}
return parsed if isinstance(parsed, dict) else {}
def _repair_tracking(item_id: int) -> tuple[Dict[str, Any], list[Dict[str, Any]]]:
activity = list_portal_item_activity(item_id, limit=500)
for entry in reversed(activity):
if str(entry.get("event_type") or "") not in _MEDIA_REPAIR_STARTED_EVENTS:
continue
tracking = _activity_metadata(entry).get("repairTracking")
if isinstance(tracking, dict):
return dict(tracking), activity
return {}, activity
def _positive_ints(value: Any) -> list[int]:
if not isinstance(value, list):
return []
return [
int(item)
for item in value
if isinstance(item, int) and not isinstance(item, bool) and item > 0
]
def _media_signature(item: Any) -> Dict[str, str]:
if not isinstance(item, dict):
return {}
result: Dict[str, str] = {}
for key in ("Id", "Etag", "Path", "DateCreated", "MediaSources"):
value = item.get(key)
if isinstance(value, (dict, list)):
result[key] = json.dumps(value, separators=(",", ":"), sort_keys=True)
elif value is not None and str(value).strip():
result[key] = str(value).strip()
return result
def _signature_changed(current: Dict[str, str], baseline: Dict[str, Any]) -> bool:
previous = _media_signature(baseline)
if not previous:
return True
return any(current.get(key) and current.get(key) != value for key, value in previous.items())
async def _media_repair_evidence(tracking: Dict[str, Any]) -> Dict[str, Any]:
request_id = str(tracking.get("requestId") or "").strip()
action_id = str(tracking.get("actionId") or "").strip()
media_type = str(tracking.get("mediaType") or "").strip().lower()
collector_id = tracking.get("collectorId")
if not request_id.isdigit() or media_type not in {"movie", "tv"} or not isinstance(collector_id, int):
return {"complete": False, "phase": "invalid", "message": "Repair tracking information is incomplete."}
snapshot = await build_snapshot(request_id)
raw = snapshot.raw if isinstance(snapshot.raw, dict) else {}
arr = raw.get("arr") if isinstance(raw.get("arr"), dict) else {}
arr_item = arr.get("item") if isinstance(arr, dict) else None
jellyfin = raw.get("jellyfin") if isinstance(raw.get("jellyfin"), dict) else {}
jellyfin_item = jellyfin.get("item") if isinstance(jellyfin, dict) else None
original_file_ids = set(_positive_ints(tracking.get("originalFileIds")))
baselines = tracking.get("jellyfinBaseline")
baselines = [item for item in baselines if isinstance(item, dict)] if isinstance(baselines, list) else []
found_at_start = tracking.get("jellyfinFoundAtStart") is True
if media_type == "movie":
movie_file = arr_item.get("movieFile") if isinstance(arr_item, dict) else None
current_file_id = movie_file.get("id") if isinstance(movie_file, dict) else None
imported = isinstance(current_file_id, int) and current_file_id > 0 and current_file_id not in original_file_ids
if not imported:
return {"complete": False, "phase": "collecting", "message": "Waiting for Radarr to import the repaired movie file."}
if not jellyfin.get("found") or not isinstance(jellyfin_item, dict):
return {"complete": False, "phase": "indexing", "message": "Radarr imported the repaired movie. Waiting for Jellyfin to index it."}
current_signature = _media_signature(jellyfin_item)
if action_id == "replace_media" and found_at_start:
if not baselines or not _signature_changed(current_signature, baselines[0]):
return {"complete": False, "phase": "indexing", "message": "Radarr imported the repaired movie. Waiting for Jellyfin to refresh the existing title."}
return {"complete": True, "phase": "complete", "message": "Radarr imported the repaired movie and Jellyfin has indexed the updated file."}
target_rows = tracking.get("episodes")
targets = [item for item in target_rows if isinstance(item, dict)] if isinstance(target_rows, list) else []
target_ids = {
int(item["id"])
for item in targets
if isinstance(item.get("id"), int) and int(item["id"]) > 0
}
target_pairs = {
(int(item["seasonNumber"]), int(item["episodeNumber"]))
for item in targets
if isinstance(item.get("seasonNumber"), int) and isinstance(item.get("episodeNumber"), int)
}
if not target_ids or not target_pairs:
return {"complete": False, "phase": "invalid", "message": "No exact Sonarr episodes were recorded for this repair."}
runtime = get_runtime_settings()
sonarr = SonarrClient(runtime.sonarr_base_url, runtime.sonarr_api_key)
episodes = await sonarr.get_episodes(collector_id)
episode_map = {
int(item["id"]): item
for item in episodes
if isinstance(item, dict) and isinstance(item.get("id"), int)
} if isinstance(episodes, list) else {}
imported = all(
episode_id in episode_map
and (
episode_map[episode_id].get("hasFile") is True
or (
isinstance(episode_map[episode_id].get("episodeFileId"), int)
and episode_map[episode_id]["episodeFileId"] > 0
)
)
and episode_map[episode_id].get("episodeFileId") not in original_file_ids
for episode_id in target_ids
)
if not imported:
return {"complete": False, "phase": "collecting", "message": "Waiting for Sonarr to import every repaired episode."}
jellyfin_series_id = jellyfin_item.get("Id") if isinstance(jellyfin_item, dict) else None
jellyfin_client = JellyfinClient(runtime.jellyfin_base_url, runtime.jellyfin_api_key)
if not jellyfin.get("found") or not jellyfin_series_id or not jellyfin_client.configured():
return {"complete": False, "phase": "indexing", "message": "Sonarr imported the repaired episodes. Waiting for Jellyfin to index them."}
jellyfin_episodes = await jellyfin_client.get_series_episodes(str(jellyfin_series_id))
current_by_pair = {
(int(item["ParentIndexNumber"]), int(item["IndexNumber"])): item
for item in jellyfin_episodes
if isinstance(item.get("ParentIndexNumber"), int) and isinstance(item.get("IndexNumber"), int)
}
if not all(pair in current_by_pair for pair in target_pairs):
return {"complete": False, "phase": "indexing", "message": "Sonarr imported the repaired episodes. Waiting for the exact episodes to appear in Jellyfin."}
if action_id == "replace_media" and found_at_start:
baseline_by_pair = {
(int(item["seasonNumber"]), int(item["episodeNumber"])): item
for item in baselines
if isinstance(item.get("seasonNumber"), int) and isinstance(item.get("episodeNumber"), int)
}
if any(pair not in baseline_by_pair for pair in target_pairs):
return {"complete": False, "phase": "indexing", "message": "Sonarr imported the repaired episodes. Waiting for Jellyfin to confirm the existing entries changed."}
if not all(
_signature_changed(_media_signature(current_by_pair[pair]), baseline_by_pair[pair])
for pair in target_pairs
):
return {"complete": False, "phase": "indexing", "message": "Sonarr imported the repaired episodes. Waiting for Jellyfin to refresh every affected episode."}
return {"complete": True, "phase": "complete", "message": "Sonarr imported every repaired episode and Jellyfin has indexed the updated files."}
def _close_issue(
item: Dict[str, Any],
*,
reason: str,
confirmed: bool,
actor_username: str = _SYSTEM_USER,
actor_role: str = "system",
) -> Dict[str, Any]:
now = _now().isoformat()
state = issue_resolution_state(item)
state.update(
{
"status": "confirmed" if confirmed else "auto_closed",
"confirmedAt": now if confirmed else state.get("confirmedAt"),
"closedAt": now,
"nextContactAt": None,
"closedReason": reason,
}
)
updated = update_portal_item(
int(item["id"]),
status="closed",
issue_resolved_at=now,
metadata_json=_metadata_with_resolution(item, state),
)
if not updated:
raise RuntimeError("Issue could not be closed")
_activity(
int(item["id"]),
"resolution_confirmed" if confirmed else "issue_auto_closed",
reason,
actor_username=actor_username,
actor_role=actor_role,
)
return updated
async def _contact_reporter(item: Dict[str, Any]) -> Dict[str, Any]:
maximum, interval_value, interval_unit = _workflow_settings()
state = issue_resolution_state(item)
attempts = max(0, int(state.get("attemptsSent") or 0))
if maximum <= 0:
return _close_issue(
item,
reason="Issue closed automatically because reporter confirmation emails are disabled.",
confirmed=False,
)
if attempts >= maximum:
return _close_issue(
item,
reason=f"Issue closed automatically after {attempts} confirmation email attempt(s) without a response.",
confirmed=False,
)
attempt_number = attempts + 1
reporter = get_user_by_username(str(item.get("created_by_username") or ""))
recipient = resolve_user_delivery_email(reporter)
issue_url = _issue_url(int(item["id"]))
sent = False
delivery_error: Optional[str] = None
if recipient:
subject = f"Is your issue fixed? #{item['id']} {item.get('title') or ''}".strip()
body_text = (
f"We have marked issue #{item['id']} as fixed and need your confirmation.\n\n"
f"Issue: {item.get('title') or 'Untitled issue'}\n"
f"Confirmation request: {attempt_number} of {maximum}\n\n"
f"Open the issue and choose whether it is fixed or still happening:\n{issue_url}\n\n"
"If you do not respond, Magent will close the issue automatically after the configured confirmation period."
)
body_html = (
'<div style="font-family:Segoe UI,Arial,sans-serif;color:#132033;">'
'<h2 style="margin:0 0 12px;">Is your issue fixed?</h2>'
f'<p style="line-height:1.6;">We have marked issue <strong>#{int(item["id"])}</strong> as fixed and need your confirmation.</p>'
f'<p style="line-height:1.6;"><strong>{escape(str(item.get("title") or "Untitled issue"))}</strong><br>'
f'Confirmation request {attempt_number} of {maximum}</p>'
f'<a href="{escape(issue_url)}" style="display:inline-block;padding:11px 18px;border-radius:8px;background:#1c6bff;color:#fff;text-decoration:none;font-weight:700;">Confirm the outcome</a>'
'<p style="margin-top:18px;color:#64748b;line-height:1.6;">If you do not respond, Magent will close the issue automatically after the configured confirmation period.</p>'
'</div>'
)
try:
await send_generic_email(
recipient_email=recipient,
subject=subject,
body_text=body_text,
body_html=body_html,
)
sent = True
except Exception as exc:
delivery_error = str(exc)
logger.exception("issue confirmation email failed item_id=%s attempt=%s", item.get("id"), attempt_number)
else:
delivery_error = "No email address is stored for the reporter."
now = _now()
state.update(
{
"status": "awaiting_confirmation",
"attemptsSent": attempt_number,
"maximumAttempts": maximum,
"lastContactAt": now.isoformat(),
"nextContactAt": (now + _interval_delta(interval_value, interval_unit)).isoformat(),
"intervalValue": interval_value,
"intervalUnit": interval_unit,
"lastDeliverySucceeded": sent,
"lastDeliveryError": delivery_error,
}
)
updated = update_portal_item(
int(item["id"]),
metadata_json=_metadata_with_resolution(item, state),
)
if not updated:
raise RuntimeError("Issue confirmation schedule could not be saved")
if sent:
message = f"Confirmation email {attempt_number} of {maximum} was sent to the reporter."
else:
message = f"Confirmation email {attempt_number} of {maximum} could not be delivered."
_activity(
int(item["id"]),
"confirmation_email_sent" if sent else "confirmation_email_failed",
message,
metadata={
"attempt": attempt_number,
"maximum": maximum,
"nextContactAt": state["nextContactAt"],
"deliveryError": delivery_error,
},
)
return updated
async def begin_issue_confirmation(
item_id: int,
*,
actor_username: str,
actor_role: str,
) -> Dict[str, Any]:
item = get_portal_item(item_id)
if not item or str(item.get("kind") or "").lower() != "issue":
raise ValueError("Issue not found")
now = _now().isoformat()
maximum, interval_value, interval_unit = _workflow_settings()
state = {
"status": "awaiting_confirmation",
"startedAt": now,
"attemptsSent": 0,
"maximumAttempts": maximum,
"lastContactAt": None,
"nextContactAt": now,
"intervalValue": interval_value,
"intervalUnit": interval_unit,
"confirmedAt": None,
"closedAt": None,
}
updated = update_portal_item(
item_id,
status="awaiting_confirmation",
issue_resolved_at=None,
metadata_json=_metadata_with_resolution(item, state),
)
if not updated:
raise RuntimeError("Issue confirmation workflow could not be started")
_activity(
item_id,
"resolution_proposed",
"The issue was marked fixed and sent to the reporter for confirmation.",
actor_username=actor_username,
actor_role=actor_role,
metadata={"maximumAttempts": maximum, "intervalValue": interval_value, "intervalUnit": interval_unit},
)
return await _contact_reporter(updated)
def respond_to_issue_confirmation(
item_id: int,
*,
resolved: bool,
actor_username: str,
actor_role: str,
) -> Dict[str, Any]:
item = get_portal_item(item_id)
if not item or str(item.get("kind") or "").lower() != "issue":
raise ValueError("Issue not found")
if str(item.get("status") or "").lower() != "awaiting_confirmation":
raise ValueError("This issue is not waiting for resolution confirmation")
if resolved:
return _close_issue(
item,
reason="The reporter confirmed that the issue is fixed.",
confirmed=True,
actor_username=actor_username,
actor_role=actor_role,
)
now = _now().isoformat()
state = issue_resolution_state(item)
state.update(
{
"status": "reported_still_broken",
"reporterResponseAt": now,
"nextContactAt": None,
"closedAt": None,
}
)
updated = update_portal_item(
item_id,
status="in_progress",
issue_resolved_at=None,
metadata_json=_metadata_with_resolution(item, state),
)
if not updated:
raise RuntimeError("Issue could not be reopened")
_activity(
item_id,
"resolution_rejected",
"The reporter said the issue is still happening. The issue was returned to In progress.",
actor_username=actor_username,
actor_role=actor_role,
)
return updated
async def process_active_media_repairs() -> Dict[str, int]:
items = list_portal_items(kind="issue", status="in_progress", limit=500)
result = {"checked": 0, "waiting": 0, "completed": 0, "failed": 0}
for item in items:
tracking, activity = _repair_tracking(int(item["id"]))
if not tracking:
continue
result["checked"] += 1
try:
evidence = await _media_repair_evidence(tracking)
if evidence.get("complete"):
_activity(
int(item["id"]),
"repair_verified",
str(evidence.get("message") or "Magent verified the repaired media in Jellyfin."),
metadata={
"requestId": tracking.get("requestId"),
"actionId": tracking.get("actionId"),
},
)
await begin_issue_confirmation(
int(item["id"]),
actor_username=_SYSTEM_USER,
actor_role="system",
)
result["completed"] += 1
continue
result["waiting"] += 1
if evidence.get("phase") == "indexing" and not any(
str(entry.get("event_type") or "") == "repair_imported"
for entry in activity
):
_activity(
int(item["id"]),
"repair_imported",
str(evidence.get("message") or "The repaired file was imported and is waiting for Jellyfin."),
metadata={
"requestId": tracking.get("requestId"),
"actionId": tracking.get("actionId"),
},
)
except Exception:
result["failed"] += 1
logger.exception("automatic media repair check failed item_id=%s", item.get("id"))
return result
async def process_due_issue_confirmations(now: Optional[datetime] = None) -> Dict[str, int]:
current = (now or _now()).astimezone(timezone.utc)
items = list_portal_items(kind="issue", status="awaiting_confirmation", limit=500)
result = {"checked": len(items), "contacted": 0, "closed": 0, "failed": 0}
maximum, _, _ = _workflow_settings()
for item in items:
state = issue_resolution_state(item)
due_at = _parse_datetime(state.get("nextContactAt"))
if due_at and due_at > current:
continue
try:
attempts = max(0, int(state.get("attemptsSent") or 0))
if maximum <= 0 or attempts >= maximum:
_close_issue(
item,
reason=(
"Issue closed automatically because reporter confirmation emails are disabled."
if maximum <= 0
else f"Issue closed automatically after {attempts} confirmation email attempt(s) without a response."
),
confirmed=False,
)
result["closed"] += 1
else:
await _contact_reporter(item)
result["contacted"] += 1
except Exception:
result["failed"] += 1
logger.exception("issue confirmation processing failed item_id=%s", item.get("id"))
return result
async def run_issue_confirmation_loop() -> None:
while True:
try:
repair_result = await process_active_media_repairs()
if repair_result["completed"] or repair_result["failed"]:
logger.info("automatic media repair sweep complete result=%s", repair_result)
result = await process_due_issue_confirmations()
if result["contacted"] or result["closed"] or result["failed"]:
logger.info("issue confirmation sweep complete result=%s", result)
except asyncio.CancelledError:
raise
except Exception:
logger.exception("issue confirmation sweep failed")
await asyncio.sleep(60)