feat(release): publish minimal self-contained Magent source

2026-09-19 16:58:12 +12:00
commit 5fa5d45535
272 changed files with 79305 additions and 0 deletions
@@ -0,0 +1,378 @@
+import asyncio
+import logging
+import os
+import time
+import uuid
+from typing import Awaitable, Callable
+
+from fastapi import FastAPI, Request
+from fastapi.exceptions import RequestValidationError
+from fastapi.exception_handlers import request_validation_exception_handler
+from fastapi.responses import JSONResponse
+
+from .config import settings
+from .db import has_admin_user, init_db
+from .routers.requests import (
+    router as requests_router,
+    startup_warmup_requests_cache,
+    run_local_request_stage_loop,
+    run_requests_delta_loop,
+    run_daily_requests_full_sync,
+    run_daily_db_cleanup,
+)
+from .routers.auth import router as auth_router
+from .routers.admin import router as admin_router, events_router as admin_events_router
+from .routers.images import router as images_router
+from .routers.branding import router as branding_router
+from .routers.status import router as status_router
+from .routers.feedback import router as feedback_router
+from .routers.site import router as site_router
+from .routers.events import router as events_router
+from .routers.portal import router as portal_router
+from .routers.operations import router as operations_router
+from .routers.insights import router as insights_router
+from .routers.identities import router as identities_router
+from .routers.recaps import router as recaps_router
+from .routers.newsletters import router as newsletters_router
+from .routers.backups import router as backups_router
+from .routers.setup import router as setup_router
+from .services.backups import apply_pending_restore
+from .services.setup import initialize_setup_state, is_setup_required, setup_token_configured
+from .services.jellyfin_sync import run_daily_jellyfin_sync
+from .services.issue_resolution import run_issue_confirmation_loop
+from .services.email_recaps import run_email_recap_loop
+from .services.newsletters import run_newsletter_loop
+from .services.operation_progress import (
+    begin_operation,
+    finish_operation,
+    normalize_operation_id,
+    reset_operation,
+)
+from .logging_config import (
+    bind_request_id,
+    configure_logging,
+    reset_request_id,
+    sanitize_headers,
+    sanitize_path,
+)
+from .runtime import get_runtime_settings
+from .metrics import record_api, start_metrics
+from .request_limits import InstallationBodyLimitMiddleware
+from .secret_storage import validate_secret_storage_configuration
+from .services.request_origins import ConfiguredOriginCORSMiddleware, can_claim_initial_origin, is_allowed_request_origin
+
+logger = logging.getLogger(__name__)
+_background_tasks: list[asyncio.Task[None]] = []
+_background_started = False
+
+app = FastAPI(
+    title=settings.app_name,
+    docs_url="/docs" if settings.api_docs_enabled else None,
+    redoc_url=None,
+    openapi_url="/openapi.json" if settings.api_docs_enabled else None,
+)
+
+app.add_middleware(
+    ConfiguredOriginCORSMiddleware,
+    allow_origins=[settings.cors_allow_origin],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+app.add_middleware(InstallationBodyLimitMiddleware)
+
+
+@app.exception_handler(RequestValidationError)
+async def installation_validation_error(request: Request, exc: RequestValidationError):
+    if request.url.path.rstrip("/") == "/setup/bootstrap" or request.url.path.startswith("/admin/backups"):
+        # Pydantic SecretStr masks parsed values, but FastAPI's default 422 body
+        # includes rejected raw input. Never echo tokens/passwords/passphrases.
+        return JSONResponse(
+            status_code=422,
+            content={"detail": [
+                {key: error[key] for key in ("type", "loc", "msg") if key in error}
+                for error in exc.errors()
+            ]},
+            headers={"Cache-Control": "no-store"},
+        )
+    return await request_validation_exception_handler(request, exc)
+
+
+@app.middleware("http")
+async def log_requests_and_add_security_headers(request: Request, call_next):
+    request_id = request.headers.get("X-Request-ID") or uuid.uuid4().hex[:12]
+    token = bind_request_id(request_id)
+    operation_id = normalize_operation_id(request.headers.get("X-Magent-Operation-ID"))
+    operation_token = None
+    if operation_id and request.method.upper() not in {"GET", "HEAD", "OPTIONS"}:
+        operation_token = begin_operation(
+            operation_id,
+            label=request.headers.get("X-Magent-Operation-Label"),
+            path=sanitize_path(request.url.path),
+        )
+    request.state.request_id = request_id
+    if request.method.upper() not in {"GET", "HEAD", "OPTIONS"}:
+        origin = str(request.headers.get("origin") or "")
+        initial_origin_claim = (
+            request.method.upper() == "POST" and request.url.path == "/setup/bootstrap"
+            and can_claim_initial_origin()
+        )
+        if origin and not is_allowed_request_origin(origin) and not initial_origin_claim:
+            record_api(request, 403, 0.0)
+            if operation_id and operation_token is not None:
+                finish_operation(operation_id, success=False, status_code=403)
+                reset_operation(operation_token)
+            reset_request_id(token)
+            return JSONResponse(
+                status_code=403,
+                content={"detail": "Cross-origin state change rejected"},
+                headers={"X-Request-ID": request_id},
+            )
+    started_at = time.perf_counter()
+    body_summary = {
+        "content_type": (request.headers.get("content-type") or "").split(";", 1)[0],
+        "declared_bytes": request.headers.get("content-length"),
+    }
+    logger.info(
+        "request started method=%s path=%s query_keys=%s client=%s headers=%s body=%s",
+        request.method,
+        sanitize_path(request.url.path),
+        sorted(set(request.query_params.keys())),
+        request.client.host if request.client else "-",
+        sanitize_headers(
+            {
+                key: value
+                for key, value in request.headers.items()
+                if key.lower()
+                in {
+                    "content-type",
+                    "content-length",
+                    "user-agent",
+                    "x-forwarded-for",
+                    "x-forwarded-proto",
+                    "x-request-id",
+                }
+            }
+        ),
+        body_summary,
+    )
+    try:
+        response = await call_next(request)
+    except Exception:
+        duration_ms = round((time.perf_counter() - started_at) * 1000, 2)
+        record_api(request, 500, time.perf_counter() - started_at)
+        logger.exception(
+            "request failed method=%s path=%s duration_ms=%s",
+            request.method,
+            sanitize_path(request.url.path),
+            duration_ms,
+        )
+        if operation_id and operation_token is not None:
+            finish_operation(operation_id, success=False, status_code=500)
+            reset_operation(operation_token)
+        reset_request_id(token)
+        raise
+
+    duration_ms = round((time.perf_counter() - started_at) * 1000, 2)
+    record_api(request, response.status_code, time.perf_counter() - started_at)
+    response.headers.setdefault("X-Request-ID", request_id)
+    response.headers.setdefault("X-Content-Type-Options", "nosniff")
+    response.headers.setdefault("X-Frame-Options", "DENY")
+    response.headers.setdefault("Referrer-Policy", "no-referrer")
+    response.headers.setdefault("Permissions-Policy", "geolocation=(), microphone=(), camera=()")
+    response.headers.setdefault("Strict-Transport-Security", "max-age=31536000; includeSubDomains")
+    # Keep API responses non-executable and non-embeddable by default.
+    if request.url.path not in {"/docs", "/redoc"} and not request.url.path.startswith("/openapi"):
+        response.headers.setdefault(
+            "Content-Security-Policy",
+            "default-src 'none'; frame-ancestors 'none'; base-uri 'none'",
+        )
+    logger.info(
+        "request completed method=%s path=%s status=%s duration_ms=%s response_headers=%s",
+        request.method,
+        sanitize_path(request.url.path),
+        response.status_code,
+        duration_ms,
+        sanitize_headers(
+            {
+                key: value
+                for key, value in response.headers.items()
+                if key.lower() in {"content-type", "content-length", "x-request-id"}
+            }
+        ),
+    )
+    if operation_id and operation_token is not None:
+        finish_operation(
+            operation_id,
+            success=response.status_code < 400,
+            status_code=response.status_code,
+        )
+        reset_operation(operation_token)
+    reset_request_id(token)
+    return response
+
+
+@app.get("/health")
+async def health() -> dict:
+    return {"status": "ok"}
+
+
+async def _run_background_task(
+    name: str, coroutine_factory: Callable[[], Awaitable[None]]
+) -> None:
+    token = bind_request_id(f"task-{name}")
+    logger.info("background task started task=%s", name)
+    try:
+        await coroutine_factory()
+        logger.warning("background task exited task=%s", name)
+    except asyncio.CancelledError:
+        logger.info("background task cancelled task=%s", name)
+        raise
+    except Exception:
+        logger.exception("background task crashed task=%s", name)
+        raise
+    finally:
+        reset_request_id(token)
+
+
+def _launch_background_task(name: str, coroutine_factory: Callable[[], Awaitable[None]]) -> None:
+    task = asyncio.create_task(
+        _run_background_task(name, coroutine_factory), name=f"magent:{name}"
+    )
+    _background_tasks.append(task)
+
+
+def _log_security_configuration_warnings() -> None:
+    jwt_secret = str(settings.jwt_secret or "").strip()
+    if len(jwt_secret) < 32 or jwt_secret == "change-me":
+        logger.warning(
+            "security configuration warning: JWT_SECRET is missing, short, or still set to the default value"
+        )
+    admin_password = str(settings.admin_password or "")
+    if admin_password == "adminadmin":
+        logger.warning(
+            "security configuration warning: ADMIN_PASSWORD is still set to the bootstrap default"
+        )
+    if bool(settings.api_docs_enabled):
+        logger.warning(
+            "security configuration warning: API docs are enabled; disable API_DOCS_ENABLED outside controlled environments"
+        )
+
+
+def _enforce_secret_configuration() -> None:
+    jwt_secret = str(settings.jwt_secret or "").strip()
+    if len(jwt_secret) < 32 or jwt_secret == "change-me":
+        raise RuntimeError(
+            "JWT_SECRET must be a strong, non-default value of at least 32 characters before startup."
+        )
+    validate_secret_storage_configuration()
+
+
+def _enforce_secure_startup_configuration() -> None:
+    _enforce_secret_configuration()
+    admin_password = str(settings.admin_password or "")
+    if not has_admin_user() and (not admin_password or admin_password == "adminadmin"):
+        if is_setup_required() and setup_token_configured():
+            return
+        raise RuntimeError(
+            "First startup requires a strong SETUP_TOKEN (at least 32 characters) for the setup wizard, "
+            "or a secure ADMIN_PASSWORD, until an admin account exists."
+        )
+
+
+@app.on_event("startup")
+async def startup() -> None:
+    start_metrics()
+    configure_logging(
+        settings.log_level,
+        settings.log_file,
+        log_file_max_bytes=settings.log_file_max_bytes,
+        log_file_backup_count=settings.log_file_backup_count,
+        log_http_client_level=settings.log_http_client_level,
+        log_background_sync_level=settings.log_background_sync_level,
+        log_format=settings.log_format,
+    )
+    logger.info("startup begin app=%s build=%s", settings.app_name, settings.site_build_number)
+    _log_security_configuration_warnings()
+    _enforce_secret_configuration()
+    # Restore offline, before any schema migration, database reader or worker.
+    apply_pending_restore()
+    initialize_setup_state()
+    init_db()
+    _enforce_secure_startup_configuration()
+    runtime = get_runtime_settings()
+    configure_logging(
+        runtime.log_level,
+        runtime.log_file,
+        log_file_max_bytes=runtime.log_file_max_bytes,
+        log_file_backup_count=runtime.log_file_backup_count,
+        log_http_client_level=runtime.log_http_client_level,
+        log_background_sync_level=runtime.log_background_sync_level,
+        log_format=runtime.log_format,
+    )
+    logger.info(
+        "runtime settings applied log_level=%s log_file=%s log_file_max_bytes=%s log_file_backup_count=%s log_http_client_level=%s log_background_sync_level=%s request_source=%s",
+        runtime.log_level,
+        runtime.log_file,
+        runtime.log_file_max_bytes,
+        runtime.log_file_backup_count,
+        runtime.log_http_client_level,
+        runtime.log_background_sync_level,
+        runtime.requests_data_source,
+    )
+    app.state.on_setup_complete = _start_background_tasks
+    await _start_background_tasks()
+    logger.info("startup complete")
+
+
+async def _start_background_tasks() -> None:
+    global _background_started
+    if _background_started:
+        return
+    if is_setup_required():
+        logger.info("Background imports and automation paused until setup is complete")
+        return
+    if os.environ.get("BACKGROUND_TASKS_ENABLED", "true").lower() == "false":
+        logger.info("Background imports and automation disabled by configuration")
+        return
+    _background_started = True
+    _launch_background_task("jellyfin-sync", run_daily_jellyfin_sync)
+    _launch_background_task("requests-warmup", startup_warmup_requests_cache)
+    _launch_background_task("request-local-stages", run_local_request_stage_loop)
+    _launch_background_task("requests-delta-loop", run_requests_delta_loop)
+    _launch_background_task("requests-full-sync", run_daily_requests_full_sync)
+    _launch_background_task("db-cleanup", run_daily_db_cleanup)
+    _launch_background_task("issue-confirmation", run_issue_confirmation_loop)
+    _launch_background_task("email-recaps", run_email_recap_loop)
+    _launch_background_task("newsletters", run_newsletter_loop)
+
+
+@app.on_event("shutdown")
+async def shutdown() -> None:
+    global _background_started
+    for task in _background_tasks:
+        task.cancel()
+    if _background_tasks:
+        await asyncio.gather(*_background_tasks, return_exceptions=True)
+    _background_tasks.clear()
+    _background_started = False
+
+
+app.include_router(requests_router)
+app.include_router(auth_router)
+app.include_router(admin_router)
+app.include_router(admin_events_router)
+app.include_router(images_router)
+app.include_router(branding_router)
+app.include_router(status_router)
+app.include_router(feedback_router)
+app.include_router(site_router)
+app.include_router(events_router)
+app.include_router(portal_router)
+app.include_router(operations_router)
+app.include_router(insights_router)
+app.include_router(identities_router)
+app.include_router(recaps_router)
+app.include_router(newsletters_router)
+app.include_router(backups_router)
+app.include_router(setup_router)