376 lines
14 KiB
Python
376 lines
14 KiB
Python
import asyncio
|
|
import logging
|
|
import os
|
|
import time
|
|
import uuid
|
|
from typing import Awaitable, Callable
|
|
|
|
from fastapi import FastAPI, Request
|
|
from fastapi.exceptions import RequestValidationError
|
|
from fastapi.exception_handlers import request_validation_exception_handler
|
|
from fastapi.middleware.cors import CORSMiddleware
|
|
from fastapi.responses import JSONResponse
|
|
|
|
from .config import settings
|
|
from .db import has_admin_user, init_db
|
|
from .routers.requests import (
|
|
router as requests_router,
|
|
startup_warmup_requests_cache,
|
|
run_local_request_stage_loop,
|
|
run_requests_delta_loop,
|
|
run_daily_requests_full_sync,
|
|
run_daily_db_cleanup,
|
|
)
|
|
from .routers.auth import router as auth_router
|
|
from .routers.admin import router as admin_router, events_router as admin_events_router
|
|
from .routers.images import router as images_router
|
|
from .routers.branding import router as branding_router
|
|
from .routers.status import router as status_router
|
|
from .routers.feedback import router as feedback_router
|
|
from .routers.site import router as site_router
|
|
from .routers.events import router as events_router
|
|
from .routers.portal import router as portal_router
|
|
from .routers.operations import router as operations_router
|
|
from .routers.insights import router as insights_router
|
|
from .routers.identities import router as identities_router
|
|
from .routers.recaps import router as recaps_router
|
|
from .routers.newsletters import router as newsletters_router
|
|
from .routers.backups import router as backups_router
|
|
from .routers.setup import router as setup_router
|
|
from .services.backups import apply_pending_restore
|
|
from .services.setup import initialize_setup_state, is_setup_required, setup_token_configured
|
|
from .services.jellyfin_sync import run_daily_jellyfin_sync
|
|
from .services.issue_resolution import run_issue_confirmation_loop
|
|
from .services.email_recaps import run_email_recap_loop
|
|
from .services.newsletters import run_newsletter_loop
|
|
from .services.operation_progress import (
|
|
begin_operation,
|
|
finish_operation,
|
|
normalize_operation_id,
|
|
reset_operation,
|
|
)
|
|
from .logging_config import (
|
|
bind_request_id,
|
|
configure_logging,
|
|
reset_request_id,
|
|
sanitize_headers,
|
|
sanitize_path,
|
|
)
|
|
from .runtime import get_runtime_settings
|
|
from .metrics import record_api, start_metrics
|
|
from .request_limits import InstallationBodyLimitMiddleware
|
|
from .secret_storage import validate_secret_storage_configuration
|
|
from .services.request_origins import is_allowed_request_origin
|
|
|
|
logger = logging.getLogger(__name__)
|
|
_background_tasks: list[asyncio.Task[None]] = []
|
|
_background_started = False
|
|
|
|
app = FastAPI(
|
|
title=settings.app_name,
|
|
docs_url="/docs" if settings.api_docs_enabled else None,
|
|
redoc_url=None,
|
|
openapi_url="/openapi.json" if settings.api_docs_enabled else None,
|
|
)
|
|
|
|
app.add_middleware(
|
|
CORSMiddleware,
|
|
allow_origins=[settings.cors_allow_origin],
|
|
allow_credentials=True,
|
|
allow_methods=["*"],
|
|
allow_headers=["*"],
|
|
)
|
|
app.add_middleware(InstallationBodyLimitMiddleware)
|
|
|
|
|
|
@app.exception_handler(RequestValidationError)
|
|
async def installation_validation_error(request: Request, exc: RequestValidationError):
|
|
if request.url.path.rstrip("/") == "/setup/bootstrap" or request.url.path.startswith("/admin/backups"):
|
|
# Pydantic SecretStr masks parsed values, but FastAPI's default 422 body
|
|
# includes rejected raw input. Never echo tokens/passwords/passphrases.
|
|
return JSONResponse(
|
|
status_code=422,
|
|
content={"detail": [
|
|
{key: error[key] for key in ("type", "loc", "msg") if key in error}
|
|
for error in exc.errors()
|
|
]},
|
|
headers={"Cache-Control": "no-store"},
|
|
)
|
|
return await request_validation_exception_handler(request, exc)
|
|
|
|
|
|
@app.middleware("http")
|
|
async def log_requests_and_add_security_headers(request: Request, call_next):
|
|
request_id = request.headers.get("X-Request-ID") or uuid.uuid4().hex[:12]
|
|
token = bind_request_id(request_id)
|
|
operation_id = normalize_operation_id(request.headers.get("X-Magent-Operation-ID"))
|
|
operation_token = None
|
|
if operation_id and request.method.upper() not in {"GET", "HEAD", "OPTIONS"}:
|
|
operation_token = begin_operation(
|
|
operation_id,
|
|
label=request.headers.get("X-Magent-Operation-Label"),
|
|
path=sanitize_path(request.url.path),
|
|
)
|
|
request.state.request_id = request_id
|
|
if request.method.upper() not in {"GET", "HEAD", "OPTIONS"}:
|
|
origin = str(request.headers.get("origin") or "")
|
|
if origin and not is_allowed_request_origin(origin):
|
|
record_api(request, 403, 0.0)
|
|
if operation_id and operation_token is not None:
|
|
finish_operation(operation_id, success=False, status_code=403)
|
|
reset_operation(operation_token)
|
|
reset_request_id(token)
|
|
return JSONResponse(
|
|
status_code=403,
|
|
content={"detail": "Cross-origin state change rejected"},
|
|
headers={"X-Request-ID": request_id},
|
|
)
|
|
started_at = time.perf_counter()
|
|
body_summary = {
|
|
"content_type": (request.headers.get("content-type") or "").split(";", 1)[0],
|
|
"declared_bytes": request.headers.get("content-length"),
|
|
}
|
|
logger.info(
|
|
"request started method=%s path=%s query_keys=%s client=%s headers=%s body=%s",
|
|
request.method,
|
|
sanitize_path(request.url.path),
|
|
sorted(set(request.query_params.keys())),
|
|
request.client.host if request.client else "-",
|
|
sanitize_headers(
|
|
{
|
|
key: value
|
|
for key, value in request.headers.items()
|
|
if key.lower()
|
|
in {
|
|
"content-type",
|
|
"content-length",
|
|
"user-agent",
|
|
"x-forwarded-for",
|
|
"x-forwarded-proto",
|
|
"x-request-id",
|
|
}
|
|
}
|
|
),
|
|
body_summary,
|
|
)
|
|
try:
|
|
response = await call_next(request)
|
|
except Exception:
|
|
duration_ms = round((time.perf_counter() - started_at) * 1000, 2)
|
|
record_api(request, 500, time.perf_counter() - started_at)
|
|
logger.exception(
|
|
"request failed method=%s path=%s duration_ms=%s",
|
|
request.method,
|
|
sanitize_path(request.url.path),
|
|
duration_ms,
|
|
)
|
|
if operation_id and operation_token is not None:
|
|
finish_operation(operation_id, success=False, status_code=500)
|
|
reset_operation(operation_token)
|
|
reset_request_id(token)
|
|
raise
|
|
|
|
duration_ms = round((time.perf_counter() - started_at) * 1000, 2)
|
|
record_api(request, response.status_code, time.perf_counter() - started_at)
|
|
response.headers.setdefault("X-Request-ID", request_id)
|
|
response.headers.setdefault("X-Content-Type-Options", "nosniff")
|
|
response.headers.setdefault("X-Frame-Options", "DENY")
|
|
response.headers.setdefault("Referrer-Policy", "no-referrer")
|
|
response.headers.setdefault("Permissions-Policy", "geolocation=(), microphone=(), camera=()")
|
|
response.headers.setdefault("Strict-Transport-Security", "max-age=31536000; includeSubDomains")
|
|
# Keep API responses non-executable and non-embeddable by default.
|
|
if request.url.path not in {"/docs", "/redoc"} and not request.url.path.startswith("/openapi"):
|
|
response.headers.setdefault(
|
|
"Content-Security-Policy",
|
|
"default-src 'none'; frame-ancestors 'none'; base-uri 'none'",
|
|
)
|
|
logger.info(
|
|
"request completed method=%s path=%s status=%s duration_ms=%s response_headers=%s",
|
|
request.method,
|
|
sanitize_path(request.url.path),
|
|
response.status_code,
|
|
duration_ms,
|
|
sanitize_headers(
|
|
{
|
|
key: value
|
|
for key, value in response.headers.items()
|
|
if key.lower() in {"content-type", "content-length", "x-request-id"}
|
|
}
|
|
),
|
|
)
|
|
if operation_id and operation_token is not None:
|
|
finish_operation(
|
|
operation_id,
|
|
success=response.status_code < 400,
|
|
status_code=response.status_code,
|
|
)
|
|
reset_operation(operation_token)
|
|
reset_request_id(token)
|
|
return response
|
|
|
|
|
|
@app.get("/health")
|
|
async def health() -> dict:
|
|
return {"status": "ok"}
|
|
|
|
|
|
async def _run_background_task(
|
|
name: str, coroutine_factory: Callable[[], Awaitable[None]]
|
|
) -> None:
|
|
token = bind_request_id(f"task-{name}")
|
|
logger.info("background task started task=%s", name)
|
|
try:
|
|
await coroutine_factory()
|
|
logger.warning("background task exited task=%s", name)
|
|
except asyncio.CancelledError:
|
|
logger.info("background task cancelled task=%s", name)
|
|
raise
|
|
except Exception:
|
|
logger.exception("background task crashed task=%s", name)
|
|
raise
|
|
finally:
|
|
reset_request_id(token)
|
|
|
|
|
|
def _launch_background_task(name: str, coroutine_factory: Callable[[], Awaitable[None]]) -> None:
|
|
task = asyncio.create_task(
|
|
_run_background_task(name, coroutine_factory), name=f"magent:{name}"
|
|
)
|
|
_background_tasks.append(task)
|
|
|
|
|
|
def _log_security_configuration_warnings() -> None:
|
|
jwt_secret = str(settings.jwt_secret or "").strip()
|
|
if len(jwt_secret) < 32 or jwt_secret == "change-me":
|
|
logger.warning(
|
|
"security configuration warning: JWT_SECRET is missing, short, or still set to the default value"
|
|
)
|
|
admin_password = str(settings.admin_password or "")
|
|
if admin_password == "adminadmin":
|
|
logger.warning(
|
|
"security configuration warning: ADMIN_PASSWORD is still set to the bootstrap default"
|
|
)
|
|
if bool(settings.api_docs_enabled):
|
|
logger.warning(
|
|
"security configuration warning: API docs are enabled; disable API_DOCS_ENABLED outside controlled environments"
|
|
)
|
|
|
|
|
|
def _enforce_secret_configuration() -> None:
|
|
jwt_secret = str(settings.jwt_secret or "").strip()
|
|
if len(jwt_secret) < 32 or jwt_secret == "change-me":
|
|
raise RuntimeError(
|
|
"JWT_SECRET must be a strong, non-default value of at least 32 characters before startup."
|
|
)
|
|
validate_secret_storage_configuration()
|
|
|
|
|
|
def _enforce_secure_startup_configuration() -> None:
|
|
_enforce_secret_configuration()
|
|
admin_password = str(settings.admin_password or "")
|
|
if not has_admin_user() and (not admin_password or admin_password == "adminadmin"):
|
|
if is_setup_required() and setup_token_configured():
|
|
return
|
|
raise RuntimeError(
|
|
"First startup requires a strong SETUP_TOKEN (at least 32 characters) for the setup wizard, "
|
|
"or a secure ADMIN_PASSWORD, until an admin account exists."
|
|
)
|
|
|
|
|
|
@app.on_event("startup")
|
|
async def startup() -> None:
|
|
start_metrics()
|
|
configure_logging(
|
|
settings.log_level,
|
|
settings.log_file,
|
|
log_file_max_bytes=settings.log_file_max_bytes,
|
|
log_file_backup_count=settings.log_file_backup_count,
|
|
log_http_client_level=settings.log_http_client_level,
|
|
log_background_sync_level=settings.log_background_sync_level,
|
|
log_format=settings.log_format,
|
|
)
|
|
logger.info("startup begin app=%s build=%s", settings.app_name, settings.site_build_number)
|
|
_log_security_configuration_warnings()
|
|
_enforce_secret_configuration()
|
|
# Restore offline, before any schema migration, database reader or worker.
|
|
apply_pending_restore()
|
|
initialize_setup_state()
|
|
init_db()
|
|
_enforce_secure_startup_configuration()
|
|
runtime = get_runtime_settings()
|
|
configure_logging(
|
|
runtime.log_level,
|
|
runtime.log_file,
|
|
log_file_max_bytes=runtime.log_file_max_bytes,
|
|
log_file_backup_count=runtime.log_file_backup_count,
|
|
log_http_client_level=runtime.log_http_client_level,
|
|
log_background_sync_level=runtime.log_background_sync_level,
|
|
log_format=runtime.log_format,
|
|
)
|
|
logger.info(
|
|
"runtime settings applied log_level=%s log_file=%s log_file_max_bytes=%s log_file_backup_count=%s log_http_client_level=%s log_background_sync_level=%s request_source=%s",
|
|
runtime.log_level,
|
|
runtime.log_file,
|
|
runtime.log_file_max_bytes,
|
|
runtime.log_file_backup_count,
|
|
runtime.log_http_client_level,
|
|
runtime.log_background_sync_level,
|
|
runtime.requests_data_source,
|
|
)
|
|
app.state.on_setup_complete = _start_background_tasks
|
|
await _start_background_tasks()
|
|
logger.info("startup complete")
|
|
|
|
|
|
async def _start_background_tasks() -> None:
|
|
global _background_started
|
|
if _background_started:
|
|
return
|
|
if is_setup_required():
|
|
logger.info("Background imports and automation paused until setup is complete")
|
|
return
|
|
if os.environ.get("BACKGROUND_TASKS_ENABLED", "true").lower() == "false":
|
|
logger.info("Background imports and automation disabled by configuration")
|
|
return
|
|
_background_started = True
|
|
_launch_background_task("jellyfin-sync", run_daily_jellyfin_sync)
|
|
_launch_background_task("requests-warmup", startup_warmup_requests_cache)
|
|
_launch_background_task("request-local-stages", run_local_request_stage_loop)
|
|
_launch_background_task("requests-delta-loop", run_requests_delta_loop)
|
|
_launch_background_task("requests-full-sync", run_daily_requests_full_sync)
|
|
_launch_background_task("db-cleanup", run_daily_db_cleanup)
|
|
_launch_background_task("issue-confirmation", run_issue_confirmation_loop)
|
|
_launch_background_task("email-recaps", run_email_recap_loop)
|
|
_launch_background_task("newsletters", run_newsletter_loop)
|
|
|
|
|
|
@app.on_event("shutdown")
|
|
async def shutdown() -> None:
|
|
global _background_started
|
|
for task in _background_tasks:
|
|
task.cancel()
|
|
if _background_tasks:
|
|
await asyncio.gather(*_background_tasks, return_exceptions=True)
|
|
_background_tasks.clear()
|
|
_background_started = False
|
|
|
|
|
|
app.include_router(requests_router)
|
|
app.include_router(auth_router)
|
|
app.include_router(admin_router)
|
|
app.include_router(admin_events_router)
|
|
app.include_router(images_router)
|
|
app.include_router(branding_router)
|
|
app.include_router(status_router)
|
|
app.include_router(feedback_router)
|
|
app.include_router(site_router)
|
|
app.include_router(events_router)
|
|
app.include_router(portal_router)
|
|
app.include_router(operations_router)
|
|
app.include_router(insights_router)
|
|
app.include_router(identities_router)
|
|
app.include_router(recaps_router)
|
|
app.include_router(newsletters_router)
|
|
app.include_router(backups_router)
|
|
app.include_router(setup_router)
|