feat: add backup recovery, setup wizard and user-view guards
This commit is contained in:
@@ -51,6 +51,7 @@ class Settings(BaseSettings):
|
||||
)
|
||||
admin_username: str = Field(default="admin", validation_alias=AliasChoices("ADMIN_USERNAME"))
|
||||
admin_password: str = Field(default="", validation_alias=AliasChoices("ADMIN_PASSWORD"))
|
||||
setup_token: str = Field(default="", validation_alias=AliasChoices("SETUP_TOKEN"))
|
||||
auth_cookie_name: str = Field(
|
||||
default="magent_auth", validation_alias=AliasChoices("AUTH_COOKIE_NAME")
|
||||
)
|
||||
|
||||
@@ -929,6 +929,10 @@ def get_request_download_evidence(request_id: str, limit: int = 100) -> Dict[str
|
||||
def ensure_admin_user() -> None:
|
||||
if not settings.admin_username or not _has_secure_bootstrap_admin_credentials():
|
||||
return
|
||||
# Environment credentials bootstrap only the first administrator. In
|
||||
# particular, do not inject a destination host's account into a restored DB.
|
||||
if has_admin_user():
|
||||
return
|
||||
existing = get_user_by_username(settings.admin_username)
|
||||
if existing:
|
||||
return
|
||||
|
||||
+62
-6
@@ -6,6 +6,8 @@ import uuid
|
||||
from typing import Awaitable, Callable
|
||||
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.exception_handlers import request_validation_exception_handler
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
@@ -33,6 +35,10 @@ from .routers.insights import router as insights_router
|
||||
from .routers.identities import router as identities_router
|
||||
from .routers.recaps import router as recaps_router
|
||||
from .routers.newsletters import router as newsletters_router
|
||||
from .routers.backups import router as backups_router
|
||||
from .routers.setup import router as setup_router
|
||||
from .services.backups import apply_pending_restore
|
||||
from .services.setup import initialize_setup_state, is_setup_required, setup_token_configured
|
||||
from .services.jellyfin_sync import run_daily_jellyfin_sync
|
||||
from .services.issue_resolution import run_issue_confirmation_loop
|
||||
from .services.email_recaps import run_email_recap_loop
|
||||
@@ -52,10 +58,12 @@ from .logging_config import (
|
||||
)
|
||||
from .runtime import get_runtime_settings
|
||||
from .metrics import record_api, start_metrics
|
||||
from .request_limits import InstallationBodyLimitMiddleware
|
||||
from .secret_storage import validate_secret_storage_configuration
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_background_tasks: list[asyncio.Task[None]] = []
|
||||
_background_started = False
|
||||
|
||||
app = FastAPI(
|
||||
title=settings.app_name,
|
||||
@@ -71,6 +79,23 @@ app.add_middleware(
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
app.add_middleware(InstallationBodyLimitMiddleware)
|
||||
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def installation_validation_error(request: Request, exc: RequestValidationError):
|
||||
if request.url.path.rstrip("/") == "/setup/bootstrap" or request.url.path.startswith("/admin/backups"):
|
||||
# Pydantic SecretStr masks parsed values, but FastAPI's default 422 body
|
||||
# includes rejected raw input. Never echo tokens/passwords/passphrases.
|
||||
return JSONResponse(
|
||||
status_code=422,
|
||||
content={"detail": [
|
||||
{key: error[key] for key in ("type", "loc", "msg") if key in error}
|
||||
for error in exc.errors()
|
||||
]},
|
||||
headers={"Cache-Control": "no-store"},
|
||||
)
|
||||
return await request_validation_exception_handler(request, exc)
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
@@ -221,9 +246,9 @@ def _log_security_configuration_warnings() -> None:
|
||||
"security configuration warning: JWT_SECRET is missing, short, or still set to the default value"
|
||||
)
|
||||
admin_password = str(settings.admin_password or "")
|
||||
if not admin_password or admin_password == "adminadmin":
|
||||
if admin_password == "adminadmin":
|
||||
logger.warning(
|
||||
"security configuration warning: ADMIN_PASSWORD is unset or still set to the bootstrap default"
|
||||
"security configuration warning: ADMIN_PASSWORD is still set to the bootstrap default"
|
||||
)
|
||||
if bool(settings.api_docs_enabled):
|
||||
logger.warning(
|
||||
@@ -244,8 +269,11 @@ def _enforce_secure_startup_configuration() -> None:
|
||||
_enforce_secret_configuration()
|
||||
admin_password = str(settings.admin_password or "")
|
||||
if not has_admin_user() and (not admin_password or admin_password == "adminadmin"):
|
||||
if is_setup_required() and setup_token_configured():
|
||||
return
|
||||
raise RuntimeError(
|
||||
"A secure ADMIN_PASSWORD is required on first startup until an admin account exists."
|
||||
"First startup requires a strong SETUP_TOKEN (at least 32 characters) for the setup wizard, "
|
||||
"or a secure ADMIN_PASSWORD, until an admin account exists."
|
||||
)
|
||||
|
||||
|
||||
@@ -264,6 +292,9 @@ async def startup() -> None:
|
||||
logger.info("startup begin app=%s build=%s", settings.app_name, settings.site_build_number)
|
||||
_log_security_configuration_warnings()
|
||||
_enforce_secret_configuration()
|
||||
# Restore offline, before any schema migration, database reader or worker.
|
||||
apply_pending_restore()
|
||||
initialize_setup_state()
|
||||
init_db()
|
||||
_enforce_secure_startup_configuration()
|
||||
runtime = get_runtime_settings()
|
||||
@@ -286,9 +317,22 @@ async def startup() -> None:
|
||||
runtime.log_background_sync_level,
|
||||
runtime.requests_data_source,
|
||||
)
|
||||
if os.environ.get("BACKGROUND_TASKS_ENABLED", "true").lower() == "false":
|
||||
logger.info("Background imports and automation paused for initial setup")
|
||||
app.state.on_setup_complete = _start_background_tasks
|
||||
await _start_background_tasks()
|
||||
logger.info("startup complete")
|
||||
|
||||
|
||||
async def _start_background_tasks() -> None:
|
||||
global _background_started
|
||||
if _background_started:
|
||||
return
|
||||
if is_setup_required():
|
||||
logger.info("Background imports and automation paused until setup is complete")
|
||||
return
|
||||
if os.environ.get("BACKGROUND_TASKS_ENABLED", "true").lower() == "false":
|
||||
logger.info("Background imports and automation disabled by configuration")
|
||||
return
|
||||
_background_started = True
|
||||
_launch_background_task("jellyfin-sync", run_daily_jellyfin_sync)
|
||||
_launch_background_task("requests-warmup", startup_warmup_requests_cache)
|
||||
_launch_background_task("request-local-stages", run_local_request_stage_loop)
|
||||
@@ -298,7 +342,17 @@ async def startup() -> None:
|
||||
_launch_background_task("issue-confirmation", run_issue_confirmation_loop)
|
||||
_launch_background_task("email-recaps", run_email_recap_loop)
|
||||
_launch_background_task("newsletters", run_newsletter_loop)
|
||||
logger.info("startup complete")
|
||||
|
||||
|
||||
@app.on_event("shutdown")
|
||||
async def shutdown() -> None:
|
||||
global _background_started
|
||||
for task in _background_tasks:
|
||||
task.cancel()
|
||||
if _background_tasks:
|
||||
await asyncio.gather(*_background_tasks, return_exceptions=True)
|
||||
_background_tasks.clear()
|
||||
_background_started = False
|
||||
|
||||
|
||||
app.include_router(requests_router)
|
||||
@@ -317,3 +371,5 @@ app.include_router(insights_router)
|
||||
app.include_router(identities_router)
|
||||
app.include_router(recaps_router)
|
||||
app.include_router(newsletters_router)
|
||||
app.include_router(backups_router)
|
||||
app.include_router(setup_router)
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Bound security-sensitive request bodies before JSON/multipart parsing."""
|
||||
|
||||
from starlette.exceptions import HTTPException
|
||||
from starlette.responses import JSONResponse
|
||||
from starlette.types import ASGIApp, Message, Receive, Scope, Send
|
||||
|
||||
|
||||
# Encrypted backup limit is 32 MiB. Allow a bounded margin for the multipart
|
||||
# envelope; count streamed chunks as well as checking the untrusted header.
|
||||
RESTORE_BODY_LIMIT = 34 * 1024 * 1024
|
||||
BOOTSTRAP_BODY_LIMIT = 16 * 1024
|
||||
|
||||
|
||||
class InstallationBodyLimitMiddleware:
|
||||
def __init__(self, app: ASGIApp) -> None:
|
||||
self.app = app
|
||||
|
||||
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
|
||||
if scope["type"] != "http" or scope.get("method") != "POST":
|
||||
await self.app(scope, receive, send)
|
||||
return
|
||||
path = scope.get("path", "").rstrip("/")
|
||||
limit = {
|
||||
"/admin/backups/restore": RESTORE_BODY_LIMIT,
|
||||
"/admin/backups/export": BOOTSTRAP_BODY_LIMIT,
|
||||
"/setup/bootstrap": BOOTSTRAP_BODY_LIMIT,
|
||||
}.get(path)
|
||||
if limit is None:
|
||||
await self.app(scope, receive, send)
|
||||
return
|
||||
headers = dict(scope.get("headers", []))
|
||||
try:
|
||||
length = int(headers.get(b"content-length", b"0"))
|
||||
except ValueError:
|
||||
length = -1
|
||||
if length < 0 or length > limit:
|
||||
await JSONResponse({"detail": "Request body is too large or has an invalid length."}, status_code=413)(scope, receive, send)
|
||||
return
|
||||
received = 0
|
||||
|
||||
async def bounded_receive() -> Message:
|
||||
nonlocal received
|
||||
message = await receive()
|
||||
if message["type"] == "http.request":
|
||||
received += len(message.get("body", b""))
|
||||
if received > limit:
|
||||
raise HTTPException(status_code=413, detail="Request body is too large.")
|
||||
return message
|
||||
|
||||
await self.app(scope, bounded_receive, send)
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Administrator-only encrypted backup downloads and staged restores."""
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile
|
||||
from fastapi.responses import Response
|
||||
from pydantic import BaseModel, ConfigDict, Field, SecretStr
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
|
||||
from ..auth import require_admin
|
||||
from ..db import get_rate_limit_status, record_rate_limit_event
|
||||
from ..services import backups
|
||||
|
||||
def _no_store(response: Response) -> None:
|
||||
response.headers["Cache-Control"] = "no-store"
|
||||
response.headers["Pragma"] = "no-cache"
|
||||
|
||||
|
||||
router = APIRouter(
|
||||
prefix="/admin/backups", tags=["backups"],
|
||||
dependencies=[Depends(require_admin), Depends(_no_store)],
|
||||
)
|
||||
|
||||
|
||||
class ExportRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
passphrase: SecretStr = Field(min_length=12, max_length=1024)
|
||||
include_cache: bool = False
|
||||
|
||||
|
||||
def _rate_limit(user: dict) -> None:
|
||||
key = str(user["username"])
|
||||
exceeded, retry = get_rate_limit_status("backups", key, 300, 3)
|
||||
if exceeded:
|
||||
raise HTTPException(429, "Too many backup operations; try again shortly", headers={"Retry-After": str(retry)})
|
||||
record_rate_limit_event("backups", key)
|
||||
|
||||
|
||||
@router.get("")
|
||||
def status() -> dict:
|
||||
return backups.backup_status()
|
||||
|
||||
|
||||
@router.post("/export")
|
||||
def export(payload: ExportRequest, user: dict = Depends(require_admin)) -> Response:
|
||||
_rate_limit(user)
|
||||
try:
|
||||
content, filename = backups.create_backup(payload.passphrase.get_secret_value(), payload.include_cache)
|
||||
except backups.BackupError as exc:
|
||||
raise HTTPException(400, str(exc)) from exc
|
||||
return Response(content, media_type="application/octet-stream", headers={
|
||||
"Content-Disposition": f'attachment; filename="{filename}"',
|
||||
"Cache-Control": "no-store", "Pragma": "no-cache",
|
||||
})
|
||||
|
||||
|
||||
@router.post("/restore", status_code=202)
|
||||
async def restore(
|
||||
file: UploadFile = File(...),
|
||||
passphrase: str = Form(..., min_length=12, max_length=1024),
|
||||
confirmation: Literal["RESTORE"] = Form(...),
|
||||
user: dict = Depends(require_admin),
|
||||
) -> dict:
|
||||
_rate_limit(user)
|
||||
try:
|
||||
if file.size is not None and file.size > backups.MAX_UPLOAD_BYTES:
|
||||
raise HTTPException(413, "Backup exceeds the 32 MiB upload limit")
|
||||
metadata = await run_in_threadpool(backups.stage_restore, file.file, passphrase)
|
||||
except backups.BackupError as exc:
|
||||
raise HTTPException(400, str(exc)) from exc
|
||||
finally:
|
||||
await file.close()
|
||||
return {
|
||||
"status": "staged", "restart_required": True, "backup": metadata,
|
||||
"message": "Backup validated. Restart Magent to apply it. Current data remains active until restart.",
|
||||
}
|
||||
|
||||
|
||||
@router.delete("/restore")
|
||||
def cancel() -> dict:
|
||||
try:
|
||||
backups.cancel_restore()
|
||||
except backups.BackupError as exc:
|
||||
raise HTTPException(409, str(exc)) from exc
|
||||
return {"status": "cancelled"}
|
||||
@@ -0,0 +1,80 @@
|
||||
"""Initial install bootstrap and authenticated setup wizard endpoints."""
|
||||
|
||||
from inspect import isawaitable
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, Response
|
||||
from pydantic import Field, SecretStr
|
||||
|
||||
from ..api_models import COMMON_ERROR_RESPONSES, StrictRequest
|
||||
from ..auth import _extract_client_ip, require_admin
|
||||
from ..services import setup as setup_service
|
||||
|
||||
|
||||
router = APIRouter(prefix="/setup", tags=["setup"], responses=COMMON_ERROR_RESPONSES)
|
||||
|
||||
|
||||
class BootstrapRequest(StrictRequest):
|
||||
setup_token: SecretStr = Field(min_length=1, max_length=1024)
|
||||
username: str = Field(min_length=1, max_length=100)
|
||||
password: SecretStr = Field(min_length=1, max_length=1024)
|
||||
|
||||
|
||||
class SetupProgress(StrictRequest):
|
||||
step: setup_service.SetupStep
|
||||
|
||||
|
||||
@router.get("/status")
|
||||
def public_status(response: Response) -> dict:
|
||||
response.headers["Cache-Control"] = "no-store"
|
||||
return setup_service.get_public_setup_status()
|
||||
|
||||
|
||||
@router.post("/bootstrap", status_code=201)
|
||||
def bootstrap(payload: BootstrapRequest, request: Request) -> dict:
|
||||
status = setup_service.get_public_setup_status()
|
||||
if not status["needs_admin"]:
|
||||
raise HTTPException(status_code=409, detail="Initial administrator setup is no longer available.")
|
||||
retry_after = setup_service.consume_bootstrap_attempt(_extract_client_ip(request))
|
||||
if retry_after is not None:
|
||||
raise HTTPException(
|
||||
status_code=429,
|
||||
detail="Too many setup attempts. Try again later.",
|
||||
headers={"Retry-After": str(retry_after)},
|
||||
)
|
||||
try:
|
||||
setup_service.bootstrap_administrator(
|
||||
payload.setup_token.get_secret_value(), payload.username, payload.password.get_secret_value()
|
||||
)
|
||||
except setup_service.InvalidSetupTokenError as exc:
|
||||
raise HTTPException(status_code=403, detail=str(exc)) from exc
|
||||
except setup_service.SetupUnavailableError as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
return {"status": "created", "username": payload.username.strip()}
|
||||
|
||||
|
||||
@router.get("/state", dependencies=[Depends(require_admin)])
|
||||
def get_state() -> dict:
|
||||
return setup_service.get_setup_state()
|
||||
|
||||
|
||||
@router.put("/state", dependencies=[Depends(require_admin)])
|
||||
def update_state(payload: SetupProgress) -> dict:
|
||||
return setup_service.update_setup_step(payload.step)
|
||||
|
||||
|
||||
@router.post("/complete", dependencies=[Depends(require_admin)])
|
||||
async def finish_setup(request: Request) -> dict:
|
||||
try:
|
||||
state = setup_service.complete_setup()
|
||||
except setup_service.SetupUnavailableError as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
||||
# Startup owns worker lifecycle. Its callback must be idempotent so retries
|
||||
# after a network interruption cannot start duplicate import/automation jobs.
|
||||
callback = getattr(request.app.state, "on_setup_complete", None)
|
||||
if callback is not None:
|
||||
result = callback()
|
||||
if isawaitable(result):
|
||||
await result
|
||||
return state
|
||||
@@ -15,7 +15,7 @@ SENSITIVE_SETTING_KEYS = frozenset(
|
||||
"magent_notify_telegram_bot_token", "magent_notify_push_token",
|
||||
"magent_notify_push_user_key", "magent_notify_webhook_url", "jellyseerr_api_key",
|
||||
"jellyfin_api_key", "sonarr_api_key", "radarr_api_key", "bazarr_api_key",
|
||||
"prowlarr_api_key", "qbittorrent_password",
|
||||
"prowlarr_api_key", "qbittorrent_password", "discord_webhook_url",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -0,0 +1,631 @@
|
||||
"""Encrypted, portable backups and restart-only SQLite restores.
|
||||
|
||||
Restore is deliberately a two-step operation: the authenticated request validates
|
||||
and stages it, then a single backend process applies it before opening the DB.
|
||||
A durable journal and a private rollback copy protect interrupted installations.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import closing, contextmanager
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path, PurePosixPath
|
||||
import re
|
||||
import secrets
|
||||
import shutil
|
||||
import sqlite3
|
||||
import stat
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, BinaryIO, Iterator
|
||||
import uuid
|
||||
import zipfile
|
||||
import zlib
|
||||
|
||||
from cryptography.exceptions import InvalidTag
|
||||
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
|
||||
from cryptography.hazmat.primitives.kdf.scrypt import Scrypt
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from ..config import Settings, settings
|
||||
from ..db import _db_path
|
||||
from ..schema_migrations import MIGRATIONS
|
||||
from ..secret_storage import SENSITIVE_SETTING_KEYS, decrypt_setting_value, encrypt_setting_value
|
||||
|
||||
FORMAT_VERSION = 1
|
||||
MAX_UPLOAD_BYTES = 32 * 1024 * 1024
|
||||
MAX_EXPANDED_BYTES = 128 * 1024 * 1024
|
||||
MAX_ENTRIES = 20_000
|
||||
MAGIC = b"MAGENT-BACKUP\x00\x01"
|
||||
_LOCK = threading.Lock()
|
||||
_ASSET_NAME = re.compile(r"^[A-Za-z0-9_.-]+$")
|
||||
_TMDB_SIZES = {"w92", "w154", "w185", "w342", "w500", "w780", "original"}
|
||||
# Host identity, process controls and local file locations belong to the target.
|
||||
_LOCAL_FIELDS = {
|
||||
"sqlite_path", "sqlite_journal_mode", "jwt_secret", "settings_encryption_key",
|
||||
"admin_username", "admin_password", "setup_token", "app_name", "cors_allow_origin",
|
||||
"auth_cookie_name", "auth_cookie_secure", "auth_cookie_samesite", "auth_cookie_domain",
|
||||
"auth_state_cookie_name", "jwt_issuer", "jwt_audience", "api_docs_enabled",
|
||||
"log_file", "magent_application_port", "magent_api_port", "magent_bind_host",
|
||||
"magent_proxy_trusted_proxies", "magent_proxy_trust_forwarded_headers",
|
||||
"magent_ssl_bind_enabled", "magent_ssl_certificate_path", "magent_ssl_private_key_path",
|
||||
"magent_ssl_certificate_pem", "magent_ssl_private_key_pem",
|
||||
"site_build_number", "site_changelog", "magent_allow_private_notification_targets",
|
||||
}
|
||||
|
||||
|
||||
class BackupError(ValueError):
|
||||
"""A safe-to-display backup validation or state error."""
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def _assets_root() -> Path:
|
||||
# Matches the image and branding routers, independently of SQLITE_PATH.
|
||||
return Path.cwd() / "data"
|
||||
|
||||
|
||||
def _control_root() -> Path:
|
||||
return Path(_db_path()).absolute().parent / "backups"
|
||||
|
||||
|
||||
def _private_dir(path: Path) -> None:
|
||||
if path.is_symlink():
|
||||
raise BackupError("Backup directories must not be symbolic links")
|
||||
path.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
path.chmod(0o700)
|
||||
|
||||
|
||||
def _write_private(path: Path, content: bytes) -> None:
|
||||
with path.open("xb") as handle:
|
||||
path.chmod(0o600)
|
||||
handle.write(content)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
|
||||
|
||||
def _write_json(path: Path, data: dict) -> None:
|
||||
temporary = path.with_name(path.name + ".tmp-" + uuid.uuid4().hex)
|
||||
try:
|
||||
_write_private(temporary, json.dumps(data, separators=(",", ":")).encode())
|
||||
os.replace(temporary, path)
|
||||
_sync_directory(path.parent)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _sync_directory(path: Path) -> None:
|
||||
if os.name != "nt":
|
||||
descriptor = os.open(path, os.O_RDONLY | os.O_DIRECTORY)
|
||||
try:
|
||||
os.fsync(descriptor)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
def _sync_tree(path: Path) -> None:
|
||||
for parent, _directories, files in os.walk(path, topdown=False):
|
||||
for filename in files:
|
||||
with (Path(parent) / filename).open("r+b") as handle:
|
||||
os.fsync(handle.fileno())
|
||||
_sync_directory(Path(parent))
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _exclusive_operation() -> Iterator[None]:
|
||||
if not _LOCK.acquire(blocking=False):
|
||||
raise BackupError("Another backup or restore operation is in progress")
|
||||
handle = None
|
||||
locked = False
|
||||
try:
|
||||
root = _control_root()
|
||||
_private_dir(root)
|
||||
handle = (root / "operation.lock").open("a+b")
|
||||
os.chmod(handle.name, 0o600)
|
||||
# OS locks are released even if a process crashes; support the dev host too.
|
||||
if os.name == "nt":
|
||||
import msvcrt
|
||||
handle.seek(0)
|
||||
if not handle.read(1):
|
||||
handle.write(b"0")
|
||||
handle.flush()
|
||||
handle.seek(0)
|
||||
try:
|
||||
msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
|
||||
except OSError as exc:
|
||||
raise BackupError("Another backup or restore operation is in progress") from exc
|
||||
else:
|
||||
import fcntl
|
||||
try:
|
||||
fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except OSError as exc:
|
||||
raise BackupError("Another backup or restore operation is in progress") from exc
|
||||
locked = True
|
||||
yield
|
||||
finally:
|
||||
if handle is not None:
|
||||
if locked:
|
||||
if os.name == "nt":
|
||||
import msvcrt
|
||||
handle.seek(0)
|
||||
msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
|
||||
else:
|
||||
import fcntl
|
||||
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
|
||||
handle.close()
|
||||
_LOCK.release()
|
||||
|
||||
|
||||
def validate_passphrase(passphrase: str) -> None:
|
||||
if not isinstance(passphrase, str) or not 12 <= len(passphrase) <= 1024:
|
||||
raise BackupError("Use a backup passphrase between 12 and 1024 characters")
|
||||
|
||||
|
||||
def _key(passphrase: str, salt: bytes) -> bytes:
|
||||
validate_passphrase(passphrase)
|
||||
return Scrypt(salt=salt, length=32, n=2**15, r=8, p=1).derive(passphrase.encode("utf-8"))
|
||||
|
||||
|
||||
def _encrypt(content: bytes, passphrase: str) -> bytes:
|
||||
salt, nonce = os.urandom(16), os.urandom(12)
|
||||
header = MAGIC + salt + nonce
|
||||
return header + AESGCM(_key(passphrase, salt)).encrypt(nonce, content, header)
|
||||
|
||||
|
||||
def _decrypt(content: bytes, passphrase: str) -> bytes:
|
||||
header_size = len(MAGIC) + 28
|
||||
if len(content) > MAX_UPLOAD_BYTES:
|
||||
raise BackupError("Backup exceeds the 32 MiB upload limit")
|
||||
if len(content) < header_size + 16 or not content.startswith(MAGIC):
|
||||
raise BackupError("This is not a supported encrypted Magent backup")
|
||||
salt = content[len(MAGIC):len(MAGIC) + 16]
|
||||
nonce = content[len(MAGIC) + 16:header_size]
|
||||
try:
|
||||
return AESGCM(_key(passphrase, salt)).decrypt(nonce, content[header_size:], content[:header_size])
|
||||
except InvalidTag as exc:
|
||||
raise BackupError("Incorrect passphrase or damaged backup") from exc
|
||||
|
||||
|
||||
def _database_copy(source: Path, destination: Path) -> None:
|
||||
if not source.is_file() or source.is_symlink():
|
||||
raise BackupError("The configured database is unavailable or is a symbolic link")
|
||||
deadline = time.monotonic() + 60
|
||||
|
||||
def progress(_status: int, _remaining: int, _total: int) -> None:
|
||||
if time.monotonic() > deadline:
|
||||
raise BackupError("Database is too busy to back up; try again shortly")
|
||||
|
||||
with closing(sqlite3.connect(source.as_uri() + "?mode=ro", uri=True)) as src:
|
||||
with closing(sqlite3.connect(destination)) as dst:
|
||||
destination.chmod(0o600)
|
||||
src.backup(dst, pages=256, progress=progress, sleep=0.05)
|
||||
dst.execute("PRAGMA journal_mode=DELETE")
|
||||
|
||||
|
||||
def _portable_database(path: Path) -> None:
|
||||
"""Materialize env-backed settings and remove source-specific encryption."""
|
||||
with closing(sqlite3.connect(path)) as conn, conn:
|
||||
conn.execute("PRAGMA secure_delete=ON")
|
||||
# init_db recreates application-owned triggers after restoration; never
|
||||
# distribute executable schema objects in a data backup.
|
||||
for (trigger,) in conn.execute("SELECT name FROM sqlite_master WHERE type='trigger'").fetchall():
|
||||
quoted = str(trigger).replace('"', '""')
|
||||
conn.execute(f'DROP TRIGGER "{quoted}"')
|
||||
overrides = dict(conn.execute("SELECT key, value FROM settings"))
|
||||
for key, default in settings.model_dump().items():
|
||||
if key in _LOCAL_FIELDS:
|
||||
continue
|
||||
value = overrides.get(key)
|
||||
value = default if value is None else decrypt_setting_value(key, value)
|
||||
conn.execute(
|
||||
"INSERT INTO settings(key,value,updated_at) VALUES (?,?,?) "
|
||||
"ON CONFLICT(key) DO UPDATE SET value=excluded.value,updated_at=excluded.updated_at",
|
||||
(key, "" if value is None else str(value), _now()),
|
||||
)
|
||||
for key in _LOCAL_FIELDS:
|
||||
conn.execute("DELETE FROM settings WHERE key=?", (key,))
|
||||
# Future secret keys may not yet be exposed through Settings.
|
||||
for key, value in conn.execute("SELECT key,value FROM settings").fetchall():
|
||||
if key in SENSITIVE_SETTING_KEYS:
|
||||
conn.execute("UPDATE settings SET value=? WHERE key=?", (decrypt_setting_value(key, value), key))
|
||||
conn.commit()
|
||||
conn.execute("VACUUM")
|
||||
|
||||
|
||||
def _asset_allowed(name: str, include_cache: bool) -> bool:
|
||||
parts = PurePosixPath(name).parts
|
||||
if name in {"files/branding/logo.png", "files/branding/favicon.ico"}:
|
||||
return True
|
||||
return bool(
|
||||
include_cache and len(parts) == 5 and parts[:3] == ("files", "artwork", "tmdb")
|
||||
and parts[3] in _TMDB_SIZES and _ASSET_NAME.fullmatch(parts[4])
|
||||
and parts[4] not in {".", ".."}
|
||||
)
|
||||
|
||||
|
||||
def _asset_files(include_cache: bool) -> Iterator[tuple[Path, str]]:
|
||||
root = _assets_root()
|
||||
for directory in ("branding", "artwork") if include_cache else ("branding",):
|
||||
base = root / directory
|
||||
if not base.exists():
|
||||
continue
|
||||
if base.is_symlink() or root.is_symlink():
|
||||
raise BackupError("Asset directories must not be symbolic links")
|
||||
for parent, directories, files in os.walk(base, followlinks=False):
|
||||
if any((Path(parent) / name).is_symlink() for name in directories + files):
|
||||
raise BackupError("Symbolic links are not supported in backup assets")
|
||||
for filename in files:
|
||||
path = Path(parent) / filename
|
||||
archive_name = "files/" + path.relative_to(root).as_posix()
|
||||
if _asset_allowed(archive_name, include_cache):
|
||||
yield path, archive_name
|
||||
|
||||
|
||||
def create_backup(passphrase: str, include_cache: bool = False) -> tuple[bytes, str]:
|
||||
validate_passphrase(passphrase)
|
||||
with _exclusive_operation(), tempfile.TemporaryDirectory(prefix="export-", dir=_control_root()) as temporary:
|
||||
directory = Path(temporary)
|
||||
directory.chmod(0o700)
|
||||
database = directory / "database.sqlite3"
|
||||
_database_copy(Path(_db_path()).absolute(), database)
|
||||
_portable_database(database)
|
||||
files = [(database, "database.sqlite3"), *_asset_files(include_cache)]
|
||||
if len(files) > MAX_ENTRIES - 1 or sum(path.stat().st_size for path, _ in files) > MAX_EXPANDED_BYTES:
|
||||
raise BackupError("Backup is too large; retry without the artwork cache")
|
||||
archive_path = directory / "payload.zip"
|
||||
manifest = {
|
||||
"format_version": FORMAT_VERSION, "created_at": _now(),
|
||||
"build": str(settings.site_build_number or "unknown"), "include_cache": include_cache,
|
||||
"files": {},
|
||||
}
|
||||
with zipfile.ZipFile(archive_path, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=6) as archive:
|
||||
archive_path.chmod(0o600)
|
||||
total = 0
|
||||
for path, name in files:
|
||||
digest = hashlib.sha256()
|
||||
size = 0
|
||||
with path.open("rb") as source, archive.open(name, "w") as destination:
|
||||
while chunk := source.read(1024 * 1024):
|
||||
total += len(chunk)
|
||||
size += len(chunk)
|
||||
if total > MAX_EXPANDED_BYTES:
|
||||
raise BackupError("Backup is too large; retry without the artwork cache")
|
||||
digest.update(chunk)
|
||||
destination.write(chunk)
|
||||
manifest["files"][name] = {"bytes": size, "sha256": digest.hexdigest()}
|
||||
archive.writestr("manifest.json", json.dumps(manifest))
|
||||
if archive_path.stat().st_size > MAX_UPLOAD_BYTES - 128:
|
||||
raise BackupError("Backup exceeds the 32 MiB limit; retry without the artwork cache")
|
||||
encrypted = _encrypt(archive_path.read_bytes(), passphrase)
|
||||
stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
||||
return encrypted, f"magent-backup-{stamp}.magent-backup"
|
||||
|
||||
|
||||
def _validate_database(path: Path, *, verify_settings_encryption: bool = False) -> None:
|
||||
try:
|
||||
with closing(sqlite3.connect(path.as_uri() + "?mode=ro", uri=True)) as conn:
|
||||
conn.execute("PRAGMA trusted_schema=OFF")
|
||||
deadline = time.monotonic() + 30
|
||||
conn.set_progress_handler(lambda: int(time.monotonic() > deadline), 10_000)
|
||||
if conn.execute("PRAGMA integrity_check").fetchall() != [("ok",)]:
|
||||
raise BackupError("Backup database failed its integrity check")
|
||||
schema = conn.execute("SELECT type,name,sql FROM sqlite_master").fetchall()
|
||||
if len(schema) > 500 or any(
|
||||
kind in {"trigger", "view"} or "VIRTUAL TABLE" in str(sql).upper()
|
||||
for kind, _name, sql in schema
|
||||
):
|
||||
raise BackupError("Backup contains an unsupported database schema")
|
||||
if conn.execute("PRAGMA foreign_key_check").fetchone() is not None:
|
||||
raise BackupError("Backup database contains broken references")
|
||||
required = {
|
||||
"settings": {"key", "value", "updated_at"},
|
||||
"users": {"id", "username", "password_hash", "role", "is_blocked", "auth_version"},
|
||||
"signup_invites": {"id", "code", "enabled"},
|
||||
"requests_cache": {"request_id", "payload_json"},
|
||||
"schema_migrations": {"version", "name", "applied_at"},
|
||||
"password_reset_tokens": {"id", "token_hash"},
|
||||
}
|
||||
for table, fields in required.items():
|
||||
columns = {row[1] for row in conn.execute(f'PRAGMA table_info("{table}")')}
|
||||
if not fields <= columns:
|
||||
raise BackupError("Backup does not contain a compatible Magent database")
|
||||
optional = {
|
||||
"installation_setup": {"id", "completed", "step", "completed_at"},
|
||||
"installation_setup_attempts": {"scope", "key_hash", "occurred_at"},
|
||||
}
|
||||
table_names = {name for kind, name, _sql in schema if kind == "table"}
|
||||
for table, fields in optional.items():
|
||||
if table in table_names:
|
||||
columns = {row[1] for row in conn.execute(f'PRAGMA table_info("{table}")')}
|
||||
if not fields <= columns:
|
||||
raise BackupError("Backup setup state has an incompatible schema")
|
||||
# An admin can stage a restore only after target initialization. Its
|
||||
# schema is a trusted reference for *all* runtime columns, including
|
||||
# versioned migrations that init_db will not rerun on a restored DB.
|
||||
target = Path(_db_path()).absolute()
|
||||
if target.is_file() and target != path:
|
||||
with closing(sqlite3.connect(target.as_uri() + "?mode=ro", uri=True)) as reference:
|
||||
tables = [row[0] for row in reference.execute("SELECT name FROM sqlite_master WHERE type='table'")]
|
||||
for table in tables:
|
||||
if table.startswith("sqlite_") or table in {"installation_setup", "installation_setup_attempts"}:
|
||||
continue
|
||||
quoted = str(table).replace('"', '""')
|
||||
expected = {
|
||||
row[1]: (row[2].upper(), bool(row[3]), row[5])
|
||||
for row in reference.execute(f'PRAGMA table_info("{quoted}")')
|
||||
}
|
||||
actual = {
|
||||
row[1]: (row[2].upper(), bool(row[3]), row[5])
|
||||
for row in conn.execute(f'PRAGMA table_info("{quoted}")')
|
||||
}
|
||||
if expected != actual:
|
||||
raise BackupError("Backup is missing database columns required by this installation")
|
||||
versions = {int(row[0]) for row in conn.execute("SELECT version FROM schema_migrations")}
|
||||
if versions != {migration.version for migration in MIGRATIONS}:
|
||||
raise BackupError("Backup schema is incompatible; restore using the same Magent version")
|
||||
if not conn.execute(
|
||||
"SELECT 1 FROM users WHERE role='admin' AND is_blocked=0 AND password_hash IS NOT NULL LIMIT 1"
|
||||
).fetchone():
|
||||
raise BackupError("Backup must contain an active administrator account")
|
||||
values = dict(conn.execute("SELECT key,value FROM settings"))
|
||||
if _LOCAL_FIELDS.intersection(values):
|
||||
raise BackupError("Backup contains host-specific configuration")
|
||||
# Pydantic checks the types of portable settings without reading env values.
|
||||
for key, value in values.items():
|
||||
if verify_settings_encryption and key in SENSITIVE_SETTING_KEYS:
|
||||
value = decrypt_setting_value(key, value)
|
||||
if key in Settings.model_fields and value not in {None, ""}:
|
||||
field = Settings.model_fields[key]
|
||||
TypeAdapter(field.rebuild_annotation()).validate_python(value)
|
||||
except (sqlite3.DatabaseError, TypeError, ValueError, RuntimeError) as exc:
|
||||
if isinstance(exc, BackupError):
|
||||
raise
|
||||
raise BackupError("Backup database or configuration is invalid") from exc
|
||||
|
||||
|
||||
def _extract_archive(payload: bytes, directory: Path) -> dict[str, Any]:
|
||||
try:
|
||||
with zipfile.ZipFile(io.BytesIO(payload)) as archive:
|
||||
entries = archive.infolist()
|
||||
if not entries or len(entries) > MAX_ENTRIES:
|
||||
raise BackupError("Backup contains too many files")
|
||||
names = [entry.filename for entry in entries]
|
||||
if len(set(names)) != len(names) or "manifest.json" not in names or "database.sqlite3" not in names:
|
||||
raise BackupError("Backup manifest is missing or contains duplicate files")
|
||||
if sum(entry.file_size for entry in entries) > MAX_EXPANDED_BYTES:
|
||||
raise BackupError("Expanded backup exceeds the 128 MiB limit")
|
||||
for entry in entries:
|
||||
parts = PurePosixPath(entry.filename).parts
|
||||
mode = entry.external_attr >> 16
|
||||
if (
|
||||
entry.is_dir() or entry.filename.startswith("/") or "\\" in entry.filename
|
||||
or str(PurePosixPath(entry.filename)) != entry.filename
|
||||
or ":" in entry.filename or any(part in {".", ".."} for part in parts)
|
||||
or (stat.S_IFMT(mode) not in {0, stat.S_IFREG}) or entry.flag_bits & 1
|
||||
or entry.compress_type not in {zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED}
|
||||
):
|
||||
raise BackupError("Backup contains an unsafe archive entry")
|
||||
if archive.getinfo("manifest.json").file_size > 4 * 1024 * 1024:
|
||||
raise BackupError("Backup manifest is too large")
|
||||
manifest = json.loads(archive.read("manifest.json"))
|
||||
if (
|
||||
not isinstance(manifest, dict) or manifest.get("format_version") != FORMAT_VERSION
|
||||
or not isinstance(manifest.get("include_cache"), bool)
|
||||
or not isinstance(manifest.get("created_at"), str) or len(manifest["created_at"]) > 64
|
||||
or not isinstance(manifest.get("build"), str) or len(manifest["build"]) > 100
|
||||
or not isinstance(manifest.get("files"), dict)
|
||||
or set(manifest["files"]) != set(names) - {"manifest.json"}
|
||||
):
|
||||
raise BackupError("Backup manifest is invalid or unsupported")
|
||||
extracted_bytes = 0
|
||||
for entry in entries:
|
||||
name = entry.filename
|
||||
if name == "manifest.json":
|
||||
continue
|
||||
if name != "database.sqlite3" and not _asset_allowed(name, manifest["include_cache"]):
|
||||
raise BackupError("Backup contains an unsupported file")
|
||||
expected = manifest["files"][name]
|
||||
if not isinstance(expected, dict) or expected.get("bytes") != entry.file_size:
|
||||
raise BackupError("Backup file does not match its manifest")
|
||||
target = directory.joinpath(*PurePosixPath(name).parts)
|
||||
_private_dir(target.parent)
|
||||
digest = hashlib.sha256()
|
||||
with archive.open(entry) as source, target.open("xb") as destination:
|
||||
target.chmod(0o600)
|
||||
while chunk := source.read(1024 * 1024):
|
||||
extracted_bytes += len(chunk)
|
||||
if extracted_bytes > MAX_EXPANDED_BYTES:
|
||||
raise BackupError("Expanded backup exceeds the 128 MiB limit")
|
||||
digest.update(chunk)
|
||||
destination.write(chunk)
|
||||
destination.flush()
|
||||
os.fsync(destination.fileno())
|
||||
if digest.hexdigest() != expected.get("sha256"):
|
||||
raise BackupError("Backup file failed its checksum")
|
||||
_validate_database(directory / "database.sqlite3")
|
||||
return manifest
|
||||
except (zipfile.BadZipFile, KeyError, TypeError, ValueError, RuntimeError, zlib.error) as exc:
|
||||
if isinstance(exc, BackupError):
|
||||
raise
|
||||
raise BackupError("Backup archive is invalid or damaged") from exc
|
||||
|
||||
|
||||
def stage_restore(source: BinaryIO, passphrase: str) -> dict[str, Any]:
|
||||
validate_passphrase(passphrase)
|
||||
with _exclusive_operation():
|
||||
root = _control_root()
|
||||
pending = root / "pending"
|
||||
if pending.exists():
|
||||
raise BackupError("A restore is already staged; cancel it before uploading another")
|
||||
payload = _decrypt(source.read(MAX_UPLOAD_BYTES + 1), passphrase)
|
||||
with tempfile.TemporaryDirectory(prefix="validate-", dir=root) as temporary:
|
||||
stage = Path(temporary)
|
||||
stage.chmod(0o700)
|
||||
manifest = _extract_archive(payload, stage)
|
||||
with closing(sqlite3.connect(stage / "database.sqlite3")) as conn, conn:
|
||||
conn.execute("PRAGMA secure_delete=ON")
|
||||
for key, value in conn.execute("SELECT key,value FROM settings").fetchall():
|
||||
if key in SENSITIVE_SETTING_KEYS:
|
||||
if value and str(value).startswith("enc:v1:"):
|
||||
raise BackupError("Backup settings are not portable")
|
||||
conn.execute("UPDATE settings SET value=? WHERE key=?", (encrypt_setting_value(key, value), key))
|
||||
# Do not revive reset links or existing browser sessions. Invites remain intact.
|
||||
conn.execute("DELETE FROM password_reset_tokens")
|
||||
conn.execute("UPDATE users SET auth_version=?", (secrets.randbelow(2**52) + 1_000_000,))
|
||||
if not manifest["include_cache"]:
|
||||
conn.execute("UPDATE artwork_cache_status SET poster_cached=0,backdrop_cached=0")
|
||||
conn.commit()
|
||||
# Remove plaintext secret remnants from replaced/free SQLite pages.
|
||||
conn.execute("VACUUM")
|
||||
metadata = {key: manifest[key] for key in ("created_at", "build", "include_cache")}
|
||||
metadata["staged_at"] = _now()
|
||||
_write_json(stage / "metadata.json", metadata)
|
||||
# Stage survives reboot; it contains only secrets encrypted for this installation.
|
||||
os.replace(stage, pending)
|
||||
_sync_directory(root)
|
||||
return metadata
|
||||
|
||||
|
||||
def backup_status() -> dict[str, Any]:
|
||||
root = _control_root()
|
||||
pending_path = root / "pending" / "metadata.json"
|
||||
last_path = root / "last-restore.json"
|
||||
return {
|
||||
"format_version": FORMAT_VERSION, "max_upload_bytes": MAX_UPLOAD_BYTES,
|
||||
"max_expanded_bytes": MAX_EXPANDED_BYTES,
|
||||
"include_cache_default": False,
|
||||
"pending_restore": json.loads(pending_path.read_text()) if pending_path.is_file() else None,
|
||||
"last_restore": json.loads(last_path.read_text()) if last_path.is_file() else None,
|
||||
}
|
||||
|
||||
|
||||
def cancel_restore() -> None:
|
||||
with _exclusive_operation():
|
||||
pending = _control_root() / "pending"
|
||||
if pending.is_symlink():
|
||||
raise BackupError("Invalid staged restore directory")
|
||||
if pending.exists():
|
||||
shutil.rmtree(pending)
|
||||
|
||||
|
||||
def _replace_file(source: Path, target: Path) -> None:
|
||||
_private_dir(target.parent)
|
||||
temporary = target.with_name(target.name + ".restore-" + uuid.uuid4().hex)
|
||||
try:
|
||||
shutil.copyfile(source, temporary)
|
||||
temporary.chmod(0o600)
|
||||
with temporary.open("r+b") as handle:
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(temporary, target)
|
||||
_sync_directory(target.parent)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _replace_assets(source: Path, target: Path) -> None:
|
||||
if target.is_symlink():
|
||||
raise BackupError("Asset directories must not be symbolic links")
|
||||
if target.exists():
|
||||
shutil.rmtree(target)
|
||||
if source.exists():
|
||||
shutil.copytree(source, target, copy_function=shutil.copyfile)
|
||||
for parent, _directories, files in os.walk(target):
|
||||
Path(parent).chmod(0o700)
|
||||
for filename in files:
|
||||
(Path(parent) / filename).chmod(0o600)
|
||||
_sync_tree(target)
|
||||
if target.parent.exists():
|
||||
_sync_directory(target.parent)
|
||||
|
||||
|
||||
def _recover(journal: dict, root: Path) -> None:
|
||||
rollback_name = journal.get("rollback_directory", "")
|
||||
if not re.fullmatch(r"rollback-[0-9a-f]{32}", rollback_name):
|
||||
raise BackupError("Restore recovery journal is invalid")
|
||||
rollback = root / rollback_name
|
||||
database = Path(_db_path()).absolute()
|
||||
if journal["had_database"]:
|
||||
_replace_file(rollback / "database.sqlite3", database)
|
||||
else:
|
||||
database.unlink(missing_ok=True)
|
||||
for suffix in ("-wal", "-shm", "-journal"):
|
||||
Path(str(database) + suffix).unlink(missing_ok=True)
|
||||
for name in journal["asset_roots"]:
|
||||
if name not in {"branding", "artwork"}:
|
||||
raise BackupError("Restore recovery journal is invalid")
|
||||
_replace_assets(rollback / "files" / name, _assets_root() / name)
|
||||
_write_json(root / "last-restore.json", {
|
||||
"status": "rolled_back", "restored_at": _now(), "rollback_directory": rollback.name,
|
||||
"message": "An interrupted or failed restore was rolled back automatically.",
|
||||
})
|
||||
_write_json(root / "restore-journal.json", {**journal, "phase": "rolled_back"})
|
||||
pending = root / "pending"
|
||||
if pending.exists():
|
||||
shutil.rmtree(pending)
|
||||
(root / "restore-journal.json").unlink()
|
||||
_sync_directory(root)
|
||||
|
||||
|
||||
def apply_pending_restore() -> bool:
|
||||
"""Call once before init_db, with no other backend processes using the DB."""
|
||||
with _exclusive_operation():
|
||||
root = _control_root()
|
||||
journal_path = root / "restore-journal.json"
|
||||
if journal_path.exists():
|
||||
journal = json.loads(journal_path.read_text())
|
||||
if journal.get("phase") in {"complete", "rolled_back"}:
|
||||
if (root / "pending").exists():
|
||||
shutil.rmtree(root / "pending")
|
||||
journal_path.unlink()
|
||||
_sync_directory(root)
|
||||
return journal["phase"] == "complete"
|
||||
_recover(journal, root)
|
||||
return False
|
||||
pending = root / "pending"
|
||||
if not pending.exists():
|
||||
return False
|
||||
if pending.is_symlink():
|
||||
raise BackupError("Invalid staged restore directory")
|
||||
metadata = json.loads((pending / "metadata.json").read_text())
|
||||
_validate_database(pending / "database.sqlite3", verify_settings_encryption=True)
|
||||
database = Path(_db_path()).absolute()
|
||||
rollback = root / ("rollback-" + uuid.uuid4().hex)
|
||||
_private_dir(rollback)
|
||||
# Ensure all disk-space/permission failures in backup happen before replacement.
|
||||
if database.exists():
|
||||
_database_copy(database, rollback / "database.sqlite3")
|
||||
names = ["branding", "artwork"] if metadata["include_cache"] else ["branding"]
|
||||
# Reject links anywhere before copying or deleting the controlled asset trees.
|
||||
list(_asset_files(metadata["include_cache"]))
|
||||
for name in names:
|
||||
source = _assets_root() / name
|
||||
if source.exists():
|
||||
shutil.copytree(source, rollback / "files" / name)
|
||||
_sync_tree(rollback)
|
||||
journal = {"rollback_directory": rollback.name, "had_database": database.exists(), "asset_roots": names}
|
||||
_write_json(journal_path, journal)
|
||||
try:
|
||||
for suffix in ("-wal", "-shm", "-journal"):
|
||||
Path(str(database) + suffix).unlink(missing_ok=True)
|
||||
_replace_file(pending / "database.sqlite3", database)
|
||||
for name in names:
|
||||
_replace_assets(pending / "files" / name, _assets_root() / name)
|
||||
_write_json(root / "last-restore.json", {
|
||||
"status": "restored", "restored_at": _now(), "rollback_directory": rollback.name,
|
||||
"backup_created_at": metadata["created_at"],
|
||||
})
|
||||
_write_json(journal_path, {**journal, "phase": "complete"})
|
||||
except Exception:
|
||||
_recover(journal, root)
|
||||
raise
|
||||
shutil.rmtree(pending)
|
||||
journal_path.unlink()
|
||||
_sync_directory(root)
|
||||
return True
|
||||
@@ -0,0 +1,197 @@
|
||||
"""Persistent, operator-authorized first-install setup.
|
||||
|
||||
Initialize the marker before the main schema: an existing users table identifies
|
||||
an upgraded installation, while a new database must finish the setup wizard.
|
||||
The marker and first administrator are protected by SQLite write transactions.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
import hmac
|
||||
from math import ceil
|
||||
from time import time
|
||||
from typing import Literal
|
||||
|
||||
from .. import db
|
||||
from ..config import settings
|
||||
from ..security import hash_password, validate_password_policy
|
||||
|
||||
|
||||
SetupStep = Literal["administrator", "apps", "preferences", "review"]
|
||||
SETUP_STEPS = ("administrator", "apps", "preferences", "review")
|
||||
BOOTSTRAP_WINDOW_SECONDS = 15 * 60
|
||||
BOOTSTRAP_IP_ATTEMPTS = 5
|
||||
BOOTSTRAP_GLOBAL_ATTEMPTS = 30
|
||||
|
||||
|
||||
class SetupUnavailableError(ValueError):
|
||||
"""Setup has finished, or another administrator already exists."""
|
||||
|
||||
|
||||
class InvalidSetupTokenError(ValueError):
|
||||
"""The operator's setup token was absent or did not match."""
|
||||
|
||||
|
||||
def initialize_setup_state() -> None:
|
||||
"""Run once before init_db; subsequent calls preserve progress."""
|
||||
with db._connect() as conn:
|
||||
conn.execute("BEGIN IMMEDIATE")
|
||||
existing_install = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'users'"
|
||||
).fetchone() is not None
|
||||
conn.execute(
|
||||
"""CREATE TABLE IF NOT EXISTS installation_setup (
|
||||
id INTEGER PRIMARY KEY CHECK (id = 1),
|
||||
completed INTEGER NOT NULL CHECK (completed IN (0, 1)),
|
||||
step TEXT NOT NULL,
|
||||
completed_at TEXT
|
||||
)"""
|
||||
)
|
||||
conn.execute(
|
||||
"""CREATE TABLE IF NOT EXISTS installation_setup_attempts (
|
||||
scope TEXT NOT NULL,
|
||||
key_hash TEXT NOT NULL,
|
||||
occurred_at REAL NOT NULL
|
||||
)"""
|
||||
)
|
||||
conn.execute(
|
||||
"""INSERT OR IGNORE INTO installation_setup (id, completed, step, completed_at)
|
||||
VALUES (1, ?, ?, ?)""",
|
||||
(
|
||||
int(existing_install),
|
||||
"review" if existing_install else "administrator",
|
||||
datetime.now(timezone.utc).isoformat() if existing_install else None,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def get_setup_state() -> dict:
|
||||
with db._connect() as conn:
|
||||
# Old databases and isolated callers without startup initialization are
|
||||
# already installed. A missing marker must never open public bootstrap.
|
||||
table = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'installation_setup'"
|
||||
).fetchone()
|
||||
row = conn.execute(
|
||||
"SELECT completed, step, completed_at FROM installation_setup WHERE id = 1"
|
||||
).fetchone() if table else None
|
||||
if row is None:
|
||||
return {"completed": True, "step": "review", "completed_at": None}
|
||||
return {"completed": bool(row[0]), "step": row[1], "completed_at": row[2]}
|
||||
|
||||
|
||||
def is_setup_required() -> bool:
|
||||
return not get_setup_state()["completed"]
|
||||
|
||||
|
||||
def get_public_setup_status() -> dict:
|
||||
required = is_setup_required()
|
||||
return {"setup_required": required, "needs_admin": required and not db.has_admin_user()}
|
||||
|
||||
|
||||
def setup_token_configured() -> bool:
|
||||
"""Reject missing values and obvious examples, without claiming to measure entropy."""
|
||||
token = str(getattr(settings, "setup_token", "") or "").strip()
|
||||
placeholder = token.casefold().replace("_", "-")
|
||||
return (
|
||||
len(token) >= 32
|
||||
and len(set(token)) > 1
|
||||
and not placeholder.startswith(("replace-with-", "replace-me", "change-me", "changeme", "your-setup-token"))
|
||||
)
|
||||
|
||||
|
||||
def consume_bootstrap_attempt(client_ip: str) -> int | None:
|
||||
"""Atomically reserve one attempt; return Retry-After when limited.
|
||||
|
||||
The IP is keyed using the existing HMAC helper, never stored in clear text.
|
||||
A shared cap limits distributed attempts and expensive password hashing.
|
||||
"""
|
||||
now = time()
|
||||
cutoff = now - BOOTSTRAP_WINDOW_SECONDS
|
||||
limits = (
|
||||
("setup-ip", db._rate_limit_key_hash(client_ip), BOOTSTRAP_IP_ATTEMPTS),
|
||||
("setup-global", db._rate_limit_key_hash("bootstrap"), BOOTSTRAP_GLOBAL_ATTEMPTS),
|
||||
)
|
||||
with db._connect() as conn:
|
||||
conn.execute("BEGIN IMMEDIATE")
|
||||
conn.execute(
|
||||
"DELETE FROM installation_setup_attempts WHERE occurred_at < ?",
|
||||
(cutoff,),
|
||||
)
|
||||
retry_after = 0
|
||||
for scope, key, maximum in limits:
|
||||
count, oldest = conn.execute(
|
||||
"""SELECT COUNT(*), MIN(occurred_at) FROM installation_setup_attempts
|
||||
WHERE scope = ? AND key_hash = ? AND occurred_at >= ?""",
|
||||
(scope, key, cutoff),
|
||||
).fetchone()
|
||||
if count >= maximum:
|
||||
retry_after = max(retry_after, ceil(BOOTSTRAP_WINDOW_SECONDS - (now - oldest)), 1)
|
||||
if retry_after:
|
||||
return retry_after
|
||||
conn.executemany(
|
||||
"INSERT INTO installation_setup_attempts (scope, key_hash, occurred_at) VALUES (?, ?, ?)",
|
||||
[(scope, key, now) for scope, key, _ in limits],
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def bootstrap_administrator(setup_token: str, username: str, password: str) -> None:
|
||||
"""Claim fresh setup exactly once using the deployment's setup token."""
|
||||
expected = str(getattr(settings, "setup_token", "") or "")
|
||||
if not setup_token_configured() or not hmac.compare_digest(
|
||||
setup_token.encode("utf-8"), expected.encode("utf-8")
|
||||
):
|
||||
raise InvalidSetupTokenError("Invalid setup token.")
|
||||
username = username.strip()
|
||||
if not username or len(username) > 100 or any(
|
||||
character.isspace() or ord(character) < 32 or ord(character) == 127 for character in username
|
||||
):
|
||||
raise ValueError("Username must contain 1 to 100 characters without spaces or control characters.")
|
||||
if len(password) > 1024:
|
||||
raise ValueError("Password must contain no more than 1024 characters.")
|
||||
password = validate_password_policy(password)
|
||||
if not is_setup_required() or db.has_admin_user():
|
||||
raise SetupUnavailableError("Initial administrator setup is no longer available.")
|
||||
|
||||
password_hash = hash_password(password)
|
||||
with db._connect() as conn:
|
||||
conn.execute("BEGIN IMMEDIATE")
|
||||
setup = conn.execute("SELECT completed FROM installation_setup WHERE id = 1").fetchone()
|
||||
admin = conn.execute("SELECT 1 FROM users WHERE LOWER(role) = 'admin' LIMIT 1").fetchone()
|
||||
if setup is None or setup[0] or admin:
|
||||
raise SetupUnavailableError("Initial administrator setup is no longer available.")
|
||||
if any(str(row[0]).strip().casefold() == username.casefold() for row in conn.execute("SELECT username FROM users")):
|
||||
raise SetupUnavailableError("That username already exists.")
|
||||
conn.execute(
|
||||
"""INSERT INTO users (username, password_hash, role, auth_provider, created_at)
|
||||
VALUES (?, ?, 'admin', 'local', ?)""",
|
||||
(username, password_hash, datetime.now(timezone.utc).isoformat()),
|
||||
)
|
||||
conn.execute("UPDATE installation_setup SET step = 'apps' WHERE id = 1")
|
||||
|
||||
|
||||
def update_setup_step(step: SetupStep) -> dict:
|
||||
if step not in SETUP_STEPS:
|
||||
raise ValueError("Invalid setup step.")
|
||||
if not is_setup_required():
|
||||
return get_setup_state()
|
||||
with db._connect() as conn:
|
||||
conn.execute(
|
||||
"UPDATE installation_setup SET step = ? WHERE id = 1 AND completed = 0", (step,)
|
||||
)
|
||||
return get_setup_state()
|
||||
|
||||
|
||||
def complete_setup() -> dict:
|
||||
if not is_setup_required():
|
||||
return get_setup_state()
|
||||
with db._connect() as conn:
|
||||
conn.execute("BEGIN IMMEDIATE")
|
||||
if not conn.execute("SELECT 1 FROM users WHERE LOWER(role) = 'admin' LIMIT 1").fetchone():
|
||||
raise SetupUnavailableError("Create an administrator before completing setup.")
|
||||
conn.execute(
|
||||
"""UPDATE installation_setup SET completed = 1, step = 'review', completed_at = ?
|
||||
WHERE id = 1 AND completed = 0""",
|
||||
(datetime.now(timezone.utc).isoformat(),),
|
||||
)
|
||||
return get_setup_state()
|
||||
Reference in New Issue
Block a user