diff --git a/.build_number b/.build_number index 29a5d8a..043dbd5 100644 --- a/.build_number +++ b/.build_number @@ -1 +1 @@ -0303261413 +0303261502 diff --git a/backend/app/build_info.py b/backend/app/build_info.py index fe6d761..35d55c0 100644 --- a/backend/app/build_info.py +++ b/backend/app/build_info.py @@ -1,2 +1,2 @@ -BUILD_NUMBER = "0303261413" -CHANGELOG = '2026-03-03|Hotfix: expand landing-page search to all requests\n2026-03-02|Hotfix: add logged-out password reset flow\n2026-03-02|Process 1 build 0203261953\n2026-03-02|Process 1 build 0203261610\n2026-03-02|Process 1 build 0203261608\n2026-03-02|Add dedicated profile invites page and fix mobile admin layout\n2026-03-01|Persist Seerr media failure suppression and reduce sync error noise\n2026-03-01|Add repository line ending policy\n2026-03-01|Finalize diagnostics, logging controls, and email test support\n2026-03-01|Add invite email templates and delivery workflow\n2026-02-28|Finalize dev-1.3 upgrades and Seerr updates\n2026-02-27|admin docs and layout refresh, build 2702261314\n2026-02-27|Build 2702261153: fix jellyfin sync user visibility\n2026-02-26|Build 2602262241: live request page updates\n2026-02-26|Build 2602262204\n2026-02-26|Build 2602262159: restore jellyfin-first user source\n2026-02-26|Build 2602262049: split magent settings and harden local login\n2026-02-26|Build 2602262030: add magent settings and hardening\n2026-02-26|Build 2602261731: fix user resync after nuclear wipe\n2026-02-26|Build 2602261717: master invite policy and self-service invite controls\n2026-02-26|Build 2602261636: self-service invites and count fixes\n2026-02-26|Build 2602261605: invite trace and cross-system user lifecycle\n2026-02-26|Build 2602261536: refine invite layouts and tighten UI\n2026-02-26|Build 2602261523: live updates, invite cleanup and nuclear resync\n2026-02-26|Build 2602261442: tidy users and invite layouts\n2026-02-26|Build 2602261409: unify invite management controls\n2026-02-26|Build 2602260214: invites profiles and expiry admin controls\n2026-02-26|Build 2602260022: enterprise UI refresh and users bulk auto-search\n2026-02-25|Build 2502262321: fix auto-search quality and per-user toggle\n2026-02-02|Build 0202261541: allow FQDN service URLs\n2026-01-30|Build 3001262148: single container\n2026-01-29|Build 2901262244: format changelog\n2026-01-29|Build 2901262240: cache users\n2026-01-29|Tidy full changelog\n2026-01-29|Update full changelog\n2026-01-29|Bake build number and changelog\n2026-01-29|Hardcode build number in backend\n2026-01-29|release: 2901262102\n2026-01-29|release: 2901262044\n2026-01-29|release: 2901262036\n2026-01-27|Hydrate missing artwork from Jellyseerr (build 271261539)\n2026-01-27|Fallback to TMDB when artwork cache fails (build 271261524)\n2026-01-27|Add service test buttons (build 271261335)\n2026-01-27|Bump build number (process 2) 271261322\n2026-01-27|Add cache load spinner (build 271261238)\n2026-01-27|Fix snapshot title fallback (build 271261228)\n2026-01-27|Fix request titles in snapshots (build 271261219)\n2026-01-27|Bump build number to 271261202\n2026-01-27|Clarify request sync settings (build 271261159)\n2026-01-27|Fix backend cache stats import (build 271261149)\n2026-01-27|Improve cache stats performance (build 271261145)\n2026-01-27|Add cache control artwork stats\n2026-01-26|Fix sync progress bar animation\n2026-01-26|Fix cache title hydration\n2026-01-25|Build 2501262041\n2026-01-25|Harden request cache titles and cache-only reads\n2026-01-25|Serve bundled branding assets by default\n2026-01-25|Seed branding logo from bundled assets\n2026-01-25|Tidy request sync controls\n2026-01-25|Add Jellyfin login cache and admin-only stats\n2026-01-25|Add user stats and activity tracking\n2026-01-25|Move account actions into avatar menu\n2026-01-25|Improve mobile header layout\n2026-01-25|Automate build number tagging and sync\n2026-01-25|Add site banner, build number, and changelog\n2026-01-24|Improve request handling and qBittorrent categories\n2026-01-24|Map Prowlarr releases to Arr indexers for manual grab\n2026-01-24|Clarify how-it-works steps and fixes\n2026-01-24|Document fix buttons in how-it-works\n2026-01-24|Route grabs through Sonarr/Radarr only\n2026-01-23|Use backend branding assets for logo and favicon\n2026-01-23|Copy public assets into frontend image\n2026-01-23|Fix backend Dockerfile paths for root context\n2026-01-23|Add Docker Hub compose override\n2026-01-23|Remove password fields from users page\n2026-01-23|Use bundled branding assets\n2026-01-23|Add default branding assets when missing\n2026-01-23|Show available status on landing when in Jellyfin\n2026-01-23|Fix cache titles and move feedback link\n2026-01-23|Add feedback form and webhook\n2026-01-23|Hide header actions when signed out\n2026-01-23|Fallback manual grab to qBittorrent\n2026-01-23|Split search actions and improve download options\n2026-01-23|Fix cache titles via Jellyseerr media lookup\n2026-01-22|Update README with Docker-first guide\n2026-01-22|Update README\n2026-01-22|Ignore build artifacts\n2026-01-22|Initial commit' +BUILD_NUMBER = "0303261502" +CHANGELOG = '2026-03-03|Add login page visibility controls\n2026-03-03|Hotfix: expand landing-page search to all requests\n2026-03-02|Hotfix: add logged-out password reset flow\n2026-03-02|Process 1 build 0203261953\n2026-03-02|Process 1 build 0203261610\n2026-03-02|Process 1 build 0203261608\n2026-03-02|Add dedicated profile invites page and fix mobile admin layout\n2026-03-01|Persist Seerr media failure suppression and reduce sync error noise\n2026-03-01|Add repository line ending policy\n2026-03-01|Finalize diagnostics, logging controls, and email test support\n2026-03-01|Add invite email templates and delivery workflow\n2026-02-28|Finalize dev-1.3 upgrades and Seerr updates\n2026-02-27|admin docs and layout refresh, build 2702261314\n2026-02-27|Build 2702261153: fix jellyfin sync user visibility\n2026-02-26|Build 2602262241: live request page updates\n2026-02-26|Build 2602262204\n2026-02-26|Build 2602262159: restore jellyfin-first user source\n2026-02-26|Build 2602262049: split magent settings and harden local login\n2026-02-26|Build 2602262030: add magent settings and hardening\n2026-02-26|Build 2602261731: fix user resync after nuclear wipe\n2026-02-26|Build 2602261717: master invite policy and self-service invite controls\n2026-02-26|Build 2602261636: self-service invites and count fixes\n2026-02-26|Build 2602261605: invite trace and cross-system user lifecycle\n2026-02-26|Build 2602261536: refine invite layouts and tighten UI\n2026-02-26|Build 2602261523: live updates, invite cleanup and nuclear resync\n2026-02-26|Build 2602261442: tidy users and invite layouts\n2026-02-26|Build 2602261409: unify invite management controls\n2026-02-26|Build 2602260214: invites profiles and expiry admin controls\n2026-02-26|Build 2602260022: enterprise UI refresh and users bulk auto-search\n2026-02-25|Build 2502262321: fix auto-search quality and per-user toggle\n2026-02-02|Build 0202261541: allow FQDN service URLs\n2026-01-30|Build 3001262148: single container\n2026-01-29|Build 2901262244: format changelog\n2026-01-29|Build 2901262240: cache users\n2026-01-29|Tidy full changelog\n2026-01-29|Update full changelog\n2026-01-29|Bake build number and changelog\n2026-01-29|Hardcode build number in backend\n2026-01-29|release: 2901262102\n2026-01-29|release: 2901262044\n2026-01-29|release: 2901262036\n2026-01-27|Hydrate missing artwork from Jellyseerr (build 271261539)\n2026-01-27|Fallback to TMDB when artwork cache fails (build 271261524)\n2026-01-27|Add service test buttons (build 271261335)\n2026-01-27|Bump build number (process 2) 271261322\n2026-01-27|Add cache load spinner (build 271261238)\n2026-01-27|Fix snapshot title fallback (build 271261228)\n2026-01-27|Fix request titles in snapshots (build 271261219)\n2026-01-27|Bump build number to 271261202\n2026-01-27|Clarify request sync settings (build 271261159)\n2026-01-27|Fix backend cache stats import (build 271261149)\n2026-01-27|Improve cache stats performance (build 271261145)\n2026-01-27|Add cache control artwork stats\n2026-01-26|Fix sync progress bar animation\n2026-01-26|Fix cache title hydration\n2026-01-25|Build 2501262041\n2026-01-25|Harden request cache titles and cache-only reads\n2026-01-25|Serve bundled branding assets by default\n2026-01-25|Seed branding logo from bundled assets\n2026-01-25|Tidy request sync controls\n2026-01-25|Add Jellyfin login cache and admin-only stats\n2026-01-25|Add user stats and activity tracking\n2026-01-25|Move account actions into avatar menu\n2026-01-25|Improve mobile header layout\n2026-01-25|Automate build number tagging and sync\n2026-01-25|Add site banner, build number, and changelog\n2026-01-24|Improve request handling and qBittorrent categories\n2026-01-24|Map Prowlarr releases to Arr indexers for manual grab\n2026-01-24|Clarify how-it-works steps and fixes\n2026-01-24|Document fix buttons in how-it-works\n2026-01-24|Route grabs through Sonarr/Radarr only\n2026-01-23|Use backend branding assets for logo and favicon\n2026-01-23|Copy public assets into frontend image\n2026-01-23|Fix backend Dockerfile paths for root context\n2026-01-23|Add Docker Hub compose override\n2026-01-23|Remove password fields from users page\n2026-01-23|Use bundled branding assets\n2026-01-23|Add default branding assets when missing\n2026-01-23|Show available status on landing when in Jellyfin\n2026-01-23|Fix cache titles and move feedback link\n2026-01-23|Add feedback form and webhook\n2026-01-23|Hide header actions when signed out\n2026-01-23|Fallback manual grab to qBittorrent\n2026-01-23|Split search actions and improve download options\n2026-01-23|Fix cache titles via Jellyseerr media lookup\n2026-01-22|Update README with Docker-first guide\n2026-01-22|Update README\n2026-01-22|Ignore build artifacts\n2026-01-22|Initial commit' diff --git a/backend/app/db.py b/backend/app/db.py index 1317a4b..49909ae 100644 --- a/backend/app/db.py +++ b/backend/app/db.py @@ -4,6 +4,7 @@ import sqlite3 import logging from hashlib import sha256 from datetime import datetime, timezone, timedelta +from time import perf_counter from typing import Any, Dict, Optional from .config import settings @@ -16,6 +17,9 @@ SEERR_MEDIA_FAILURE_SHORT_SUPPRESS_HOURS = 6 SEERR_MEDIA_FAILURE_RETRY_SUPPRESS_HOURS = 24 SEERR_MEDIA_FAILURE_PERSISTENT_SUPPRESS_DAYS = 30 SEERR_MEDIA_FAILURE_PERSISTENT_THRESHOLD = 3 +SQLITE_BUSY_TIMEOUT_MS = 5_000 +SQLITE_CACHE_SIZE_KIB = 32_768 +SQLITE_MMAP_SIZE_BYTES = 256 * 1024 * 1024 def _db_path() -> str: @@ -26,8 +30,30 @@ def _db_path() -> str: return path +def _apply_connection_pragmas(conn: sqlite3.Connection) -> None: + pragmas = ( + ("journal_mode", "WAL"), + ("synchronous", "NORMAL"), + ("temp_store", "MEMORY"), + ("cache_size", -SQLITE_CACHE_SIZE_KIB), + ("mmap_size", SQLITE_MMAP_SIZE_BYTES), + ("busy_timeout", SQLITE_BUSY_TIMEOUT_MS), + ) + for pragma, value in pragmas: + try: + conn.execute(f"PRAGMA {pragma} = {value}") + except sqlite3.DatabaseError: + logger.debug("sqlite pragma skipped: %s=%s", pragma, value, exc_info=True) + + def _connect() -> sqlite3.Connection: - return sqlite3.connect(_db_path()) + conn = sqlite3.connect( + _db_path(), + timeout=SQLITE_BUSY_TIMEOUT_MS / 1000, + cached_statements=512, + ) + _apply_connection_pragmas(conn) + return conn def _parse_datetime_value(value: Optional[str]) -> Optional[datetime]: @@ -321,6 +347,30 @@ def init_db() -> None: ON requests_cache (requested_by_norm) """ ) + conn.execute( + """ + CREATE INDEX IF NOT EXISTS idx_requests_cache_updated_at + ON requests_cache (updated_at DESC, request_id DESC) + """ + ) + conn.execute( + """ + CREATE INDEX IF NOT EXISTS idx_requests_cache_requested_by_id_created_at + ON requests_cache (requested_by_id, created_at DESC, request_id DESC) + """ + ) + conn.execute( + """ + CREATE INDEX IF NOT EXISTS idx_requests_cache_requested_by_norm_created_at + ON requests_cache (requested_by_norm, created_at DESC, request_id DESC) + """ + ) + conn.execute( + """ + CREATE INDEX IF NOT EXISTS idx_requests_cache_status_created_at + ON requests_cache (status, created_at DESC, request_id DESC) + """ + ) conn.execute( """ CREATE INDEX IF NOT EXISTS idx_artwork_cache_status_updated_at @@ -441,6 +491,15 @@ def init_db() -> None: ) except sqlite3.OperationalError: pass + try: + conn.execute( + """ + CREATE INDEX IF NOT EXISTS idx_users_username_nocase + ON users (username COLLATE NOCASE) + """ + ) + except sqlite3.OperationalError: + pass try: conn.execute("ALTER TABLE requests_cache ADD COLUMN requested_by_id INTEGER") except sqlite3.OperationalError: @@ -454,6 +513,10 @@ def init_db() -> None: ) except sqlite3.OperationalError: pass + try: + conn.execute("PRAGMA optimize") + except sqlite3.OperationalError: + pass _backfill_auth_providers() ensure_admin_user() @@ -1619,41 +1682,44 @@ def get_user_request_stats(username_norm: str, requested_by_id: Optional[int] = "last_request_at": None, } with _connect() as conn: - total_row = conn.execute( + row = conn.execute( """ - SELECT COUNT(*) + SELECT + COUNT(*) AS total, + SUM(CASE WHEN status = 4 THEN 1 ELSE 0 END) AS ready, + SUM(CASE WHEN status = 1 THEN 1 ELSE 0 END) AS pending, + SUM(CASE WHEN status = 2 THEN 1 ELSE 0 END) AS approved, + SUM(CASE WHEN status = 5 THEN 1 ELSE 0 END) AS working, + SUM(CASE WHEN status = 6 THEN 1 ELSE 0 END) AS partial, + SUM(CASE WHEN status = 3 THEN 1 ELSE 0 END) AS declined, + MAX(created_at) AS last_request_at FROM requests_cache WHERE requested_by_id = ? """, (requested_by_id,), ).fetchone() - status_rows = conn.execute( - """ - SELECT status, COUNT(*) - FROM requests_cache - WHERE requested_by_id = ? - GROUP BY status - """, - (requested_by_id,), - ).fetchall() - last_row = conn.execute( - """ - SELECT MAX(created_at) - FROM requests_cache - WHERE requested_by_id = ? - """, - (requested_by_id,), - ).fetchone() - counts = {int(row[0]): int(row[1]) for row in status_rows if row[0] is not None} - pending = counts.get(1, 0) - approved = counts.get(2, 0) - declined = counts.get(3, 0) - ready = counts.get(4, 0) - working = counts.get(5, 0) - partial = counts.get(6, 0) + if not row: + return { + "total": 0, + "ready": 0, + "pending": 0, + "approved": 0, + "working": 0, + "partial": 0, + "declined": 0, + "in_progress": 0, + "last_request_at": None, + } + total = int(row[0] or 0) + ready = int(row[1] or 0) + pending = int(row[2] or 0) + approved = int(row[3] or 0) + working = int(row[4] or 0) + partial = int(row[5] or 0) + declined = int(row[6] or 0) in_progress = approved + working + partial return { - "total": int(total_row[0] or 0) if total_row else 0, + "total": total, "ready": ready, "pending": pending, "approved": approved, @@ -1661,7 +1727,7 @@ def get_user_request_stats(username_norm: str, requested_by_id: Optional[int] = "partial": partial, "declined": declined, "in_progress": in_progress, - "last_request_at": last_row[0] if last_row else None, + "last_request_at": row[7], } @@ -1688,6 +1754,143 @@ def get_global_request_total() -> int: return int(row[0] or 0) +_REQUESTS_CACHE_UPSERT_SQL = """ + INSERT INTO requests_cache ( + request_id, + media_id, + media_type, + status, + title, + year, + requested_by, + requested_by_norm, + requested_by_id, + created_at, + updated_at, + payload_json + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(request_id) DO UPDATE SET + media_id = excluded.media_id, + media_type = excluded.media_type, + status = excluded.status, + title = excluded.title, + year = excluded.year, + requested_by = excluded.requested_by, + requested_by_norm = excluded.requested_by_norm, + requested_by_id = excluded.requested_by_id, + created_at = excluded.created_at, + updated_at = excluded.updated_at, + payload_json = excluded.payload_json +""" + + +def get_request_cache_lookup(request_ids: list[int]) -> Dict[int, Dict[str, Any]]: + normalized_ids = sorted({int(request_id) for request_id in request_ids if isinstance(request_id, int)}) + if not normalized_ids: + return {} + placeholders = ", ".join("?" for _ in normalized_ids) + query = f""" + SELECT request_id, updated_at, title, year + FROM requests_cache + WHERE request_id IN ({placeholders}) + """ + with _connect() as conn: + rows = conn.execute(query, tuple(normalized_ids)).fetchall() + return { + int(row[0]): { + "request_id": int(row[0]), + "updated_at": row[1], + "title": row[2], + "year": row[3], + } + for row in rows + } + + +def _prepare_requests_cache_upsert_rows( + records: list[Dict[str, Any]], conn: sqlite3.Connection +) -> list[tuple[Any, ...]]: + if not records: + return [] + existing_rows: Dict[int, tuple[Optional[str], Optional[int]]] = {} + ids_needing_existing = [ + int(record["request_id"]) + for record in records + if isinstance(record.get("request_id"), int) + and ( + not _normalize_title_value(record.get("title")) + or _normalize_year_value(record.get("year")) is None + ) + ] + if ids_needing_existing: + placeholders = ", ".join("?" for _ in sorted(set(ids_needing_existing))) + query = f""" + SELECT request_id, title, year + FROM requests_cache + WHERE request_id IN ({placeholders}) + """ + for row in conn.execute(query, tuple(sorted(set(ids_needing_existing)))).fetchall(): + existing_rows[int(row[0])] = (row[1], row[2]) + + prepared: list[tuple[Any, ...]] = [] + for record in records: + request_id = int(record["request_id"]) + media_id = record.get("media_id") + media_type = record.get("media_type") + status = record.get("status") + requested_by = record.get("requested_by") + requested_by_norm = record.get("requested_by_norm") + requested_by_id = record.get("requested_by_id") + created_at = record.get("created_at") + updated_at = record.get("updated_at") + payload_json = str(record.get("payload_json") or "") + + normalized_title = _normalize_title_value(record.get("title")) + normalized_year = _normalize_year_value(record.get("year")) + derived_title = None + derived_year = None + if not normalized_title or normalized_year is None: + derived_title, derived_year = _extract_title_year_from_payload(payload_json) + if _is_placeholder_title(normalized_title, request_id): + normalized_title = None + if derived_title and not normalized_title: + normalized_title = derived_title + if normalized_year is None and derived_year is not None: + normalized_year = derived_year + + existing_title = None + existing_year = None + if normalized_title is None or normalized_year is None: + existing = existing_rows.get(request_id) + if existing: + existing_title, existing_year = existing + if _is_placeholder_title(existing_title, request_id): + existing_title = None + if normalized_title is None and existing_title: + normalized_title = existing_title + if normalized_year is None and existing_year is not None: + normalized_year = existing_year + + prepared.append( + ( + request_id, + media_id, + media_type, + status, + normalized_title, + normalized_year, + requested_by, + requested_by_norm, + requested_by_id, + created_at, + updated_at, + payload_json, + ) + ) + return prepared + + def upsert_request_cache( request_id: int, media_id: Optional[int], @@ -1702,79 +1905,28 @@ def upsert_request_cache( updated_at: Optional[str], payload_json: str, ) -> None: - normalized_title = _normalize_title_value(title) - normalized_year = _normalize_year_value(year) - derived_title = None - derived_year = None - if not normalized_title or normalized_year is None: - derived_title, derived_year = _extract_title_year_from_payload(payload_json) - if _is_placeholder_title(normalized_title, request_id): - normalized_title = None - if derived_title and not normalized_title: - normalized_title = derived_title - if normalized_year is None and derived_year is not None: - normalized_year = derived_year with _connect() as conn: - existing_title = None - existing_year = None - if normalized_title is None or normalized_year is None: - row = conn.execute( - "SELECT title, year FROM requests_cache WHERE request_id = ?", - (request_id,), - ).fetchone() - if row: - existing_title, existing_year = row[0], row[1] - if _is_placeholder_title(existing_title, request_id): - existing_title = None - if normalized_title is None and existing_title: - normalized_title = existing_title - if normalized_year is None and existing_year is not None: - normalized_year = existing_year - conn.execute( - """ - INSERT INTO requests_cache ( - request_id, - media_id, - media_type, - status, - title, - year, - requested_by, - requested_by_norm, - requested_by_id, - created_at, - updated_at, - payload_json - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(request_id) DO UPDATE SET - media_id = excluded.media_id, - media_type = excluded.media_type, - status = excluded.status, - title = excluded.title, - year = excluded.year, - requested_by = excluded.requested_by, - requested_by_norm = excluded.requested_by_norm, - requested_by_id = excluded.requested_by_id, - created_at = excluded.created_at, - updated_at = excluded.updated_at, - payload_json = excluded.payload_json - """, - ( - request_id, - media_id, - media_type, - status, - normalized_title, - normalized_year, - requested_by, - requested_by_norm, - requested_by_id, - created_at, - updated_at, - payload_json, - ), + rows = _prepare_requests_cache_upsert_rows( + [ + { + "request_id": request_id, + "media_id": media_id, + "media_type": media_type, + "status": status, + "title": title, + "year": year, + "requested_by": requested_by, + "requested_by_norm": requested_by_norm, + "requested_by_id": requested_by_id, + "created_at": created_at, + "updated_at": updated_at, + "payload_json": payload_json, + } + ], + conn, ) + if rows: + conn.execute(_REQUESTS_CACHE_UPSERT_SQL, rows[0]) logger.debug( "requests_cache upsert: request_id=%s media_id=%s status=%s updated_at=%s", request_id, @@ -1784,6 +1936,17 @@ def upsert_request_cache( ) +def upsert_request_cache_many(records: list[Dict[str, Any]]) -> int: + if not records: + return 0 + with _connect() as conn: + rows = _prepare_requests_cache_upsert_rows(records, conn) + if rows: + conn.executemany(_REQUESTS_CACHE_UPSERT_SQL, rows) + logger.debug("requests_cache bulk upsert: rows=%s", len(records)) + return len(records) + + def get_request_cache_last_updated() -> Optional[str]: with _connect() as conn: row = conn.execute( @@ -2017,9 +2180,45 @@ def upsert_artwork_cache_status( poster_cached: bool, backdrop_cached: bool, ) -> None: + upsert_artwork_cache_status_many( + [ + { + "request_id": request_id, + "tmdb_id": tmdb_id, + "media_type": media_type, + "poster_path": poster_path, + "backdrop_path": backdrop_path, + "has_tmdb": has_tmdb, + "poster_cached": poster_cached, + "backdrop_cached": backdrop_cached, + } + ] + ) + + +def upsert_artwork_cache_status_many(records: list[Dict[str, Any]]) -> int: + if not records: + return 0 updated_at = datetime.now(timezone.utc).isoformat() + params = [ + ( + record["request_id"], + record.get("tmdb_id"), + record.get("media_type"), + record.get("poster_path"), + record.get("backdrop_path"), + 1 if record.get("has_tmdb") else 0, + 1 if record.get("poster_cached") else 0, + 1 if record.get("backdrop_cached") else 0, + updated_at, + ) + for record in records + if isinstance(record.get("request_id"), int) + ] + if not params: + return 0 with _connect() as conn: - conn.execute( + conn.executemany( """ INSERT INTO artwork_cache_status ( request_id, @@ -2043,18 +2242,9 @@ def upsert_artwork_cache_status( backdrop_cached = excluded.backdrop_cached, updated_at = excluded.updated_at """, - ( - request_id, - tmdb_id, - media_type, - poster_path, - backdrop_path, - 1 if has_tmdb else 0, - 1 if poster_cached else 0, - 1 if backdrop_cached else 0, - updated_at, - ), + params, ) + return len(params) def get_artwork_cache_status_count() -> int: @@ -2638,6 +2828,73 @@ def run_integrity_check() -> str: return str(row[0]) +def get_database_diagnostics() -> Dict[str, Any]: + db_path = _db_path() + wal_path = f"{db_path}-wal" + shm_path = f"{db_path}-shm" + + def _size(path: str) -> int: + try: + return os.path.getsize(path) + except OSError: + return 0 + + started = perf_counter() + with _connect() as conn: + integrity_started = perf_counter() + integrity_row = conn.execute("PRAGMA integrity_check").fetchone() + integrity_ms = round((perf_counter() - integrity_started) * 1000, 1) + integrity = str(integrity_row[0]) if integrity_row else "unknown" + + pragma_started = perf_counter() + page_size_row = conn.execute("PRAGMA page_size").fetchone() + page_count_row = conn.execute("PRAGMA page_count").fetchone() + freelist_row = conn.execute("PRAGMA freelist_count").fetchone() + pragma_ms = round((perf_counter() - pragma_started) * 1000, 1) + + row_count_started = perf_counter() + table_counts = { + "users": int(conn.execute("SELECT COUNT(*) FROM users").fetchone()[0] or 0), + "requests_cache": int(conn.execute("SELECT COUNT(*) FROM requests_cache").fetchone()[0] or 0), + "artwork_cache_status": int(conn.execute("SELECT COUNT(*) FROM artwork_cache_status").fetchone()[0] or 0), + "signup_invites": int(conn.execute("SELECT COUNT(*) FROM signup_invites").fetchone()[0] or 0), + "settings": int(conn.execute("SELECT COUNT(*) FROM settings").fetchone()[0] or 0), + "actions": int(conn.execute("SELECT COUNT(*) FROM actions").fetchone()[0] or 0), + "snapshots": int(conn.execute("SELECT COUNT(*) FROM snapshots").fetchone()[0] or 0), + "seerr_media_failures": int(conn.execute("SELECT COUNT(*) FROM seerr_media_failures").fetchone()[0] or 0), + "password_reset_tokens": int(conn.execute("SELECT COUNT(*) FROM password_reset_tokens").fetchone()[0] or 0), + } + row_count_ms = round((perf_counter() - row_count_started) * 1000, 1) + + page_size = int(page_size_row[0] or 0) if page_size_row else 0 + page_count = int(page_count_row[0] or 0) if page_count_row else 0 + freelist_pages = int(freelist_row[0] or 0) if freelist_row else 0 + + db_size_bytes = _size(db_path) + wal_size_bytes = _size(wal_path) + shm_size_bytes = _size(shm_path) + + return { + "integrity_check": integrity, + "database_path": db_path, + "database_size_bytes": db_size_bytes, + "wal_size_bytes": wal_size_bytes, + "shm_size_bytes": shm_size_bytes, + "page_size_bytes": page_size, + "page_count": page_count, + "freelist_pages": freelist_pages, + "allocated_bytes": page_size * page_count, + "free_bytes": page_size * freelist_pages, + "row_counts": table_counts, + "timings_ms": { + "integrity_check": integrity_ms, + "pragmas": pragma_ms, + "row_counts": row_count_ms, + "total": round((perf_counter() - started) * 1000, 1), + }, + } + + def vacuum_db() -> None: with _connect() as conn: conn.execute("VACUUM") diff --git a/backend/app/routers/requests.py b/backend/app/routers/requests.py index 1e826cd..2fc750a 100644 --- a/backend/app/routers/requests.py +++ b/backend/app/routers/requests.py @@ -26,7 +26,7 @@ from ..db import ( get_cached_requests, get_cached_requests_since, get_cached_request_by_media_id, - get_request_cache_by_id, + get_request_cache_lookup, get_request_cache_payload, get_request_cache_last_updated, get_request_cache_count, @@ -35,7 +35,9 @@ from ..db import ( repair_request_cache_titles, prune_duplicate_requests_cache, upsert_request_cache, + upsert_request_cache_many, upsert_artwork_cache_status, + upsert_artwork_cache_status_many, get_artwork_cache_missing_count, get_artwork_cache_status_count, get_setting, @@ -411,26 +413,55 @@ def _upsert_artwork_status( poster_cached: Optional[bool] = None, backdrop_cached: Optional[bool] = None, ) -> None: + record = _build_artwork_status_record(payload, cache_mode, poster_cached, backdrop_cached) + if not record: + return + upsert_artwork_cache_status(**record) + + +def _build_request_cache_record(payload: Dict[str, Any], request_payload: Dict[str, Any]) -> Dict[str, Any]: + return { + "request_id": payload.get("request_id"), + "media_id": payload.get("media_id"), + "media_type": payload.get("media_type"), + "status": payload.get("status"), + "title": payload.get("title"), + "year": payload.get("year"), + "requested_by": payload.get("requested_by"), + "requested_by_norm": payload.get("requested_by_norm"), + "requested_by_id": payload.get("requested_by_id"), + "created_at": payload.get("created_at"), + "updated_at": payload.get("updated_at"), + "payload_json": json.dumps(request_payload, ensure_ascii=True), + } + + +def _build_artwork_status_record( + payload: Dict[str, Any], + cache_mode: str, + poster_cached: Optional[bool] = None, + backdrop_cached: Optional[bool] = None, +) -> Optional[Dict[str, Any]]: parsed = _parse_request_payload(payload) request_id = parsed.get("request_id") if not isinstance(request_id, int): - return + return None tmdb_id, media_type = _extract_tmdb_lookup(payload) poster_path, backdrop_path = _extract_artwork_paths(payload) has_tmdb = bool(tmdb_id and media_type) poster_cached_flag, backdrop_cached_flag = _compute_cached_flags( poster_path, backdrop_path, cache_mode, poster_cached, backdrop_cached ) - upsert_artwork_cache_status( - request_id=request_id, - tmdb_id=tmdb_id, - media_type=media_type, - poster_path=poster_path, - backdrop_path=backdrop_path, - has_tmdb=has_tmdb, - poster_cached=poster_cached_flag, - backdrop_cached=backdrop_cached_flag, - ) + return { + "request_id": request_id, + "tmdb_id": tmdb_id, + "media_type": media_type, + "poster_path": poster_path, + "backdrop_path": backdrop_path, + "has_tmdb": has_tmdb, + "poster_cached": poster_cached_flag, + "backdrop_cached": backdrop_cached_flag, + } def _collect_artwork_cache_disk_stats() -> tuple[int, int]: @@ -631,6 +662,16 @@ async def _sync_all_requests(client: JellyseerrClient) -> int: if not isinstance(items, list) or not items: logger.info("Seerr sync completed: no more results at skip=%s", skip) break + page_request_ids = [ + payload.get("request_id") + for item in items + if isinstance(item, dict) + for payload in [_parse_request_payload(item)] + if isinstance(payload.get("request_id"), int) + ] + cached_by_request_id = get_request_cache_lookup(page_request_ids) + page_cache_records: list[Dict[str, Any]] = [] + page_artwork_records: list[Dict[str, Any]] = [] for item in items: if not isinstance(item, dict): continue @@ -638,10 +679,9 @@ async def _sync_all_requests(client: JellyseerrClient) -> int: request_id = payload.get("request_id") cached_title = None if isinstance(request_id, int): - if not payload.get("title"): - cached = get_request_cache_by_id(request_id) - if cached and cached.get("title"): - cached_title = cached.get("title") + cached = cached_by_request_id.get(request_id) + if not payload.get("title") and cached and cached.get("title"): + cached_title = cached.get("title") needs_details = ( not payload.get("title") or not payload.get("media_id") @@ -672,25 +712,17 @@ async def _sync_all_requests(client: JellyseerrClient) -> int: payload["title"] = cached_title if not isinstance(payload.get("request_id"), int): continue - payload_json = json.dumps(item, ensure_ascii=True) - upsert_request_cache( - request_id=payload.get("request_id"), - media_id=payload.get("media_id"), - media_type=payload.get("media_type"), - status=payload.get("status"), - title=payload.get("title"), - year=payload.get("year"), - requested_by=payload.get("requested_by"), - requested_by_norm=payload.get("requested_by_norm"), - requested_by_id=payload.get("requested_by_id"), - created_at=payload.get("created_at"), - updated_at=payload.get("updated_at"), - payload_json=payload_json, - ) + page_cache_records.append(_build_request_cache_record(payload, item)) if isinstance(item, dict): - _upsert_artwork_status(item, cache_mode) + artwork_record = _build_artwork_status_record(item, cache_mode) + if artwork_record: + page_artwork_records.append(artwork_record) stored += 1 _sync_state["stored"] = stored + if page_cache_records: + upsert_request_cache_many(page_cache_records) + if page_artwork_records: + upsert_artwork_cache_status_many(page_artwork_records) if len(items) < take: logger.info("Seerr sync completed: stored=%s", stored) break @@ -749,6 +781,16 @@ async def _sync_delta_requests(client: JellyseerrClient) -> int: if not isinstance(items, list) or not items: logger.info("Seerr delta sync completed: no more results at skip=%s", skip) break + page_request_ids = [ + payload.get("request_id") + for item in items + if isinstance(item, dict) + for payload in [_parse_request_payload(item)] + if isinstance(payload.get("request_id"), int) + ] + cached_by_request_id = get_request_cache_lookup(page_request_ids) + page_cache_records: list[Dict[str, Any]] = [] + page_artwork_records: list[Dict[str, Any]] = [] page_changed = False for item in items: if not isinstance(item, dict): @@ -756,7 +798,7 @@ async def _sync_delta_requests(client: JellyseerrClient) -> int: payload = _parse_request_payload(item) request_id = payload.get("request_id") if isinstance(request_id, int): - cached = get_request_cache_by_id(request_id) + cached = cached_by_request_id.get(request_id) incoming_updated = payload.get("updated_at") cached_title = cached.get("title") if cached else None if cached and incoming_updated and cached.get("updated_at") == incoming_updated and cached.get("title"): @@ -790,26 +832,18 @@ async def _sync_delta_requests(client: JellyseerrClient) -> int: payload["title"] = cached_title if not isinstance(payload.get("request_id"), int): continue - payload_json = json.dumps(item, ensure_ascii=True) - upsert_request_cache( - request_id=payload.get("request_id"), - media_id=payload.get("media_id"), - media_type=payload.get("media_type"), - status=payload.get("status"), - title=payload.get("title"), - year=payload.get("year"), - requested_by=payload.get("requested_by"), - requested_by_norm=payload.get("requested_by_norm"), - requested_by_id=payload.get("requested_by_id"), - created_at=payload.get("created_at"), - updated_at=payload.get("updated_at"), - payload_json=payload_json, - ) + page_cache_records.append(_build_request_cache_record(payload, item)) if isinstance(item, dict): - _upsert_artwork_status(item, cache_mode) + artwork_record = _build_artwork_status_record(item, cache_mode) + if artwork_record: + page_artwork_records.append(artwork_record) stored += 1 page_changed = True _sync_state["stored"] = stored + if page_cache_records: + upsert_request_cache_many(page_cache_records) + if page_artwork_records: + upsert_artwork_cache_status_many(page_artwork_records) if not page_changed: unchanged_pages += 1 else: @@ -894,6 +928,8 @@ async def _prefetch_artwork_cache( batch = get_request_cache_payloads(limit=limit, offset=offset) if not batch: break + page_cache_records: list[Dict[str, Any]] = [] + page_artwork_records: list[Dict[str, Any]] = [] for row in batch: payload = row.get("payload") if not isinstance(payload, dict): @@ -921,20 +957,7 @@ async def _prefetch_artwork_cache( parsed = _parse_request_payload(payload) request_id = parsed.get("request_id") if isinstance(request_id, int): - upsert_request_cache( - request_id=request_id, - media_id=parsed.get("media_id"), - media_type=parsed.get("media_type"), - status=parsed.get("status"), - title=parsed.get("title"), - year=parsed.get("year"), - requested_by=parsed.get("requested_by"), - requested_by_norm=parsed.get("requested_by_norm"), - requested_by_id=parsed.get("requested_by_id"), - created_at=parsed.get("created_at"), - updated_at=parsed.get("updated_at"), - payload_json=json.dumps(payload, ensure_ascii=True), - ) + page_cache_records.append(_build_request_cache_record(parsed, payload)) poster_cached_flag = False backdrop_cached_flag = False if poster_path: @@ -949,17 +972,23 @@ async def _prefetch_artwork_cache( backdrop_cached_flag = bool(await cache_tmdb_image(backdrop_path, "w780")) except httpx.HTTPError: backdrop_cached_flag = False - _upsert_artwork_status( + artwork_record = _build_artwork_status_record( payload, cache_mode, poster_cached=poster_cached_flag if poster_path else None, backdrop_cached=backdrop_cached_flag if backdrop_path else None, ) + if artwork_record: + page_artwork_records.append(artwork_record) processed += 1 if processed % 25 == 0: _artwork_prefetch_state.update( {"processed": processed, "message": f"Cached artwork for {processed} requests"} ) + if page_cache_records: + upsert_request_cache_many(page_cache_records) + if page_artwork_records: + upsert_artwork_cache_status_many(page_artwork_records) offset += limit total_requests = get_request_cache_count() diff --git a/backend/app/services/diagnostics.py b/backend/app/services/diagnostics.py index 5a07113..9184045 100644 --- a/backend/app/services/diagnostics.py +++ b/backend/app/services/diagnostics.py @@ -16,7 +16,7 @@ from ..clients.qbittorrent import QBittorrentClient from ..clients.radarr import RadarrClient from ..clients.sonarr import SonarrClient from ..config import settings as env_settings -from ..db import run_integrity_check +from ..db import get_database_diagnostics from ..runtime import get_runtime_settings from .invite_email import send_test_email, smtp_email_config_ready, smtp_email_delivery_warning @@ -205,12 +205,16 @@ async def _run_http_post( async def _run_database_check() -> Dict[str, Any]: - integrity = await asyncio.to_thread(run_integrity_check) + detail = await asyncio.to_thread(get_database_diagnostics) + integrity = _clean_text(detail.get("integrity_check"), "unknown") + requests_cached = detail.get("row_counts", {}).get("requests_cache", 0) if isinstance(detail, dict) else 0 + wal_size_bytes = detail.get("wal_size_bytes", 0) if isinstance(detail, dict) else 0 + wal_size_megabytes = round((float(wal_size_bytes or 0) / (1024 * 1024)), 2) status = "up" if integrity == "ok" else "degraded" return { "status": status, - "message": f"SQLite integrity_check returned {integrity}", - "detail": integrity, + "message": f"SQLite {integrity} · {requests_cached} cached requests · WAL {wal_size_megabytes:.2f} MB", + "detail": detail, } diff --git a/frontend/app/admin/SettingsPage.tsx b/frontend/app/admin/SettingsPage.tsx index 27cb922..274805b 100644 --- a/frontend/app/admin/SettingsPage.tsx +++ b/frontend/app/admin/SettingsPage.tsx @@ -243,6 +243,31 @@ const MAGENT_GROUPS_BY_SECTION: Record> = { ]), } +const SITE_SECTION_GROUPS: Array<{ + key: string + title: string + description: string + keys: string[] +}> = [ + { + key: 'site-banner', + title: 'Site Banner', + description: 'Control the sitewide banner message, tone, and visibility.', + keys: ['site_banner_enabled', 'site_banner_tone', 'site_banner_message'], + }, + { + key: 'site-login', + title: 'Login Page Behaviour', + description: 'Control which sign-in and recovery options are shown on the logged-out login page.', + keys: [ + 'site_login_show_jellyfin_login', + 'site_login_show_local_login', + 'site_login_show_forgot_password', + 'site_login_show_signup_link', + ], + }, +] + const SETTING_LABEL_OVERRIDES: Record = { jellyseerr_base_url: 'Seerr base URL', jellyseerr_api_key: 'Seerr API key', @@ -559,6 +584,7 @@ export default function SettingsPage({ section }: SettingsPageProps) { const settingsSection = SETTINGS_SECTION_MAP[section] ?? null const isMagentGroupedSection = section === 'magent' || section === 'general' || section === 'notifications' + const isSiteGroupedSection = section === 'site' const visibleSections = settingsSection ? [settingsSection] : [] const isCacheSection = section === 'cache' const cacheSettingKeys = new Set(['requests_sync_ttl_minutes', 'requests_data_source']) @@ -620,6 +646,22 @@ export default function SettingsPage({ section }: SettingsPageProps) { }) return groups })() + : isSiteGroupedSection + ? (() => { + const siteItems = groupedSettings.site ?? [] + const byKey = new Map(siteItems.map((item) => [item.key, item])) + return SITE_SECTION_GROUPS.map((group) => { + const items = group.keys + .map((key) => byKey.get(key)) + .filter((item): item is AdminSetting => Boolean(item)) + return { + key: group.key, + title: group.title, + description: group.description, + items, + } + }) + })() : visibleSections.map((sectionKey) => ({ key: sectionKey, title: SECTION_LABELS[sectionKey] ?? sectionKey, @@ -1696,7 +1738,7 @@ export default function SettingsPage({ section }: SettingsPageProps) { )} {(sectionGroup.description || SECTION_DESCRIPTIONS[sectionGroup.key]) && - (!settingsSection || isMagentGroupedSection) && ( + (!settingsSection || isMagentGroupedSection || isSiteGroupedSection) && (

{sectionGroup.description || SECTION_DESCRIPTIONS[sectionGroup.key]}

@@ -2172,11 +2214,12 @@ export default function SettingsPage({ section }: SettingsPageProps) { const isPemField = setting.key === 'magent_ssl_certificate_pem' || setting.key === 'magent_ssl_private_key_pem' + const shouldSpanFull = isPemField || setting.key === 'site_banner_message' return (