kryten-webqueue 0.46.1__tar.gz → 0.48.0__tar.gz

{kryten_webqueue-0.46.1 → kryten_webqueue-0.48.0}/.gitignore

@@ -62,6 +62,10 @@ config-*.json
 *.db
 *.db-wal
 *.db-shm
+*.sqlite3
+*.sqlite3-wal
+*.sqlite3-shm
+data/
 
 # OS
 .DS_Store

{kryten_webqueue-0.46.1 → kryten_webqueue-0.48.0}/CHANGELOG.md

@@ -1,5 +1,40 @@
 # Changelog
 
+## [0.48.0] - 2026-09-23
+
+### Added
+
+- **PostgreSQL & Chandra-1 Podman Migration Foundations (`postgres-migration`)**:
+  - Added PostgreSQL backend support and configuration in `kryten_webqueue/config.py` with `PostgresConfig` and `DatabaseConfig(backend="sqlite"|"postgres")`.
+  - Added SQLAlchemy 2.0 async engine and session factory builder in `kryten_webqueue/catalog/db/engine.py` (`create_pg_engine`) utilizing `asyncpg` with connection pooling (`pool_size=10`, `max_overflow=20`, `pool_pre_ping=True`).
+  - Added schema DDL `kryten_webqueue/catalog/db/sql/001_initial_schema.sql` defining 40 tables across 5 logical schemas (`catalog`, `queue`, `jobs`, `users`, `tmdb`), English `tsvector` generated search vector, and `pg_trgm` fuzzy trigram matching.
+  - Implemented automated 30-day job run log pruning task (`job_log_prune_job`) in `kryten_webqueue/jobs/tasks.py` and `kryten_webqueue/catalog/db/_jobs_db.py` with strict exemptions for financial, chat, feedback, and audit history.
+  - Implemented one-shot ETL streaming script `kryten_webqueue/migrate_sqlite_to_pg.py` (`kryten-webqueue-migrate-pg`) with batched row streaming, identity sequence synchronization (`setval`), type normalization, and verification reporting.
+  - Added rootful Podman Quadlet specifications (`deploy/podman/webqueue/webqueue.network` and `webqueue-app.container`) and Containerfile for deployment on `chandra-1`.
+  - Added comprehensive operational cutover runbook `docs/postgres-cutover.md`.
+
+## [0.47.0] - 2026-09-23
+
+### Added
+
+- **SQLite Domain Partitioning (`sqlite-domain-separation`)**:
+  - Split database architecture isolating heavy background write jobs from real-time pollers across 4 discrete SQLite databases:
+    - `catalog.sqlite3` — Catalog items, FTS5 index, categories, tags, people, studios, enrichment state, MOTD overrides.
+    - `queue.sqlite3` — Queue shadow, spend requests, queue history, saved playlists, schedules, play completions, blackouts.
+    - `jobs.sqlite3` — Background job runs, per-line job run logs, schedules, and fetch queue.
+    - `users.sqlite3` — OTPs, device link codes, device API keys, watchlists, feedback, title suggestions.
+  - Multi-database connection facade (`Database`) managing domain sub-connections (`self.catalog`, `self.queue`, `self.jobs`, `self.users`) with independent WAL, checkpoints, and busy timeouts.
+  - Configurable `DatabaseConfig` supporting `layout="monolith"` and `layout="partitioned"` with safe path resolution and guards against accidental silent bypass of existing monolith databases.
+  - Automated database migration ETL script `scripts/split_databases.py` (`kryten-webqueue-split-db`) with checkpoint-safe reading, foreign key verification, and deterministic SHA-256 data parity validation.
+  - Concurrency stress test suite (`tests/test_concurrency_split.py`) verifying zero cross-domain lock contention between high-frequency queue polling, burst job logging, and web browse requests.
+
+### Changed
+
+- Decoupled cross-domain SQL queries:
+  - User watchlist ("My List") queries `users.sqlite3` for ordered tokens and hydrates item metadata from `catalog.sqlite3`.
+  - Recently-played hiding and weekend blackout exclusions resolved across domain boundaries without cross-file SQL joins.
+  - Promo-pool hide state purge orchestrated across `queue.sqlite3` and `catalog.sqlite3`.
+
 ## [0.46.1] - 2026-09-10
 
 ### Changed

kryten_webqueue-0.48.0/Containerfile

@@ -0,0 +1,23 @@
+FROM python:3.12-slim
+
+WORKDIR /app
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends build-essential ffmpeg curl \
+    && rm -rf /var/lib/apt/lists/* \
+    && useradd --system --create-home --uid 1001 kryten
+
+COPY . /tmp/kryten-webqueue
+
+RUN pip install --no-cache-dir --upgrade pip \
+    && pip install --no-cache-dir /tmp/kryten-webqueue
+
+RUN mkdir -p /etc/kryten-webqueue /var/lib/kryten-webqueue /var/log/kryten-webqueue \
+    && chown -R kryten:kryten /etc/kryten-webqueue /var/lib/kryten-webqueue /var/log/kryten-webqueue
+
+USER kryten
+
+EXPOSE 2010
+
+ENTRYPOINT ["kryten-webqueue"]
+CMD ["--config", "/etc/kryten-webqueue/config.json"]

{kryten_webqueue-0.46.1 → kryten_webqueue-0.48.0}/PKG-INFO

@@ -1,13 +1,15 @@
 Metadata-Version: 2.5
 Name: kryten-webqueue
-Version: 0.46.1
+Version: 0.48.0
 Summary: Netflix/Tubi-style catalog browser and pay-to-play queue management for CyTube
 Author: grobertson
 License-Expression: MIT
 Requires-Python: >=3.12
 Requires-Dist: aiosqlite>=0.20
 Requires-Dist: apscheduler>=3.10
+Requires-Dist: asyncpg>=0.29.0
 Requires-Dist: fastapi>=0.115
+Requires-Dist: greenlet>=3.0
 Requires-Dist: httpx>=0.27
 Requires-Dist: jinja2>=3.1
 Requires-Dist: msal>=1.28
@@ -21,6 +23,7 @@ Requires-Dist: python-multipart>=0.0.9
 Requires-Dist: python-slugify>=8.0
 Requires-Dist: pyyaml>=6.0
 Requires-Dist: requests>=2.31
+Requires-Dist: sqlalchemy>=2.0.30
 Requires-Dist: uvicorn[standard]>=0.30
 Requires-Dist: websockets>=12.0
 Requires-Dist: yt-dlp>=2024.1

{kryten_webqueue-0.46.1 → kryten_webqueue-0.48.0}/config.example.json

@@ -62,6 +62,27 @@
     }
   },
 
+  "database": {
+    "backend": "sqlite",
+    "layout": "monolith",
+    "db_path": "/var/lib/kryten-webqueue/webqueue.db",
+    "data_dir": "/var/lib/kryten-webqueue/data",
+    "catalog_db_path": null,
+    "queue_db_path": null,
+    "jobs_db_path": null,
+    "users_db_path": null,
+    "postgres": {
+      "host": "host.containers.internal",
+      "port": 5432,
+      "user": "kryten",
+      "dbname": "webqueue",
+      "password_env": "KRYTEN_WEBQUEUE_PG_PASSWORD",
+      "dsn_env": null,
+      "dsn": null,
+      "pool_size": 10,
+      "max_overflow": 20
+    }
+  },
   "db_path": "/var/lib/kryten-webqueue/webqueue.db",
 
   "image_dir": "/var/lib/kryten-webqueue/images",

kryten_webqueue-0.48.0/deploy/podman/webqueue/webqueue-app.container

@@ -0,0 +1,24 @@
+[Unit]
+Description=kryten-webqueue - CyTube Queue Management (FastAPI app)
+After=network-online.target
+
+[Container]
+ContainerName=webqueue-app
+Image=localhost/kryten-webqueue:latest
+Network=podman-webqueue
+PublishPort=2010:2010
+AddHost=host.containers.internal:host-gateway
+
+EnvironmentFile=/etc/kryten/webqueue/webqueue.env
+Volume=/etc/kryten/webqueue/config.json:/etc/kryten-webqueue/config.json:ro,z
+Volume=/var/lib/kryten/media:/var/lib/kryten-webqueue:Z
+Volume=/var/log/kryten/webqueue:/var/log/kryten-webqueue:Z
+
+Exec=uvicorn kryten_webqueue.app:app --host 0.0.0.0 --port 2010
+
+[Service]
+Restart=always
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target default.target

kryten_webqueue-0.48.0/deploy/podman/webqueue/webqueue.network

@@ -0,0 +1,3 @@
+[Network]
+NetworkName=podman-webqueue
+Subnet=10.89.30.0/24

kryten_webqueue-0.48.0/docs/DATABASE_ARCHITECTURE_PLAN.md

@@ -0,0 +1,206 @@
+# Database Architecture & Evolution Plan: Kryten-WebQueue
+
+**Status**: Approved Architecture Plan  
+**Target System**: `kryten-webqueue`  
+**Author**: Senior Microservice Architect (Kryten)  
+**Last Updated**: September 2026  
+
+---
+
+## Executive Summary
+
+`kryten-webqueue` has evolved from a lightweight catalog viewer into a multi-faceted web application hosting the CyTube channel catalog, real-time queue shadows, automated playlist schedulers, background media ingestion, TMDB enrichment pipelines, user watchlists, device API authentication, and per-line job logging.
+
+Under heavy concurrent operation, the application suffers from SQLite database write-lock contention (`sqlite3.OperationalError: database is locked`) because all domains share a single SQLite database file and a single async worker connection.
+
+This document outlines the **two-part architectural roadmap**:
+
+1. **Part 1 (Immediate / Stopgap)**: **SQLite Domain Partitioning** — Split the monolithic SQLite database into 4 logical database files (`catalog.db`, `queue.db`, `jobs.db`, `users.db`) to isolate high-throughput writers (job logs, enrichment syncs) from real-time pollers and public browse traffic. This removes cross-domain contention; it does not remove writer serialization within a single domain file.
+2. **Part 2 (Strategic / Chandra-1 Migration)**: **PostgreSQL on Chandra-1 via Podman** — Migrate all partitioned domains into one PostgreSQL database (`webqueue`) on `chandra-1` utilizing schemas (`catalog`, `queue`, `jobs`, `users`, `tmdb`), powered by **SQLAlchemy 2.0 (`asyncpg`)**, PostgreSQL Full-Text Search (`tsvector` + GIN) + `pg_trgm` fuzzy matching, and automated 30-day job log pruning (strictly exempting all financial, chat, and purchase records).
+
+```
+┌────────────────────────────────────────────────────────────────────────────────────────┐
+│                                 TWO-PART ROADMAP                                       │
+├────────────────────────────────────────┬───────────────────────────────────────────────┤
+│ PART 1: SQLite Domain Partitioning     │ PART 2: Chandra-1 PostgreSQL & Podman Pod     │
+├────────────────────────────────────────┼───────────────────────────────────────────────┤
+│ • Split single DB into 4 files:        │ • Single Postgres database `webqueue` on      │
+│   - catalog.db (MediaCMS mirror, art)  │   chandra-1 with logical schemas.             │
+│   - queue.db (shadow, schedules, play) │ • SQLAlchemy 2.0 (asyncpg) connection pool.   │
+│   - jobs.db (job runs, logs, fetch)    │ • English `tsvector` + GIN + `pg_trgm` search.│
+│   - users.db (OTPs, devices, list)     │ • 30-day job run log pruning policy (strictly │
+│ • Decouple cross-table JOINs in Python │   exempting economy, spend, and chat data).   │
+│ • Zero lock contention between batch   │ • Podman container deployment on chandra-1    │
+│   jobs and real-time playback pollers. │   with shared media asset volumes.            │
+└────────────────────────────────────────┴───────────────────────────────────────────────┘
+```
+
+---
+
+## Part 1: SQLite Domain Partitioning & Lock Elimination
+
+### 1.1 Root Cause of Current Contention
+
+1. **Single Writer Serialization**: In SQLite WAL mode, any number of readers can proceed concurrently, but only one write transaction can execute at a time per database file.
+2. **Single Connection Dispatcher**: `_connection.py` instantiates a single `aiosqlite.Connection`. In `aiosqlite`, all calls queue onto a single background OS thread, serializing even concurrent read operations.
+3. **Collision of Asynchronous Workloads**:
+   - **Heavy Writers**: `catalog_enrich` (updating thousands of item records), `catalog_sync`, `fetch_queue_drain` (yt-dlp ingestion), and `log_capture.py` (bulk inserting hundreds of log rows per job run).
+   - **Real-Time Pollers & API**: `StatePoller` (every 2–5s), `RacePoller`, `PlaylistScheduler`, `CompletionRecorder`, `PresenceRefundMonitor`, and device API key authentication (updating `last_used_at` per HTTP request).
+   - When a batch job holds the WAL write lock or issues rapid write transactions, real-time pollers and web requests exceed the 5000ms `busy_timeout` and crash with `database is locked`.
+
+### 1.2 Domain Partitioning Strategy
+
+We partition tables into 4 dedicated SQLite databases:
+
+```
+┌──────────────────────────┐   ┌──────────────────────────┐
+│        catalog.db        │   │         queue.db         │
+├──────────────────────────┤   ├──────────────────────────┤
+│ • catalog                │   │ • queue_shadow           │
+│ • catalog_fts (FTS5)     │   │ • spend_requests         │
+│ • categories             │   │ • queue_history          │
+│ • catalog_categories     │   │ • saved_playlists        │
+│ • tags                   │   │ • saved_playlist_items   │
+│ • catalog_tags           │   │ • playlist_schedules     │
+│ • people                 │   │ • active_schedule        │
+│ • catalog_people         │   │ • play_completions       │
+│ • studios                │   │ • playlist_item_played   │
+│ • catalog_studios        │   │ • catalog_blackouts      │
+│ • item_enrichment_state  │   └──────────────────────────┘
+│ • item_edit_log          │
+│ • sync_log               │   ┌──────────────────────────┐
+│ • motd_overrides         │   │         users.db         │
+└──────────────────────────┘   ├──────────────────────────┤
+                               │ • user_watchlist         │
+┌──────────────────────────┐   │ • otps                   │
+│         jobs.db          │   │ • device_link_codes      │
+├──────────────────────────┤   │ • device_api_keys        │
+│ • job_runs               │   │ • feedback               │
+│ • job_run_logs           │   │ • title_suggestions      │
+│ • job_schedules          │   └──────────────────────────┘
+│ • fetch_queue            │
+└──────────────────────────┘
+```
+
+### 1.3 Cross-Domain Query Decoupling
+
+Separating SQLite database files removes the ability to perform single SQL `JOIN`s across tables in different files. We resolve the existing cross-table queries in the application layer:
+
+1. **Recently-Played Catalog Suppression Filter**:
+   - *Previous*: `catalog` query joined `play_completions` and `playlist_item_played`.
+   - *Partitioned*: `queue.db` executes `get_active_hidden_media_ids() -> set[str]`. The browse query in `catalog.db` filters with `WHERE friendly_token NOT IN (...)`.
+2. **User Watchlist ("My List")**:
+   - *Previous*: `user_watchlist JOIN catalog`.
+   - *Partitioned*: Query `users.db` for `friendly_tokens` for the user, then query `catalog.db` with `get_catalog_items_by_tokens(tokens)`.
+3. **Title Suggestions Catalog Match**:
+   - *Previous*: `title_suggestions` lookup checking if `catalog_token` exists.
+   - *Partitioned*: Check `catalog.db` by token/title independently during triage or submission.
+4. **Job Run Logs & Job History**:
+   - `JobManager` writes exclusively to `jobs.db`. Background enrichment tasks write their metadata updates to `catalog.db` without locking the job log tables.
+
+### 1.4 Partitioning Limits and Safety
+
+Partitioning removes only cross-domain SQLite contention. `catalog` still contains browse reads
+and enrichment writes, while `queue` still contains poller, scheduler, and queue-history writes.
+Each domain therefore uses short, bounded transactions, retry/latency metrics, and domain-scoped
+concurrency tests. A `busy_timeout` is a wait budget, not a correctness mechanism.
+
+The layout is selected explicitly as `monolith` or `partitioned`. Startup never chooses a new
+default data directory when a legacy database exists, and migration history is never copied from
+the monolith: each partition has a fresh baseline schema version after ETL validation.
+
+### 1.5 Part 1 Implementation Artifacts
+
+- **Sprint Specs**: Located in `docs/sqlite-domain-separation/`
+  - `PRD-sqlite-domain-separation.md`
+  - `SPEC-Sortie-1-schema-partitioning-and-config.md`
+  - `SPEC-Sortie-2-multi-db-connection-layer.md`
+  - `SPEC-Sortie-3-cross-domain-query-decoupling.md`
+  - `SPEC-Sortie-4-etl-split-script-and-validation.md`
+
+---
+
+## Part 2: Chandra-1 PostgreSQL & Podman Migration
+
+### 2.1 Target Architecture
+
+Upon completion of Part 1, the partitioned domain model directly maps to PostgreSQL. Instead of maintaining multiple SQLite files, the application connects to a **single PostgreSQL database (`webqueue`)** on `chandra-1`, structured with logical schemas:
+
+- `catalog.*` — MediaCMS catalog mirror, cast/crew, tags, categories, enrichment state, MOTD overrides.
+- `queue.*` — Queue shadow, spend requests, saved playlists, schedules, play completions, blackouts.
+- `jobs.*` — Job schedules, job runs, per-line job logs, fetch queue.
+- `users.*` — OTP authentication, device link codes & API keys, user watchlists, feedback, title suggestions.
+- `tmdb.*` — Persistent, queryable local TMDB dump index.
+
+```
+┌────────────────────────────────────────────────────────────────────────────────────────┐
+│                                 chandra-1 Host System                                  │
+│                                                                                        │
+│  ┌──────────────────────────────────────────────────────────────────────────────────┐  │
+│  │                            Podman Pod: kryten-webqueue                           │  │
+│  │                                                                                  │  │
+│  │  ┌─────────────────────────────┐           ┌──────────────────────────────────┐  │  │
+│  │  │    kryten-webqueue-app      │           │           nginx proxy            │  │  │
+│  │  │   (FastAPI + uvicorn)       │◄─────────►│     (Reverse Proxy + SSL)        │  │  │
+│  │  │   SQLAlchemy 2.0 (asyncpg)  │           │                                  │  │  │
+│  │  └──────────────┬──────────────┘           └────────────────┬─────────────────┘  │  │
+│  └─────────────────┼───────────────────────────────────────────┼────────────────────┘  │
+│                    │                                           │                       │
+│                    ▼                                           ▼                       │
+│  ┌─────────────────────────────────────┐     ┌──────────────────────────────────────┐  │
+│  │        PostgreSQL on chandra-1      │     │         Shared Media Volume          │  │
+│  │        Database: webqueue           │     │        /var/lib/kryten/media         │  │
+│  │        Schemas: catalog, queue,     │     │        - cover art / posters         │  │
+│  │                 jobs, users, tmdb   │     │        - TMDB dump files             │  │
+│  └─────────────────────────────────────┘     └──────────────────────────────────────┘  │
+└────────────────────────────────────────────────────────────────────────────────────────┘
+```
+
+### 2.2 Core Technical Specifications
+
+1. **ORM & Driver**: **SQLAlchemy 2.0 with `asyncpg`** (`create_async_engine`, `async_sessionmaker`, `AsyncSession`).
+   - Retains clean async repository layer with type-safe query construction and parameterized execution.
+   - Automatic connection pooling (`pool_size=10, max_overflow=20, pool_pre_ping=True`).
+2. **Search Engine**:
+   - **Full-Text Search**: English `tsvector` generated column + `GIN` index on `catalog.title` and `catalog.description`.
+   - **Typo-Tolerant Matching**: `pg_trgm` extension with GIN/GiST index on `catalog.title` for fuzzy `similarity()` matching (e.g., matching "terminatr" to "The Terminator").
+   - Eliminates all SQLite FTS5 query-sanitization crashes.
+3. **Automated Log Pruning Policy**:
+   - A scheduled background job cleans up rows in `jobs.job_run_logs` older than **30 days**.
+   - **CRITICAL COMPLIANCE CONSTRAINT**: This pruning policy applies **EXCLUSIVELY** to `jobs.job_run_logs`. It must **NEVER** touch or prune:
+     - Chat logs or user communication records
+     - Economy or z-coin purchase history (`spend_requests`, `queue_history`)
+     - User feedback or moderation audit logs (`item_edit_log`, `feedback`)
+4. **ETL & Data Migration**:
+   - Streaming migration script (`migrate_sqlite_to_pg.py`) reading from the 4 SQLite databases and writing into PostgreSQL with foreign key verification, sequence resets, and JSONB conversion.
+
+### 2.3 Cutover Contract
+
+WebQueue runs as a rootful Quadlet application container on `chandra-1`; PostgreSQL is
+host-managed on the same host and reached through an explicit container host gateway, never
+container-localhost. The PostgreSQL cutover uses a maintenance window and forward-fix recovery:
+the SQLite snapshot remains read-only for one release, but it is not an automatic rollback target
+after PostgreSQL accepts writes.
+
+### 2.4 Part 2 Implementation Artifacts
+
+- **Sprint Specs**: Located in `docs/postgres-migration/`
+  - `PRD-postgres-migration.md`
+  - `SPEC-Sortie-1-db-config-and-pool.md`
+  - `SPEC-Sortie-2-connection-layer-port.md`
+  - `SPEC-Sortie-3-fts5-to-tsvector.md`
+  - `SPEC-Sortie-4-tmdb-index-database.md`
+  - `SPEC-Sortie-5-etl-migration.md`
+  - `SPEC-Sortie-6-tests-cutover-release.md`
+
+---
+
+## Sequencing & Next Steps
+
+1. **Phase 1 Execution (Immediate)**:
+   - Implement Part 1 (SQLite Domain Partitioning) to resolve production write-lock crashes immediately without requiring infrastructure migration on chandra-1.
+   - Run `split_databases.py` in staging/production to migrate single SQLite DB to `catalog.sqlite3`, `queue.sqlite3`, `jobs.sqlite3`, `users.sqlite3`.
+2. **Phase 2 Execution (Subsequent)**:
+   - Bootstrap PostgreSQL `webqueue` database on `chandra-1`.
+   - Deploy Podman container on `chandra-1` with SQLAlchemy 2.0 + asyncpg.
+   - Execute ETL migration and switch production traffic.

kryten_webqueue-0.48.0/docs/postgres-cutover.md

@@ -0,0 +1,94 @@
+# PostgreSQL Cutover Runbook: Kryten-WebQueue
+
+**Target Version**: `0.48.0`  
+**Host Target**: `chandra-1` (PostgreSQL 16 + Podman Quadlet)  
+**Source Host**: `grindhouse.local` (pipx systemd / SQLite partitioned data)  
+**Parent Plan**: [DATABASE_ARCHITECTURE_PLAN.md](DATABASE_ARCHITECTURE_PLAN.md)  
+**PRD**: [postgres-migration/PRD-postgres-migration.md](postgres-migration/PRD-postgres-migration.md)  
+
+---
+
+## 1. Pre-Flight Checklist
+
+- [ ] `chandra-1` has PostgreSQL 16+ active and `pg_trgm` extension installed.
+- [ ] Database `webqueue` exists with owner `kryten`.
+- [ ] Schema `001_initial_schema.sql` applied cleanly across `catalog`, `queue`, `jobs`, `users`, `tmdb`.
+- [ ] Privileges granted to `kryten`:
+  ```sql
+  GRANT USAGE, CREATE ON SCHEMA catalog, queue, jobs, users, tmdb TO kryten;
+  GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA catalog, queue, jobs, users, tmdb, public TO kryten;
+  GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA catalog, queue, jobs, users, tmdb, public TO kryten;
+  ALTER DEFAULT PRIVILEGES IN SCHEMA catalog, queue, jobs, users, tmdb GRANT ALL ON TABLES TO kryten;
+  ALTER DEFAULT PRIVILEGES IN SCHEMA catalog, queue, jobs, users, tmdb GRANT ALL ON SEQUENCES TO kryten;
+  ```
+- [ ] Podman image built on `chandra-1`:
+  ```bash
+  sudo podman build -f Containerfile -t localhost/kryten-webqueue:latest .
+  ```
+- [ ] Environment file `/etc/kryten/webqueue/webqueue.env` created on `chandra-1`:
+  ```env
+  KRYTEN_WEBQUEUE_PG_PASSWORD=kryten_secret_password
+  ```
+
+---
+
+## 2. Maintenance Window Cutover (Step-by-Step)
+
+### Step 1: Stop Source Service & Freeze Writes
+On `grindhouse.local`:
+```bash
+sudo systemctl stop kryten-webqueue.service
+sudo systemctl status kryten-webqueue.service --no-pager
+```
+
+### Step 2: Checkpoint SQLite WAL & Create Final Snapshot
+On `grindhouse.local`:
+```bash
+SNAPSHOT_DIR="/var/lib/kryten-webqueue/backups/cutover_$(date +%Y%m%d_%H%M%S)"
+sudo mkdir -p "$SNAPSHOT_DIR"
+
+for db in catalog queue jobs users; do
+  sudo sqlite3 "/var/lib/kryten-webqueue/data/$db.sqlite3" "PRAGMA wal_checkpoint(TRUNCATE);"
+done
+
+sudo cp -av /var/lib/kryten-webqueue/data "$SNAPSHOT_DIR/"
+sudo cp -av /etc/kryten-webqueue/config.json "$SNAPSHOT_DIR/"
+```
+
+### Step 3: Stream SQLite Data to PostgreSQL on Chandra-1
+Run the ETL migration script with `--verify`:
+```bash
+sudo -u kryten /home/kryten/.local/pipx/venvs/kryten-webqueue/bin/python -m kryten_webqueue.migrate_sqlite_to_pg \
+  --data-dir /var/lib/kryten-webqueue/data \
+  --pg-dsn "postgresql://kryten:kryten_secret_password@chandra-1.local:5432/webqueue"
+```
+Verify `postgres_migration_report.json` indicates `"status": "success"` and 0 errors.
+
+### Step 4: Transfer Media & Image Assets to Chandra-1
+Sync cover art images to `/var/lib/kryten/media`:
+```bash
+rsync -avz /var/lib/kryten-webqueue/images/ groberts@chandra-1.local:/var/lib/kryten/media/images/
+```
+
+### Step 5: Start Podman Quadlet on Chandra-1
+On `chandra-1`:
+```bash
+sudo cp deploy/podman/webqueue/webqueue.network /etc/containers/systemd/
+sudo cp deploy/podman/webqueue/webqueue-app.container /etc/containers/systemd/
+sudo systemctl daemon-reload
+sudo systemctl start webqueue-app.service
+sudo systemctl status webqueue-app.service --no-pager
+```
+
+### Step 6: Smoke Testing & Health Verification
+```bash
+curl -I http://127.0.0.1:2010/auth/login
+curl -s http://127.0.0.1:2010/queue/next-schedule
+```
+
+---
+
+## 3. Rollback & Forward Repair Policy
+
+- **Pre-Cutover Failure**: If ETL validation fails or container startup fails before DNS/traffic cutover, immediately restart `kryten-webqueue.service` on `grindhouse.local`. SQLite databases remain untouched and consistent.
+- **Post-Cutover Failure**: Once traffic is switched and PostgreSQL accepts writes (new spend requests, logins, completions), recovery is a **forward repair in PostgreSQL**. The retained SQLite snapshot is preserved read-only for forensics.