kryten-webqueue 0.47.0__tar.gz → 0.48.0__tar.gz

{kryten_webqueue-0.47.0 → kryten_webqueue-0.48.0}/CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## [0.48.0] - 2026-09-23
+
+### Added
+
+- **PostgreSQL & Chandra-1 Podman Migration Foundations (`postgres-migration`)**:
+  - Added PostgreSQL backend support and configuration in `kryten_webqueue/config.py` with `PostgresConfig` and `DatabaseConfig(backend="sqlite"|"postgres")`.
+  - Added SQLAlchemy 2.0 async engine and session factory builder in `kryten_webqueue/catalog/db/engine.py` (`create_pg_engine`) utilizing `asyncpg` with connection pooling (`pool_size=10`, `max_overflow=20`, `pool_pre_ping=True`).
+  - Added schema DDL `kryten_webqueue/catalog/db/sql/001_initial_schema.sql` defining 40 tables across 5 logical schemas (`catalog`, `queue`, `jobs`, `users`, `tmdb`), English `tsvector` generated search vector, and `pg_trgm` fuzzy trigram matching.
+  - Implemented automated 30-day job run log pruning task (`job_log_prune_job`) in `kryten_webqueue/jobs/tasks.py` and `kryten_webqueue/catalog/db/_jobs_db.py` with strict exemptions for financial, chat, feedback, and audit history.
+  - Implemented one-shot ETL streaming script `kryten_webqueue/migrate_sqlite_to_pg.py` (`kryten-webqueue-migrate-pg`) with batched row streaming, identity sequence synchronization (`setval`), type normalization, and verification reporting.
+  - Added rootful Podman Quadlet specifications (`deploy/podman/webqueue/webqueue.network` and `webqueue-app.container`) and Containerfile for deployment on `chandra-1`.
+  - Added comprehensive operational cutover runbook `docs/postgres-cutover.md`.
+
 ## [0.47.0] - 2026-09-23
 
 ### Added

kryten_webqueue-0.48.0/Containerfile

@@ -0,0 +1,23 @@
+FROM python:3.12-slim
+
+WORKDIR /app
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends build-essential ffmpeg curl \
+    && rm -rf /var/lib/apt/lists/* \
+    && useradd --system --create-home --uid 1001 kryten
+
+COPY . /tmp/kryten-webqueue
+
+RUN pip install --no-cache-dir --upgrade pip \
+    && pip install --no-cache-dir /tmp/kryten-webqueue
+
+RUN mkdir -p /etc/kryten-webqueue /var/lib/kryten-webqueue /var/log/kryten-webqueue \
+    && chown -R kryten:kryten /etc/kryten-webqueue /var/lib/kryten-webqueue /var/log/kryten-webqueue
+
+USER kryten
+
+EXPOSE 2010
+
+ENTRYPOINT ["kryten-webqueue"]
+CMD ["--config", "/etc/kryten-webqueue/config.json"]

{kryten_webqueue-0.47.0 → kryten_webqueue-0.48.0}/PKG-INFO

@@ -1,13 +1,15 @@
 Metadata-Version: 2.5
 Name: kryten-webqueue
-Version: 0.47.0
+Version: 0.48.0
 Summary: Netflix/Tubi-style catalog browser and pay-to-play queue management for CyTube
 Author: grobertson
 License-Expression: MIT
 Requires-Python: >=3.12
 Requires-Dist: aiosqlite>=0.20
 Requires-Dist: apscheduler>=3.10
+Requires-Dist: asyncpg>=0.29.0
 Requires-Dist: fastapi>=0.115
+Requires-Dist: greenlet>=3.0
 Requires-Dist: httpx>=0.27
 Requires-Dist: jinja2>=3.1
 Requires-Dist: msal>=1.28
@@ -21,6 +23,7 @@ Requires-Dist: python-multipart>=0.0.9
 Requires-Dist: python-slugify>=8.0
 Requires-Dist: pyyaml>=6.0
 Requires-Dist: requests>=2.31
+Requires-Dist: sqlalchemy>=2.0.30
 Requires-Dist: uvicorn[standard]>=0.30
 Requires-Dist: websockets>=12.0
 Requires-Dist: yt-dlp>=2024.1

{kryten_webqueue-0.47.0 → kryten_webqueue-0.48.0}/config.example.json

@@ -63,13 +63,25 @@
   },
 
   "database": {
+    "backend": "sqlite",
     "layout": "monolith",
     "db_path": "/var/lib/kryten-webqueue/webqueue.db",
     "data_dir": "/var/lib/kryten-webqueue/data",
     "catalog_db_path": null,
     "queue_db_path": null,
     "jobs_db_path": null,
-    "users_db_path": null
+    "users_db_path": null,
+    "postgres": {
+      "host": "host.containers.internal",
+      "port": 5432,
+      "user": "kryten",
+      "dbname": "webqueue",
+      "password_env": "KRYTEN_WEBQUEUE_PG_PASSWORD",
+      "dsn_env": null,
+      "dsn": null,
+      "pool_size": 10,
+      "max_overflow": 20
+    }
   },
   "db_path": "/var/lib/kryten-webqueue/webqueue.db",
 

kryten_webqueue-0.48.0/deploy/podman/webqueue/webqueue-app.container

@@ -0,0 +1,24 @@
+[Unit]
+Description=kryten-webqueue - CyTube Queue Management (FastAPI app)
+After=network-online.target
+
+[Container]
+ContainerName=webqueue-app
+Image=localhost/kryten-webqueue:latest
+Network=podman-webqueue
+PublishPort=2010:2010
+AddHost=host.containers.internal:host-gateway
+
+EnvironmentFile=/etc/kryten/webqueue/webqueue.env
+Volume=/etc/kryten/webqueue/config.json:/etc/kryten-webqueue/config.json:ro,z
+Volume=/var/lib/kryten/media:/var/lib/kryten-webqueue:Z
+Volume=/var/log/kryten/webqueue:/var/log/kryten-webqueue:Z
+
+Exec=uvicorn kryten_webqueue.app:app --host 0.0.0.0 --port 2010
+
+[Service]
+Restart=always
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target default.target

kryten_webqueue-0.48.0/deploy/podman/webqueue/webqueue.network

@@ -0,0 +1,3 @@
+[Network]
+NetworkName=podman-webqueue
+Subnet=10.89.30.0/24

kryten_webqueue-0.48.0/docs/postgres-cutover.md

@@ -0,0 +1,94 @@
+# PostgreSQL Cutover Runbook: Kryten-WebQueue
+
+**Target Version**: `0.48.0`  
+**Host Target**: `chandra-1` (PostgreSQL 16 + Podman Quadlet)  
+**Source Host**: `grindhouse.local` (pipx systemd / SQLite partitioned data)  
+**Parent Plan**: [DATABASE_ARCHITECTURE_PLAN.md](DATABASE_ARCHITECTURE_PLAN.md)  
+**PRD**: [postgres-migration/PRD-postgres-migration.md](postgres-migration/PRD-postgres-migration.md)  
+
+---
+
+## 1. Pre-Flight Checklist
+
+- [ ] `chandra-1` has PostgreSQL 16+ active and `pg_trgm` extension installed.
+- [ ] Database `webqueue` exists with owner `kryten`.
+- [ ] Schema `001_initial_schema.sql` applied cleanly across `catalog`, `queue`, `jobs`, `users`, `tmdb`.
+- [ ] Privileges granted to `kryten`:
+  ```sql
+  GRANT USAGE, CREATE ON SCHEMA catalog, queue, jobs, users, tmdb TO kryten;
+  GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA catalog, queue, jobs, users, tmdb, public TO kryten;
+  GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA catalog, queue, jobs, users, tmdb, public TO kryten;
+  ALTER DEFAULT PRIVILEGES IN SCHEMA catalog, queue, jobs, users, tmdb GRANT ALL ON TABLES TO kryten;
+  ALTER DEFAULT PRIVILEGES IN SCHEMA catalog, queue, jobs, users, tmdb GRANT ALL ON SEQUENCES TO kryten;
+  ```
+- [ ] Podman image built on `chandra-1`:
+  ```bash
+  sudo podman build -f Containerfile -t localhost/kryten-webqueue:latest .
+  ```
+- [ ] Environment file `/etc/kryten/webqueue/webqueue.env` created on `chandra-1`:
+  ```env
+  KRYTEN_WEBQUEUE_PG_PASSWORD=kryten_secret_password
+  ```
+
+---
+
+## 2. Maintenance Window Cutover (Step-by-Step)
+
+### Step 1: Stop Source Service & Freeze Writes
+On `grindhouse.local`:
+```bash
+sudo systemctl stop kryten-webqueue.service
+sudo systemctl status kryten-webqueue.service --no-pager
+```
+
+### Step 2: Checkpoint SQLite WAL & Create Final Snapshot
+On `grindhouse.local`:
+```bash
+SNAPSHOT_DIR="/var/lib/kryten-webqueue/backups/cutover_$(date +%Y%m%d_%H%M%S)"
+sudo mkdir -p "$SNAPSHOT_DIR"
+
+for db in catalog queue jobs users; do
+  sudo sqlite3 "/var/lib/kryten-webqueue/data/$db.sqlite3" "PRAGMA wal_checkpoint(TRUNCATE);"
+done
+
+sudo cp -av /var/lib/kryten-webqueue/data "$SNAPSHOT_DIR/"
+sudo cp -av /etc/kryten-webqueue/config.json "$SNAPSHOT_DIR/"
+```
+
+### Step 3: Stream SQLite Data to PostgreSQL on Chandra-1
+Run the ETL migration script with `--verify`:
+```bash
+sudo -u kryten /home/kryten/.local/pipx/venvs/kryten-webqueue/bin/python -m kryten_webqueue.migrate_sqlite_to_pg \
+  --data-dir /var/lib/kryten-webqueue/data \
+  --pg-dsn "postgresql://kryten:kryten_secret_password@chandra-1.local:5432/webqueue"
+```
+Verify `postgres_migration_report.json` indicates `"status": "success"` and 0 errors.
+
+### Step 4: Transfer Media & Image Assets to Chandra-1
+Sync cover art images to `/var/lib/kryten/media`:
+```bash
+rsync -avz /var/lib/kryten-webqueue/images/ groberts@chandra-1.local:/var/lib/kryten/media/images/
+```
+
+### Step 5: Start Podman Quadlet on Chandra-1
+On `chandra-1`:
+```bash
+sudo cp deploy/podman/webqueue/webqueue.network /etc/containers/systemd/
+sudo cp deploy/podman/webqueue/webqueue-app.container /etc/containers/systemd/
+sudo systemctl daemon-reload
+sudo systemctl start webqueue-app.service
+sudo systemctl status webqueue-app.service --no-pager
+```
+
+### Step 6: Smoke Testing & Health Verification
+```bash
+curl -I http://127.0.0.1:2010/auth/login
+curl -s http://127.0.0.1:2010/queue/next-schedule
+```
+
+---
+
+## 3. Rollback & Forward Repair Policy
+
+- **Pre-Cutover Failure**: If ETL validation fails or container startup fails before DNS/traffic cutover, immediately restart `kryten-webqueue.service` on `grindhouse.local`. SQLite databases remain untouched and consistent.
+- **Post-Cutover Failure**: Once traffic is switched and PostgreSQL accepts writes (new spend requests, logins, completions), recovery is a **forward repair in PostgreSQL**. The retained SQLite snapshot is preserved read-only for forensics.

{kryten_webqueue-0.47.0 → kryten_webqueue-0.48.0}/kryten_webqueue/app.py

@@ -111,6 +111,7 @@ async def lifespan(app: FastAPI):
         motd_posters_job,
         tmdb_index_refresh_job,
         tmdb_coverage_report_job,
+        job_log_prune_job,
         CATALOG_ENRICH_SCHEMA,
         CATALOG_BLACKOUT_SCHEMA,
         DEVICE_KEY_BAN_RECONCILE_SCHEMA,
@@ -121,6 +122,7 @@ async def lifespan(app: FastAPI):
         MOTD_POSTERS_SCHEMA,
         TMDB_INDEX_REFRESH_SCHEMA,
         TMDB_COVERAGE_REPORT_SCHEMA,
+        JOB_LOG_PRUNE_SCHEMA,
     )
 
     job_manager.register(
@@ -168,6 +170,12 @@ async def lifespan(app: FastAPI):
         label="MOTD Posters (weekend poster grid)",
         schema=MOTD_POSTERS_SCHEMA,
     )
+    job_manager.register(
+        "job_log_prune",
+        job_log_prune_job,
+        label="Job Run Log Pruner (30-day retention)",
+        schema=JOB_LOG_PRUNE_SCHEMA,
+    )
     from .jobs.motd_publish import motd_publish_job, MOTD_PUBLISH_SCHEMA
 
     job_manager.register(

{kryten_webqueue-0.47.0 → kryten_webqueue-0.48.0}/kryten_webqueue/catalog/db/__init__.py

@@ -107,17 +107,17 @@ _DOMAIN_METHOD_MAP: dict[str, str] = {
     "get_job_schedule": "jobs",
     "upsert_job_schedule": "jobs",
     "delete_job_schedule": "jobs",
-    "fetch_queue_add": "jobs",
-    "fetch_queue_get": "jobs",
-    "fetch_queue_update": "jobs",
-    "fetch_queue_delete": "jobs",
-    "fetch_queue_list": "jobs",
-    "fetch_queue_count": "jobs",
-    "fetch_queue_get_next_pending": "jobs",
-    "fetch_queue_mark_started": "jobs",
-    "fetch_queue_mark_finished": "jobs",
-    "fetch_queue_requeue": "jobs",
-    "fetch_queue_cleanup": "jobs",
+    "prune_job_run_logs": "jobs",
+    "enqueue_fetch": "jobs",
+    "claim_next_fetch_item": "jobs",
+    "finish_fetch_item": "jobs",
+    "requeue_fetch_item": "jobs",
+    "requeue_fetch_item_for_retry": "jobs",
+    "reset_running_fetch_items": "jobs",
+    "get_fetch_queue": "jobs",
+    "count_fetch_queue": "jobs",
+    "count_fetch_queue_pending": "jobs",
+    "delete_fetch_queue_item": "jobs",
     # users
     "watchlist_add": "users",
     "watchlist_remove": "users",
@@ -303,10 +303,22 @@ class Database(
         except AttributeError:
             layout = "monolith"
 
-        if layout == "partitioned" and name in _DOMAIN_METHOD_MAP:
-            domain_name = _DOMAIN_METHOD_MAP[name]
-            domain_obj = object.__getattribute__(self, domain_name)
-            return getattr(domain_obj, name)
+        if layout == "partitioned":
+            if name in Database.__dict__ or name.startswith("_"):
+                return object.__getattribute__(self, name)
+
+            if name in _DOMAIN_METHOD_MAP:
+                domain_name = _DOMAIN_METHOD_MAP[name]
+                domain_obj = object.__getattribute__(self, domain_name)
+                return getattr(domain_obj, name)
+
+            for domain_name in ("jobs", "queue", "users", "catalog"):
+                try:
+                    domain_obj = object.__getattribute__(self, domain_name)
+                    if hasattr(domain_obj, name) and not hasattr(_DBBase, name):
+                        return getattr(domain_obj, name)
+                except AttributeError:
+                    pass
 
         return object.__getattribute__(self, name)
 

{kryten_webqueue-0.47.0 → kryten_webqueue-0.48.0}/kryten_webqueue/catalog/db/_catalog.py

@@ -1331,6 +1331,30 @@ class _CatalogMixin:
     async def delete_job_schedule(self, job_name: str) -> None:
         await self._execute("DELETE FROM job_schedules WHERE job_name=?", [job_name])
 
+    async def prune_job_run_logs(
+        self, retention_days: int = 30, batch_size: int = 5000
+    ) -> int:
+        total_deleted = 0
+        cutoff = f"-{int(retention_days)} days"
+        while True:
+            cursor = await self._db.execute(
+                """
+                DELETE FROM job_run_logs
+                WHERE id IN (
+                    SELECT id FROM job_run_logs
+                    WHERE logged_at < datetime('now', ?)
+                    LIMIT ?
+                )
+                """,
+                [cutoff, batch_size],
+            )
+            await self._db.commit()
+            count = cursor.rowcount or 0
+            total_deleted += count
+            if count < batch_size:
+                break
+        return total_deleted
+
     # --- OTP ---
 
     async def store_otp(self, username: str, code: str, expires_at: str):

{kryten_webqueue-0.47.0 → kryten_webqueue-0.48.0}/kryten_webqueue/catalog/db/_jobs_db.py

@@ -121,3 +121,31 @@ class _JobsDB(_FetchQueueMixin, _DomainDB):
 
     async def delete_job_schedule(self, job_name: str) -> None:
         await self._execute("DELETE FROM job_schedules WHERE job_name=?", [job_name])
+
+    async def prune_job_run_logs(
+        self, retention_days: int = 30, batch_size: int = 5000
+    ) -> int:
+        """Prune job_run_logs older than retention_days in bounded batches.
+
+        Strictly touches ONLY job_run_logs.
+        """
+        total_deleted = 0
+        cutoff = f"-{int(retention_days)} days"
+        while True:
+            cursor = await self._db.execute(
+                """
+                DELETE FROM job_run_logs
+                WHERE id IN (
+                    SELECT id FROM job_run_logs
+                    WHERE logged_at < datetime('now', ?)
+                    LIMIT ?
+                )
+                """,
+                [cutoff, batch_size],
+            )
+            await self._db.commit()
+            count = cursor.rowcount or 0
+            total_deleted += count
+            if count < batch_size:
+                break
+        return total_deleted

kryten_webqueue-0.48.0/kryten_webqueue/catalog/db/engine.py

@@ -0,0 +1,29 @@
+"""SQLAlchemy 2.0 async engine and session factory for PostgreSQL."""
+
+from sqlalchemy.ext.asyncio import (
+    AsyncEngine,
+    AsyncSession,
+    async_sessionmaker,
+    create_async_engine,
+)
+
+from ...config import PostgresConfig
+
+
+def create_pg_engine(
+    config: PostgresConfig,
+) -> tuple[AsyncEngine, async_sessionmaker[AsyncSession]]:
+    """Create a SQLAlchemy 2.0 async engine and sessionmaker from PostgresConfig."""
+    url = config.get_async_url()
+    engine = create_async_engine(
+        url,
+        pool_size=config.pool_size,
+        max_overflow=config.max_overflow,
+        pool_pre_ping=True,
+    )
+    session_factory = async_sessionmaker(
+        bind=engine,
+        class_=AsyncSession,
+        expire_on_commit=False,
+    )
+    return engine, session_factory

{kryten_webqueue-0.47.0 → kryten_webqueue-0.48.0}/kryten_webqueue/catalog/db/schemas/catalog_schema.py

@@ -78,7 +78,7 @@ CATALOG_MIGRATIONS: list[str] = [
     CREATE INDEX IF NOT EXISTS idx_people_name ON people(name);
 
     CREATE TABLE IF NOT EXISTS catalog_people (
-        friendly_token TEXT NOT NULL REFERENCES catalog(friendly_token) ON DELETE CASCADE,
+        friendly_token TEXT NOT NULL,
         person_id      INTEGER NOT NULL REFERENCES people(id) ON DELETE CASCADE,
         role           TEXT NOT NULL,
         position       INTEGER NOT NULL DEFAULT 0,
@@ -93,14 +93,14 @@ CATALOG_MIGRATIONS: list[str] = [
     );
 
     CREATE TABLE IF NOT EXISTS catalog_studios (
-        friendly_token TEXT NOT NULL REFERENCES catalog(friendly_token) ON DELETE CASCADE,
+        friendly_token TEXT NOT NULL,
         studio_id      INTEGER NOT NULL REFERENCES studios(id) ON DELETE CASCADE,
         PRIMARY KEY (friendly_token, studio_id)
     );
     CREATE INDEX IF NOT EXISTS idx_catalog_studios_token ON catalog_studios(friendly_token);
 
     CREATE TABLE IF NOT EXISTS item_enrichment_state (
-        friendly_token     TEXT PRIMARY KEY REFERENCES catalog(friendly_token) ON DELETE CASCADE,
+        friendly_token     TEXT PRIMARY KEY,
         content_type       TEXT,
         hosted_show        TEXT,
         lookup_title       TEXT,