kryten-webqueue 0.34.16__tar.gz → 0.35.0__tar.gz

{kryten_webqueue-0.34.16 → kryten_webqueue-0.35.0}/CHANGELOG.md

@@ -2,6 +2,42 @@
 
 ## [Unreleased]
 
+## [0.35.0] - 2026-08-13
+
+### Added
+
+- **Unified Catalog Enrichment Pipeline** (`kryten_webqueue/catalog/enrichment/`).
+  Consolidates `enrichtitles`, `enrichmeta`, `enrichtv`, and the sync-time art
+  resolver into a single `CatalogEnrichmentPipeline` with composable, named steps.
+
+  **Steps:** `sync` → `classify` → `title` → `meta` → `art` → `tags` →
+  `categories` (stub, deferred).
+
+  **Key fixes:**
+  - Art step uses `classification.lookup_title` (the extracted film title, e.g.
+    `"Phantom of the Mall (1989)"`) instead of the raw CMS title (`"Phantom of
+    the Mall Svengoolie"`). Svengoolie, MonsterVision, Last Drive-In, Rifftrax,
+    and MST3K items can now resolve TMDB posters correctly.
+  - Single `HOSTED_SHOW_REGISTRY` replaces three scattered `_HOSTED_PATTERNS`
+    lists; hosted-show detection is now authoritative in `classify.py`.
+  - Single `normalise.py` replaces five separate title-cleaning implementations.
+  - `meta_json` cached in `item_enrichment_state` carries `poster_url` so art
+    and meta share one TMDB call (no double lookup).
+  - Meta step populates `catalog_people` / `catalog_studios` tables from TMDB
+    credits automatically.
+
+  **New job:** `catalog_enrich` — full or partial pipeline; params: `steps`,
+  `tokens`, `force`, `dry_run`, `limit`, `min_score`.
+
+  **Backward-compat wrappers:** `enrichtitles`, `enrichmeta`, `enrichtv` now
+  delegate to the new pipeline (`steps=classify,title` / `classify,meta`).
+
+  **Re-enrich button** on the item detail admin panel — force-runs all
+  enrichment steps for a single item without touching the cron schedule.
+
+  **DB migration v19:** `item_enrichment_state` table — per-item cache of
+  classification, TMDB/OMDB result, and per-step timestamps.
+
 ## [0.34.16] - 2026-08-13
 
 ### Added

{kryten_webqueue-0.34.16 → kryten_webqueue-0.35.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: kryten-webqueue
-Version: 0.34.16
+Version: 0.35.0
 Summary: Netflix/Tubi-style catalog browser and pay-to-play queue management for CyTube
 Author: grobertson
 License-Expression: MIT

{kryten_webqueue-0.34.16 → kryten_webqueue-0.35.0}/kryten_webqueue/app.py

@@ -92,7 +92,7 @@ async def lifespan(app: FastAPI):
     app.state.catalog_sync = catalog_sync
 
     # Generic background job runner (records run history to job_runs)
-    job_manager = JobManager(db, api_gate=api_gate, config=config)
+    job_manager = JobManager(db, api_gate=api_gate, config=config, cover_art=cover_art)
 
     async def _catalog_sync_job(params, ctx):
         # catalog_sync registers with no schema; params/ctx are ignored. The
@@ -112,9 +112,11 @@ async def lifespan(app: FastAPI):
         enrichtitles_job,
         enrichmeta_job,
         enrichtv_job,
+        catalog_enrich_job,
         fetch_job,
         fetchurls_job,
         motd_posters_job,
+        CATALOG_ENRICH_SCHEMA,
         ENRICHTITLES_SCHEMA,
         ENRICHMETA_SCHEMA,
         ENRICHTV_SCHEMA,
@@ -123,6 +125,12 @@ async def lifespan(app: FastAPI):
         MOTD_POSTERS_SCHEMA,
     )
 
+    job_manager.register(
+        "catalog_enrich",
+        catalog_enrich_job,
+        label="Catalog Enrichment Pipeline",
+        schema=CATALOG_ENRICH_SCHEMA,
+    )
     job_manager.register(
         "enrichtitles",
         enrichtitles_job,

{kryten_webqueue-0.34.16 → kryten_webqueue-0.35.0}/kryten_webqueue/catalog/db/__init__.py

@@ -10,6 +10,7 @@ from ._queue import _QueueMixin
 from ._feedback import _FeedbackMixin
 from ._watchlist import _WatchlistMixin
 from ._people import _PeopleMixin
+from ._enrichment import _EnrichmentMixin
 
 
 class Database(
@@ -19,6 +20,7 @@ class Database(
     _FeedbackMixin,
     _WatchlistMixin,
     _PeopleMixin,
+    _EnrichmentMixin,
     _DBBase,
 ):
     """Async SQLite database wrapper."""
@@ -30,3 +32,11 @@ __all__ = [
     "HIDDEN_CATEGORY_NAMES",
     "HIDDEN_TAG_NAMES",
 ]
+
+
+__all__ = [
+    "Database",
+    "HIDDEN_ITEM_TAG",
+    "HIDDEN_CATEGORY_NAMES",
+    "HIDDEN_TAG_NAMES",
+]

{kryten_webqueue-0.34.16 → kryten_webqueue-0.35.0}/kryten_webqueue/catalog/db/_connection.py

@@ -373,6 +373,28 @@ MIGRATIONS = [
     );
     CREATE INDEX IF NOT EXISTS idx_catalog_studios_token ON catalog_studios(friendly_token);
     """,
+    # v19: Per-item enrichment pipeline state cache.
+    """
+    CREATE TABLE IF NOT EXISTS item_enrichment_state (
+        friendly_token    TEXT PRIMARY KEY,
+        content_type      TEXT,
+        hosted_show       TEXT,
+        lookup_title      TEXT,
+        lookup_year       TEXT,
+        tv_show           TEXT,
+        tv_season         INTEGER,
+        tv_episode_num    INTEGER,
+        description_score INTEGER,
+        tmdb_id           TEXT,
+        imdb_id           TEXT,
+        meta_json         TEXT,
+        last_classify_at  TEXT,
+        last_title_at     TEXT,
+        last_meta_at      TEXT,
+        last_art_at       TEXT,
+        last_tags_at      TEXT
+    );
+    """,
 ]
 
 

kryten_webqueue-0.35.0/kryten_webqueue/catalog/db/_enrichment.py

@@ -0,0 +1,114 @@
+"""Enrichment pipeline state — per-item caching of classification + metadata."""
+
+import json
+
+
+class _EnrichmentMixin:
+    """item_enrichment_state CRUD methods."""
+
+    async def ensure_enrichment_state(self, token: str) -> None:
+        """Create a skeleton enrichment-state row if none exists."""
+        await self._db.execute(
+            "INSERT OR IGNORE INTO item_enrichment_state (friendly_token) VALUES (?)",
+            [token],
+        )
+        await self._db.commit()
+
+    async def get_enrichment_state(self, token: str) -> dict | None:
+        return await self._fetch_one(
+            "SELECT * FROM item_enrichment_state WHERE friendly_token = ?", [token]
+        )
+
+    async def save_enrichment_state(self, token: str, **fields) -> None:
+        """Upsert enrichment state fields for a token.
+
+        Safely handles any subset of columns; unknown keys are silently ignored.
+        """
+        if not fields:
+            return
+        allowed = {
+            "content_type",
+            "hosted_show",
+            "lookup_title",
+            "lookup_year",
+            "tv_show",
+            "tv_season",
+            "tv_episode_num",
+            "description_score",
+            "tmdb_id",
+            "imdb_id",
+            "meta_json",
+            "last_classify_at",
+            "last_title_at",
+            "last_meta_at",
+            "last_art_at",
+            "last_tags_at",
+        }
+        safe = {k: v for k, v in fields.items() if k in allowed}
+        if not safe:
+            return
+        await self.ensure_enrichment_state(token)
+        sets = ", ".join(f"{k} = ?" for k in safe)
+        await self._execute(
+            f"UPDATE item_enrichment_state SET {sets} WHERE friendly_token = ?",
+            [*safe.values(), token],
+        )
+
+    async def get_catalog_for_enrichment(
+        self,
+        *,
+        step: str,
+        tokens: list[str] | None = None,
+        force: bool = False,
+        limit: int | None = None,
+    ) -> list[dict]:
+        """Return catalog rows joined with enrichment state, filtered to items that
+        need the given step.
+
+        In normal mode an item needs a step when ``last_{step}_at IS NULL``.
+        ``force=True`` returns all items regardless.
+        """
+        step_col = f"last_{step}_at"
+        sql = """
+            SELECT c.friendly_token, c.title, c.duration_sec,
+                   c.cover_art_path, c.cover_art_source, c.thumbnail_url,
+                   c.description, c.manifest_url,
+                   e.content_type, e.hosted_show, e.lookup_title, e.lookup_year,
+                   e.tv_show, e.tv_season, e.tv_episode_num,
+                   e.description_score, e.tmdb_id, e.imdb_id, e.meta_json,
+                   e.last_classify_at, e.last_title_at, e.last_meta_at,
+                   e.last_art_at, e.last_tags_at
+            FROM catalog c
+            LEFT JOIN item_enrichment_state e ON e.friendly_token = c.friendly_token
+        """
+        params: list = []
+        conditions: list[str] = []
+
+        if tokens:
+            ph = ",".join("?" * len(tokens))
+            conditions.append(f"c.friendly_token IN ({ph})")
+            params.extend(tokens)
+
+        if not force:
+            conditions.append(f"(e.{step_col} IS NULL)")
+
+        if conditions:
+            sql += " WHERE " + " AND ".join(conditions)
+
+        sql += " ORDER BY c.title"
+
+        if limit:
+            sql += " LIMIT ?"
+            params.append(limit)
+
+        return await self._fetch_all(sql, params)
+
+    def parse_meta_json(self, row: dict) -> dict | None:
+        """Deserialise the cached meta_json field from an enrichment row."""
+        raw = (row or {}).get("meta_json")
+        if not raw:
+            return None
+        try:
+            return json.loads(raw)
+        except (ValueError, TypeError):
+            return None

kryten_webqueue-0.35.0/kryten_webqueue/catalog/enrichment/__init__.py

@@ -0,0 +1,30 @@
+from .pipeline import CatalogEnrichmentPipeline, ALL_STEPS
+from .classify import (
+    classify_item,
+    ItemClassification,
+    HostedInfo,
+    HostedShowEntry,
+    HOSTED_SHOW_REGISTRY,
+)
+from .providers import TMDBProvider, OMDBProvider, MovieMetadata, merge_metadata
+from .normalise import normalize_and_clean, normalize_leading_year, strip_extension
+from .report import EnrichmentReport, StepResult
+
+__all__ = [
+    "CatalogEnrichmentPipeline",
+    "ALL_STEPS",
+    "classify_item",
+    "ItemClassification",
+    "HostedInfo",
+    "HostedShowEntry",
+    "HOSTED_SHOW_REGISTRY",
+    "TMDBProvider",
+    "OMDBProvider",
+    "MovieMetadata",
+    "merge_metadata",
+    "normalize_and_clean",
+    "normalize_leading_year",
+    "strip_extension",
+    "EnrichmentReport",
+    "StepResult",
+]

kryten_webqueue-0.35.0/kryten_webqueue/catalog/enrichment/classify.py

@@ -0,0 +1,343 @@
+"""Item classification — hosted-show detection, content-type determination.
+
+Single authoritative source for HOSTED_SHOW_REGISTRY and all classification
+logic, replacing scattered _HOSTED_PATTERNS lists in enrichmeta.py and images.py.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+
+from .normalise import normalize_and_clean, strip_extension
+
+# ---------------------------------------------------------------------------
+# Hosted show registry
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class HostedShowEntry:
+    pattern: re.Pattern
+    show_name: str
+    content_type: str  # "hosted_movie" | "riffed_movie"
+    title_treatment: str  # "keep" | "reformat"
+    reformat_template: str | None
+    cms_tag: str
+
+
+HOSTED_SHOW_REGISTRY: list[HostedShowEntry] = [
+    HostedShowEntry(
+        re.compile(r"Joe\s*Bob'?s?\s+Drive[\s\-]*In\s+Theater", re.I),
+        "Joe Bob's Drive-In Theater",
+        "hosted_movie",
+        "keep",
+        None,
+        "joebobbriggs",
+    ),
+    HostedShowEntry(
+        re.compile(r"JBBTLDI|Joe\s*Bob\s+TLDI", re.I),
+        "The Last Drive-In with Joe Bob Briggs",
+        "hosted_movie",
+        "keep",
+        None,
+        "lastdrivein",
+    ),
+    HostedShowEntry(
+        re.compile(r"(?:The\s+)?Last\s+Drive[\s\-]*In", re.I),
+        "The Last Drive-In with Joe Bob Briggs",
+        "hosted_movie",
+        "keep",
+        None,
+        "lastdrivein",
+    ),
+    HostedShowEntry(
+        re.compile(r"Monster\s*Vision", re.I),
+        "MonsterVision with Joe Bob Briggs",
+        "hosted_movie",
+        "keep",
+        None,
+        "monstervision",
+    ),
+    HostedShowEntry(
+        re.compile(r"Svengoolie", re.I),
+        "Svengoolie",
+        "hosted_movie",
+        "keep",
+        None,
+        "svengoolie",
+    ),
+    HostedShowEntry(
+        re.compile(r"Riff\s*Trax(?:\s+Live)?", re.I),
+        "RiffTrax",
+        "riffed_movie",
+        "reformat",
+        "RiffTrax Presents: {title} ({year})",
+        "rifftrax",
+    ),
+    HostedShowEntry(
+        re.compile(r"MST3K|Mystery\s*Science\s*Theater", re.I),
+        "Mystery Science Theater 3000",
+        "riffed_movie",
+        "keep",
+        None,
+        "mst3k",
+    ),
+]
+
+# ---------------------------------------------------------------------------
+# TV episode detection
+# ---------------------------------------------------------------------------
+
+_TV_RE = re.compile(
+    r"(?:[Ss]\d{1,2}\s*[Ee]\d{1,2}"
+    r"|\b\d{1,2}[Xx]\d{2,3}\b"
+    r"|\b[Ss]eason\s*\d+"
+    r"|\b[Ee]pisode\s*\d+"
+    r"|\b[Ee][Pp]\.?\s*\d+"
+    r")"
+)
+_SXE_RE = re.compile(r"[Ss](\d{1,2})[Ee](\d{1,2})")
+
+# ---------------------------------------------------------------------------
+# Archive detection (broadcast dates, wrestling, etc.)
+# ---------------------------------------------------------------------------
+
+_ARCHIVE_RE = re.compile(
+    r"(?:\b(?:19|20)\d{2}-\d{2}-\d{2}\b"  # ISO date
+    r"|\b\d{1,2}-\d{1,2}-(?:19|20)?\d{2}\b"  # MM-DD-YY
+    r"|\bWWF\b|\bWCW\b|\bNWA\b|\bAWA\b"  # wrestling promotions
+    r"|Complete\s+Broadcast"
+    r"|Original\s+Broadcast"
+    r")",
+    re.I,
+)
+
+# ---------------------------------------------------------------------------
+# Hosted extraction helpers (adapted from enrichmeta.py detect_hosted)
+# ---------------------------------------------------------------------------
+
+_EXT_RE = re.compile(r"\.\w{2,5}$")
+_YEAR_PAREN_RE = re.compile(r"\((\d{4})\)")
+_YEAR_BRACKET_RE = re.compile(r"[\[\]{}]\s*(\d{4})\s*[\[\]{}]")
+_YEAR_BARE_RE = re.compile(r"\b((?:19|20)\d{2})\b")
+_DATE_RE = re.compile(
+    r"(?:(?:19|20)\d{2}-\d{1,2}-\d{1,2})|(?:\d{1,2}-\d{1,2}-(?:19|20)?\d{2})"
+)
+_EPISODE_CODE_RE = re.compile(r"[SW]\d+[eE]\d+", re.I)
+_SHOW_YEAR_PREFIX_RE = re.compile(
+    r"^\s*\(\d{4}\)\s*S\d+-Wk\s*\d+\s*Film\s*\d+\s*[-\u2013\u2014]?\s*", re.I
+)
+_WEEK_PREFIX_RE = re.compile(
+    r"(?:S\d+-)?(?:Wk|Week)\s*\d+\s*[-\u2013\u2014]?\s*(?:(?:Film|Movie)\s*\d+\s*[-\u2013\u2014]?\s*)?",
+    re.I,
+)
+_LEADING_EPNUM_RE = re.compile(r"^\s*\d{1,2}\s*[-\u2013\u2014]?\s+")
+_SCENE_HOSTED_RE = re.compile(
+    r"\[?(?:720p|480p|1080p|who(?:do)?dude\w*|CG|Hybrid)\]?", re.I
+)
+
+
+@dataclass
+class HostedInfo:
+    show_name: str
+    movie_title: str
+    movie_year: str | None
+    cms_tag: str = ""
+
+
+def _extract_hosted_title(raw: str, entry: HostedShowEntry) -> HostedInfo:
+    """Strip show name from raw title and extract the underlying movie title + year."""
+    title = _EXT_RE.sub("", raw.strip())
+
+    # remove the matched show pattern
+    m = entry.pattern.search(title)
+    if m:
+        title = title[: m.start()] + title[m.end() :]
+
+    # remove any remaining registry patterns (handles double-show mentions)
+    for e in HOSTED_SHOW_REGISTRY:
+        title = e.pattern.sub("", title)
+
+    title = _SCENE_HOSTED_RE.sub("", title)
+    title = title.replace("_", " ")
+    title = _DATE_RE.sub("", title)
+    title = _SHOW_YEAR_PREFIX_RE.sub("", title)
+    title = _EPISODE_CODE_RE.sub("", title)
+    title = _WEEK_PREFIX_RE.sub("", title)
+    title = re.sub(r"^[\s\-\u2013\u2014:,]+", "", title)
+    title = _LEADING_EPNUM_RE.sub("", title)
+    title = re.sub(r"^[\s\-\u2013\u2014:,]+", "", title)
+    title = _LEADING_EPNUM_RE.sub("", title)
+
+    year: str | None = None
+    m = _YEAR_PAREN_RE.search(title)
+    if m:
+        year = m.group(1)
+        title = title[: m.start()] + title[m.end() :]
+    else:
+        m = _YEAR_BRACKET_RE.search(title)
+        if m:
+            year = m.group(1)
+            title = title[: m.start()] + title[m.end() :]
+        else:
+            m = _YEAR_BARE_RE.search(title)
+            if m and 1920 <= int(m.group(1)) <= 2030:
+                year = m.group(1)
+                title = title[: m.start()] + title[m.end() :]
+
+    title = re.sub(r"\[\s*\]|\(\s*\)", "", title)
+    title = re.sub(r"^[\s\-\u2013\u2014:,]+", "", title)
+    title = re.sub(r"[\s\-\u2013\u2014:,]+$", "", title)
+    title = re.sub(r"\s*-\s*\d+$", "", title)
+    title = re.sub(r"\s+", " ", title).strip()
+
+    return HostedInfo(
+        show_name=entry.show_name,
+        movie_title=title or raw,
+        movie_year=year,
+        cms_tag=entry.cms_tag,
+    )
+
+
+# ---------------------------------------------------------------------------
+# ItemClassification
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class ItemClassification:
+    friendly_token: str
+    raw_title: str
+    content_type: (
+        str  # movie | tv_episode | hosted_movie | riffed_movie | archive | unknown
+    )
+    hosted: HostedInfo | None
+    lookup_title: str  # used for TMDB/OMDB queries and art resolution
+    lookup_year: str | None
+    genre_hints: list[str] = field(default_factory=list)
+    tv_show: str | None = None
+    tv_season: int | None = None
+    tv_episode: int | None = None
+    duration_sec: int = 0
+    description_score: int = 0
+    has_real_art: bool = False
+
+
+# YouTube "Title | English Full Movie | Genre1 Genre2" pattern
+_YT_FULL_MOVIE_RE = re.compile(
+    r"^(?P<title>.+?)\s*\|\s*(?:\w+\s+)?Full\s+Movie\s*(?:\|\s*(?P<genres>.+))?$", re.I
+)
+
+
+def classify_item(
+    friendly_token: str,
+    raw_title: str,
+    duration_sec: int,
+    *,
+    cover_art_source: str | None = None,
+    description: str | None = None,
+    description_score: int = 0,
+) -> ItemClassification:
+    """Classify a single catalog item.  Pure function; no DB access."""
+    has_real_art = (cover_art_source or "") in ("tmdb", "omdb")
+
+    # 1. Hosted / riffed — check registry first
+    for entry in HOSTED_SHOW_REGISTRY:
+        if entry.pattern.search(raw_title):
+            info = _extract_hosted_title(raw_title, entry)
+            return ItemClassification(
+                friendly_token=friendly_token,
+                raw_title=raw_title,
+                content_type=entry.content_type,
+                hosted=info,
+                lookup_title=info.movie_title,
+                lookup_year=info.movie_year,
+                duration_sec=duration_sec,
+                description_score=description_score,
+                has_real_art=has_real_art,
+            )
+
+    # 2. TV episode
+    if _TV_RE.search(raw_title):
+        tv_show: str | None = None
+        tv_season: int | None = None
+        tv_ep: int | None = None
+        m = _SXE_RE.search(raw_title)
+        if m:
+            tv_season = int(m.group(1))
+            tv_ep = int(m.group(2))
+            tv_show = raw_title[: m.start()].strip(" -–—")
+        return ItemClassification(
+            friendly_token=friendly_token,
+            raw_title=raw_title,
+            content_type="tv_episode",
+            hosted=None,
+            lookup_title=tv_show or raw_title,
+            lookup_year=None,
+            tv_show=tv_show,
+            tv_season=tv_season,
+            tv_episode=tv_ep,
+            duration_sec=duration_sec,
+            description_score=description_score,
+            has_real_art=has_real_art,
+        )
+
+    # 3. Archive (broadcast recordings, wrestling)
+    if duration_sec > 1800 and _ARCHIVE_RE.search(raw_title):
+        return ItemClassification(
+            friendly_token=friendly_token,
+            raw_title=raw_title,
+            content_type="archive",
+            hosted=None,
+            lookup_title=raw_title,
+            lookup_year=None,
+            duration_sec=duration_sec,
+            description_score=description_score,
+            has_real_art=has_real_art,
+        )
+
+    # 4. Movie (>= 30 min)
+    if duration_sec >= 1800:
+        yt = _YT_FULL_MOVIE_RE.match(raw_title.strip())
+        if yt:
+            clean_title = yt.group("title").strip()
+            genres = (yt.group("genres") or "").split() if yt.group("genres") else []
+            return ItemClassification(
+                friendly_token=friendly_token,
+                raw_title=raw_title,
+                content_type="movie",
+                hosted=None,
+                lookup_title=clean_title,
+                lookup_year=None,
+                genre_hints=genres,
+                duration_sec=duration_sec,
+                description_score=description_score,
+                has_real_art=has_real_art,
+            )
+
+        clean_title, year = normalize_and_clean(raw_title)
+        return ItemClassification(
+            friendly_token=friendly_token,
+            raw_title=raw_title,
+            content_type="movie",
+            hosted=None,
+            lookup_title=clean_title,
+            lookup_year=year,
+            duration_sec=duration_sec,
+            description_score=description_score,
+            has_real_art=has_real_art,
+        )
+
+    # 5. Unknown / short
+    return ItemClassification(
+        friendly_token=friendly_token,
+        raw_title=raw_title,
+        content_type="unknown",
+        hosted=None,
+        lookup_title=raw_title,
+        lookup_year=None,
+        duration_sec=duration_sec,
+        description_score=description_score,
+        has_real_art=has_real_art,
+    )