kryten-webqueue 0.36.2__tar.gz → 0.36.4__tar.gz

{kryten_webqueue-0.36.2 → kryten_webqueue-0.36.4}/CHANGELOG.md

@@ -2,6 +2,34 @@
 
 ## [Unreleased]
 
+## [0.36.4] - 2026-08-14
+
+### Changed
+
+- **Catalog sync pulls tags/categories in bulk.** Instead of one
+  `GET /api/v1/media/{token}` per item (10k+ requests, ~30+ min), sync now
+  prefetches every item's facets from a new `GET /api/v1/media_facets` endpoint
+  in a handful of paginated calls (~35s for the full catalog). Falls back to the
+  per-item detail endpoint automatically when the bulk endpoint is unavailable
+  (stock MediaCMS). See `scripts/patch_mediacms_bulk_facets.py` for the CMS-side
+  endpoint.
+
+## [0.36.3] - 2026-08-14
+
+### Added
+
+- **`mediacms_manage_all_media` config flag.** When set (paired with a CMS whose
+  `bulk_actions` is patched to let managers/superusers tag any media), the tags
+  step pushes derived genre/MPAA/hosted tags to CMS for every item, not just
+  media the API user owns. Defaults to `false` (owner-only, safe).
+
+### Changed
+
+- **MPAA rating tags are now alphanumeric** (`mpaar`, `mpaapg13`, …) to match
+  MediaCMS's `Tag.save()` normalization (which strips non-alphanumerics), so tags
+  round-trip cleanly between the local catalog and CMS without creating hyphenated
+  duplicates on reverse-sync.
+
 ## [0.36.2] - 2026-08-14
 
 ### Fixed

{kryten_webqueue-0.36.2 → kryten_webqueue-0.36.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: kryten-webqueue
-Version: 0.36.2
+Version: 0.36.4
 Summary: Netflix/Tubi-style catalog browser and pay-to-play queue management for CyTube
 Author: grobertson
 License-Expression: MIT

{kryten_webqueue-0.36.2 → kryten_webqueue-0.36.4}/config.example.json

@@ -13,6 +13,7 @@
 
   "mediacms_url": "https://www.dropsugar.com",
   "mediacms_token": "CHANGE_ME",
+  "mediacms_manage_all_media": false,
 
   "tmdb_api_key": "",
   "omdb_api_key": "",

{kryten_webqueue-0.36.2 → kryten_webqueue-0.36.4}/kryten_webqueue/catalog/enrichment/steps/tags.py

@@ -18,15 +18,15 @@ _TIMEOUT = 20.0
 _MPAA = frozenset({"G", "PG", "PG-13", "R", "NC-17", "NR", "TV-MA", "TV-14", "TV-PG"})
 
 _MPAA_SLUG = {
-    "G": "mpaa-g",
-    "PG": "mpaa-pg",
-    "PG-13": "mpaa-pg13",
-    "R": "mpaa-r",
-    "NC-17": "mpaa-nc17",
-    "NR": "mpaa-nr",
-    "TV-MA": "mpaa-tvma",
-    "TV-14": "mpaa-tv14",
-    "TV-PG": "mpaa-tvpg",
+    "G": "mpaag",
+    "PG": "mpaapg",
+    "PG-13": "mpaapg13",
+    "R": "mpaar",
+    "NC-17": "mpaanc17",
+    "NR": "mpaanr",
+    "TV-MA": "mpaatvma",
+    "TV-14": "mpaatv14",
+    "TV-PG": "mpaatvpg",
 }
 
 
@@ -41,14 +41,17 @@ class TagsStep:
             config.mediacms_url.rstrip("/").removesuffix("/api/v1").removesuffix("/api")
         )
         self._headers = {"Authorization": f"Token {config.mediacms_token}"}
+        # When the token can manage all media (e.g. a manager/superuser on a
+        # patched CMS), push tags for every item instead of only owned media.
+        self._manage_all = bool(getattr(config, "mediacms_manage_all_media", False))
 
     async def _get_api_username(self, client: httpx.AsyncClient) -> str | None:
         """Return the API token's username, or None.
 
-        MediaCMS ``/media/user/bulk_actions`` filters ``Media.objects.filter(
-        user=request.user, ...)`` with no manager/superuser bypass, so tag
-        pushes only succeed on media this user owns. We use this to skip futile
-        pushes against media owned by others instead of flooding CMS with 400s.
+        Stock MediaCMS ``/media/user/bulk_actions`` filters ``Media.objects.filter(
+        user=request.user, ...)``, so tag pushes only succeed on media this user
+        owns. When ``mediacms_manage_all_media`` is not set we use this to skip
+        futile pushes against media owned by others instead of flooding CMS.
         """
         try:
             resp = await client.get(f"{self._base}/api/v1/whoami")
@@ -70,7 +73,11 @@ class TagsStep:
         now = datetime.now(UTC).isoformat()
 
         async with httpx.AsyncClient(headers=self._headers, timeout=_TIMEOUT) as client:
-            api_user = None if dry_run else await self._get_api_username(client)
+            api_user = (
+                None
+                if dry_run or self._manage_all
+                else await self._get_api_username(client)
+            )
             for cls in classifications:
                 result.processed += 1
                 try:
@@ -141,10 +148,10 @@ class TagsStep:
 
         The ``PUT /media/{token}`` endpoint only accepts title/description/media_file
         and silently ignores a ``tags`` field, so tag changes must go through
-        ``POST /media/user/bulk_actions`` with ``action: add_tags``. That endpoint
-        filters ``Media.objects.filter(user=request.user, ...)`` with no manager or
-        superuser bypass, so we skip the push unless the API user owns the media
-        rather than flooding CMS with 400s.
+        ``POST /media/user/bulk_actions`` with ``action: add_tags``. On stock
+        MediaCMS that endpoint is owner-scoped, so unless ``manage_all`` is set we
+        skip the push for media the API user does not own rather than flooding
+        CMS with 400s.
         """
         detail_url = f"{self._base}/api/v1/media/{token}"
         try:
@@ -153,7 +160,7 @@ class TagsStep:
                 return False
             data = resp.json()
             owner = data.get("user")
-            if api_user is None or owner != api_user:
+            if not self._manage_all and (api_user is None or owner != api_user):
                 return False
             current = {
                 (t.get("title") if isinstance(t, dict) else t)

{kryten_webqueue-0.36.2 → kryten_webqueue-0.36.4}/kryten_webqueue/catalog/sync.py

@@ -46,6 +46,9 @@ class CatalogSync:
         self._token = mediacms_token
         self._db = db
         self._cover_art = cover_art
+        # Populated once per sync by _prefetch_all_facets; None means the bulk
+        # endpoint is unavailable and we fall back to per-item detail fetches.
+        self._facets_map: dict[str, dict] | None = None
         self._client = httpx.AsyncClient(
             headers={"Authorization": f"Token {mediacms_token}"},
             timeout=30.0,
@@ -60,6 +63,10 @@ class CatalogSync:
         stats = {"seen": 0, "new": 0, "updated": 0, "errors": 0, "deleted": 0}
         sync_started_at = datetime.now(UTC).isoformat()
 
+        # Pull every item's tags/categories in a few paginated calls up front so
+        # per-item facet sync needs no HTTP (falls back to detail fetch on miss).
+        await self._prefetch_all_facets()
+
         try:
             # /manage_media returns all items (9k+); /media is capped at 1000
             next_url: str | None = f"{self._url}/api/v1/manage_media"
@@ -191,33 +198,76 @@ class CatalogSync:
             await asyncio.sleep(0.25)
 
     async def _sync_item_facets(self, token: str):
-        """Populate category/tag memberships from the media detail endpoint.
+        """Populate category/tag memberships for an item.
 
-        The list (manage_media) serializer omits categories/tags, so we fetch
-        the per-item detail (``/api/v1/media/{token}``) which exposes
-        ``categories_info`` and ``tags_info``.
+        Prefers the bulk facets pulled by :meth:`_prefetch_all_facets`; falls
+        back to the per-item detail endpoint (``/api/v1/media/{token}``) which
+        exposes ``categories_info`` and ``tags_info`` when the bulk map is
+        unavailable or missing this token.
         """
-        resp = await self._client.get(f"{self._url}/api/v1/media/{token}")
-        if resp.status_code != 200:
-            return
-        data = resp.json()
-
-        cat_names = [
-            c.get("title")
-            for c in (data.get("categories_info") or [])
-            if isinstance(c, dict) and c.get("title")
-        ]
-        tag_names = []
-        for t in data.get("tags_info") or []:
-            name = t.get("title") if isinstance(t, dict) else t
-            if name:
-                tag_names.append(name)
+        cached = self._facets_map.get(token) if self._facets_map is not None else None
+        if cached is not None:
+            cat_names = cached.get("categories") or []
+            tag_names = cached.get("tags") or []
+        else:
+            resp = await self._client.get(f"{self._url}/api/v1/media/{token}")
+            if resp.status_code != 200:
+                return
+            data = resp.json()
+            cat_names = [
+                c.get("title")
+                for c in (data.get("categories_info") or [])
+                if isinstance(c, dict) and c.get("title")
+            ]
+            tag_names = []
+            for t in data.get("tags_info") or []:
+                name = t.get("title") if isinstance(t, dict) else t
+                if name:
+                    tag_names.append(name)
 
         cat_ids = [await self._db.upsert_category(n) for n in cat_names]
         tag_ids = [await self._db.upsert_tag(n) for n in tag_names]
         await self._db.set_catalog_categories(token, cat_ids)
         await self._db.set_catalog_tags(token, tag_ids)
 
+    async def _prefetch_all_facets(self) -> None:
+        """Bulk-load every item's tags/categories from ``/api/v1/media_facets``.
+
+        One call per page (page_size=500) instead of one GET per item. Sets
+        ``self._facets_map`` to None on any failure so callers fall back to the
+        per-item detail endpoint.
+        """
+        facets: dict[str, dict] = {}
+        page = 1
+        try:
+            while True:
+                resp = await self._client.get(
+                    f"{self._url}/api/v1/media_facets",
+                    params={"page": page, "page_size": 500},
+                )
+                if resp.status_code != 200:
+                    # Endpoint absent (stock CMS) — fall back to per-item.
+                    self._facets_map = None if page == 1 else facets
+                    return
+                data = resp.json()
+                for row in data.get("results") or []:
+                    tok = row.get("friendly_token")
+                    if tok:
+                        facets[tok] = {
+                            "tags": row.get("tags") or [],
+                            "categories": row.get("categories") or [],
+                        }
+                if not data.get("has_next"):
+                    break
+                page += 1
+            self._facets_map = facets
+            logger.info(
+                "Prefetched facets for %d items in %d page(s)", len(facets), page
+            )
+        except (httpx.HTTPError, ValueError) as exc:
+            logger.debug("Facet prefetch failed (%s); using per-item fallback", exc)
+            self._facets_map = None
+
     def _build_manifest_url(self, media: dict) -> str:
         # CyTube custom media ("cm") requires the manifest JSON URL, NOT the
         # human watch page (/view?m=TOKEN). MediaCMS exposes a CyTube manifest at

{kryten_webqueue-0.36.2 → kryten_webqueue-0.36.4}/kryten_webqueue/config.py

@@ -151,6 +151,9 @@ class Config(BaseModel):
     # MediaCMS
     mediacms_url: str = "https://www.dropsugar.com"
     mediacms_token: str
+    # Set true when the token can tag any media (manager/superuser on a CMS whose
+    # bulk_actions is patched to allow it); pushes derived tags for all items.
+    mediacms_manage_all_media: bool = False
 
     # Cover art APIs
     tmdb_api_key: str = ""

{kryten_webqueue-0.36.2 → kryten_webqueue-0.36.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "kryten-webqueue"
-version = "0.36.2"
+version = "0.36.4"
 description = "Netflix/Tubi-style catalog browser and pay-to-play queue management for CyTube"
 readme = "README.md"
 license = "MIT"

kryten_webqueue-0.36.4/scripts/patch_mediacms_bulk_facets.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+"""Add a read-only bulk facets endpoint to MediaCMS so integrations can pull
+every item's friendly_token + tags + categories in a few paginated calls
+instead of one GET per media. Editor/superuser only. Idempotent."""
+
+import sys
+
+BASE = "/home/mediacms.io/mediacms/files"
+MEDIA = f"{BASE}/views/media.py"
+INIT = f"{BASE}/views/__init__.py"
+URLS = f"{BASE}/urls.py"
+
+VIEW_MARKER = "class MediaFacetsList(APIView):"
+VIEW_CODE = '''
+
+class MediaFacetsList(APIView):
+    """Bulk read-only listing of media friendly_token + tags + categories.
+
+    Lets trusted integrations (MediaCMS editors) pull every item's facets in a
+    few paginated calls instead of one GET per media. Slice pagination by id.
+    """
+
+    permission_classes = (permissions.IsAuthenticated,)
+
+    def get(self, request, format=None):
+        if not is_mediacms_editor(request.user):
+            return Response({"detail": "not allowed"}, status=status.HTTP_403_FORBIDDEN)
+        try:
+            page = max(1, int(request.query_params.get("page", 1)))
+        except (TypeError, ValueError):
+            page = 1
+        try:
+            page_size = int(request.query_params.get("page_size", 500))
+        except (TypeError, ValueError):
+            page_size = 500
+        page_size = max(1, min(2000, page_size))
+        start = (page - 1) * page_size
+        rows = Media.objects.order_by("id").prefetch_related("tags", "category")[
+            start : start + page_size
+        ]
+        results = [
+            {
+                "friendly_token": m.friendly_token,
+                "tags": [t.title for t in m.tags.all()],
+                "categories": [c.title for c in m.category.all()],
+            }
+            for m in rows
+        ]
+        return Response(
+            {
+                "page": page,
+                "page_size": page_size,
+                "has_next": len(results) == page_size,
+                "results": results,
+            }
+        )
+'''
+
+INIT_ANCHOR = "from .media import MediaBulkUserActions  # noqa: F401\n"
+INIT_ADD = "from .media import MediaFacetsList  # noqa: F401\n"
+
+URL_ANCHOR = (
+    '    re_path(r"^api/v1/media/user/bulk_actions/$", views.MediaBulkUserActions.as_view()),\n'
+)
+URL_ADD = '    re_path(r"^api/v1/media_facets$", views.MediaFacetsList.as_view()),\n'
+
+
+def patch(path, check, apply_fn):
+    with open(path, encoding="utf-8") as f:
+        src = f.read()
+    if check in src:
+        print(f"{path}: already patched")
+        return True
+    new = apply_fn(src)
+    if new is None:
+        print(f"ERROR: anchor not found in {path}")
+        return False
+    with open(path, "w", encoding="utf-8") as f:
+        f.write(new)
+    print(f"{path}: patched")
+    return True
+
+
+def main() -> int:
+    ok = True
+    ok &= patch(MEDIA, VIEW_MARKER, lambda s: s.rstrip("\n") + "\n" + VIEW_CODE)
+    ok &= patch(
+        INIT,
+        INIT_ADD.strip(),
+        lambda s: s.replace(INIT_ANCHOR, INIT_ANCHOR + INIT_ADD, 1)
+        if INIT_ANCHOR in s
+        else None,
+    )
+    ok &= patch(
+        URLS,
+        "media_facets",
+        lambda s: s.replace(URL_ANCHOR, URL_ANCHOR + URL_ADD, 1)
+        if URL_ANCHOR in s
+        else None,
+    )
+    return 0 if ok else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

kryten_webqueue-0.36.4/scripts/patch_mediacms_tags.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+"""Patch MediaCMS MediaBulkUserActions so editors/superusers can tag any media,
+and add_tags creates missing tags (normalized like Tag.save)."""
+
+import sys
+
+PATH = "/home/mediacms.io/mediacms/files/views/media.py"
+
+QS_OLD = (
+    "        media = Media.objects.filter(user=request.user, friendly_token__in=media_ids)\n"
+)
+QS_NEW = (
+    "        if is_mediacms_editor(request.user):\n"
+    "            media = Media.objects.filter(friendly_token__in=media_ids)\n"
+    "        else:\n"
+    "            media = Media.objects.filter(user=request.user, friendly_token__in=media_ids)\n"
+)
+
+TAGS_OLD = (
+    '        elif action == "add_tags":\n'
+    "            tag_titles = request.data.get('tag_titles', [])\n"
+    "            if not tag_titles:\n"
+    '                return Response({"detail": "tag_titles is required for add_tags action"}, status=status.HTTP_400_BAD_REQUEST)\n'
+    "\n"
+    "            tags = Tag.objects.filter(title__in=tag_titles)\n"
+    "            if not tags:\n"
+    '                return Response({"detail": "No matching tags found"}, status=status.HTTP_400_BAD_REQUEST)\n'
+    "\n"
+    "            added_count = 0\n"
+    "            for tag in tags:\n"
+    "                for m in media:\n"
+    "                    if not m.tags.filter(title=tag.title).exists():\n"
+    "                        m.tags.add(tag)\n"
+    "                        added_count += 1\n"
+    "\n"
+    '            return Response({"detail": f"Added {added_count} media items to {tags.count()} tags"})\n'
+)
+TAGS_NEW = (
+    '        elif action == "add_tags":\n'
+    "            tag_titles = request.data.get('tag_titles', [])\n"
+    "            if not tag_titles:\n"
+    '                return Response({"detail": "tag_titles is required for add_tags action"}, status=status.HTTP_400_BAD_REQUEST)\n'
+    "\n"
+    "            tags = []\n"
+    "            for raw in tag_titles:\n"
+    "                title = helpers.get_alphanumeric_only(raw)[:100]\n"
+    "                if not title:\n"
+    "                    continue\n"
+    "                tag, _ = Tag.objects.get_or_create(title=title, defaults={\"user\": request.user})\n"
+    "                tags.append(tag)\n"
+    "            if not tags:\n"
+    '                return Response({"detail": "No matching tags found"}, status=status.HTTP_400_BAD_REQUEST)\n'
+    "\n"
+    "            added_count = 0\n"
+    "            for tag in tags:\n"
+    "                for m in media:\n"
+    "                    if not m.tags.filter(title=tag.title).exists():\n"
+    "                        m.tags.add(tag)\n"
+    "                        added_count += 1\n"
+    "\n"
+    '            return Response({"detail": f"Added {added_count} media items to {len(tags)} tags"})\n'
+)
+
+
+def main() -> int:
+    with open(PATH, encoding="utf-8") as f:
+        src = f.read()
+
+    if QS_NEW in src and TAGS_NEW in src:
+        print("Already patched; nothing to do.")
+        return 0
+
+    if src.count(QS_OLD) != 1:
+        print(f"ERROR: queryset anchor found {src.count(QS_OLD)} times (expected 1)")
+        return 2
+    if src.count(TAGS_OLD) != 1:
+        print(f"ERROR: add_tags anchor found {src.count(TAGS_OLD)} times (expected 1)")
+        return 3
+
+    src = src.replace(QS_OLD, QS_NEW, 1)
+    src = src.replace(TAGS_OLD, TAGS_NEW, 1)
+
+    with open(PATH, "w", encoding="utf-8") as f:
+        f.write(src)
+    print("Patched OK.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

kryten_webqueue-0.36.4/scripts/trigger_enrich.py

@@ -0,0 +1,45 @@
+"""One-off: mint an admin session and trigger the catalog_enrich job locally.
+
+Usage (on the server, with the service venv python):
+    python trigger_enrich.py steps=tags force=1
+"""
+
+import json
+import sys
+import urllib.request
+from datetime import datetime, timedelta, UTC
+
+import jwt
+
+CONFIG_PATH = "/etc/kryten-webqueue/config.json"
+
+
+def main() -> int:
+    params = dict(a.split("=", 1) for a in sys.argv[1:] if "=" in a)
+    cfg = json.load(open(CONFIG_PATH))
+    secret = cfg["secret_key"]
+    port = cfg.get("port", 2010)
+    token = jwt.encode(
+        {
+            "sub": "backfill-admin",
+            "rank": 3,
+            "iat": datetime.now(UTC),
+            "exp": datetime.now(UTC) + timedelta(hours=1),
+        },
+        secret,
+        algorithm="HS256",
+    )
+    body = json.dumps({"params": params}).encode()
+    req = urllib.request.Request(
+        f"http://127.0.0.1:{port}/admin/jobs/catalog_enrich/run",
+        data=body,
+        headers={"Content-Type": "application/json", "Cookie": f"session={token}"},
+        method="POST",
+    )
+    with urllib.request.urlopen(req, timeout=30) as resp:
+        print(resp.status, resp.read().decode())
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())