kryten-webqueue 0.36.3__tar.gz → 0.36.5__tar.gz

{kryten_webqueue-0.36.3 → kryten_webqueue-0.36.5}/CHANGELOG.md

@@ -2,6 +2,29 @@
 
 ## [Unreleased]
 
+## [0.36.5] - 2026-08-14
+
+### Changed
+
+- **Tags step no longer reads every item.** With `mediacms_manage_all_media` set,
+  the tags step now issues a single idempotent `add_tags` POST per item (the
+  server skips tags already present) instead of a per-item `GET` to read current
+  tags/owner. The redundant per-item reverse-sync `GET` was also removed — catalog
+  sync already mirrors CMS tags in bulk via `/api/v1/media_facets`. Stock-CMS
+  (owner-scoped) deployments still do the ownership `GET` on the fallback path.
+
+## [0.36.4] - 2026-08-14
+
+### Changed
+
+- **Catalog sync pulls tags/categories in bulk.** Instead of one
+  `GET /api/v1/media/{token}` per item (10k+ requests, ~30+ min), sync now
+  prefetches every item's facets from a new `GET /api/v1/media_facets` endpoint
+  in a handful of paginated calls (~35s for the full catalog). Falls back to the
+  per-item detail endpoint automatically when the bulk endpoint is unavailable
+  (stock MediaCMS). See `scripts/patch_mediacms_bulk_facets.py` for the CMS-side
+  endpoint.
+
 ## [0.36.3] - 2026-08-14
 
 ### Added

{kryten_webqueue-0.36.3 → kryten_webqueue-0.36.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: kryten-webqueue
-Version: 0.36.3
+Version: 0.36.5
 Summary: Netflix/Tubi-style catalog browser and pay-to-play queue management for CyTube
 Author: grobertson
 License-Expression: MIT

{kryten_webqueue-0.36.3 → kryten_webqueue-0.36.5}/kryten_webqueue/catalog/enrichment/steps/tags.py

@@ -117,8 +117,6 @@ class TagsStep:
                             desired,
                             api_user=api_user,
                         )
-                        # Reverse sync: pull any CMS-only tags back to local DB
-                        await self._reverse_sync(client, cls.friendly_token)
                         if changed:
                             result.changed += 1
                         else:
@@ -148,50 +146,37 @@ class TagsStep:
 
         The ``PUT /media/{token}`` endpoint only accepts title/description/media_file
         and silently ignores a ``tags`` field, so tag changes must go through
-        ``POST /media/user/bulk_actions`` with ``action: add_tags``. On stock
-        MediaCMS that endpoint is owner-scoped, so unless ``manage_all`` is set we
-        skip the push for media the API user does not own rather than flooding
-        CMS with 400s.
+        ``POST /media/user/bulk_actions`` with ``action: add_tags``.
+
+        When ``manage_all`` is set the server-side ``add_tags`` is idempotent
+        (it skips tags already present), so we POST directly with no per-item GET.
+        On stock MediaCMS the endpoint is owner-scoped, so we must first GET the
+        item to check ownership and skip the push for media we do not own.
         """
-        detail_url = f"{self._base}/api/v1/media/{token}"
         try:
-            resp = await client.get(detail_url)
-            if resp.status_code != 200:
-                return False
-            data = resp.json()
-            owner = data.get("user")
-            if not self._manage_all and (api_user is None or owner != api_user):
-                return False
-            current = {
-                (t.get("title") if isinstance(t, dict) else t)
-                for t in (data.get("tags_info") or [])
-            }
-            to_add = [t for t in new_tags if t not in current]
-            if not to_add:
-                return False
+            if not self._manage_all:
+                resp = await client.get(f"{self._base}/api/v1/media/{token}")
+                if resp.status_code != 200:
+                    return False
+                data = resp.json()
+                owner = data.get("user")
+                if api_user is None or owner != api_user:
+                    return False
+                current = {
+                    (t.get("title") if isinstance(t, dict) else t)
+                    for t in (data.get("tags_info") or [])
+                }
+                new_tags = [t for t in new_tags if t not in current]
+                if not new_tags:
+                    return False
             put = await client.post(
                 f"{self._base}/api/v1/media/user/bulk_actions",
                 json={
                     "action": "add_tags",
                     "media_ids": [token],
-                    "tag_titles": to_add,
+                    "tag_titles": new_tags,
                 },
             )
             return put.status_code in (200, 201)
         except (httpx.HTTPError, json.JSONDecodeError, ValueError):
             return False
-
-    async def _reverse_sync(self, client: httpx.AsyncClient, token: str) -> None:
-        """Insert any CMS-only tags into the local catalog_tags table."""
-        url = f"{self._base}/api/v1/media/{token}"
-        try:
-            resp = await client.get(url)
-            if resp.status_code != 200:
-                return
-            data = resp.json()
-            for t in data.get("tags_info") or []:
-                name = t.get("title") if isinstance(t, dict) else t
-                if name:
-                    await self._db.add_catalog_tag(token, str(name))
-        except (httpx.HTTPError, json.JSONDecodeError, ValueError) as exc:
-            logger.warning("[tags] Failed reverse-sync for %s: %s", token, exc)

{kryten_webqueue-0.36.3 → kryten_webqueue-0.36.5}/kryten_webqueue/catalog/sync.py

@@ -46,6 +46,9 @@ class CatalogSync:
         self._token = mediacms_token
         self._db = db
         self._cover_art = cover_art
+        # Populated once per sync by _prefetch_all_facets; None means the bulk
+        # endpoint is unavailable and we fall back to per-item detail fetches.
+        self._facets_map: dict[str, dict] | None = None
         self._client = httpx.AsyncClient(
             headers={"Authorization": f"Token {mediacms_token}"},
             timeout=30.0,
@@ -60,6 +63,10 @@ class CatalogSync:
         stats = {"seen": 0, "new": 0, "updated": 0, "errors": 0, "deleted": 0}
         sync_started_at = datetime.now(UTC).isoformat()
 
+        # Pull every item's tags/categories in a few paginated calls up front so
+        # per-item facet sync needs no HTTP (falls back to detail fetch on miss).
+        await self._prefetch_all_facets()
+
         try:
             # /manage_media returns all items (9k+); /media is capped at 1000
             next_url: str | None = f"{self._url}/api/v1/manage_media"
@@ -191,33 +198,76 @@ class CatalogSync:
             await asyncio.sleep(0.25)
 
     async def _sync_item_facets(self, token: str):
-        """Populate category/tag memberships from the media detail endpoint.
+        """Populate category/tag memberships for an item.
 
-        The list (manage_media) serializer omits categories/tags, so we fetch
-        the per-item detail (``/api/v1/media/{token}``) which exposes
-        ``categories_info`` and ``tags_info``.
+        Prefers the bulk facets pulled by :meth:`_prefetch_all_facets`; falls
+        back to the per-item detail endpoint (``/api/v1/media/{token}``) which
+        exposes ``categories_info`` and ``tags_info`` when the bulk map is
+        unavailable or missing this token.
         """
-        resp = await self._client.get(f"{self._url}/api/v1/media/{token}")
-        if resp.status_code != 200:
-            return
-        data = resp.json()
-
-        cat_names = [
-            c.get("title")
-            for c in (data.get("categories_info") or [])
-            if isinstance(c, dict) and c.get("title")
-        ]
-        tag_names = []
-        for t in data.get("tags_info") or []:
-            name = t.get("title") if isinstance(t, dict) else t
-            if name:
-                tag_names.append(name)
+        cached = self._facets_map.get(token) if self._facets_map is not None else None
+        if cached is not None:
+            cat_names = cached.get("categories") or []
+            tag_names = cached.get("tags") or []
+        else:
+            resp = await self._client.get(f"{self._url}/api/v1/media/{token}")
+            if resp.status_code != 200:
+                return
+            data = resp.json()
+            cat_names = [
+                c.get("title")
+                for c in (data.get("categories_info") or [])
+                if isinstance(c, dict) and c.get("title")
+            ]
+            tag_names = []
+            for t in data.get("tags_info") or []:
+                name = t.get("title") if isinstance(t, dict) else t
+                if name:
+                    tag_names.append(name)
 
         cat_ids = [await self._db.upsert_category(n) for n in cat_names]
         tag_ids = [await self._db.upsert_tag(n) for n in tag_names]
         await self._db.set_catalog_categories(token, cat_ids)
         await self._db.set_catalog_tags(token, tag_ids)
 
+    async def _prefetch_all_facets(self) -> None:
+        """Bulk-load every item's tags/categories from ``/api/v1/media_facets``.
+
+        One call per page (page_size=500) instead of one GET per item. Sets
+        ``self._facets_map`` to None on any failure so callers fall back to the
+        per-item detail endpoint.
+        """
+        facets: dict[str, dict] = {}
+        page = 1
+        try:
+            while True:
+                resp = await self._client.get(
+                    f"{self._url}/api/v1/media_facets",
+                    params={"page": page, "page_size": 500},
+                )
+                if resp.status_code != 200:
+                    # Endpoint absent (stock CMS) — fall back to per-item.
+                    self._facets_map = None if page == 1 else facets
+                    return
+                data = resp.json()
+                for row in data.get("results") or []:
+                    tok = row.get("friendly_token")
+                    if tok:
+                        facets[tok] = {
+                            "tags": row.get("tags") or [],
+                            "categories": row.get("categories") or [],
+                        }
+                if not data.get("has_next"):
+                    break
+                page += 1
+            self._facets_map = facets
+            logger.info(
+                "Prefetched facets for %d items in %d page(s)", len(facets), page
+            )
+        except (httpx.HTTPError, ValueError) as exc:
+            logger.debug("Facet prefetch failed (%s); using per-item fallback", exc)
+            self._facets_map = None
+
     def _build_manifest_url(self, media: dict) -> str:
         # CyTube custom media ("cm") requires the manifest JSON URL, NOT the
         # human watch page (/view?m=TOKEN). MediaCMS exposes a CyTube manifest at

{kryten_webqueue-0.36.3 → kryten_webqueue-0.36.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "kryten-webqueue"
-version = "0.36.3"
+version = "0.36.5"
 description = "Netflix/Tubi-style catalog browser and pay-to-play queue management for CyTube"
 readme = "README.md"
 license = "MIT"

kryten_webqueue-0.36.5/scripts/patch_mediacms_bulk_facets.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+"""Add a read-only bulk facets endpoint to MediaCMS so integrations can pull
+every item's friendly_token + tags + categories in a few paginated calls
+instead of one GET per media. Editor/superuser only. Idempotent."""
+
+import sys
+
+BASE = "/home/mediacms.io/mediacms/files"
+MEDIA = f"{BASE}/views/media.py"
+INIT = f"{BASE}/views/__init__.py"
+URLS = f"{BASE}/urls.py"
+
+VIEW_MARKER = "class MediaFacetsList(APIView):"
+VIEW_CODE = '''
+
+class MediaFacetsList(APIView):
+    """Bulk read-only listing of media friendly_token + tags + categories.
+
+    Lets trusted integrations (MediaCMS editors) pull every item's facets in a
+    few paginated calls instead of one GET per media. Slice pagination by id.
+    """
+
+    permission_classes = (permissions.IsAuthenticated,)
+
+    def get(self, request, format=None):
+        if not is_mediacms_editor(request.user):
+            return Response({"detail": "not allowed"}, status=status.HTTP_403_FORBIDDEN)
+        try:
+            page = max(1, int(request.query_params.get("page", 1)))
+        except (TypeError, ValueError):
+            page = 1
+        try:
+            page_size = int(request.query_params.get("page_size", 500))
+        except (TypeError, ValueError):
+            page_size = 500
+        page_size = max(1, min(2000, page_size))
+        start = (page - 1) * page_size
+        rows = Media.objects.order_by("id").prefetch_related("tags", "category")[
+            start : start + page_size
+        ]
+        results = [
+            {
+                "friendly_token": m.friendly_token,
+                "tags": [t.title for t in m.tags.all()],
+                "categories": [c.title for c in m.category.all()],
+            }
+            for m in rows
+        ]
+        return Response(
+            {
+                "page": page,
+                "page_size": page_size,
+                "has_next": len(results) == page_size,
+                "results": results,
+            }
+        )
+'''
+
+INIT_ANCHOR = "from .media import MediaBulkUserActions  # noqa: F401\n"
+INIT_ADD = "from .media import MediaFacetsList  # noqa: F401\n"
+
+URL_ANCHOR = (
+    '    re_path(r"^api/v1/media/user/bulk_actions/$", views.MediaBulkUserActions.as_view()),\n'
+)
+URL_ADD = '    re_path(r"^api/v1/media_facets$", views.MediaFacetsList.as_view()),\n'
+
+
+def patch(path, check, apply_fn):
+    with open(path, encoding="utf-8") as f:
+        src = f.read()
+    if check in src:
+        print(f"{path}: already patched")
+        return True
+    new = apply_fn(src)
+    if new is None:
+        print(f"ERROR: anchor not found in {path}")
+        return False
+    with open(path, "w", encoding="utf-8") as f:
+        f.write(new)
+    print(f"{path}: patched")
+    return True
+
+
+def main() -> int:
+    ok = True
+    ok &= patch(MEDIA, VIEW_MARKER, lambda s: s.rstrip("\n") + "\n" + VIEW_CODE)
+    ok &= patch(
+        INIT,
+        INIT_ADD.strip(),
+        lambda s: s.replace(INIT_ANCHOR, INIT_ANCHOR + INIT_ADD, 1)
+        if INIT_ANCHOR in s
+        else None,
+    )
+    ok &= patch(
+        URLS,
+        "media_facets",
+        lambda s: s.replace(URL_ANCHOR, URL_ANCHOR + URL_ADD, 1)
+        if URL_ANCHOR in s
+        else None,
+    )
+    return 0 if ok else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

kryten_webqueue-0.36.5/scripts/trigger_enrich.py

@@ -0,0 +1,45 @@
+"""One-off: mint an admin session and trigger the catalog_enrich job locally.
+
+Usage (on the server, with the service venv python):
+    python trigger_enrich.py steps=tags force=1
+"""
+
+import json
+import sys
+import urllib.request
+from datetime import datetime, timedelta, UTC
+
+import jwt
+
+CONFIG_PATH = "/etc/kryten-webqueue/config.json"
+
+
+def main() -> int:
+    params = dict(a.split("=", 1) for a in sys.argv[1:] if "=" in a)
+    cfg = json.load(open(CONFIG_PATH))
+    secret = cfg["secret_key"]
+    port = cfg.get("port", 2010)
+    token = jwt.encode(
+        {
+            "sub": "backfill-admin",
+            "rank": 3,
+            "iat": datetime.now(UTC),
+            "exp": datetime.now(UTC) + timedelta(hours=1),
+        },
+        secret,
+        algorithm="HS256",
+    )
+    body = json.dumps({"params": params}).encode()
+    req = urllib.request.Request(
+        f"http://127.0.0.1:{port}/admin/jobs/catalog_enrich/run",
+        data=body,
+        headers={"Content-Type": "application/json", "Cookie": f"session={token}"},
+        method="POST",
+    )
+    with urllib.request.urlopen(req, timeout=30) as resp:
+        print(resp.status, resp.read().decode())
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())