kryten-webqueue 0.33.3__tar.gz → 0.33.4__tar.gz

{kryten_webqueue-0.33.3 → kryten_webqueue-0.33.4}/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## [0.33.4] - 2026-08-07
+
+### Added
+- `rehost_emotes` job: fetches the current channel emote list, downloads any
+  images not yet hosted on `dropsugar.co`, places them in
+  `/home/mediacms.io/mediacms/static/emotes/{bare_name}{ext}` with `www-data`
+  group ownership, updates each emote URL via api-gate, and saves timestamped
+  before/after JSON backups in `~/emote_backups/`. Runnable on demand from
+  Admin → Jobs or automatically on a configurable interval (default 24 h).
+  New config section `emote_rehost` (see `config.example.json`).
+- `ApiGateClient.get_emotes()` and `ApiGateClient.update_emote()` methods.
+
 All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),

{kryten_webqueue-0.33.3 → kryten_webqueue-0.33.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kryten-webqueue
-Version: 0.33.3
+Version: 0.33.4
 Summary: Netflix/Tubi-style catalog browser and pay-to-play queue management for CyTube
 Author: grobertson
 License-Expression: MIT

{kryten_webqueue-0.33.3 → kryten_webqueue-0.33.4}/config.example.json

@@ -65,5 +65,16 @@
   "playlist_bulk_add_delay_sec": 0.5,
   "playlist_bulk_add_max_retries": 2,
 
+  "emote_rehost": {
+    "enabled": true,
+    "rehost_domain": "dropsugar.co",
+    "static_dir": "/home/mediacms.io/mediacms/static/emotes",
+    "base_url": "https://www.dropsugar.co/static/emotes",
+    "backup_dir": "/home/kryten/emote_backups",
+    "check_interval_hours": 24.0,
+    "download_max_retries": 5,
+    "inter_emote_delay_sec": 2.0
+  },
+
   "prometheus_port": 28292
 }

{kryten_webqueue-0.33.3 → kryten_webqueue-0.33.4}/kryten_webqueue/api_gate/client.py

@@ -1,4 +1,5 @@
 import httpx
+from urllib.parse import quote
 
 
 class ApiGateClient:
@@ -189,6 +190,18 @@ class ApiGateClient:
     async def moderator_ping(self) -> dict:
         return await self.get("/moderator/ping")
 
+    # --- Emotes ---
+
+    async def get_emotes(self) -> list[dict]:
+        result = await self.get("/emotes/")
+        return result.get("emotes", [])
+
+    async def update_emote(self, name: str, image: str) -> dict:
+        """Update a single emote URL. name may include the '#' prefix."""
+        return await self.put(
+            f"/emotes/{quote(name, safe='')}", json={"image": image}
+        )
+
     async def moderator_health(self) -> dict:
         return await self.get("/moderator/health")
 

{kryten_webqueue-0.33.3 → kryten_webqueue-0.33.4}/kryten_webqueue/app.py

@@ -144,6 +144,14 @@ async def lifespan(app: FastAPI):
         label="Fetch URLs (weekend workbook)",
         schema=FETCHURLS_SCHEMA,
     )
+    from .jobs.rehost_emotes import rehost_emotes_job, REHOST_EMOTES_SCHEMA
+
+    job_manager.register(
+        "rehost_emotes",
+        rehost_emotes_job,
+        label="Rehost Emotes (download & serve from dropsugar.co)",
+        schema=REHOST_EMOTES_SCHEMA,
+    )
     app.state.job_manager = job_manager
 
     # WebSocket manager
@@ -271,10 +279,24 @@ async def lifespan(app: FastAPI):
             except Exception as e:
                 logger.error(f"Immutability expiry error: {e}")
 
+    async def _emote_rehost_loop():
+        interval_hours = config.emote_rehost.check_interval_hours
+        if not config.emote_rehost.enabled or interval_hours <= 0:
+            return
+        interval = interval_hours * 3600
+        # Mirror catalog_sync pattern: don't run immediately on startup.
+        while True:
+            await asyncio.sleep(interval)
+            try:
+                await job_manager.run("rehost_emotes", triggered_by="scheduler")
+            except Exception as e:
+                logger.exception("Emote rehost periodic run error: %s", e)
+
     bg_tasks = [
         asyncio.create_task(_catalog_sync_loop()),
         asyncio.create_task(_otp_cleanup_loop()),
         asyncio.create_task(_immutability_expiry_loop()),
+        asyncio.create_task(_emote_rehost_loop()),
     ]
 
     logger.info(

{kryten_webqueue-0.33.3 → kryten_webqueue-0.33.4}/kryten_webqueue/config.py

@@ -3,6 +3,24 @@ from pydantic import BaseModel, PrivateAttr
 import json
 
 
+class EmoteRehostConfig(BaseModel):
+    """Settings for the emote rehost job."""
+
+    enabled: bool = True
+    # Emotes whose image URL contains this string are considered already rehosted.
+    rehost_domain: str = "dropsugar.co"
+    # Local directory where rehosted image files are written.
+    static_dir: str = "/home/mediacms.io/mediacms/static/emotes"
+    # Public base URL served from static_dir; final URL: {base_url}/{bare_name}{ext}
+    base_url: str = "https://www.dropsugar.co/static/emotes"
+    # Directory for timestamped backup JSON files (created if absent).
+    backup_dir: str = "/home/kryten/emote_backups"
+    # Background check interval in hours; 0 disables the periodic loop.
+    check_interval_hours: float = 24.0
+    download_max_retries: int = 5
+    inter_emote_delay_sec: float = 2.0
+
+
 class FetchUrlsConfig(BaseModel):
     """Settings for the fetchurls job.
 
@@ -130,6 +148,7 @@ class Config(BaseModel):
     # Jobs (optional; jobs whose config/deps are absent fail fast at run time)
     fetch_cookies_path: str = ""  # optional yt-dlp cookies for gated sources
     fetchurls: FetchUrlsConfig = FetchUrlsConfig()
+    emote_rehost: EmoteRehostConfig = EmoteRehostConfig()
 
     # Presence-based cancel/refund of pending paid items
     presence_refund: PresenceRefundConfig = PresenceRefundConfig()

kryten_webqueue-0.33.4/kryten_webqueue/jobs/rehost_emotes.py

@@ -0,0 +1,289 @@
+"""Emote rehost job — download externally-hosted emotes and serve from dropsugar.co.
+
+Processes emotes whose image URLs are not already on the configured rehost
+domain.  For each, aggressively downloads the image, places it at
+``{static_dir}/{bare_name}{ext}`` with www-data group ownership, and pushes
+the new URL back to CyTube via api-gate.
+
+Emote names in the channel JSON include the ``#`` prefix (e.g. ``#behold``);
+filenames always use the bare name without ``#`` (e.g. ``behold.gif``).
+"""
+
+import asyncio
+import json
+import logging
+import os
+import random
+import shutil
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+from urllib.parse import unquote, urlparse
+
+import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+
+from .manager import JobError
+
+logger = logging.getLogger(__name__)
+
+REHOST_EMOTES_SCHEMA: list[dict] = []
+
+_USER_AGENTS = [
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Edge/120.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0",
+]
+
+
+def _detect_ext(url: str, content_type: str | None) -> str:
+    """Determine file extension from Content-Type or URL path; default .gif."""
+    if content_type:
+        ct = content_type.lower()
+        if "gif" in ct:
+            return ".gif"
+        if "png" in ct:
+            return ".png"
+        if "jpeg" in ct or "jpg" in ct:
+            return ".jpg"
+        if "webp" in ct:
+            return ".webp"
+    path = unquote(urlparse(url).path).lower()
+    for ext in (".gif", ".png", ".jpg", ".jpeg", ".webp"):
+        if path.endswith(ext):
+            return ".jpg" if ext == ".jpeg" else ext
+    return ".gif"
+
+
+def _make_session() -> requests.Session:
+    session = requests.Session()
+    retry = Retry(
+        total=5,
+        backoff_factor=2,
+        status_forcelist=[429, 500, 502, 503, 504],
+        allowed_methods=["HEAD", "GET"],
+    )
+    adapter = HTTPAdapter(max_retries=retry)
+    session.mount("https://", adapter)
+    session.mount("http://", adapter)
+    return session
+
+
+def _set_permissions(path: Path) -> None:
+    """Set mode 0644 and group to www-data (best-effort; logs on failure)."""
+    path.chmod(0o644)
+    try:
+        import grp  # type: ignore[import]  # Unix only
+
+        gid = grp.getgrnam("www-data").gr_gid  # type: ignore[attr-defined]
+        os.chown(path, -1, gid)  # type: ignore[attr-defined]
+    except (ImportError, KeyError, PermissionError) as exc:
+        logger.warning("Could not set www-data group on %s: %s", path, exc)
+
+
+def _place_emote(
+    url: str, bare_name: str, static_dir: Path, max_retries: int
+) -> str | None:
+    """Download url → static_dir/{bare_name}{ext} with correct permissions.
+
+    Returns the file extension (e.g. ``.gif``) on success, ``None`` if all
+    strategies are exhausted.  Blocking — must be called via asyncio.to_thread.
+    """
+    base = static_dir / bare_name
+    tmp = base.parent / f"{base.name}.tmp"
+    session = _make_session()
+
+    try:
+        # Primary strategy: resumable, rotating User-Agent, progressive timeout
+        for attempt in range(max_retries):
+            try:
+                headers: dict[str, str] = {
+                    "User-Agent": random.choice(_USER_AGENTS),
+                    "Accept": "image/webp,image/apng,image/*,*/*;q=0.8",
+                    "DNT": "1",
+                }
+                if tmp.exists():
+                    headers["Range"] = f"bytes={tmp.stat().st_size}-"
+
+                timeout = min(60 + attempt * 30, 180)
+                resp = session.get(url, headers=headers, timeout=timeout, stream=True)
+                resp.raise_for_status()
+
+                ext = _detect_ext(url, resp.headers.get("content-type"))
+                mode = "ab" if "Range" in headers and resp.status_code == 206 else "wb"
+                with open(tmp, mode) as fh:
+                    for chunk in resp.iter_content(chunk_size=8192):
+                        if chunk:
+                            fh.write(chunk)
+
+                final = base.with_suffix(ext)
+                shutil.move(str(tmp), str(final))
+                _set_permissions(final)
+                return ext
+
+            except requests.exceptions.RequestException as exc:
+                logger.debug(
+                    "Attempt %d/%d for %s: %s", attempt + 1, max_retries, url, exc
+                )
+                time.sleep(2**attempt + random.uniform(0, 1))
+            except Exception as exc:
+                logger.debug(
+                    "Unexpected error attempt %d/%d for %s: %s",
+                    attempt + 1,
+                    max_retries,
+                    url,
+                    exc,
+                )
+                tmp.unlink(missing_ok=True)
+                time.sleep(2**attempt + random.uniform(0, 1))
+
+        # Fallback: minimal headers, non-streaming
+        for attempt in range(2):
+            try:
+                resp = requests.get(
+                    url,
+                    headers={"User-Agent": "curl/7.68.0", "Accept": "*/*"},
+                    timeout=120,
+                )
+                resp.raise_for_status()
+                ext = _detect_ext(url, resp.headers.get("content-type"))
+                final = base.with_suffix(ext)
+                final.write_bytes(resp.content)
+                _set_permissions(final)
+                return ext
+            except Exception as exc:
+                logger.debug("Fallback attempt %d for %s: %s", attempt + 1, url, exc)
+                time.sleep(2)
+
+        return None
+
+    finally:
+        session.close()
+        tmp.unlink(missing_ok=True)
+
+
+def _write_json(path: Path, data: list[dict]) -> None:
+    """Atomically write data to path as indented JSON."""
+    tmp = path.with_suffix(".tmp")
+    tmp.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
+    tmp.replace(path)
+
+
+async def rehost_emotes_job(params: dict, ctx) -> dict:
+    """Rehost externally-hosted channel emotes to dropsugar.co."""
+    cfg = ctx.config.emote_rehost
+    api = ctx.api_gate
+
+    try:
+        emotes = await api.get_emotes()
+    except Exception as exc:
+        raise JobError(f"Failed to fetch emotes from api-gate: {exc}") from exc
+
+    if not emotes:
+        return {
+            "total_emotes": 0,
+            "already_rehosted": 0,
+            "attempted": 0,
+            "succeeded": 0,
+            "failed": 0,
+            "pushed": 0,
+            "failed_emotes": [],
+        }
+
+    # Back up current state before making any changes
+    backup_dir = Path(cfg.backup_dir)
+    await asyncio.to_thread(backup_dir.mkdir, parents=True, exist_ok=True)
+    stamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S")
+    await asyncio.to_thread(
+        _write_json, backup_dir / f"emotes-{stamp}-before.json", emotes
+    )
+
+    to_rehost = [e for e in emotes if cfg.rehost_domain not in e.get("image", "")]
+    already = len(emotes) - len(to_rehost)
+    await ctx.progress(
+        {"step": "fetched", "total": len(emotes), "to_rehost": len(to_rehost)}
+    )
+
+    if not to_rehost:
+        result: dict = {
+            "total_emotes": len(emotes),
+            "already_rehosted": already,
+            "attempted": 0,
+            "succeeded": 0,
+            "failed": 0,
+            "pushed": 0,
+            "failed_emotes": [],
+        }
+        await ctx.progress({"step": "complete", **result})
+        return result
+
+    static_dir = Path(cfg.static_dir)
+    await asyncio.to_thread(static_dir.mkdir, parents=True, exist_ok=True)
+
+    succeeded: list[str] = []
+    failed: list[str] = []
+    # mutable working copy keyed by name for building the after-backup
+    updated = {e["name"]: dict(e) for e in emotes}
+
+    for i, emote in enumerate(to_rehost, 1):
+        name = emote["name"]  # e.g. "#behold" (includes #)
+        url = emote["image"]
+        bare = name.lstrip("#")  # "behold" → used as filename root
+
+        ext = await asyncio.to_thread(
+            _place_emote, url, bare, static_dir, cfg.download_max_retries
+        )
+
+        if ext is None:
+            logger.warning(
+                "All download strategies failed for emote %s (%s)", name, url
+            )
+            failed.append(name)
+            await ctx.progress(
+                {"step": "failed", "emote": name, "done": i, "total": len(to_rehost)}
+            )
+            await asyncio.sleep(cfg.inter_emote_delay_sec)
+            continue
+
+        new_url = f"{cfg.base_url.rstrip('/')}/{bare}{ext}"
+        updated[name]["image"] = new_url
+
+        try:
+            await api.update_emote(name, new_url)
+            succeeded.append(name)
+            await ctx.progress(
+                {"step": "pushed", "emote": name, "done": i, "total": len(to_rehost)}
+            )
+        except Exception as exc:
+            logger.error("Failed to push emote %s to api-gate: %s", name, exc)
+            failed.append(name)
+            await ctx.progress(
+                {
+                    "step": "push_failed",
+                    "emote": name,
+                    "done": i,
+                    "total": len(to_rehost),
+                    "error": str(exc),
+                }
+            )
+
+        await asyncio.sleep(cfg.inter_emote_delay_sec)
+
+    await asyncio.to_thread(
+        _write_json, backup_dir / f"emotes-{stamp}-after.json", list(updated.values())
+    )
+
+    result = {
+        "total_emotes": len(emotes),
+        "already_rehosted": already,
+        "attempted": len(to_rehost),
+        "succeeded": len(succeeded),
+        "failed": len(failed),
+        "pushed": len(succeeded),
+        "failed_emotes": failed,
+    }
+    await ctx.progress({"step": "complete", **result})
+    return result

{kryten_webqueue-0.33.3 → kryten_webqueue-0.33.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "kryten-webqueue"
-version = "0.33.3"
+version = "0.33.4"
 description = "Netflix/Tubi-style catalog browser and pay-to-play queue management for CyTube"
 readme = "README.md"
 license = "MIT"