unit3dprep 1.0.4__py3-none-any.whl

unit3dprep/web/webup_client.py

@@ -0,0 +1,190 @@
+"""HTTP client for Unit3DWebUp FastAPI bot.
+
+Wraps the endpoints exposed by https://github.com/31December99/Unit3DWebUp.
+Singleton pattern (one client per app lifespan) — held in `app.state.webup`.
+Base URL resolved at instantiation time from `runtime_setting("WEBUP_URL")`.
+"""
+from __future__ import annotations
+
+import hashlib
+import time
+from pathlib import Path
+from typing import Any
+
+import httpx
+
+from .config import runtime_setting
+
+
+DEFAULT_BASE_URL = "http://127.0.0.1:8000"
+_HEALTH_TTL = 5.0
+
+
+def base_url() -> str:
+    return runtime_setting("WEBUP_URL", DEFAULT_BASE_URL).rstrip("/")
+
+
+def compute_job_id(file_or_folder_path: str) -> str:
+    """Replicate Unit3DWebUp's job_id derivation: sha256(str(folder/subfolder)).
+
+    Webup normalizes its scan_path with `os.path.normpath` before deriving
+    job_ids, so we match that here.
+    """
+    import os as _os
+    return hashlib.sha256(_os.path.normpath(str(file_or_folder_path)).encode()).hexdigest()
+
+
+def compute_job_list_id(scan_path: str) -> str:
+    return hashlib.sha256(str(scan_path).encode()).hexdigest()
+
+
+class WebupError(RuntimeError):
+    pass
+
+
+class WebupClient:
+    def __init__(self, base: str | None = None) -> None:
+        self._base = (base or base_url()).rstrip("/")
+        self._client = httpx.AsyncClient(
+            base_url=self._base,
+            timeout=httpx.Timeout(connect=5.0, read=180.0, write=30.0, pool=5.0),
+        )
+        self._health_at: float = 0.0
+        self._health_ok: bool = False
+        self._health_payload: dict[str, Any] = {}
+
+    @property
+    def base(self) -> str:
+        return self._base
+
+    async def aclose(self) -> None:
+        await self._client.aclose()
+
+    async def _post(self, path: str, payload: dict | None = None) -> httpx.Response:
+        try:
+            return await self._client.post(path, json=payload or {})
+        except httpx.HTTPError as e:
+            raise WebupError(f"webup {path} request failed: {e}") from e
+
+    async def _get(self, path: str) -> httpx.Response:
+        try:
+            return await self._client.get(path)
+        except httpx.HTTPError as e:
+            raise WebupError(f"webup {path} request failed: {e}") from e
+
+    async def health(self, force: bool = False) -> dict[str, Any]:
+        """Cheap reachability check + version readout. Cached for 5 s."""
+        now = time.time()
+        if not force and (now - self._health_at) < _HEALTH_TTL:
+            return {"ok": self._health_ok, **self._health_payload}
+        t0 = time.perf_counter()
+        try:
+            r = await self._client.post("/setting", timeout=5.0)
+            r.raise_for_status()
+            data = r.json()
+            prefs = data.get("userPreferences") or {}
+            payload = {
+                "version": prefs.get("UNIT3DWEBUP__VERSION") or prefs.get("VERSION") or "",
+                "scan_path": prefs.get("SCAN_PATH"),
+                "torrent_archive_path": prefs.get("TORRENT_ARCHIVE_PATH"),
+                "torrent_client": prefs.get("TORRENT_CLIENT"),
+                "latency_ms": int((time.perf_counter() - t0) * 1000),
+            }
+            self._health_ok = True
+            self._health_payload = payload
+        except Exception as e:
+            self._health_ok = False
+            self._health_payload = {"error": str(e), "latency_ms": int((time.perf_counter() - t0) * 1000)}
+        self._health_at = now
+        return {"ok": self._health_ok, **self._health_payload}
+
+    async def setting(self) -> dict[str, Any]:
+        r = await self._post("/setting")
+        r.raise_for_status()
+        return r.json()
+
+    async def setenv(self, key: str, value: str) -> dict[str, Any]:
+        r = await self._post("/setenv", {"key": key, "value": value})
+        r.raise_for_status()
+        return r.json()
+
+    async def scan(self) -> dict[str, Any]:
+        # `path` field is required by the Pydantic model but ignored by the
+        # endpoint — it reads `app.state.scan_path` (set via PREFS__SCAN_PATH).
+        r = await self._post("/scan", {"path": "ignored"})
+        r.raise_for_status()
+        return r.json()
+
+    async def maketorrent(self, job_id: str) -> dict[str, Any] | None:
+        r = await self._post("/maketorrent", {"job_id": job_id})
+        r.raise_for_status()
+        return r.json() if r.text else None
+
+    async def upload(self, job_id: str) -> dict[str, Any] | None:
+        r = await self._post("/upload", {"job_id": job_id})
+        r.raise_for_status()
+        return r.json() if r.text else None
+
+    async def seed(self, job_id: str) -> tuple[int, dict[str, Any] | None]:
+        r = await self._post("/seed", {"job_id": job_id})
+        body: dict[str, Any] | None = None
+        if r.text:
+            try:
+                body = r.json()
+            except ValueError:
+                body = None
+        return r.status_code, body
+
+    async def processall(self, job_list_id: str) -> dict[str, Any] | None:
+        r = await self._post("/processall", {"job_list_id": job_list_id})
+        r.raise_for_status()
+        return r.json() if r.text else None
+
+    async def set_tmdbid(self, job_id: str, new_id: str) -> None:
+        r = await self._post("/settmdbid", {"job_id": job_id, "field_id": "tmdb_id", "new_id": str(new_id)})
+        r.raise_for_status()
+
+    async def set_tvdbid(self, job_id: str, new_id: str) -> None:
+        r = await self._post("/settvdbid", {"job_id": job_id, "field_id": "tvdb_id", "new_id": str(new_id)})
+        r.raise_for_status()
+
+    async def set_imdbid(self, job_id: str, new_id: str) -> None:
+        r = await self._post("/setimdbid", {"job_id": job_id, "field_id": "imdb_id", "new_id": str(new_id)})
+        r.raise_for_status()
+
+    async def set_poster_url(self, job_id: str, new_url: str) -> None:
+        r = await self._post("/setposterurl", {"job_id": job_id, "field_id": "backdrop_path", "new_id": new_url})
+        r.raise_for_status()
+
+    async def set_poster_dname(self, job_id: str, new_name: str) -> None:
+        r = await self._post("/setposterdname", {"job_id": job_id, "field_id": "display_name", "new_id": new_name})
+        r.raise_for_status()
+
+    async def filter_search(self, title: str) -> dict[str, Any]:
+        r = await self._post("/filter", {"title": title})
+        r.raise_for_status()
+        return r.json()
+
+    async def cjoblist(self, job_list_id: str) -> dict[str, Any] | None:
+        r = await self._post("/cjoblist", {"job_list_id": job_list_id})
+        r.raise_for_status()
+        return r.json() if r.text else None
+
+
+_singleton: WebupClient | None = None
+
+
+def get_client() -> WebupClient:
+    """Lazy singleton. Recreated when WEBUP_URL changes."""
+    global _singleton
+    target = base_url()
+    if _singleton is None or _singleton.base != target:
+        _singleton = WebupClient(target)
+    return _singleton
+
+
+async def shutdown_client() -> None:
+    global _singleton
+    if _singleton is not None:
+        await _singleton.aclose()
+        _singleton = None

unit3dprep/web/webup_job_fix.py

@@ -0,0 +1,231 @@
+"""Workarounds for Unit3DwebUp 0.0.25 job-state bugs.
+
+This module patches the Redis-backed Media record that webup builds in
+its `/scan` step, BEFORE we invoke `/maketorrent` and `/upload`. We do
+this because some webup logic mis-flags otherwise-uploadable media and
+the upstream fix lives on the wrong branch / unreleased.
+
+Currently it patches a single bug:
+
+  Audio-language gate (`tags_service.py:281`) leaves `media.can_upload`
+  permanently False if the FIRST audio track is not in the preferred
+  language. The check runs inside the per-track loop and only ever
+  flips the flag to False, never back to True — so a movie with [eng,
+  ita] tracks fails the gate even when `PREFERRED_LANG=it`. We detect
+  that case here (preferred language IS present in some track) and
+  force `can_upload = True` back.
+
+If/when webup ships a fix, this whole module can be removed.
+"""
+from __future__ import annotations
+
+import json
+import logging
+import re
+from typing import Any
+
+import redis.asyncio as aioredis
+
+log = logging.getLogger("unit3dprep.webup_job_fix")
+
+# Webup hardcodes Redis on localhost:6379 — see CLAUDE.md.
+_REDIS_URL = "redis://127.0.0.1:6379/0"
+
+# Season/episode token as written by the bridge's `build_name`: S01, S01E01,
+# S01E01-E12, S01E01-12, S01E01E12.
+_SEASON_TOKEN = re.compile(r"S\d{2}(?:E\d{2}(?:-?E?\d{2})?)?", re.IGNORECASE)
+
+
+def _season_label(*sources: str | None) -> str:
+    """First S##(E##) token from any bridge-built name, upper-cased."""
+    for src in sources:
+        if not src:
+            continue
+        m = _SEASON_TOKEN.search(src)
+        if m:
+            return m.group(0).upper()
+    return ""
+
+
+def _languages_for_track(track: dict[str, Any]) -> set[str]:
+    """All language codes mentioned by mediainfo for a single audio track."""
+    out: set[str] = set()
+    lang = track.get("language")
+    if isinstance(lang, str) and lang:
+        out.add(lang.strip().lower())
+    others = track.get("other_language")
+    if isinstance(others, (list, tuple)):
+        for v in others:
+            if isinstance(v, str) and v.strip():
+                out.add(v.strip().lower())
+    return out
+
+
+def _all_audio_languages(media: dict[str, Any]) -> set[str]:
+    out: set[str] = set()
+    mediafile = media.get("mediafile") or {}
+    tracks = mediafile.get("audio_tracks") or []
+    if not isinstance(tracks, list):
+        return out
+    for t in tracks:
+        if isinstance(t, dict):
+            out |= _languages_for_track(t)
+    return out
+
+
+def _preferred_present(preferred: str, langs: set[str]) -> bool:
+    """Is `preferred` (e.g. 'it') matched by any code in `langs`?
+
+    Mediainfo emits 2-letter ('it') and 3-letter ('ita') codes plus
+    plain-text names ('Italian'). We do a case-insensitive substring
+    match in either direction so 'it' matches 'ita', 'italian',
+    'italiano' — without false positives across unrelated codes
+    (we require an exact start-with).
+    """
+    p = (preferred or "").strip().lower()
+    if not p:
+        return False
+    for code in langs:
+        if code == p:
+            return True
+        if code.startswith(p) or p.startswith(code):
+            return True
+    return False
+
+
+async def maybe_force_can_upload(job_id: str, preferred_lang: str) -> dict[str, Any]:
+    """If webup wrongly set `can_upload=False`, force it back to True.
+
+    Returns a small dict describing what happened — useful for logs:
+    ``{"patched": bool, "reason": str, "preferred": str,
+       "languages": [list]}``.
+
+    Never raises: on any failure returns ``{"patched": False,
+    "reason": "<error>"}`` so the caller can keep the upload flow
+    going.
+    """
+    pref = (preferred_lang or "").strip().lower()
+    result: dict[str, Any] = {
+        "patched": False,
+        "preferred": pref,
+        "languages": [],
+        "reason": "",
+    }
+    if not pref:
+        result["reason"] = "no preferred lang set"
+        return result
+    try:
+        client = aioredis.from_url(_REDIS_URL, decode_responses=True)
+    except Exception as e:
+        result["reason"] = f"redis connect failed: {e!r}"
+        return result
+    try:
+        try:
+            raw = await client.hget(job_id, "data")
+        except Exception as e:
+            result["reason"] = f"redis hget failed: {e!r}"
+            return result
+        if not raw:
+            result["reason"] = "no job data in redis"
+            return result
+        try:
+            data = json.loads(raw)
+        except Exception as e:
+            result["reason"] = f"json parse failed: {e!r}"
+            return result
+        if data.get("can_upload"):
+            result["reason"] = "can_upload already true"
+            return result
+        langs = sorted(_all_audio_languages(data))
+        result["languages"] = langs
+        if not _preferred_present(pref, set(langs)):
+            result["reason"] = f"preferred '{pref}' not in audio tracks"
+            return result
+        data["can_upload"] = True
+        try:
+            await client.hset(job_id, mapping={"data": json.dumps(data)})
+        except Exception as e:
+            result["reason"] = f"redis hset failed: {e!r}"
+            return result
+        result["patched"] = True
+        result["reason"] = (
+            f"forced can_upload=true (preferred '{pref}' present in audio tracks {langs})"
+        )
+        return result
+    finally:
+        try:
+            await client.aclose()
+        except Exception:
+            pass
+
+
+async def maybe_inject_season(job_id: str) -> dict[str, Any]:
+    """Ensure webup's tracker ``display_name`` carries the season label.
+
+    Webup builds ``display_name`` from ``PREFS__TAG_POSITION_SERIE`` but reads
+    that setting from a module-global ``settings`` captured at import time
+    (``media.py`` / ``tags_service.py``), so a corrected tag order in the
+    ``.env`` only takes effect after a webup *restart*. To make the season
+    appear without restarting webup, patch the Redis job's ``display_name``
+    directly: take the ``S##(E##)`` token from the bridge-built folder/torrent
+    name and insert it right after the title.
+
+    Series only. Idempotent: skips if the label is already present. Never
+    raises — returns ``{"patched": bool, "reason": str, "display_name": str}``.
+    """
+    result: dict[str, Any] = {"patched": False, "reason": "", "display_name": ""}
+    try:
+        client = aioredis.from_url(_REDIS_URL, decode_responses=True)
+    except Exception as e:
+        result["reason"] = f"redis connect failed: {e!r}"
+        return result
+    try:
+        try:
+            raw = await client.hget(job_id, "data")
+        except Exception as e:
+            result["reason"] = f"redis hget failed: {e!r}"
+            return result
+        if not raw:
+            result["reason"] = "no job data in redis"
+            return result
+        try:
+            data = json.loads(raw)
+        except Exception as e:
+            result["reason"] = f"json parse failed: {e!r}"
+            return result
+        if data.get("category") != "series":
+            result["reason"] = "not a series"
+            return result
+        display = (data.get("display_name") or "").strip()
+        if not display:
+            result["reason"] = "no display_name"
+            return result
+        label = _season_label(
+            data.get("torrent_name"), data.get("title"), data.get("title_sanitized")
+        )
+        if not label:
+            result["reason"] = "no season token in source name"
+            return result
+        if re.search(rf"(?<![A-Za-z0-9]){re.escape(label)}(?![A-Za-z0-9])", display, re.IGNORECASE):
+            result["reason"] = f"season '{label}' already present"
+            return result
+        guess_title = (data.get("guess_title") or "").strip()
+        if guess_title and guess_title in display:
+            new_display = display.replace(guess_title, f"{guess_title} {label}", 1)
+        else:
+            new_display = f"{label} {display}"
+        data["display_name"] = new_display
+        try:
+            await client.hset(job_id, mapping={"data": json.dumps(data)})
+        except Exception as e:
+            result["reason"] = f"redis hset failed: {e!r}"
+            return result
+        result["patched"] = True
+        result["display_name"] = new_display
+        result["reason"] = f"inserted season '{label}' into display_name"
+        return result
+    finally:
+        try:
+            await client.aclose()
+        except Exception:
+            pass

unit3dprep/web/webup_logclass.py

@@ -0,0 +1,107 @@
+"""Classify WebSocket messages from Unit3DWebUp /ws into (kind, event) pairs.
+
+Replaces the regex-on-stdout classifier (`logclass.py`) used for the unit3dup
+CLI. Webup messages are already structured: `{type, level, message, job_id?}`.
+We map level → log kind and the message text to a stable event slug used by
+log filters/UI.
+"""
+from __future__ import annotations
+
+import re
+from typing import Any
+
+
+_LEVEL_TO_KIND = {
+    "success": "ok",
+    "ok": "ok",
+    "info": "info",
+    "warn": "warn",
+    "warning": "warn",
+    "error": "error",
+    "debug": "debug",
+}
+
+_RX_TORRENT_CREATED = re.compile(r"torrent.*(created|file exists)", re.I)
+_RX_UPLOAD_OK = re.compile(r"\b(uploaded|successful|success)\b", re.I)
+# Match: explicit fail words OR Laravel-style validation error dicts that
+# webup forwards verbatim (e.g. "{'episode_number': ['The episode number
+# field is required.']}").
+_RX_UPLOAD_FAIL = re.compile(
+    r"\b(fail|failure|error|exception|denied|invalid|required|rejected)\b",
+    re.I,
+)
+_RX_SEED_OK = re.compile(r"added to (qbittorrent|transmission|rtorrent)", re.I)
+_RX_SEED_FAIL = re.compile(r"(login failed|file not found)", re.I)
+_RX_SCAN_DONE = re.compile(r"scan completato|scan complete|scan done", re.I)
+_RX_TRACKER_ONLINE = re.compile(r"tracker.*online", re.I)
+_RX_QBIT = re.compile(r"qbittorrent|qbit\b", re.I)
+_RX_IMAGES = re.compile(r"(screenshot|image host|imgbb|passima|ptscreens)", re.I)
+_RX_TMDB = re.compile(r"\b(tmdb|tvdb|imdb)\b", re.I)
+
+
+def kind_for_level(level: str | None) -> str:
+    return _LEVEL_TO_KIND.get((level or "info").lower(), "info")
+
+
+def classify_msg(msg: dict[str, Any]) -> tuple[str, str | None, str]:
+    """
+    Returns (kind, event, text) for a webup WS message.
+
+    - kind: 'info'|'ok'|'warn'|'error'|'debug' for the log buffer.
+    - event: stable slug like 'upload.done', 'upload.maketorrent', 'scan.done', or None.
+    - text: the message string (already concatenated; use as the log line).
+    """
+    mtype = (msg.get("type") or "").lower()
+    level = msg.get("level")
+    text = str(msg.get("message") or "")
+    kind = kind_for_level(level)
+
+    if mtype == "posterlogmessage":
+        if _RX_UPLOAD_OK.search(text):
+            return ("ok", "upload.done", text)
+        if _RX_UPLOAD_FAIL.search(text):
+            return ("error", "upload.done", text)
+        if _RX_TORRENT_CREATED.search(text):
+            return (kind, "upload.maketorrent", text)
+        if _RX_SEED_OK.search(text):
+            return ("ok", "upload.qbit", text)
+        if _RX_SEED_FAIL.search(text):
+            return ("error", "upload.qbit", text)
+        return (kind, "upload.target", text)
+
+    if mtype == "log":
+        if _RX_SCAN_DONE.search(text):
+            return ("ok", "scan.done", text)
+        if _RX_TRACKER_ONLINE.search(text):
+            return ("ok", "tracker.online", text)
+        if _RX_QBIT.search(text):
+            return (kind, "upload.qbit", text)
+        if _RX_IMAGES.search(text):
+            return (kind, "upload.images", text)
+        if _RX_TMDB.search(text):
+            return (kind, "upload.tmdb", text)
+        return (kind, None, text)
+
+    return (kind, None, text)
+
+
+def is_terminal_success(msg: dict[str, Any]) -> bool:
+    """True if this message indicates the upload succeeded for its job_id."""
+    if (msg.get("type") or "").lower() != "posterlogmessage":
+        return False
+    text = str(msg.get("message") or "")
+    return bool(_RX_UPLOAD_OK.search(text))
+
+
+def is_terminal_failure(msg: dict[str, Any]) -> bool:
+    if (msg.get("level") or "").lower() == "error":
+        return True
+    if (msg.get("type") or "").lower() == "posterlogmessage":
+        text = str(msg.get("message") or "")
+        # "None" (string) means tracker rejected with no 'data' field returned
+        # by itt_tracker_service (response.get('data', None) when Unit3D API
+        # returns {'message':..., 'errors':{...}} without a 'data' key).
+        if not text or text == "None":
+            return True
+        return bool(_RX_UPLOAD_FAIL.search(text))
+    return False