kd-engine 1.0.0__py3-none-any.whl

ytk/cache/__init__.py

@@ -0,0 +1,34 @@
+"""v2 layered cache (T12.1).
+
+The legacy transcript cache (:mod:`ytk.ingestion.transcript_cache`) keeps its
+own v1 namespace and continues to *not* cache ASR/MiniMax output.  This package
+is the independent v2 layer cache; the two never share files.
+
+Public API::
+
+    from ytk.cache import LayeredCache
+
+    cache = LayeredCache()
+    cache.put("asr", {"source_id": "...", "material_digest": "...",
+                      "language": "zh", "model": "large-v3", "params": {"beam": 5}},
+              payload_bytes)
+    payload = cache.get("asr", {...})           # None on MISS, never raises
+"""
+
+from __future__ import annotations
+
+from ytk.cache.v2 import (
+    CACHE_SCHEMA_VERSION,
+    DEFAULT_MAX_BYTES,
+    NAMESPACES,
+    LayeredCache,
+    sanitize_key_parts,
+)
+
+__all__ = [
+    "CACHE_SCHEMA_VERSION",
+    "DEFAULT_MAX_BYTES",
+    "NAMESPACES",
+    "LayeredCache",
+    "sanitize_key_parts",
+]

ytk/cache/v2.py

@@ -0,0 +1,562 @@
+"""T12.1 — v2 layered cache (media / subtitle / asr / ocr / generation).
+
+Design rules frozen by ``docs/implementation/five-platform/contracts-v2.md`` §1.5:
+
+* the fingerprint is computed from **business identity only**: source id,
+  material digest, language, model/version, parameters and (for generation)
+  the prompt hash.  A signed/expiring URL is transport, never identity — every
+  URL-ish value or query token is stripped before hashing, so rotating a
+  ``xsec_token`` can never fork the cache;
+* the five namespaces are isolated: the same key parts in ``subtitle`` and
+  ``asr`` produce different fingerprints and live in different directories;
+* a corrupt entry (bad JSON, wrong digest, truncated payload) is a MISS and is
+  removed — the cache must never make extraction less reliable than it is;
+* an unwritable cache root degrades to a no-op (log + ``None``), it never
+  raises into the extraction path;
+* capacity is bounded with LRU eviction by last access time
+  (``max_bytes`` default ``YTK_CACHE_MAX_BYTES`` or 2 GiB);
+* the v1 :mod:`ytk.ingestion.transcript_cache` is untouched and uses different
+  file names/dirs, so v2 can never overwrite a v1 entry.
+
+The on-disk entry is a single self-validating file: a one-line JSON header
+(schema, namespace, fingerprint, payload sha256/size) followed by the raw
+payload bytes.  Writes go to a same-directory temp file and are ``os.replace``-d
+into place, so a concurrent reader sees either the previous entry or the new
+one, never a half-written file.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import os
+import tempfile
+import time
+from collections.abc import Iterable, Mapping
+from pathlib import Path
+from typing import Any
+from urllib.parse import parse_qsl, urlsplit, urlunsplit
+
+LOGGER = logging.getLogger(__name__)
+
+#: Bump when the fingerprint recipe or envelope changes (invalidates entries).
+CACHE_SCHEMA_VERSION = 2
+
+#: The five cache layers.  Anything else is a programming error.
+NAMESPACES: frozenset[str] = frozenset({"media", "subtitle", "asr", "ocr", "generation"})
+
+#: Default capacity.  ``YTK_CACHE_MAX_BYTES`` overrides it; invalid values are
+#: ignored (never crash the CLI over a bad env var).
+DEFAULT_MAX_BYTES = 2 * 1024 * 1024 * 1024
+MAX_BYTES_ENV_VAR = "YTK_CACHE_MAX_BYTES"
+CACHE_DIR_ENV_VAR = "YTK_V2_CACHE_DIR"
+CACHE_DIR_ENV_VAR_FALLBACK = "YTK_CACHE_DIR"
+
+_ENTRY_SUFFIX = ".entry"
+_TMP_SUFFIX = ".tmp"
+_MAGIC = b"YTK-CACHE-V2\n"
+
+#: Query parameters that express access rights / expiry, never business identity.
+_SENSITIVE_QUERY_PARAMS: frozenset[str] = frozenset(
+    {
+        "xsec_token",
+        "xsec_source",
+        "xsec_app_secuboid",
+        "mstoken",
+        "mstoken_v2",
+        "share_token",
+        "share_source",
+        "sign",
+        "signature",
+        "expires",
+        "expire",
+        "expiration",
+        "token",
+        "access_token",
+        "refresh_token",
+        "sessionid",
+        "session_id",
+        "sid",
+        "cookie",
+        "auth_key",
+        "authkey",
+        "w_rid",
+        "wts",
+        "sig",
+        "key",
+        "hdnts",
+        "policy",
+        "playauth",
+        "nsec_token",
+        "appkey",
+        "api_key",
+        "apikey",
+    }
+)
+
+#: Key names that carry transport/credentials rather than identity.  Values for
+#: these names are kept only after URL sanitisation; secret-like ones are
+#: dropped outright so they can never influence the fingerprint.
+_URL_KEY_NAMES: frozenset[str] = frozenset(
+    {
+        "url",
+        "uri",
+        "href",
+        "fetch_url",
+        "download_url",
+        "media_url",
+        "signed_url",
+        "source_url",
+        "canonical_url",
+        "request_url",
+        "subtitle_url",
+        "thumbnail_url",
+    }
+)
+_SECRET_KEY_NAMES: frozenset[str] = frozenset(
+    {
+        "cookie",
+        "cookies",
+        "cookie_header",
+        "authorization",
+        "auth",
+        "token",
+        "access_token",
+        "refresh_token",
+        "api_key",
+        "apikey",
+        "secret",
+        "password",
+        "proxy",
+        "headers",
+        "signed_url",
+    }
+)
+
+
+# --------------------------------------------------------------------------- #
+# Key sanitisation / canonicalisation
+# --------------------------------------------------------------------------- #
+def _looks_like_http_url(value: object) -> bool:
+    return isinstance(value, str) and value.strip().lower().startswith(("http://", "https://"))
+
+
+def _strip_sensitive_query(url: str) -> str:
+    """Remove access/expiry query params and embedded credentials from *url*."""
+    try:
+        parts = urlsplit(url.strip())
+    except ValueError:
+        return url
+    if parts.scheme not in {"http", "https"} or not parts.hostname:
+        return url
+    try:
+        port = parts.port
+    except ValueError:
+        port = None
+    host = parts.hostname.lower()
+    netloc = host if port in (None, 80, 443) else f"{host}:{port}"
+    kept = [
+        (name, value)
+        for name, value in parse_qsl(parts.query, keep_blank_values=True)
+        if name.lower() not in _SENSITIVE_QUERY_PARAMS and name not in _SENSITIVE_QUERY_PARAMS
+    ]
+    try:
+        return urlunsplit((parts.scheme, netloc, parts.path or "/", _encode_query(kept), ""))
+    except ValueError:
+        return url
+
+
+def _encode_query(pairs: Iterable[tuple[str, str]]) -> str:
+    from urllib.parse import urlencode
+
+    return urlencode(list(pairs))
+
+
+def _sanitize_value(key: str | None, value: object) -> object:
+    """Sanitise one value before it can reach the fingerprint.
+
+    * dict/list/tuple/set are walked recursively;
+    * secret-named keys are dropped by :func:`sanitize_key_parts` before this;
+    * every http(s) URL — or any string that parses as one — loses its
+      access/expiry query parameters;
+    * ``bytes``/``Path`` are normalised to digest/string.
+    """
+    if isinstance(value, str):
+        if _looks_like_http_url(value):
+            return _strip_sensitive_query(value)
+        return value
+    if isinstance(value, bytes | bytearray | memoryview):
+        return {"sha256": hashlib.sha256(bytes(value)).hexdigest()}
+    if isinstance(value, Path):
+        return str(value)
+    if isinstance(value, Mapping):
+        return {
+            str(key2): _sanitize_value(None if key2 is None else str(key2), value2)
+            for key2, value2 in value.items()
+        }
+    if isinstance(value, (list, tuple)):
+        return [_sanitize_value(key, item) for item in value]
+    if isinstance(value, (set, frozenset)):
+        normalised = [_sanitize_value(key, item) for item in value]
+        return sorted(normalised, key=lambda item: _canonical_json(item))
+    if isinstance(value, float) and value.is_integer():
+        return int(value)
+    if isinstance(value, (int, float, bool)) or value is None:
+        return value
+    return str(value)
+
+
+def sanitize_key_parts(key_parts: Mapping[str, object] | None) -> dict[str, object]:
+    """Return a credential-free, canonical view of *key_parts*.
+
+    - secret-like key names are dropped;
+    - values of URL-like key names are stripped of access/expiry parameters;
+    - every http(s) URL value is stripped, whatever the key;
+    - non-JSON values (bytes/Path/set) are normalised deterministically.
+    """
+    if key_parts is None:
+        return {}
+    if not isinstance(key_parts, Mapping):
+        raise TypeError("key_parts must be a mapping")
+    clean: dict[str, object] = {}
+    for raw_key, raw_value in key_parts.items():
+        key = str(raw_key)
+        lowered = key.lower()
+        if lowered in _SECRET_KEY_NAMES:
+            continue
+        if lowered in _URL_KEY_NAMES and raw_value is None:
+            continue
+        clean[key] = _sanitize_value(key, raw_value)
+    return clean
+
+
+def _canonical_json(payload: object) -> str:
+    return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
+
+
+# --------------------------------------------------------------------------- #
+# Defaults
+# --------------------------------------------------------------------------- #
+def _default_root() -> Path:
+    override = os.environ.get(CACHE_DIR_ENV_VAR) or os.environ.get(CACHE_DIR_ENV_VAR_FALLBACK)
+    if override:
+        return Path(override).expanduser()
+    xdg = os.environ.get("XDG_CACHE_HOME")
+    if xdg:
+        return Path(xdg).expanduser() / "ytk" / "v2"
+    return Path.home() / ".cache" / "ytk" / "v2"
+
+
+def _max_bytes_from_env() -> int:
+    raw = os.environ.get(MAX_BYTES_ENV_VAR)
+    if raw in (None, ""):
+        return DEFAULT_MAX_BYTES
+    try:
+        value = int(raw)
+    except (TypeError, ValueError):
+        LOGGER.warning("ignoring invalid %s=%r (using default capacity)", MAX_BYTES_ENV_VAR, raw)
+        return DEFAULT_MAX_BYTES
+    if value < 0:
+        LOGGER.warning("ignoring negative %s=%r (using default capacity)", MAX_BYTES_ENV_VAR, raw)
+        return DEFAULT_MAX_BYTES
+    return value
+
+
+# --------------------------------------------------------------------------- #
+# Cache
+# --------------------------------------------------------------------------- #
+class LayeredCache:
+    """File-per-fingerprint layered cache with atomic writes and LRU capacity."""
+
+    def __init__(
+        self,
+        root: Path | str | None = None,
+        enabled: bool = True,
+        *,
+        max_bytes: int | None = None,
+    ) -> None:
+        self._root = Path(root).expanduser() if root is not None else _default_root()
+        self._enabled = bool(enabled)
+        self._max_bytes = _max_bytes_from_env() if max_bytes is None else int(max_bytes)
+
+    # -------------------------------------------------------------- properties
+    @property
+    def root(self) -> Path:
+        return self._root
+
+    @property
+    def enabled(self) -> bool:
+        return self._enabled
+
+    @property
+    def max_bytes(self) -> int:
+        return self._max_bytes
+
+    # ------------------------------------------------------------- fingerprint
+    def fingerprint(self, namespace: str, key_parts: Mapping[str, object] | None) -> str:
+        """Stable business fingerprint (``sha256`` hex) for one cache layer."""
+        self._validate_namespace(namespace)
+        clean = sanitize_key_parts(key_parts)
+        payload = {
+            "schema": CACHE_SCHEMA_VERSION,
+            "namespace": namespace,
+            "parts": clean,
+        }
+        return hashlib.sha256(_canonical_json(payload).encode("utf-8")).hexdigest()
+
+    # ------------------------------------------------------------------- get
+    def get(self, namespace: str, key_parts: Mapping[str, object] | None) -> bytes | None:
+        """Return cached bytes, or ``None`` for a MISS / corrupt / unreadable entry."""
+        if not self._enabled:
+            return None
+        try:
+            fingerprint = self.fingerprint(namespace, key_parts)
+        except (TypeError, ValueError):  # bad caller input is a MISS, never a crash
+            LOGGER.warning("cache get with invalid key parts (namespace=%s)", namespace)
+            return None
+        path = self._entry_path(namespace, fingerprint)
+        try:
+            raw = path.read_bytes()
+        except FileNotFoundError:
+            return None
+        except OSError as exc:
+            LOGGER.warning("cache read failed for %s: %s", path, exc)
+            return None
+        payload = self._decode_entry(path, raw, namespace=namespace, fingerprint=fingerprint)
+        if payload is None:
+            return None
+        self._touch(path)
+        return payload
+
+    # ------------------------------------------------------------------- put
+    def put(self, namespace: str, key_parts: Mapping[str, object] | None, payload: bytes) -> None:
+        """Store *payload* under the fingerprint; failures degrade to a no-op."""
+        if not self._enabled:
+            return
+        self._validate_namespace(namespace)
+        if not isinstance(payload, (bytes, bytearray, memoryview)):
+            raise TypeError("cache payload must be bytes")
+        data = bytes(payload)
+        try:
+            fingerprint = self.fingerprint(namespace, key_parts)
+        except (TypeError, ValueError):
+            LOGGER.warning("cache put with invalid key parts (namespace=%s)", namespace)
+            return
+        path = self._entry_path(namespace, fingerprint)
+        envelope = self._encode_entry(
+            namespace=namespace, fingerprint=fingerprint, data=data
+        )
+        try:
+            path.parent.mkdir(parents=True, exist_ok=True)
+            fd, tmp_name = tempfile.mkstemp(
+                dir=path.parent, prefix=f".{fingerprint[:16]}.", suffix=_TMP_SUFFIX
+            )
+            try:
+                with os.fdopen(fd, "wb") as handle:
+                    handle.write(envelope)
+                    handle.flush()
+                os.replace(tmp_name, path)
+            except BaseException:
+                try:
+                    os.unlink(tmp_name)
+                except OSError:
+                    pass
+                raise
+        except OSError as exc:
+            # Unwritable cache must never break extraction: log + drop the write.
+            LOGGER.warning("cache write failed for %s (ignored): %s", path, exc)
+            return
+        self._evict_if_needed()
+
+    # ----------------------------------------------------------------- stats
+    def stats(self) -> dict[str, Any]:
+        """Best-effort introspection for ``ytk cache status``; never raises."""
+        per_namespace: dict[str, dict[str, int]] = {
+            namespace: {"entries": 0, "bytes": 0} for namespace in sorted(NAMESPACES)
+        }
+        total_bytes = 0
+        total_entries = 0
+        try:
+            entries = list(self._iter_entries())
+        except OSError:
+            entries = []
+        for namespace, path in entries:
+            try:
+                size = path.stat().st_size
+            except OSError:
+                continue
+            bucket = per_namespace.setdefault(namespace, {"entries": 0, "bytes": 0})
+            bucket["entries"] += 1
+            bucket["bytes"] += size
+            total_entries += 1
+            total_bytes += size
+        return {
+            "root": str(self._root),
+            "enabled": self._enabled,
+            "entries": total_entries,
+            "total_bytes": total_bytes,
+            "max_bytes": self._max_bytes,
+            "namespaces": per_namespace,
+        }
+
+    # ----------------------------------------------------------------- clear
+    def clear(self, namespace: str | None = None) -> int:
+        """Remove entries (and crashed-write temp files); return how many went."""
+        if namespace is not None:
+            self._validate_namespace(namespace)
+        removed = 0
+        try:
+            if namespace is None:
+                paths = [path for _, path in self._iter_entries()]
+                strays = list(self._root.glob(f"*/*{_TMP_SUFFIX}")) if self._root.exists() else []
+            else:
+                ns_dir = self._root / namespace
+                paths = list(ns_dir.glob(f"*{_ENTRY_SUFFIX}")) if ns_dir.exists() else []
+                strays = list(ns_dir.glob(f"*{_TMP_SUFFIX}")) if ns_dir.exists() else []
+        except OSError:
+            return 0
+        for path in paths:
+            try:
+                path.unlink()
+                removed += 1
+            except OSError:
+                LOGGER.warning("cache: failed to remove %s", path)
+        for stray in strays:
+            try:
+                stray.unlink()
+            except OSError:
+                pass
+        return removed
+
+    # --------------------------------------------------------------- helpers
+    def _validate_namespace(self, namespace: str) -> None:
+        if namespace not in NAMESPACES:
+            raise ValueError(
+                f"unknown cache namespace {namespace!r}; expected one of: "
+                f"{', '.join(sorted(NAMESPACES))}"
+            )
+
+    def _entry_path(self, namespace: str, fingerprint: str) -> Path:
+        return self._root / namespace / f"{fingerprint}{_ENTRY_SUFFIX}"
+
+    def _iter_entries(self) -> list[tuple[str, Path]]:
+        found: list[tuple[str, Path]] = []
+        if not self._root.exists():
+            return found
+        for namespace in sorted(NAMESPACES):
+            directory = self._root / namespace
+            if not directory.is_dir():
+                continue
+            for path in directory.glob(f"*{_ENTRY_SUFFIX}"):
+                found.append((namespace, path))
+        return found
+
+    @staticmethod
+    def _encode_entry(*, namespace: str, fingerprint: str, data: bytes) -> bytes:
+        header = {
+            "schema": CACHE_SCHEMA_VERSION,
+            "namespace": namespace,
+            "fingerprint": fingerprint,
+            "sha256": hashlib.sha256(data).hexdigest(),
+            "size": len(data),
+            "created_at": int(time.time()),
+        }
+        return _MAGIC + _canonical_json(header).encode("utf-8") + b"\n" + data
+
+    def _decode_entry(
+        self, path: Path, raw: bytes, *, namespace: str, fingerprint: str
+    ) -> bytes | None:
+        def corrupt(reason: str) -> None:
+            LOGGER.warning("cache entry %s is corrupt (%s): treating as miss", path, reason)
+            try:
+                path.unlink()
+            except OSError:
+                pass
+
+        if not raw.startswith(_MAGIC):
+            corrupt("bad magic")
+            return None
+        body = raw[len(_MAGIC) :]
+        header_raw, separator, data = body.partition(b"\n")
+        if not separator:
+            corrupt("missing header separator")
+            return None
+        try:
+            header = json.loads(header_raw.decode("utf-8"))
+        except (UnicodeDecodeError, json.JSONDecodeError):
+            corrupt("unreadable header")
+            return None
+        if not isinstance(header, dict):
+            corrupt("header is not an object")
+            return None
+        if header.get("schema") != CACHE_SCHEMA_VERSION:
+            corrupt(f"schema {header.get('schema')!r} != {CACHE_SCHEMA_VERSION}")
+            return None
+        if header.get("namespace") != namespace:
+            corrupt("namespace mismatch")
+            return None
+        if header.get("fingerprint") != fingerprint:
+            corrupt("fingerprint mismatch")
+            return None
+        try:
+            expected_size = int(header.get("size", -1))
+        except (TypeError, ValueError):
+            corrupt("invalid size")
+            return None
+        if expected_size != len(data):
+            corrupt(f"size mismatch ({expected_size} != {len(data)})")
+            return None
+        expected_digest = header.get("sha256")
+        if not isinstance(expected_digest, str) or hashlib.sha256(data).hexdigest() != expected_digest:
+            corrupt("payload digest mismatch")
+            return None
+        return data
+
+    @staticmethod
+    def _touch(path: Path) -> None:
+        """Mark an entry as recently used (LRU bookkeeping); best-effort."""
+        try:
+            os.utime(path, None)
+        except OSError:
+            pass
+
+    def _evict_if_needed(self) -> None:
+        """Bound the cache by total bytes, evicting least-recently-used entries."""
+        if self._max_bytes < 0:
+            return
+        try:
+            entries = self._iter_entries()
+        except OSError:
+            return
+        sized: list[tuple[float, int, Path]] = []
+        total = 0
+        for _, path in entries:
+            try:
+                stat = path.stat()
+            except OSError:
+                continue
+            sized.append((stat.st_mtime, stat.st_size, path))
+            total += stat.st_size
+        if total <= self._max_bytes:
+            return
+        sized.sort(key=lambda item: item[0])  # oldest access first
+        for _, size, path in sized:
+            if total <= self._max_bytes:
+                break
+            try:
+                path.unlink()
+                total -= size
+            except OSError:
+                continue
+
+
+__all__ = [
+    "CACHE_DIR_ENV_VAR",
+    "CACHE_SCHEMA_VERSION",
+    "DEFAULT_MAX_BYTES",
+    "MAX_BYTES_ENV_VAR",
+    "NAMESPACES",
+    "LayeredCache",
+    "sanitize_key_parts",
+]

ytk/cleaning/density.py

@@ -0,0 +1,40 @@
+import re
+
+TECHNICAL_TERMS = {
+    "api",
+    "async",
+    "await",
+    "class",
+    "cli",
+    "container",
+    "coroutine",
+    "def",
+    "docker",
+    "function",
+    "http",
+    "json",
+    "method",
+    "module",
+    "python",
+    "sql",
+}
+
+PATHISH_PATTERN = re.compile(r"[A-Za-z0-9]+(?:[/_-][A-Za-z0-9.]+)+")
+TOKEN_PATTERN = re.compile(r"[A-Za-z0-9]+")
+FLAG_PATTERNS = (
+    re.compile(r"`[^`]+`"),
+    re.compile(r"\w+\(\)"),
+    re.compile(r"\w+::\w+"),
+)
+
+
+def technical_density_score(text: str) -> float:
+    lowered = text.lower()
+    tokens = {token.lower() for token in TOKEN_PATTERN.findall(lowered)}
+    score = 0.0
+    score += float(len(tokens & TECHNICAL_TERMS))
+    score += 0.5 * len(PATHISH_PATTERN.findall(text))
+    for pattern in FLAG_PATTERNS:
+        if pattern.search(text):
+            score += 0.5
+    return score

ytk/cleaning/fillers.py

@@ -0,0 +1,18 @@
+import re
+
+FILLER_PREFIX_PATTERN = re.compile(
+    r"^\s*(?:so basically|let'?s go ahead and|you know|"
+    r"okay so|alright so|right so|"
+    r"um+|uh+|ah+|"
+    r"like basically|i mean|you see|"
+    r"basically|actually|literally|honestly|"
+    r"now let'?s|so let'?s|"
+    r"and then|so then|"
+    r"right,?\s*so|okay,?\s*so|alright,?\s*so)"
+    r"(?:,\s*|\s+)",
+    re.IGNORECASE,
+)
+
+
+def strip_fillers(text: str) -> str:
+    return FILLER_PREFIX_PATTERN.sub("", text, count=1)

ytk/cleaning/sentence_boundary.py

@@ -0,0 +1,30 @@
+from typing import Protocol
+
+from ytk.cleaning.fillers import strip_fillers
+from ytk.models.transcript import TranscriptSegment
+from ytk.text_normalization import normalize_known_term_noise
+
+
+class SentenceBoundaryBackend(Protocol):
+    def clean_segment(self, segment: TranscriptSegment) -> TranscriptSegment:
+        """Return a cleaned transcript segment while preserving evidence mapping."""
+
+
+class RuleBasedSentenceBoundaryBackend:
+    def normalize_text(self, text: str) -> str:
+        cleaned_text = normalize_known_term_noise(strip_fillers(text)).strip()
+        if cleaned_text:
+            cleaned_text = cleaned_text[0].upper() + cleaned_text[1:]
+        if cleaned_text and cleaned_text[-1] not in ".!?。！？":
+            if any("\u4e00" <= char <= "\u9fff" for char in cleaned_text):
+                cleaned_text = f"{cleaned_text}。"
+            else:
+                cleaned_text = f"{cleaned_text}."
+        return cleaned_text
+
+    def clean_segment(self, segment: TranscriptSegment) -> TranscriptSegment:
+        return segment.model_copy(
+            update={
+                "cleaned_text": self.normalize_text(segment.text),
+            }
+        )