theus-cli-linux-x64 1.2.1 → 1.3.0

package/vendor/higiene-motor/synthid_score_server.py

@@ -0,0 +1,176 @@
+#!/usr/bin/env python3
+"""Tiny stdlib HTTP sidecar exposing the reverse-SynthID pixel scorer.
+
+Runs inside the local-only wr-synthid heavy image so the published core
+image never bundles the non-commercial reverse-SynthID code. The core
+service calls this sidecar for SynthID image scoring when
+WATERMARKS_SYNTHID_SCORER_URL is set (see compose.yaml / .env.example).
+
+Endpoints:
+    GET  /health  -> {"ok": true, "version": ...}
+    POST /score   -> {"file": <base64>} -> score_synthid payload
+
+Hardening mirrors server.py: optional bearer key, input size caps,
+unprivileged user, read-only rootfs with a /tmp tmpfs. Intended for the
+compose network or a trusted network only.
+"""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import binascii
+import json
+import os
+import sys
+import tempfile
+from http import HTTPStatus
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlparse
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+from score_synthid import score_file
+
+VERSION = os.environ.get("WATERMARKS_SYNTHID_SERVER_VERSION", "dev")
+
+# Mirror common.MAX_INPUT_BYTES (env-overridable) with the base64 envelope
+# headroom. Read at import; the sidecar image does not copy common.py, so the
+# default is repeated here.
+MAX_INPUT_BYTES = int(os.environ.get("WATERMARKS_MAX_INPUT_BYTES", str(256 << 20)))
+MAX_BODY_BYTES = MAX_INPUT_BYTES + (MAX_INPUT_BYTES >> 1)
+
+API_KEY = os.environ.get("WATERMARKS_SYNTHID_SCORER_API_KEY", "").strip()
+MODEL = os.environ.get("WATERMARKS_SYNTHID_MODEL", "").strip() or None
+
+
+def _json_ok(payload: dict[str, Any]) -> bytes:
+    return json.dumps(payload, ensure_ascii=False, indent=2).encode("utf-8")
+
+
+class Handler(BaseHTTPRequestHandler):
+    server_version = f"watermarks-remover-synthid/{VERSION}"
+
+    def log_message(self, fmt: str, *args: object) -> None:
+        print(f"{self.address_string()} - {fmt % args}", file=sys.stderr)
+
+    def _authorized(self) -> bool:
+        if not API_KEY:
+            return True
+        return self.headers.get("Authorization", "") == f"Bearer {API_KEY}"
+
+    def _read_json(self) -> dict[str, Any] | None:
+        raw = self.headers.get("Content-Length")
+        if raw is None or not raw.isdigit():
+            return None
+        length = int(raw)
+        if length > MAX_BODY_BYTES:
+            return None
+        try:
+            body = json.loads(self.rfile.read(length).decode("utf-8"))
+        except (UnicodeDecodeError, json.JSONDecodeError, OSError):
+            return None
+        return body if isinstance(body, dict) else None
+
+    def _respond(self, status: int, payload: dict[str, Any]) -> None:
+        data = _json_ok(payload)
+        self.send_response(status)
+        self.send_header("Content-Type", "application/json; charset=utf-8")
+        self.send_header("Content-Length", str(len(data)))
+        self.send_header("Cache-Control", "no-store")
+        self.end_headers()
+        self.wfile.write(data)
+
+    def do_GET(self) -> None:
+        if not self._authorized():
+            self._respond(HTTPStatus.UNAUTHORIZED, {"ok": False, "error": "unauthorized"})
+            return
+        if urlparse(self.path).path == "/health":
+            self._respond(HTTPStatus.OK, {"ok": True, "version": VERSION})
+        else:
+            self._respond(HTTPStatus.NOT_FOUND, {"ok": False, "error": "not found"})
+
+    def do_POST(self) -> None:
+        if not self._authorized():
+            self._respond(HTTPStatus.UNAUTHORIZED, {"ok": False, "error": "unauthorized"})
+            return
+        if urlparse(self.path).path != "/score":
+            self._respond(HTTPStatus.NOT_FOUND, {"ok": False, "error": "not found"})
+            return
+        body = self._read_json()
+        if body is None:
+            raw_len = self.headers.get("Content-Length")
+            oversized = raw_len is not None and raw_len.isdigit() and int(raw_len) > MAX_BODY_BYTES
+            self._respond(
+                HTTPStatus.REQUEST_ENTITY_TOO_LARGE if oversized else HTTPStatus.BAD_REQUEST,
+                {"ok": False, "error": "invalid request body"},
+            )
+            return
+
+        raw = body.get("file")
+        if not isinstance(raw, str):
+            self._respond(HTTPStatus.BAD_REQUEST, {"ok": False, "error": "missing 'file' field"})
+            return
+        try:
+            data = base64.b64decode(raw, validate=True)
+        except (binascii.Error, ValueError):
+            self._respond(
+                HTTPStatus.BAD_REQUEST, {"ok": False, "error": "'file' is not valid base64"}
+            )
+            return
+        if len(data) > MAX_INPUT_BYTES:
+            self._respond(
+                HTTPStatus.REQUEST_ENTITY_TOO_LARGE, {"ok": False, "error": "file too large"}
+            )
+            return
+
+        with tempfile.TemporaryDirectory(prefix="wm-synthid-") as tmp:
+            path = Path(tmp) / "input.png"
+            try:
+                path.write_bytes(data)
+            except OSError as e:
+                self._respond(HTTPStatus.INTERNAL_SERVER_ERROR, {"ok": False, "error": str(e)})
+                return
+            code, payload = score_file(path, model=MODEL)
+
+        if code == 0 and payload is not None:
+            self._respond(HTTPStatus.OK, payload)
+        elif code == 2:
+            self._respond(HTTPStatus.BAD_REQUEST, {"ok": False, "error": "could not load image"})
+        else:
+            # exit 1 (runtime error) or 3 (unavailable) -> fail-soft payload,
+            # matching the shape image_meta.run_synthid_score expects.
+            self._respond(
+                HTTPStatus.OK,
+                {"available": False, "error": "scorer unavailable (see sidecar stderr)"},
+            )
+
+
+def main() -> int:
+    global API_KEY  # noqa: PLW0603 — CLI overrides env
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument("--host", default=os.environ.get("WATERMARKS_SYNTHID_SERVER_HOST", "127.0.0.1"))
+    p.add_argument(
+        "--port", type=int, default=int(os.environ.get("WATERMARKS_SYNTHID_SERVER_PORT", "8766"))
+    )
+    p.add_argument("--api-key", default=API_KEY, help="require this bearer token (default: none)")
+    args = p.parse_args()
+
+    if args.host not in ("127.0.0.1", "localhost", "::1"):
+        print(
+            f"warning: binding {args.host} — intended for a trusted network only", file=sys.stderr
+        )
+    API_KEY = args.api_key
+    print(f"synthid scorer sidecar {VERSION} on http://{args.host}:{args.port}", file=sys.stderr)
+    server = ThreadingHTTPServer((args.host, args.port), Handler)
+    try:
+        server.serve_forever()
+    except KeyboardInterrupt:
+        server.shutdown()
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/vendor/higiene-motor/text_detectors.py

@@ -0,0 +1,433 @@
+#!/usr/bin/env python3
+"""Research text-watermark detectors behind one interface.
+
+Detects statistical (Layer B) text watermarks using research or vendor
+detectors. Every detector implements the same small protocol:
+
+    name: str                 stable identifier (surfaced in /capabilities)
+    available() -> bool       configured and usable right now
+    detect(text) -> dict      JSON-safe report; never raises
+
+Reports follow the fail-soft contract: a detector that is unconfigured,
+times out, or errors returns {"available": False, "error": ...} and can
+never block cleaning.
+
+Detectors:
+
+- markllm — research harness (KGW / SynthID schemes) via
+  detect_text_watermark.py, activated by MARKLLM_DIR. Same-config-only
+  detection; not a vendor oracle.
+- gumbel — model-free same-key replay of the keyed-Gumbel (Aaronson EXP)
+  scheme (detect_gumbel.py), activated by WATERMARKS_GUMBEL_KEY. Stdlib-only;
+  valid only against the same key, tokenizer, and PRF layout used at
+  generation (self-hosted engines such as arbi-serve); not a vendor oracle.
+- claude-text — placeholder for Anthropic's announced text-watermark
+  detection API. Reports unavailable until a public endpoint exists; the
+  interface it must implement is already defined here.
+
+Vendor note (Aug 2026): Google retired SynthID text watermarking on the
+Generative Language API — API text output is no longer watermarked and
+DETECT_TEXT_WATERMARK is rejected on current (3.x) models. The former
+gemini-synthid-text detector was removed for this reason; a vendor seam can
+be re-added if Google exposes detection again (e.g. via Vertex AI).
+"""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import os
+import subprocess
+import sys
+import tempfile
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any, Protocol
+
+from common import subprocess_creationflags
+from detect_gumbel import DEFAULT_THRESHOLD, DEFAULT_WINDOW, detect_text
+
+DEFAULT_MARKLLM_SCHEME = "kgw"
+DEFAULT_MARKLLM_TIMEOUT = 600.0
+
+
+class TextDetector(Protocol):
+    name: str
+
+    def available(self) -> bool: ...
+
+    def detect(self, text: str) -> dict[str, Any]: ...
+
+
+def _env_float(name: str, default: float) -> float:
+    try:
+        return float(os.environ.get(name, str(default)))
+    except ValueError:
+        return default
+
+
+def _worker_port() -> int | None:
+    """Loopback port of a resident MarkLLM serve worker (WATERMARKS_MARKLLM_PORT)."""
+    raw = os.environ.get("WATERMARKS_MARKLLM_PORT", "").strip()
+    if not raw:
+        return None
+    try:
+        port = int(raw)
+    except ValueError:
+        return None
+    return port if 0 < port < 65536 else None
+
+
+def _detect_via_worker(port: int, text: str, timeout: float) -> dict[str, Any]:
+    """One detect request to a resident MarkLLM serve worker over loopback TCP."""
+    import socket as _socket
+
+    with _socket.create_connection(("127.0.0.1", port), timeout=timeout) as conn:
+        conn.sendall((json.dumps({"op": "detect", "text": text}) + "\n").encode("utf-8"))
+        f = conn.makefile("r", encoding="utf-8")
+        line = f.readline()
+    if not line:
+        raise RuntimeError("worker closed without a response")
+    try:
+        resp = json.loads(line)
+    except json.JSONDecodeError as e:
+        raise RuntimeError(f"worker emitted non-JSON: {line[:120]!r}") from e
+    if not isinstance(resp, dict) or not resp.get("ok"):
+        raise RuntimeError(resp.get("error") or "worker detect failed")
+    return resp
+
+
+# ---------------------------------------------------------------------------
+# MarkLLM (open-source research harness: KGW / SynthID schemes)
+# ---------------------------------------------------------------------------
+
+
+def _venv_python(upstream: Path) -> Path | None:
+    """Prefer the MarkLLM checkout's venv interpreter, if it exists."""
+    if os.name == "nt":
+        candidate = upstream / ".venv" / "Scripts" / "python.exe"
+    else:
+        candidate = upstream / ".venv" / "bin" / "python"
+    return candidate if candidate.is_file() else None
+
+
+def _markllm_preexec() -> Callable[[], None] | None:
+    """Optional RLIMIT_AS guard for the MarkLLM child; None means "no limit".
+
+    torch/CUDA usually needs large address spaces, so this is opt-in via
+    WATERMARKS_MARKLLM_RLIMIT_AS (byte count, hex/octal allowed). POSIX only;
+    on Windows preexec_fn must stay None.
+    """
+    raw = os.environ.get("WATERMARKS_MARKLLM_RLIMIT_AS")
+    if not raw or os.name != "posix":
+        return None
+    try:
+        limit = int(raw, 0)
+    except ValueError:
+        return None
+
+    def _apply() -> None:
+        import resource
+
+        resource.setrlimit(resource.RLIMIT_AS, (limit, limit))
+
+    return _apply
+
+
+class MarkLLMTextDetector:
+    """Same-config-only research detection via detect_text_watermark.py.
+
+    Constructor overrides (scheme, upstream_dir, model, timeout) take
+    precedence over the environment, so callers such as rewrite_text.py can
+    keep CLI flags driving the harness. When the MarkLLM checkout has a
+    venv, its interpreter runs the child process; otherwise the current
+    interpreter is used (the service image bundles the harness deps).
+    """
+
+    name = "markllm"
+
+    def __init__(
+        self,
+        *,
+        scheme: str | None = None,
+        upstream_dir: str | None = None,
+        model: str | None = None,
+        timeout: float | None = None,
+    ) -> None:
+        self._scheme = scheme
+        self._upstream_dir = upstream_dir
+        self._model = model
+        self._timeout = timeout
+
+    def available(self) -> bool:
+        upstream = self._upstream_dir or os.environ.get("MARKLLM_DIR", "").strip()
+        return bool(upstream)
+
+    def detect(self, text: str) -> dict[str, Any]:
+        upstream = self._upstream_dir or os.environ.get("MARKLLM_DIR", "").strip()
+        scheme = (
+            self._scheme
+            or os.environ.get("WATERMARKS_MARKLLM_SCHEME", "")
+            or DEFAULT_MARKLLM_SCHEME
+        )
+        report: dict[str, Any] = {
+            "detector": self.name,
+            "scheme": scheme,
+            "vendor": "open-llm",
+            "available": False,
+        }
+        if not upstream:
+            report["error"] = "MARKLLM_DIR not set"
+            return report
+
+        timeout = (
+            self._timeout
+            if self._timeout is not None
+            else _env_float("WATERMARKS_MARKLLM_TIMEOUT", DEFAULT_MARKLLM_TIMEOUT)
+        )
+
+        # Reuse a resident serve worker (WATERMARKS_MARKLLM_PORT) when one is
+        # up — avoids a ~20s torch+model cold start per detect. Falls back to
+        # a one-shot subprocess if the worker is unreachable.
+        port = _worker_port()
+        if port is not None:
+            try:
+                resp = _detect_via_worker(port, text, timeout)
+                return {
+                    **report,
+                    "available": True,
+                    "is_watermarked": bool(resp["is_watermarked"]),
+                    "score": resp.get("score"),
+                    "threshold": resp.get("threshold"),
+                    "note": "detected via resident MarkLLM serve worker",
+                }
+            except Exception as e:
+                report["error"] = f"MarkLLM worker detect failed ({e}); falling back"
+
+        script = Path(__file__).resolve().parent / "detect_text_watermark.py"
+        venv_python = _venv_python(Path(upstream).expanduser().resolve())
+        python = str(venv_python) if venv_python is not None else sys.executable
+
+        with tempfile.NamedTemporaryFile("w", suffix=".txt", encoding="utf-8", delete=False) as f:
+            f.write(text)
+            tmp = f.name
+
+        cmd = [python, str(script), "detect", tmp, "--scheme", scheme, "--json"]
+        if self._model:
+            cmd += ["--model", self._model]
+        if self._upstream_dir:
+            cmd += ["--upstream-dir", str(Path(upstream).expanduser().resolve())]
+
+        try:
+            try:
+                r = subprocess.run(
+                    cmd,
+                    capture_output=True,
+                    text=True,
+                    timeout=timeout,
+                    preexec_fn=_markllm_preexec(),
+                    check=False,
+                    creationflags=subprocess_creationflags,
+                )
+            except subprocess.TimeoutExpired:
+                report["error"] = "MarkLLM detection timed out"
+                return report
+            if r.returncode == 3:
+                report["error"] = (r.stderr or "").strip()[:400] or "MarkLLM unavailable"
+                return report
+            if r.returncode != 0:
+                report["error"] = (r.stderr or "").strip()[:400] or f"MarkLLM exit {r.returncode}"
+                return report
+            try:
+                payload = json.loads(r.stdout or "{}")
+            except json.JSONDecodeError as e:
+                report["error"] = f"bad MarkLLM JSON: {e}"
+                return report
+        finally:
+            with contextlib.suppress(OSError):
+                Path(tmp).unlink()
+
+        if not isinstance(payload, dict):
+            report["error"] = "bad MarkLLM response"
+            return report
+        payload["available"] = True
+        payload["detector"] = self.name
+        payload["note"] = (
+            "MarkLLM is a research harness: detection is only valid against the "
+            "same scheme config and keys used at generation; not a vendor detector."
+        )
+        return payload
+
+
+# ---------------------------------------------------------------------------
+# Keyed-Gumbel (Aaronson EXP) — model-free same-key replay
+# ---------------------------------------------------------------------------
+
+
+class GumbelTextDetector:
+    """Same-key replay of the keyed-Gumbel (Aaronson EXP) text watermark.
+
+    Model-free: implements the detection arithmetic of the ARBI keyed-Gumbel
+    technical report (Sections 2-3) — replay u = PRF(Hash(key, window), token)
+    from the text alone and test the Gamma tail — so it needs no GPU, model,
+    or logits. Detection is valid only against the SAME key, tokenizer, and
+    PRF layout used at generation (self-hosted engines such as arbi-serve);
+    it is not a vendor oracle. Key from WATERMARKS_GUMBEL_KEY (env) or the
+    constructor override.
+    """
+
+    name = "gumbel"
+    vendor = "self-hosted"
+
+    def __init__(
+        self,
+        *,
+        key: str | None = None,
+        window: int | None = None,
+        threshold: float | None = None,
+    ) -> None:
+        self._key = key
+        self._window = window
+        self._threshold = threshold
+
+    def _key_env(self) -> str | None:
+        if self._key:
+            return self._key
+        return os.environ.get("WATERMARKS_GUMBEL_KEY", "").strip() or None
+
+    def available(self) -> bool:
+        return self._key_env() is not None
+
+    def detect(self, text: str) -> dict[str, Any]:
+        key = self._key_env()
+        report: dict[str, Any] = {
+            "detector": self.name,
+            "scheme": "exp",
+            "vendor": self.vendor,
+            "available": False,
+        }
+        if key is None:
+            report["error"] = "WATERMARKS_GUMBEL_KEY not set"
+            return report
+        try:
+            payload = detect_text(
+                text,
+                key,
+                window=self._window or DEFAULT_WINDOW,
+                threshold=self._threshold or DEFAULT_THRESHOLD,
+            )
+        except Exception as e:  # fail-soft contract: never raise
+            report["error"] = f"keyed-Gumbel detection failed: {e}"
+            return report
+        payload["detector"] = self.name
+        payload["note"] = (
+            "same-key replay of the keyed-Gumbel (Aaronson EXP) watermark: valid "
+            "only with the same key, tokenizer, and PRF layout used at generation; "
+            "not a vendor detector."
+        )
+        return payload
+
+
+# ---------------------------------------------------------------------------
+# Claude (Anthropic) — announced detector API, not yet public
+# ---------------------------------------------------------------------------
+
+
+class ClaudeTextDetector:
+    """Placeholder for Anthropic's announced text-watermark detection API.
+
+    Anthropic has announced a watermark detection API for Claude-generated
+    text; no public endpoint exists yet. When it ships, set
+    WATERMARKS_CLAUDE_API_KEY, flip available() to check it, and fill in
+    detect() against the documented endpoint.
+    """
+
+    name = "claude-text"
+    vendor = "anthropic"
+
+    def available(self) -> bool:
+        return False
+
+    def detect(self, text: str) -> dict[str, Any]:
+        return {
+            "detector": self.name,
+            "vendor": self.vendor,
+            "available": False,
+            "error": (
+                "Anthropic has announced a text-watermark detection API for "
+                "Claude; no public endpoint is available yet. When it ships, "
+                "set WATERMARKS_CLAUDE_API_KEY and implement ClaudeTextDetector."
+            ),
+        }
+
+
+# ---------------------------------------------------------------------------
+# Registry
+# ---------------------------------------------------------------------------
+
+
+def all_detectors(
+    markllm: MarkLLMTextDetector | None = None,
+    *,
+    include_markllm: bool = True,
+    gumbel: GumbelTextDetector | None = None,
+    include_gumbel: bool = True,
+) -> list[TextDetector]:
+    detectors: list[TextDetector] = []
+    if include_markllm:
+        detectors.append(markllm or MarkLLMTextDetector())
+    if include_gumbel:
+        detectors.append(gumbel or GumbelTextDetector())
+    detectors.append(ClaudeTextDetector())
+    return detectors
+
+
+def detector_status() -> dict[str, bool]:
+    """Configured/usable status per detector (for /capabilities)."""
+    return {d.name: d.available() for d in all_detectors()}
+
+
+def run_all_text_detectors(
+    text: str,
+    *,
+    markllm: MarkLLMTextDetector | None = None,
+    include_markllm: bool = True,
+    gumbel: GumbelTextDetector | None = None,
+    include_gumbel: bool = True,
+) -> list[dict[str, Any]]:
+    """Run every detector (including unavailable ones, with reasons).
+
+    markllm injects a caller-parameterized MarkLLM detector (e.g. one
+    driven by rewrite_text.py CLI flags); pass include_markllm=False to
+    exclude the MarkLLM harness entirely. Same for gumbel.
+    """
+    return [
+        d.detect(text)
+        for d in all_detectors(
+            markllm,
+            include_markllm=include_markllm,
+            gumbel=gumbel,
+            include_gumbel=include_gumbel,
+        )
+    ]
+
+
+def run_text_detectors(
+    text: str,
+    *,
+    markllm: MarkLLMTextDetector | None = None,
+    include_markllm: bool = True,
+    gumbel: GumbelTextDetector | None = None,
+    include_gumbel: bool = True,
+) -> list[dict[str, Any]]:
+    """Run only the detectors that are configured and usable."""
+    return [
+        d.detect(text)
+        for d in all_detectors(
+            markllm,
+            include_markllm=include_markllm,
+            gumbel=gumbel,
+            include_gumbel=include_gumbel,
+        )
+        if d.available()
+    ]