theus-cli-linux-arm64 1.2.1 → 1.3.0

package/vendor/higiene-motor/clean_ctrlregen.py

@@ -0,0 +1,212 @@
+#!/usr/bin/env python3
+"""Optional CtrlRegen pixel-watermark remover backed by an external noai-watermark checkout.
+
+This script does NOT vendor upstream code. It imports ``CtrlRegenEngine`` from
+a user-provided checkout of ``mertizci/noai-watermark`` at runtime, using that
+environment's optional dependencies (torch, diffusers, transformers, etc.).
+
+Exit codes:
+  0  removed successfully
+  1  remover runtime error
+  2  bad input (missing/unreadable image, bad args)
+  3  remover unavailable (not configured / missing checkout / missing deps)
+"""
+
+from __future__ import annotations
+
+import argparse
+import io
+import json
+import os
+import sys
+from pathlib import Path
+
+SCRIPTS_DIR = Path(__file__).resolve().parent
+sys.path.insert(0, str(SCRIPTS_DIR))
+
+from common import cleaned_path, safe_write_bytes  # noqa: E402
+
+# Backend default guidance scale (CtrlRegenEngine.run() default). Kept
+# internal: strength is the user-facing knob, not the CFG scale.
+DEFAULT_GUIDANCE_SCALE = 2.0
+
+
+def resolve_upstream(raw: str | None) -> Path | None:
+    if not raw:
+        return None
+    upstream = Path(raw).expanduser().resolve()
+    if not upstream.is_dir():
+        return None
+    return upstream
+
+
+def resolve_device(raw: str | None) -> str:
+    """Resolve the ``auto`` device hint to a concrete torch device."""
+    if raw and raw != "auto":
+        return raw
+    try:
+        import torch
+
+        if torch.cuda.is_available():
+            return "cuda"
+        mps = getattr(torch.backends, "mps", None)
+        if mps is not None and mps.is_available():
+            return "mps"
+    except Exception:  # noqa: S110 - optional torch device detection
+        pass
+    return "cpu"
+
+
+def save_image_bytes(image, output: Path) -> bytes:
+    """Encode the regenerated image, honoring a JPEG output suffix."""
+    ext = output.suffix.lower()
+    buf = io.BytesIO()
+    if ext in (".jpg", ".jpeg", ".jpe", ".jfif"):
+        image = image.convert("RGB")
+        image.save(buf, format="JPEG", quality=95)
+    else:
+        image.save(buf, format="PNG")
+    return buf.getvalue()
+
+
+def _progress(message: str) -> None:
+    print(f"[ctrlregen] {message}", file=sys.stderr)
+
+
+def main() -> int:
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument("path", type=Path, help="Input image (PNG/JPEG/etc.)")
+    p.add_argument("-o", "--output", type=Path, help="Output path (default: *.ctrlregen.*)")
+    p.add_argument(
+        "--upstream-dir",
+        type=Path,
+        default=None,
+        help="noai-watermark checkout root (default: $NOAI_WATERMARK_DIR)",
+    )
+    p.add_argument(
+        "--strength",
+        type=float,
+        default=0.25,
+        help="Regeneration strength in (0, 1]; lower preserves more detail (default: 0.25)",
+    )
+    p.add_argument(
+        "--steps",
+        type=int,
+        default=50,
+        help="Diffusion inference steps (default: 50; effective steps ~= steps * strength)",
+    )
+    p.add_argument(
+        "--device",
+        type=str,
+        default="auto",
+        help="auto|cpu|cuda|mps (default: auto)",
+    )
+    p.add_argument("--seed", type=int, default=None, help="Optional RNG seed")
+    p.add_argument("--json", action="store_true", help="Emit JSON on stdout")
+    args = p.parse_args()
+
+    if not args.path.is_file():
+        print(f"not a file: {args.path}", file=sys.stderr)
+        return 2
+    if not 0 < args.strength <= 1:
+        print(f"strength must be in (0, 1]: {args.strength}", file=sys.stderr)
+        return 2
+    if args.steps < 1:
+        print(f"steps must be >= 1: {args.steps}", file=sys.stderr)
+        return 2
+
+    raw_upstream = args.upstream_dir or os.environ.get("NOAI_WATERMARK_DIR")
+    upstream = resolve_upstream(str(raw_upstream) if raw_upstream else None)
+    if upstream is None:
+        print(
+            "CtrlRegen not configured: set NOAI_WATERMARK_DIR or pass --upstream-dir",
+            file=sys.stderr,
+        )
+        return 3
+
+    src_dir = upstream / "src"
+    if not src_dir.is_dir():
+        print(f"upstream src dir not found: {src_dir}", file=sys.stderr)
+        return 3
+
+    sys.path.insert(0, str(src_dir))
+    try:
+        from ctrlregen.engine import CtrlRegenEngine, is_ctrlregen_available
+        from PIL import Image
+    except ImportError as e:
+        print(f"CtrlRegen dependencies missing: {e}", file=sys.stderr)
+        print("run setup_ctrlregen.sh first", file=sys.stderr)
+        return 3
+
+    if not is_ctrlregen_available():
+        print(
+            "CtrlRegen dependencies not installed; run setup_ctrlregen.sh first",
+            file=sys.stderr,
+        )
+        return 3
+
+    try:
+        image = Image.open(args.path).convert("RGB")
+        image.load()
+    except Exception as e:
+        print(f"could not load image: {e}", file=sys.stderr)
+        return 2
+
+    device = resolve_device(args.device)
+    output = args.output or cleaned_path(args.path, ".ctrlregen")
+
+    engine = CtrlRegenEngine(
+        base_model_id=None,
+        device=device,
+        torch_dtype=None,
+        hf_token=os.environ.get("HF_TOKEN"),
+        progress_callback=_progress,
+    )
+
+    try:
+        result = engine.run(
+            image,
+            strength=args.strength,
+            num_inference_steps=args.steps,
+            guidance_scale=DEFAULT_GUIDANCE_SCALE,
+            seed=args.seed,
+        )
+    except Exception as e:
+        print(f"CtrlRegen error: {e}", file=sys.stderr)
+        return 1
+
+    try:
+        data = save_image_bytes(result, output)
+        safe_write_bytes(output, data)
+    except (OSError, ValueError) as e:
+        print(f"cannot write output: {e}", file=sys.stderr)
+        return 1
+
+    payload = {
+        "available": True,
+        "upstream_dir": str(upstream),
+        "output": str(output),
+        "strength": args.strength,
+        "steps": args.steps,
+        "device": device,
+        "seed": args.seed,
+        "input_size": list(image.size),
+        "output_size": list(result.size),
+        "bytes_out": len(data),
+    }
+
+    if args.json:
+        json.dump(payload, sys.stdout, indent=2)
+        sys.stdout.write("\n")
+    else:
+        print(
+            f"CtrlRegen removed: {args.path} -> {output} "
+            f"({payload['input_size']} -> {payload['output_size']}, "
+            f"strength {args.strength}, device {device})"
+        )
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/vendor/higiene-motor/clean_file.py

@@ -0,0 +1,237 @@
+#!/usr/bin/env python3
+"""Unified clean: text Layer A, raster metadata, and document containers."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+from av_meta import clean_av
+from common import (
+    MAX_INPUT_BYTES,
+    ROUTER_ADVICE,
+    backup_path,
+    cleaned_path,
+    eprint,
+    guard_binary,
+    safe_write_text,
+)
+from container_meta import clean_container, detect_container_format
+from format_dispatch import classify
+from image_meta import clean_image
+from text_unicode import clean_text
+
+
+def main() -> int:
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument("path", type=Path)
+    p.add_argument("-o", "--output", type=Path)
+    p.add_argument("--in-place", action="store_true")
+    p.add_argument("--json", action="store_true")
+    p.add_argument("--nfkc", action="store_true", help="Text: NFKC normalize")
+    p.add_argument(
+        "--deep-images",
+        choices=("auto", "always", "lossless", "never"),
+        default="auto",
+        help=(
+            "PDF: Ghostscript re-distill that reaches metadata inside embedded "
+            "images. auto (default) only when AI/C2PA markers survive the "
+            "document strip; always also clears non-AI EXIF; lossless never "
+            "recompresses images; never skips the pass entirely"
+        ),
+    )
+    p.add_argument("--aggressive-homoglyphs", action="store_true")
+    p.add_argument(
+        "--keep-non-ai-metadata",
+        action="store_true",
+        help="Images/audio/video: only drop C2PA/AI-looking segments",
+    )
+    p.add_argument(
+        "--as",
+        dest="force_type",
+        choices=("auto", "text", "image", "container", "av"),
+        default="auto",
+    )
+    p.add_argument(
+        "--force-text",
+        action="store_true",
+        help="Clean as text even when the bytes look like a binary container",
+    )
+    args = p.parse_args()
+
+    if not args.path.is_file():
+        eprint(f"not a file: {args.path}")
+        return 2
+
+    if args.path.stat().st_size > MAX_INPUT_BYTES:
+        eprint(f"refusing input larger than {MAX_INPUT_BYTES} bytes: {args.path}")
+        return 2
+
+    kind = args.force_type if args.force_type != "auto" else classify(args.path)
+
+    # classify() reports "unknown" for bytes that match no supported format.
+    # Never mutate those in auto mode: a binary with valid UTF-8 runs would
+    # be decoded and written back mangled. --as text / --force-text are the
+    # explicit opt-ins and both route through the text pipeline below.
+    if kind == "unknown":
+        if args.force_type == "text" or args.force_text:
+            kind = "text"
+        else:
+            eprint(f"refusing to classify {args.path}: unrecognized format")
+            for line in ROUTER_ADVICE:
+                eprint(line)
+            return 2
+
+    # In-place cleaning reads from a .bak copy whose suffix would make
+    # markdown/HTML (detected by extension, not magic bytes) classify as
+    # "unknown". Pin the format from the original path so --in-place and -o
+    # route identically.
+    container_fmt = None
+    if kind == "container":
+        container_fmt = detect_container_format(args.path, args.path.read_bytes())
+
+    # classify() falls back to "text" for unrecognised bytes, so an unknown
+    # binary would otherwise be decoded, scrubbed and written back mangled.
+    # Sniff before --in-place takes a backup: refusing afterwards would leave a
+    # .bak sidecar behind for a file this run never touches.
+    raw = args.path.read_bytes() if kind == "text" else None
+    if raw is not None:
+        guard_binary(
+            raw,
+            str(args.path),
+            allow_binary=args.force_text,
+            advice=ROUTER_ADVICE,
+        )
+
+    bak = None
+    if args.in_place:
+        bak, created = backup_path(args.path)
+        if not created:
+            # PARCHE Theus (ver PARCHES.md, parche 4).
+            # Keeping the pristine .bak is right (#172), but stopping there is
+            # not: `src = bak` below would clean THAT file and rename it over
+            # the original, silently substituting its contents — and the
+            # bytes we are about to lose would be backed up nowhere. Take a
+            # free name instead: the pristine .bak survives untouched and the
+            # current contents get their own copy (.bak.2, .bak.3…).
+            eprint(f"backup {bak} already exists from an earlier run; keeping it")
+            bak, created = backup_path(args.path, unique=True)
+            if not created:
+                eprint(f"cannot back up {args.path}: too many backups beside it")
+                return 2
+            eprint(f"backed up the current contents to {bak} instead")
+        dest = args.path
+        src = bak
+    else:
+        src = args.path
+        dest = args.output or cleaned_path(args.path)
+
+    def finalize(result: dict) -> dict:
+        """Stamp the in-place truth onto a result dict before it is printed.
+
+        Two corrections. ``input``: in-place cleans FROM the backup copy, so
+        the sub-cleaners echo the sidecar's path; the caller asked about the
+        original. ``backup``: consumers cannot guess it — with a pre-existing
+        ``.bak`` the real one is ``.bak.2`` — so state it rather than let them
+        assume ``<path>.bak`` and promise a backup that isn't there.
+        """
+        if bak is not None:
+            result["input"] = str(args.path)
+            result["backup"] = str(bak)
+        return result
+
+    if kind == "text":
+        text = raw.decode("utf-8", errors="surrogateescape")
+        cleaned, stats = clean_text(
+            text,
+            nfkc=args.nfkc,
+            aggressive_homoglyphs=args.aggressive_homoglyphs,
+        )
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        safe_write_text(dest, cleaned)
+        result = finalize({
+            "kind": "text",
+            "input": str(args.path),
+            "output": str(dest),
+            "stats": stats,
+        })
+        if args.json:
+            print(json.dumps(result, indent=2, ensure_ascii=False))
+        else:
+            eprint(
+                f"wrote {dest} removed={stats['removed_count']} replaced={stats['replaced_count']}"
+            )
+        return 0
+
+    if kind == "image":
+        try:
+            result = clean_image(
+                src,
+                dest,
+                strip_all_metadata=not args.keep_non_ai_metadata,
+            )
+        except Exception as e:
+            eprint(f"error: {e}")
+            return 1
+        result = finalize({"kind": "image", **result})
+        residual = result["still_has_c2pa"] or result["still_has_ai_metadata"]
+        if args.json:
+            print(json.dumps(result, indent=2))
+        else:
+            eprint(f"wrote {result['output']} ({result['bytes_in']} -> {result['bytes_out']})")
+            for a in result["actions"]:
+                eprint(f"  - {a}")
+            if residual:
+                eprint("warning: residual C2PA/AI signals may remain")
+        return 1 if residual else 0
+
+    if kind == "av":
+        try:
+            result = clean_av(
+                src,
+                dest,
+                strip_all_metadata=not args.keep_non_ai_metadata,
+            )
+        except Exception as e:
+            eprint(f"error: {e}")
+            return 1
+        result = finalize({"kind": "av", **result})
+        residual = result["still_has_c2pa"] or result["still_has_ai_metadata"]
+        if args.json:
+            print(json.dumps(result, indent=2))
+        else:
+            eprint(f"wrote {result['output']} ({result['bytes_in']} -> {result['bytes_out']})")
+            for a in result["actions"]:
+                eprint(f"  - {a}")
+            if residual:
+                eprint("warning: residual C2PA/AI signals may remain")
+        return 1 if residual else 0
+
+    try:
+        result = clean_container(src, dest, fmt=container_fmt, deep_images=args.deep_images)
+    except Exception as e:
+        eprint(f"error: {e}")
+        return 1
+    result = finalize({"kind": "container", **result})
+    residual = result["still_has_c2pa"] or result["still_has_ai_metadata"]
+    degraded = bool(result.get("meta", {}).get("degraded"))
+    if args.json:
+        print(json.dumps(result, indent=2, ensure_ascii=False))
+    else:
+        eprint(f"wrote {result['output']} format={result['format']}")
+        for a in result["actions"]:
+            eprint(f"  - {a}")
+        if residual:
+            eprint("warning: residual C2PA/AI signals may remain")
+            for f in result.get("post_findings") or []:
+                eprint(f"  ! {f}")
+    # A degraded (best-effort) PDF copy warns but is not a hard failure.
+    return 1 if (residual and not degraded) else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/vendor/higiene-motor/clean_image.py

@@ -0,0 +1,218 @@
+#!/usr/bin/env python3
+"""Strip C2PA and AI-related metadata from raster images (PNG/JPEG/WebP/AVIF/HEIC/BMP/GIF/TIFF)."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+from common import backup_path, cleaned_path, eprint
+from image_meta import clean_image
+
+
+def main() -> int:
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument("path", type=Path, help="Input image (PNG/JPEG/WebP/AVIF/HEIC/BMP/GIF/TIFF)")
+    p.add_argument("-o", "--output", type=Path, help="Output path (default: *.cleaned.*)")
+    p.add_argument(
+        "--in-place",
+        action="store_true",
+        help="Overwrite input (writes .bak backup first)",
+    )
+    p.add_argument(
+        "--keep-non-ai-metadata",
+        action="store_true",
+        help="Only drop segments/chunks that look like C2PA/AI (less aggressive)",
+    )
+    p.add_argument("--json", action="store_true", help="JSON result on stdout")
+    p.add_argument(
+        "--synthid-dir",
+        type=str,
+        default=None,
+        help="reverse-SynthID checkout root for optional pixel SynthID scoring",
+    )
+    p.add_argument(
+        "--remove-pixel",
+        choices=["ctrlregen", "diffusion"],
+        default=None,
+        help="Run optional pixel-watermark removal after metadata cleaning "
+        "(ctrlregen = CtrlRegen regeneration; diffusion = MarkDiffusion "
+        "DiffusionPurification regeneration)",
+    )
+    p.add_argument(
+        "--ctrlregen-dir",
+        type=str,
+        default=None,
+        help="noai-watermark checkout root (default: $NOAI_WATERMARK_DIR)",
+    )
+    p.add_argument(
+        "--ctrlregen-strength",
+        type=float,
+        default=0.25,
+        help="CtrlRegen strength in (0, 1] (default: 0.25, conservative)",
+    )
+    p.add_argument(
+        "--ctrlregen-steps",
+        type=int,
+        default=50,
+        help="CtrlRegen diffusion steps (default: 50)",
+    )
+    p.add_argument(
+        "--ctrlregen-device",
+        type=str,
+        default=None,
+        help="CtrlRegen device: auto|cpu|cuda|mps (default: auto)",
+    )
+    p.add_argument(
+        "--ctrlregen-seed",
+        type=int,
+        default=None,
+        help="Optional CtrlRegen RNG seed",
+    )
+    p.add_argument(
+        "--ctrlregen-timeout",
+        type=int,
+        default=3600,
+        help="CtrlRegen subprocess timeout in seconds (default: 3600)",
+    )
+    p.add_argument(
+        "--markdiffusion-dir",
+        type=str,
+        default=None,
+        help="MarkDiffusion bootstrap dir (default: $MARKDIFFUSION_DIR)",
+    )
+    p.add_argument(
+        "--markdiffusion-strength",
+        type=float,
+        default=0.3,
+        help="DiffusionPurification strength in (0, 1] (default: 0.3)",
+    )
+    p.add_argument(
+        "--markdiffusion-model",
+        type=str,
+        default=None,
+        help="Stable Diffusion model for purification (default: SD 2.1 base)",
+    )
+    p.add_argument(
+        "--markdiffusion-size",
+        type=int,
+        default=512,
+        help="Purification working size in px (default: 512)",
+    )
+    p.add_argument(
+        "--markdiffusion-steps",
+        type=int,
+        default=50,
+        help="Purification diffusion steps (default: 50)",
+    )
+    p.add_argument(
+        "--markdiffusion-device",
+        type=str,
+        default=None,
+        help="Purification device: auto|cpu|cuda|mps (default: auto)",
+    )
+    p.add_argument(
+        "--markdiffusion-timeout",
+        type=int,
+        default=3600,
+        help="Purification subprocess timeout in seconds (default: 3600)",
+    )
+    args = p.parse_args()
+
+    if not args.path.is_file():
+        eprint(f"not a file: {args.path}")
+        return 2
+
+    src = args.path
+    if args.in_place:
+        # PARCHE Theus (ver PARCHES.md, parche 4).
+        # backup_path() returns (path, created). Unpacking it as a bare name
+        # bound the TUPLE to src. Same fallback as clean_file.py: a kept .bak
+        # is not a backup of what we are about to overwrite, and cleaning it
+        # into place would substitute the file's contents outright.
+        bak, created = backup_path(args.path)
+        if not created:
+            eprint(f"backup {bak} already exists from an earlier run; keeping it")
+            bak, created = backup_path(args.path, unique=True)
+            if not created:
+                eprint(f"cannot back up {args.path}: too many backups beside it")
+                return 2
+            eprint(f"backed up the current contents to {bak} instead")
+        src = bak
+        dest = args.path
+    else:
+        dest = args.output or cleaned_path(args.path)
+
+    try:
+        result = clean_image(
+            src,
+            dest,
+            strip_all_metadata=not args.keep_non_ai_metadata,
+            synthid_dir=args.synthid_dir,
+            remove_pixel=args.remove_pixel,
+            ctrlregen_dir=args.ctrlregen_dir,
+            ctrlregen_strength=args.ctrlregen_strength,
+            ctrlregen_steps=args.ctrlregen_steps,
+            ctrlregen_device=args.ctrlregen_device,
+            ctrlregen_seed=args.ctrlregen_seed,
+            ctrlregen_timeout=args.ctrlregen_timeout,
+            markdiffusion_dir=args.markdiffusion_dir,
+            markdiffusion_strength=args.markdiffusion_strength,
+            markdiffusion_model=args.markdiffusion_model,
+            markdiffusion_size=args.markdiffusion_size,
+            markdiffusion_steps=args.markdiffusion_steps,
+            markdiffusion_device=args.markdiffusion_device,
+            markdiffusion_timeout=args.markdiffusion_timeout,
+        )
+    except Exception as e:
+        eprint(f"error: {e}")
+        return 1
+
+    pr = result.get("pixel_removal")
+    residual = result["still_has_c2pa"] or result["still_has_ai_metadata"]
+    failed = residual or (pr is not None and not pr.get("available"))
+
+    if args.json:
+        print(json.dumps(result, indent=2))
+    else:
+        eprint(f"wrote {result['output']} ({result['bytes_in']} -> {result['bytes_out']})")
+        for a in result["actions"]:
+            eprint(f"  - {a}")
+        if result.get("synthid_before") and result["synthid_before"].get("available"):
+            label = "yes" if result["synthid_before"].get("is_watermarked") else "no"
+            eprint(
+                "SynthID before: "
+                f"confidence {result['synthid_before'].get('confidence', 0.0):.3f} "
+                f"(watermarked: {label})"
+            )
+        if result.get("synthid_after") and result["synthid_after"].get("available"):
+            label = "yes" if result["synthid_after"].get("is_watermarked") else "no"
+            eprint(
+                "SynthID after: "
+                f"confidence {result['synthid_after'].get('confidence', 0.0):.3f} "
+                f"(watermarked: {label})"
+            )
+        if pr is not None:
+            if pr.get("available"):
+                engine = (
+                    "CtrlRegen" if args.remove_pixel == "ctrlregen" else "DiffusionPurification"
+                )
+                eprint(f"{engine}: removed on {pr.get('device', 'unknown device')}")
+            else:
+                engine = (
+                    "CtrlRegen" if args.remove_pixel == "ctrlregen" else "DiffusionPurification"
+                )
+                eprint(f"{engine}: unavailable: {pr.get('error', 'unknown error')}")
+        if residual:
+            eprint("warning: residual C2PA/AI signals may remain")
+            for f in result.get("post_findings") or []:
+                eprint(f"  ! {f}")
+    return 1 if failed else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())