theus-cli-linux-x64 1.2.2 → 1.4.0

package/vendor/higiene-motor/clean_staged.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+"""Strip AI/C2PA provenance marks from given files in place.
+
+Designed to run as a pre-commit hook: the pre-commit framework invokes this
+with the staged files as positional arguments. It wraps clean_file.py
+--in-place as a subprocess per file — no cleaning logic is duplicated here,
+this is purely a batch-over-staged-files wrapper.
+
+Rewriting a file a developer just staged is a real behavior change, not a
+lint-and-continue: following the standard pre-commit auto-fixer convention
+(same as e.g. black / ruff --fix hooks), this exits 1 whenever it modified
+at least one file, so the hook framework stops the commit and the developer
+reviews the diff and re-stages. Exit 0 means every file was already clean.
+
+A file clean_file.py could not process at all exits 3 (EXIT_PARTIAL) — the
+code audit_dir.py / audit_website.py already use for an incomplete run — and
+it outranks the auto-fix 1: "this file was never cleaned" is a different
+instruction than "your files were rewritten, re-stage them". Folding the two
+into one "skipped" bucket is what let a cleaner that crashed, was killed, or
+died before writing its report exit 0 and carry an uncleaned file into the
+commit, with its traceback invisible outside pre-commit's verbose mode.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+from common import EXIT_PARTIAL, eprint, subprocess_creationflags
+
+CLEAN_FILE_PY = Path(__file__).resolve().parent / "clean_file.py"
+
+
+def _changed(result: dict) -> bool:
+    stats = result.get("stats")
+    if stats is not None:
+        return bool(stats.get("removed_count") or stats.get("replaced_count"))
+    return bool(result.get("actions"))
+
+
+def _failure_detail(proc: subprocess.CompletedProcess[str], summary: str) -> str:
+    """Reduce clean_file.py's stderr to the one line worth putting in the report.
+
+    An uncaught exception arrives as a multi-line traceback whose last line
+    names the actual cause ("OSError: refusing to write through symlink: …");
+    a killed child leaves no stderr at all, so fall back to naming the exit
+    status. Echoing the whole traceback under a path prefix, as this used to,
+    buries the reason in stack frames the developer has to read backwards.
+    """
+    said = [line.strip() for line in proc.stderr.splitlines() if line.strip()]
+    if said:
+        return f"{summary}: {said[-1]}"
+    return f"{summary} (exit {proc.returncode})"
+
+
+def _clean_one(path: Path) -> tuple[str, str]:
+    """Returns ('changed' | 'unchanged' | 'skipped' | 'failed', detail).
+
+    *detail* explains a 'failed' status and is empty for every other status.
+    """
+    proc = subprocess.run(
+        [sys.executable, str(CLEAN_FILE_PY), str(path), "--in-place", "--json"],
+        capture_output=True,
+        text=True,
+        check=False,
+        creationflags=subprocess_creationflags,
+    )
+    if proc.returncode == 2:
+        # Unrecognized format or oversized input; clean_file.py already
+        # explained why on stderr, this is a non-fatal skip for the hook.
+        return "skipped", ""
+    # Every other outcome is judged on the JSON report, not on the exit code:
+    # clean_file.py exits 1 for a *successful* clean that left residual
+    # signals behind (pinned by tests/test_json_exit_code.py), so a non-zero
+    # code is not by itself a failure. A run that produced no parsable report
+    # never reached its write, though — that file is still marked.
+    if not proc.stdout.strip():
+        return "failed", _failure_detail(proc, "clean_file.py wrote no report")
+    try:
+        result = json.loads(proc.stdout)
+    except json.JSONDecodeError:
+        return "failed", _failure_detail(proc, "clean_file.py wrote an unparsable report")
+    status = "changed" if _changed(result) else "unchanged"
+    return status, ""
+
+
+def main() -> int:
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument(
+        "paths", nargs="+", type=Path, help="Files to clean in place (e.g. staged files)"
+    )
+    args = p.parse_args()
+
+    changed_paths: list[Path] = []
+    failures: list[tuple[Path, str]] = []
+    for path in args.paths:
+        if not path.is_file():
+            eprint(f"not a file: {path}")
+            continue
+        status, detail = _clean_one(path)
+        if status == "changed":
+            changed_paths.append(path)
+        elif status == "failed":
+            failures.append((path, detail))
+
+    if changed_paths:
+        eprint(f"watermarks-remover: cleaned {len(changed_paths)} file(s) in place:")
+        for path in changed_paths:
+            eprint(f"  {path}")
+        eprint("Review the changes and re-stage before committing.")
+
+    if failures:
+        eprint(f"watermarks-remover: {len(failures)} file(s) could not be cleaned:")
+        for path, detail in failures:
+            eprint(f"  {path}")
+            eprint(f"    - {detail}")
+            # clean_file.py --in-place takes its backup before it writes, so a
+            # run that died mid-write leaves the sidecar behind. Name it rather
+            # than delete it: this wrapper cannot tell its own leftover from a
+            # .bak the developer already had, and deleting the wrong file
+            # inside an already-failing hook is the worse outcome. The original
+            # is intact either way — safe_write_bytes() replaces atomically or
+            # not at all — so the sidecar is redundant, not load-bearing.
+            bak = path.with_suffix(path.suffix + ".bak")
+            if bak.exists():
+                eprint(f"    - backup left behind: {bak}")
+        eprint("Those files still carry whatever marks they had. Re-run the cleaner by hand:")
+        eprint("  python3 service/scripts/clean_file.py <path> --in-place")
+
+    # A file that could not be processed outranks the auto-fix signal, the same
+    # precedence audit_dir.py applies: an incomplete run is the more important
+    # thing for the developer (and CI) to see, and exit 1 would send them
+    # looking for a diff to re-stage that does not exist.
+    return EXIT_PARTIAL if failures else (1 if changed_paths else 0)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/vendor/higiene-motor/clean_text.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+"""Strip invisible Unicode / normalize space homoglyphs (Layer A)."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+from common import (
+    backup_path,
+    cleaned_path,
+    eprint,
+    read_text_input,
+    write_text_output,
+)
+from text_unicode import clean_text
+
+
+def main() -> int:
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument("path", nargs="?", default="-", help="Input text file, or - for stdin")
+    p.add_argument("-o", "--output", help="Output path (default: stdout or *.cleaned.*)")
+    p.add_argument("--nfkc", action="store_true", help="Apply Unicode NFKC after scrub")
+    p.add_argument(
+        "--aggressive-homoglyphs",
+        action="store_true",
+        help="Map Cyrillic/fullwidth Latin confusables to ASCII Latin",
+    )
+    p.add_argument(
+        "--no-normalize-spaces",
+        action="store_true",
+        help="Do not rewrite exotic spaces to U+0020",
+    )
+    p.add_argument(
+        "--strip-emoji-glue",
+        action="store_true",
+        help="Paranoid: strip all load-bearing invisibles too (emoji glue, script joiners, flag tags, same-script fillers/selectors, orthographic Cf)",
+    )
+    p.add_argument(
+        "--strip-bidi",
+        action="store_true",
+        help="Also strip legitimate RTL/LTR directional marks and isolates",
+    )
+    p.add_argument("--stats", action="store_true", help="Print stats JSON to stderr")
+    p.add_argument(
+        "--force-text",
+        action="store_true",
+        help="Clean even when the input looks like a binary container "
+        "(this rewrites the bytes and will corrupt the file)",
+    )
+    p.add_argument(
+        "--in-place",
+        action="store_true",
+        help="Overwrite input file (creates .bak backup)",
+    )
+    args = p.parse_args()
+
+    text = read_text_input(args.path, allow_binary=args.force_text)
+    cleaned, stats = clean_text(
+        text,
+        nfkc=args.nfkc,
+        aggressive_homoglyphs=args.aggressive_homoglyphs,
+        normalize_spaces=not args.no_normalize_spaces,
+        strip_emoji_glue=args.strip_emoji_glue,
+        strip_bidi=args.strip_bidi,
+    )
+
+    out = args.output
+    if args.in_place:
+        if args.path in (None, "-"):
+            eprint("--in-place requires a file path")
+            return 2
+        src = Path(args.path)
+        # PARCHE Theus (ver PARCHES.md, parche 4).
+        # Side effect: keep a .bak of the original. An earlier run's .bak is
+        # preserved, not overwritten (#172) — but then it holds SOMEONE ELSE'S
+        # bytes, so the text we are about to replace would go unbacked. Fall
+        # back to a free name (.bak.2…) so both copies survive.
+        bak, created = backup_path(src)
+        if not created:
+            eprint(f"backup {bak} already exists from an earlier run; keeping it")
+            bak, created = backup_path(src, unique=True)
+            if not created:
+                eprint(f"cannot back up {src}: too many backups beside it")
+                return 2
+            eprint(f"backed up the current contents to {bak} instead")
+        out = str(src)
+    elif out is None and args.path not in (None, "-"):
+        out = str(cleaned_path(Path(args.path)))
+
+    write_text_output(cleaned, out)
+
+    if args.stats:
+        eprint(json.dumps(stats, indent=2, ensure_ascii=False))
+    else:
+        eprint(
+            f"removed={stats['removed_count']} replaced={stats['replaced_count']} "
+            f"len {stats['input_length']}->{stats['output_length']}"
+        )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())