loop-engineer 0.7.0__py3-none-any.whl

loop/_bundle/tools/anticheat_scan.py

@@ -0,0 +1,490 @@
+"""Anti-cheat trajectory scan — sweep a Succeeded claim for shortcut signatures.
+
+After a loop claims ``Succeeded``, this scans the diff and (optionally) the run
+trajectory for the ways an agent games a verifier instead of solving the task:
+
+  * gate-tampering     — the diff edits a gate script (self_eval, the gates
+                         themselves, CI config). CRITICAL -> FailedSafety.
+  * skip-injection     — added @pytest.mark.skip/xfail, *.skip(, t.Skip(, @Ignore.
+  * assert-true        — added a tautological assertion that always passes.
+  * hidden-answer-read  — the trajectory touched a holdout / answer-key / golden /
+                         oracle file. (HIGH -> FailedUnverifiable.)
+  * test-file-mutation — a test file changed. MEDIUM: a *review flag*, not an
+                         auto-fail (legitimate in TDD); surfaced, never silent.
+
+Calibration: only HIGH and CRITICAL findings auto-downgrade the verdict. MEDIUM
+findings are surfaced for review so an honest TDD loop is not punished for
+writing tests. The path/line signatures are conservative and configurable.
+
+This ships as composable tooling, not a runtime. Pair it with ``holdout_gate.py``.
+
+Run::
+
+    git diff | python3 anticheat_scan.py
+    python3 anticheat_scan.py --diff changes.diff --trajectory trace.json
+
+``--trajectory`` is a JSON array of strings (tool calls / paths the loop touched).
+Prints findings as JSON. The exit code splits the scan's *outcome* from an
+*operational* failure so a broken invocation is never read as a clean scan or as
+findings:
+
+  * 0 — clean: the scan ran and found nothing.
+  * 1 — findings: a downgrade to FailedUnverifiable, or review-only findings.
+  * 2 — gate tampering: a critical finding -> FailedSafety.
+  * 3 — operational error: the scan could not run at all (unreadable ``--diff`` /
+        ``--trajectory`` file, malformed ``--trajectory`` JSON, failed
+        ``--self-check`` git call, or a bad argument).
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import pathlib
+import re
+import subprocess
+import sys
+
+# Exit codes. Findings (1, 2) and the scan's severity live apart from an
+# operational failure (3), so a broken invocation never masquerades as a clean
+# scan (0) or as a downgrade finding (1).
+EXIT_CLEAN = 0
+EXIT_FINDINGS = 1
+EXIT_GATE_TAMPERING = 2
+EXIT_OPERATIONAL_ERROR = 3
+
+# Path markers for a gate file. A bare name (e.g. "self_eval.py") matches by
+# BASENAME / path-segment, never by raw substring — so a test file whose name
+# merely CONTAINS a gate name (test_anticheat_scan.py) is not mis-upgraded.
+# A trailing "/" entry (".github/workflows/") matches as a directory prefix.
+DEFAULT_GATE_PATHS = (
+    "self_eval.py",
+    "validate_frontmatter.py",
+    "holdout_gate.py",
+    "anticheat_scan.py",
+    "conftest.py",
+    ".github/workflows/",
+)
+
+# The scanner's own source set — excluded from gate-tampering self-flagging so a
+# diff that corrects the scanner is not graded as tampering against itself.
+_SELF_FILES = frozenset({"anticheat_scan.py", "test_anticheat_scan.py"})
+
+
+# Contract-template gate scripts: verify-fast / verify-full / verify-safety,
+# matched by basename prefix (with or without a .sh suffix).
+_VERIFY_GATE_ROOTS = frozenset({"fast", "full", "safety"})
+
+
+def _basename(path: str) -> str:
+    return path.rstrip("/").rsplit("/", 1)[-1]
+
+
+def _is_verify_gate(path: str) -> bool:
+    base = _basename(path)
+    if not base.startswith("verify-"):
+        return False
+    root = base[len("verify-"):]
+    if root.endswith(".sh"):
+        root = root[: -len(".sh")]
+    return root in _VERIFY_GATE_ROOTS
+
+
+def _is_gate_path(path: str, gate_paths: tuple[str, ...]) -> bool:
+    """Whether ``path`` is a gate file.
+
+    A directory-style entry (ends in ``/``) matches as a path prefix/segment.
+    A filename entry matches only by exact basename, never by substring, so a
+    file that merely contains a gate name in its own filename is not upgraded.
+    """
+    if _basename(path) in _SELF_FILES:
+        return False
+    if _is_verify_gate(path):
+        return True
+    segments = path.split("/")
+    for g in gate_paths:
+        if g.endswith("/"):
+            if g.rstrip("/") in segments or g in path:
+                return True
+        elif _basename(path) == g:
+            return True
+    return False
+
+# A changed file that is a test (any common language convention).
+_TEST_PATH_RE = re.compile(
+    r"(^|/)(tests?|spec|__tests__)/"
+    r"|(^|/)test_[^/]+$"
+    r"|_test\.[a-z]+$"
+    r"|\.test\.[a-z]+$"
+    r"|\.spec\.[a-z]+$"
+    r"|_spec\.rb$"
+)
+
+# Answer-key / holdout files no honest solve needs to read. PATH-SHAPED tokens
+# only (never bare prose words) so 'assert result == expected' does not flag.
+_ANSWER_PATH_RE = re.compile(
+    r"hold[_-]?out|held-out|answer[_/-]?key|/golden/|\.golden\b"
+    r"|/expected/|expected\.(json|txt|csv|yaml|out)\b|oracle|\.secret",
+    re.I,
+)
+
+# Signatures on ADDED diff lines (lines starting with '+', not the +++ header).
+_ADDED_LINE_SIGNATURES = (
+    ("skip-injection", "high", re.compile(
+        r"^\+(?!\+\+).*(@pytest\.mark\.(skip|xfail)"
+        r"|pytest\.skip\("
+        r"|\b(it|test|describe)\.skip\("
+        r"|@Ignore\b"
+        r"|\bt\.Skip\()"
+    )),
+    ("assert-true", "high", re.compile(
+        r"^\+(?!\+\+).*\bexpect\(\s*true\s*\)",
+        re.IGNORECASE,
+    )),
+    # Tautologies that always pass: `assert True`, identical numeric/identifier
+    # operands (`assert 1 == 1`, `assert x == x`), and `assertTrue(True)`.
+    # Backreference-pinned so only IDENTICAL operands match — `assert a == b`
+    # and `assert result == expected` are real assertions and stay clean.
+    ("assert-true", "high", re.compile(
+        r"^\+(?!\+\+).*\bassert\s+(?:True\b"
+        r"|(?P<lit>-?\d+(?:\.\d+)?)\s*==\s*(?P=lit)\b"
+        r"|(?P<id>[A-Za-z_]\w*)\s*==\s*(?P=id)\b)"
+    )),
+    ("assert-true", "high", re.compile(
+        r"^\+(?!\+\+).*\bassertTrue\(\s*True\s*\)"
+    )),
+    ("hardcoded-expected", "medium", re.compile(
+        r"^\+(?!\+\+).*#\s*(expected|hardcode|hack|cheat|to pass)", re.IGNORECASE
+    )),
+)
+
+_PLUS_HDR_RE = re.compile(r"^\+\+\+ (b/(?P<b>.+)|/dev/null)$")
+_MINUS_HDR_RE = re.compile(r"^--- (a/(?P<a>.+)|/dev/null)$")
+_RENAME_FROM_RE = re.compile(r"^rename from (?P<p>.+)$")
+_RENAME_TO_RE = re.compile(r"^rename to (?P<p>.+)$")
+
+_RANK = {"critical": 3, "high": 2, "medium": 1, "low": 0}
+
+
+def _iter_hunks(diff_text: str):
+    """Yield ``(file_path, added_line)`` for every added ('+') diff line.
+
+    ``file_path`` is the path the hunk's ``+++ b/<p>`` header names (or, for a
+    deletion to ``/dev/null``, the ``--- a/<p>`` path). Used to attribute added
+    lines to their file so per-file exemptions can apply.
+    """
+    current = None
+    pending_minus = None
+    for line in diff_text.splitlines():
+        m = _MINUS_HDR_RE.match(line)
+        if m:
+            pending_minus = m.group("a")
+            continue
+        m = _PLUS_HDR_RE.match(line)
+        if m:
+            current = m.group("b") or pending_minus
+            pending_minus = None
+            continue
+        if line.startswith("+") and not line.startswith("+++"):
+            yield current, line
+
+
+def parse_changed_files(diff_text: str) -> list[str]:
+    """Extract changed file paths from a unified diff.
+
+    Captures modified/added files (``+++ b/<p>``), DELETED files (``--- a/<p>``
+    paired with ``+++ /dev/null``), and RENAMED files (``rename from/to``), so a
+    gate file that is removed or moved away is still surfaced. Order-preserving,
+    deduplicated.
+    """
+    paths: list[str] = []
+    pending_minus = None
+    for line in diff_text.splitlines():
+        m = _RENAME_FROM_RE.match(line) or _RENAME_TO_RE.match(line)
+        if m:
+            paths.append(m.group("p"))
+            continue
+        m = _MINUS_HDR_RE.match(line)
+        if m:
+            pending_minus = m.group("a")
+            continue
+        m = _PLUS_HDR_RE.match(line)
+        if m:
+            b = m.group("b")
+            paths.append(b if b is not None else pending_minus)
+            pending_minus = None
+    return list(dict.fromkeys(p for p in paths if p))
+
+
+# Self-edit that empties a gate collection outright (Shape A).
+_EVISCERATE_RE = re.compile(
+    r"^\+\s*(DEFAULT_GATE_PATHS|_ADDED_LINE_SIGNATURES|_SELF_FILES|_ANSWER_PATH_RE)"
+    r"\s*=\s*(\(\s*\)|frozenset\(\)|tuple\(\)|set\(\))\s*$"
+)
+
+# A collection-entry line: a bare string literal (optionally trailing comma),
+# i.e. a member of one of the gate collections. Used to compare net add vs
+# remove (Shape B semantic shrink).
+_COLLECTION_ENTRY_RE = re.compile(r"""^\s*['"][^'"]*['"]\s*,?\s*$""")
+
+
+def _gate_evisceration_findings(diff_text: str) -> list[dict]:
+    """Detect a self-edit that REMOVES/empties gate collections (P1.1).
+
+    Shape A: an added line assigns a gate collection to an empty literal.
+    Shape B: within ``anticheat_scan.py`` hunks, more string-literal entries are
+    removed than added (net shrink). Reorder/comment-only (net-zero or net-add)
+    stays clean.
+    """
+    findings: list[dict] = []
+    removed = added = 0
+    in_self = False
+    pending_minus = None
+    for line in diff_text.splitlines():
+        m = _MINUS_HDR_RE.match(line)
+        if m:
+            pending_minus = m.group("a")
+            continue
+        m = _PLUS_HDR_RE.match(line)
+        if m:
+            target = m.group("b") or pending_minus
+            in_self = bool(target) and _basename(target) == "anticheat_scan.py"
+            pending_minus = None
+            continue
+        if not in_self:
+            continue
+        if line.startswith("+") and not line.startswith("+++"):
+            if _EVISCERATE_RE.match(line):
+                findings.append(
+                    {"signature": "gate-evisceration", "severity": "critical",
+                     "evidence": line.strip()[:120]}
+                )
+            if _COLLECTION_ENTRY_RE.match(line[1:]):
+                added += 1
+        elif line.startswith("-") and not line.startswith("---"):
+            if _COLLECTION_ENTRY_RE.match(line[1:]):
+                removed += 1
+    if removed > added:
+        findings.append(
+            {"signature": "gate-evisceration", "severity": "critical",
+             "evidence": f"net removal of {removed - added} gate collection entr"
+                         f"{'y' if removed - added == 1 else 'ies'}"}
+        )
+    return findings
+
+
+_CRITICAL_RANK_WEAKEN_RE = re.compile(
+    r'^\+\s*_RANK\s*=.*["\']critical["\']\s*:\s*[0-2]\b'
+)
+_DOWNGRADE_WEAKEN_RE = re.compile(
+    r'^\+.*downgrade\s*=\s*["\']FailedUnverifiable["\']'
+)
+
+
+def _semantic_gate_weakening_findings(diff_text: str) -> list[dict]:
+    """Detect self-edits that preserve code shape but weaken safety semantics."""
+    findings: list[dict] = []
+    in_self = False
+    pending_minus = None
+    for line in diff_text.splitlines():
+        m = _MINUS_HDR_RE.match(line)
+        if m:
+            pending_minus = m.group("a")
+            continue
+        m = _PLUS_HDR_RE.match(line)
+        if m:
+            target = m.group("b") or pending_minus
+            in_self = bool(target) and _basename(target) == "anticheat_scan.py"
+            pending_minus = None
+            continue
+        if not in_self or not line.startswith("+") or line.startswith("+++"):
+            continue
+        if _CRITICAL_RANK_WEAKEN_RE.match(line) or _DOWNGRADE_WEAKEN_RE.match(line):
+            findings.append(
+                {
+                    "signature": "semantic-gate-weakening",
+                    "severity": "critical",
+                    "evidence": line.strip()[:120],
+                }
+            )
+    return findings
+
+
+def _is_cosmetic_change(content: str) -> bool:
+    """Whether a changed scanner line carries no decision logic.
+
+    Only blank lines and ``#`` comments are cosmetic. Docstring/prose lines are
+    NOT special-cased: telling a docstring body from executable code in a diff
+    alone is unreliable, and a triple-quote-state heuristic would itself be a
+    bypass vector (a real code line like ``SEP = '\"\"\"'`` could toggle the
+    state to hide a following ``return``). So the scanner conservatively treats
+    any non-blank, non-comment change as code — a false-positive on a
+    docstring-only touch-up is acceptable, it merely asks for human review.
+    """
+    s = content.strip()
+    return (not s) or s.startswith("#")
+
+
+def _scanner_self_edit_findings(diff_text: str) -> list[dict]:
+    """Flag any self-edit to anticheat_scan.py that changes a code line (P1.6).
+
+    The collection-shape and severity-mapping self-checks only catch two known
+    shapes; a diff that rewrites the decision LOGIC itself (e.g. inserting
+    ``return False`` into ``_is_gate_path``) slips past them and certifies clean.
+    The diff-layer invariant closes this: any hunk touching the scanner's own
+    source that adds/removes a non-comment, non-blank line is a finding.
+    Cosmetic-only edits stay clean; a false-positive on genuine scanner
+    maintenance is intended — maintenance should get human eyes.
+    """
+    in_self = False
+    pending_minus = None
+    for line in diff_text.splitlines():
+        m = _MINUS_HDR_RE.match(line)
+        if m:
+            pending_minus = m.group("a")
+            continue
+        m = _PLUS_HDR_RE.match(line)
+        if m:
+            target = m.group("b") or pending_minus
+            in_self = bool(target) and _basename(target) == "anticheat_scan.py"
+            pending_minus = None
+            continue
+        if not in_self:
+            continue
+        if line.startswith("+++") or line.startswith("---"):
+            continue
+        if (line.startswith("+") or line.startswith("-")) and not _is_cosmetic_change(line[1:]):
+            return [{
+                "signature": "scanner_self_edit",
+                "severity": "high",
+                "message": "self-edits to the scanner require human review",
+                "evidence": line[1:].strip()[:120],
+            }]
+    return []
+
+
+def scan(
+    diff_text: str = "",
+    changed_files: list[str] | None = None,
+    trajectory: list[str] | None = None,
+    gate_paths: tuple[str, ...] | None = None,
+) -> dict:
+    """Scan a diff + trajectory for shortcut signatures.
+
+    Returns ``{findings, clean, downgrade_to}``. ``downgrade_to`` is
+    ``"FailedSafety"`` for any critical finding, ``"FailedUnverifiable"`` for any
+    high finding, else ``None`` (medium/low are review flags only).
+    """
+    gate_paths = gate_paths if gate_paths is not None else DEFAULT_GATE_PATHS
+    files = changed_files if changed_files is not None else parse_changed_files(diff_text)
+    trajectory = trajectory or []
+    findings: list[dict] = []
+
+    for f in files:
+        if _is_gate_path(f, gate_paths):
+            findings.append(
+                {"signature": "gate-tampering", "severity": "critical", "evidence": f}
+            )
+        elif _TEST_PATH_RE.search(f):
+            findings.append(
+                {"signature": "test-file-mutation", "severity": "medium", "evidence": f}
+            )
+
+    # Added-line signatures are attributed to their hunk's file. The scanner's
+    # own source files (_SELF_FILES) are exempt so this suite's own regression
+    # FIXTURES (e.g. `+    assert 1 == 1`) are not self-flagged — other test
+    # files are still scanned. The gate-evisceration check below is EXEMPT from
+    # this exemption (it must still scan anticheat_scan.py).
+    for file_path, line in _iter_hunks(diff_text):
+        if file_path is not None and _basename(file_path) in _SELF_FILES:
+            continue
+        for name, severity, rx in _ADDED_LINE_SIGNATURES:
+            if rx.search(line):
+                findings.append(
+                    {"signature": name, "severity": severity,
+                     "evidence": line.strip()[:120]}
+                )
+
+    findings.extend(_gate_evisceration_findings(diff_text))
+    findings.extend(_semantic_gate_weakening_findings(diff_text))
+    findings.extend(_scanner_self_edit_findings(diff_text))
+
+    for entry in trajectory:
+        if _ANSWER_PATH_RE.search(str(entry)):
+            findings.append(
+                {"signature": "hidden-answer-read", "severity": "high",
+                 "evidence": str(entry)[:120]}
+            )
+
+    severities = {f["severity"] for f in findings}
+    if "critical" in severities:
+        downgrade = "FailedSafety"
+    elif "high" in severities:
+        downgrade = "FailedUnverifiable"
+    else:
+        downgrade = None
+
+    findings.sort(key=lambda f: _RANK.get(f["severity"], 0), reverse=True)
+    return {"findings": findings, "clean": not findings, "downgrade_to": downgrade}
+
+
+class _ArgParser(argparse.ArgumentParser):
+    """Argparse variant that exits with the operational-error code on a bad
+    argument, instead of argparse's default 2 (which would collide with the
+    gate-tampering finding code)."""
+
+    def error(self, message: str):  # noqa: D401 - argparse override
+        self.print_usage(sys.stderr)
+        print(f"{self.prog}: error: {message}", file=sys.stderr)
+        raise SystemExit(EXIT_OPERATIONAL_ERROR)
+
+
+def main(argv: list[str]) -> int:
+    ap = _ArgParser(description="Anti-cheat trajectory scan.")
+    ap.add_argument("--diff", help="path to a unified diff file (else read stdin)")
+    ap.add_argument("--files", help="comma-separated changed files (overrides diff parse)")
+    ap.add_argument("--trajectory", help="path to a JSON array of trajectory strings")
+    ap.add_argument(
+        "--self-check",
+        action="store_true",
+        help="scan the scanner's own git diff (anticheat_scan.py + its test)",
+    )
+    args = ap.parse_args(argv)
+
+    # Gathering inputs is where a *broken invocation* surfaces (unreadable file,
+    # malformed JSON, failed git). Fail with the distinct operational code so it
+    # is never confused with a clean scan or a findings downgrade.
+    try:
+        if args.self_check:
+            here = pathlib.Path(__file__).resolve().parent
+            diff_text = subprocess.run(
+                ["git", "diff", "--", "scripts/anticheat_scan.py",
+                 "scripts/test_anticheat_scan.py"],
+                cwd=here.parent, capture_output=True, text=True, check=True,
+            ).stdout
+        elif args.diff:
+            with open(args.diff, encoding="utf-8") as fh:
+                diff_text = fh.read()
+        elif not sys.stdin.isatty():
+            diff_text = sys.stdin.read()
+        else:
+            diff_text = ""
+
+        files = args.files.split(",") if args.files else None
+        trajectory = None
+        if args.trajectory:
+            with open(args.trajectory, encoding="utf-8") as fh:
+                trajectory = json.load(fh)
+    except (OSError, json.JSONDecodeError, subprocess.CalledProcessError) as exc:
+        print(f"{ap.prog}: operational error: {exc}", file=sys.stderr)
+        return EXIT_OPERATIONAL_ERROR
+
+    result = scan(diff_text=diff_text, changed_files=files, trajectory=trajectory)
+    print(json.dumps(result, indent=2))
+    if result["downgrade_to"] == "FailedSafety":
+        return EXIT_GATE_TAMPERING
+    return EXIT_CLEAN if result["clean"] else EXIT_FINDINGS
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

loop/_bundle/tools/holdout_gate.py

@@ -0,0 +1,121 @@
+"""Held-out verifier split — the runnable core of *measured* false-completion.
+
+A loop can game a verifier it can see. So acceptance checks are split:
+
+  * visible  — the loop sees these and optimizes against them during the run.
+  * holdout  — withheld until terminal verification by this independent gate.
+
+A loop may declare ``Succeeded`` only if the HOLDOUT set passes. Passing the
+visible set while failing the holdout set is the measurable *false-completion*
+event (``false_completion: true``) — the signature of overfitting to the tests
+the agent could see. Aggregated across runs, that flag IS the
+false-completion-rate metric (no longer self-reported).
+
+This ships as composable tooling, not a runtime: a loop calls it at its terminal
+gate. Pair it with ``anticheat_scan.py`` for the trajectory sweep.
+
+Manifest schema (JSON)::
+
+    {
+      "visible": [{"id": "unit", "cmd": "pytest tests/unit -q"}],
+      "holdout": [{"id": "spec", "cmd": "pytest tests/holdout -q"}]
+    }
+
+The manifest is trusted, operator-authored config (like a Makefile target); its
+commands run in a shell. Do not feed it untrusted input.
+
+Run::
+
+    python3 holdout_gate.py <manifest.json> [--cwd DIR]
+
+Prints the verdict as JSON. Exit code 0 iff verdict == "Succeeded".
+"""
+
+from __future__ import annotations
+
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+SUCCEEDED = "Succeeded"
+FAILED_UNVERIFIABLE = "FailedUnverifiable"
+# Non-terminal: a precondition is unmet (keep working, or fix the gate config).
+NOT_READY = "NotReady"
+
+
+def _all_passed(results: list[dict]) -> bool:
+    return all(r["passed"] for r in results)
+
+
+def decide(visible: list[dict], holdout: list[dict]) -> dict:
+    """Pure verdict from pre-computed check results — never executes anything.
+
+    ``visible`` / ``holdout`` are lists of ``{"id": str, "passed": bool}``.
+
+    Rules:
+      * visible empty                -> NotReady  (nothing was optimized against)
+      * visible not all green        -> NotReady  (loop shouldn't certify yet)
+      * holdout empty                -> NotReady  (cannot certify without holdout)
+      * visible green + holdout green -> Succeeded
+      * visible green + holdout red   -> FailedUnverifiable (false completion)
+    """
+    passed_visible = bool(visible) and _all_passed(visible)
+    passed_holdout = bool(holdout) and _all_passed(holdout)
+
+    if not visible:
+        verdict, reason = NOT_READY, "no visible checks defined — cannot certify Succeeded"
+    elif not passed_visible:
+        verdict, reason = NOT_READY, "visible gate not green — keep working or repair"
+    elif not holdout:
+        verdict, reason = NOT_READY, "no holdout checks defined — cannot certify Succeeded"
+    elif passed_holdout:
+        verdict, reason = SUCCEEDED, "visible and holdout gates both green"
+    else:
+        verdict, reason = (
+            FAILED_UNVERIFIABLE,
+            "visible passed but holdout failed — false completion",
+        )
+
+    return {
+        "verdict": verdict,
+        "reason": reason,
+        "passed_visible": passed_visible,
+        "passed_holdout": passed_holdout,
+        "false_completion": passed_visible and bool(holdout) and not passed_holdout,
+        "visible": visible,
+        "holdout": holdout,
+    }
+
+
+def _run_one(check: dict, cwd: str | None) -> dict:
+    proc = subprocess.run(
+        check["cmd"], shell=True, cwd=cwd, capture_output=True, text=True
+    )
+    return {
+        "id": check["id"],
+        "passed": proc.returncode == 0,
+        "returncode": proc.returncode,
+    }
+
+
+def run_manifest(manifest: dict, cwd: str | None = None) -> dict:
+    """Execute a visible/holdout manifest and return the verdict dict."""
+    visible = [_run_one(c, cwd) for c in manifest.get("visible", [])]
+    holdout = [_run_one(c, cwd) for c in manifest.get("holdout", [])]
+    return decide(visible, holdout)
+
+
+def main(argv: list[str]) -> int:
+    if not argv:
+        print("usage: holdout_gate.py <manifest.json> [--cwd DIR]", file=sys.stderr)
+        return 2
+    manifest = json.loads(Path(argv[0]).read_text(encoding="utf-8"))
+    cwd = argv[argv.index("--cwd") + 1] if "--cwd" in argv else None
+    result = run_manifest(manifest, cwd)
+    print(json.dumps(result, indent=2))
+    return 0 if result["verdict"] == SUCCEEDED else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))