grudge-cli 0.1.0__py3-none-any.whl

grudge/admission/court.py

@@ -0,0 +1,239 @@
+"""Fail-closed, deterministic admission proof for regression scars.
+
+Empty sanity means that an empty trace is not failure evidence. Safety rules
+may safely hold on empty; non-vacuity additionally requires witnessed violations
+on every successful bad-agent run, alongside clean fixed and control runs.
+"""
+
+from __future__ import annotations
+
+import math
+import re
+from collections.abc import Callable, Sequence
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from grudge.admission.verdict import AdmissionVerdict
+from grudge.predicates.compiler import Checker, PredicateResult, compile_predicate
+from grudge.replay.sandbox import run_agent
+from grudge.replay.scripted_llm import ScriptedLLM
+from grudge.schema.scar import AdmissionRecord, Scar
+from grudge.schema.trace import ToolResult, Trace
+
+DEFAULT_SCARS_DIR = Path(__file__).resolve().parents[2] / "scars"
+_EMPTY_SANITY = (
+    "Empty sanity: empty is not failure evidence; safety rules may hold on empty, "
+    "and non-vacuity requires witnessed bad-run violations."
+)
+
+
+def _trace_snapshot(trace: Trace, *, allow_empty: bool = False) -> Trace:
+    """Revalidate mutable models, including nested steps, before trusting them."""
+
+    if not isinstance(trace, Trace):
+        raise TypeError("expected a Trace")
+    snapshot = Trace.model_validate(trace.model_dump(warnings="error"), strict=True)
+    if snapshot.outcome != "ok":
+        details = [
+            str(step.meta["failure"])
+            for step in snapshot.steps
+            if "failure" in step.meta
+        ]
+        suffix = ": " + "; ".join(details) if details else ""
+        raise ValueError("failed or truncated trace" + suffix)
+    if not allow_empty and not snapshot.steps:
+        raise ValueError("trace must be nonempty")
+    for step in snapshot.steps:
+        if isinstance(step, ToolResult) and step.error is not None:
+            raise ValueError(f"tool replay failed: {step.error}")
+        if step.meta.get("failure") or step.meta.get("truncated"):
+            raise ValueError("trace contains a failure or truncation marker")
+    return snapshot
+
+
+def _check(checker: Checker, trace: Trace) -> PredicateResult:
+    """Reject invalid checker output and evidence absent from the checked trace."""
+
+    result = checker.check(trace)
+    if not isinstance(result, PredicateResult):
+        raise TypeError("checker did not return a PredicateResult")
+    validated = PredicateResult.model_validate(
+        result.model_dump(warnings="error"), strict=True
+    )
+    if any(step not in trace.steps for step in validated.evidence):
+        raise ValueError("checker evidence is not present in the trace")
+    return validated
+
+
+def admit(
+    scar_candidate: Scar,
+    bad_agent: Callable[..., Any],
+    good_agent: Callable[..., Any],
+    *,
+    runs: int,
+    controls: Sequence[Trace],
+    output_dir: Path | str | None = None,
+    timeout: float = 2.0,
+    max_steps: int = 128,
+) -> AdmissionVerdict:
+    """Prove all admission gates and save a detached, proof-bearing Scar.
+
+    A new policy is constructed outside guarded replay for *each* agent run.
+    The fixed agent may set its supplied policy to defended mode without
+    contaminating another run. Rejections never persist a candidate.
+    """
+
+    if isinstance(runs, bool) or not isinstance(runs, int) or runs <= 0:
+        raise ValueError("runs must be a positive integer")
+    if (
+        not math.isfinite(timeout)
+        or timeout <= 0
+        or isinstance(max_steps, bool)
+        or not isinstance(max_steps, int)
+        or max_steps <= 0
+    ):
+        raise ValueError("timeout and max_steps must be positive finite limits")
+
+    verdict = AdmissionVerdict(admitted=False)
+    try:
+        # Round-tripping also checks models altered through model_copy/construct
+        # or in-place mutations, which assignment validation alone cannot catch.
+        candidate = Scar.model_validate(
+            scar_candidate.model_dump(warnings="error"), strict=True
+        ).model_copy(deep=True)
+        if re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}", candidate.id) is None:
+            raise ValueError("scar id must be a safe filename component")
+        checker = compile_predicate(candidate.predicate)
+    except Exception as exc:
+        verdict.reasons.append(
+            f"Invalid candidate or predicate: {type(exc).__name__}: {exc}"
+        )
+        return verdict
+
+    empty = Trace(
+        run_id="admission-empty",
+        agent_version="admission",
+        task=candidate.cassette.task,
+        steps=[],
+        outcome="ok",
+    )
+    try:
+        verdict.empty_result = _check(checker, empty)
+        if verdict.empty_result.violated:
+            verdict.reasons.append(
+                "Empty sanity failed: predicate violates on empty trace."
+            )
+    except Exception as exc:
+        verdict.reasons.append(f"Empty checker exception: {type(exc).__name__}: {exc}")
+
+    if not controls:
+        verdict.reasons.append("At least one nonempty clean control trace is required.")
+    for index, control in enumerate(controls, start=1):
+        try:
+            trace = _trace_snapshot(control)
+            result = _check(checker, trace)
+            verdict.control_results.append(result)
+            if result.violated:
+                verdict.reasons.append(
+                    f"Control {index} violated: {result.explanation}"
+                )
+        except Exception as exc:
+            verdict.reasons.append(
+                f"Control {index} failed: {type(exc).__name__}: {exc}"
+            )
+
+    for index in range(1, runs + 1):
+        for label, agent, results in (
+            ("Bad", bad_agent, verdict.bad_results),
+            ("Good", good_agent, verdict.good_results),
+        ):
+            try:
+                policy = ScriptedLLM()
+                replay = run_agent(
+                    agent,
+                    candidate.cassette.model_copy(deep=True),
+                    policy,
+                    timeout=timeout,
+                    max_steps=max_steps,
+                )
+                trace = _trace_snapshot(replay)
+                result = _check(checker, trace)
+                results.append(result)
+                if label == "Bad":
+                    if not result.violated or not result.evidence:
+                        verdict.reasons.append(
+                            f"Bad run {index} needs a violation "
+                            "with nonempty evidence: " + result.explanation
+                        )
+                    else:
+                        verdict.evidence.extend(result.evidence)
+                elif result.violated:
+                    verdict.reasons.append(
+                        f"Good run {index} violated: {result.explanation}"
+                    )
+            except Exception as exc:
+                verdict.reasons.append(
+                    f"{label} run {index} replay/checker failed: "
+                    f"{type(exc).__name__}: {exc}"
+                )
+
+    if verdict.reasons:
+        return verdict
+
+    verdict.reasons.extend(
+        [
+            f"All {runs} bad runs violated with nonempty evidence.",
+            f"All {runs} good runs and {len(controls)} controls "
+            "were successful and clean.",
+            _EMPTY_SANITY,
+        ]
+    )
+    candidate.admission = AdmissionRecord.model_validate(
+        {
+            "admitted": True,
+            "reasons": list(verdict.reasons),
+            "bad_runs": len(verdict.bad_results),
+            "good_runs": len(verdict.good_results),
+            "checked_at": datetime.now(UTC),
+            "control_runs": len(verdict.control_results),
+            "empty_sanity": _EMPTY_SANITY,
+            "evidence": [step.model_dump(mode="json") for step in verdict.evidence],
+            "bad_results": [
+                result.model_dump(mode="json") for result in verdict.bad_results
+            ],
+            "good_results": [
+                result.model_dump(mode="json") for result in verdict.good_results
+            ],
+            "control_results": [
+                result.model_dump(mode="json") for result in verdict.control_results
+            ],
+            "empty_result": (
+                verdict.empty_result.model_dump(mode="json")
+                if verdict.empty_result is not None
+                else None
+            ),
+            "timeout": timeout,
+            "max_steps": max_steps,
+        }
+    )
+    try:
+        directory = (
+            DEFAULT_SCARS_DIR if output_dir is None else Path(output_dir)
+        ).resolve()
+        target = directory / f"{candidate.id}.scar.json"
+        payload = candidate.model_dump_json(indent=2) + "\n"
+        directory.mkdir(parents=True, exist_ok=True)
+        # Exclusive creation prevents symlink escapes and overwriting an existing
+        # proof, including a target swapped between path checking and opening.
+        with target.open("x", encoding="utf-8") as handle:
+            handle.write(payload)
+    except Exception as exc:
+        verdict.reasons.append(f"Persistence failed: {type(exc).__name__}: {exc}")
+        return verdict
+    verdict.admitted = True
+    verdict.saved_path = target
+    return verdict
+
+
+__all__ = ["admit"]

grudge/admission/verdict.py

@@ -0,0 +1,26 @@
+"""Structured proof and diagnostics returned by the admission court."""
+
+from pathlib import Path
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from grudge.predicates.compiler import PredicateResult
+from grudge.schema.trace import Step
+
+
+class AdmissionVerdict(BaseModel):
+    """Admission succeeds only after every proof gate and persistence succeed."""
+
+    model_config = ConfigDict(extra="forbid", validate_assignment=True)
+
+    admitted: bool
+    reasons: list[str] = Field(default_factory=list)
+    evidence: list[Step] = Field(default_factory=list)
+    bad_results: list[PredicateResult] = Field(default_factory=list)
+    good_results: list[PredicateResult] = Field(default_factory=list)
+    control_results: list[PredicateResult] = Field(default_factory=list)
+    empty_result: PredicateResult | None = None
+    saved_path: Path | None = None
+
+
+__all__ = ["AdmissionVerdict"]

grudge/cli/__init__.py

@@ -0,0 +1 @@
+"""Command-line package for the Phase 0 Grudge surface."""

grudge/cli/_common.py

@@ -0,0 +1,126 @@
+"""Shared helpers for the Phase 9 scriptable command surface."""
+
+from __future__ import annotations
+
+import importlib
+import json
+import sys
+import traceback
+from collections.abc import Callable, Mapping
+from pathlib import Path
+from typing import Any, NoReturn, TypeVar, cast
+
+import typer
+
+from grudge.errors import GrudgeError
+
+T = TypeVar("T")
+
+
+def finish(payload: Mapping[str, Any], *, json_output: bool) -> None:
+    """Print a command result as JSON or a compact human-readable summary."""
+
+    if json_output:
+        typer.echo(json.dumps(dict(payload), ensure_ascii=False, sort_keys=True))
+        return
+    for key, value in payload.items():
+        if isinstance(value, list):
+            for item in value:
+                typer.echo(str(item))
+        else:
+            typer.echo(f"{key}: {value}")
+
+
+def fail(exc: Exception, *, json_output: bool, debug: bool) -> NoReturn:
+    """Render an expected command failure and terminate with status 2."""
+
+    if debug:
+        traceback.print_exc(file=sys.stderr)
+    message = str(exc) or type(exc).__name__
+    if json_output:
+        typer.echo(
+            json.dumps(
+                {"error": message, "error_type": type(exc).__name__},
+                ensure_ascii=False,
+                sort_keys=True,
+            )
+        )
+    else:
+        typer.echo(f"Error: {message}", err=True)
+    raise typer.Exit(code=2)
+
+
+def command(action: Callable[[], T], *, json_output: bool, debug: bool) -> T:
+    """Run a command body with the common Grudge error contract."""
+
+    try:
+        return action()
+    except typer.Exit:
+        raise
+    except Exception as exc:
+        fail(exc, json_output=json_output, debug=debug)
+    raise AssertionError("unreachable")
+
+
+def load_callable(module_name: str) -> Callable[..., Any]:
+    """Import an agent module and select its conventional run entry point."""
+
+    try:
+        module = importlib.import_module(module_name)
+    except Exception as exc:
+        raise GrudgeError(
+            f"could not import agent module {module_name!r}: {exc}"
+        ) from exc
+    for name in ("run_agent", "run"):
+        candidate = getattr(module, name, None)
+        if callable(candidate):
+            return cast(Callable[..., Any], candidate)
+    raise GrudgeError(
+        f"agent module {module_name!r} must expose a callable run_agent or run"
+    )
+
+
+def require_file(path: Path, *, label: str) -> Path:
+    """Validate a required input file without leaking a raw OS exception."""
+
+    if not path.is_file():
+        raise GrudgeError(f"{label} file does not exist: {path}")
+    return path
+
+
+def read_text(path: Path, *, label: str) -> str:
+    """Read a required UTF-8 file as a Grudge-friendly operation."""
+
+    require_file(path, label=label)
+    try:
+        return path.read_text(encoding="utf-8")
+    except (OSError, UnicodeError) as exc:
+        raise GrudgeError(f"could not read {label} file {path}: {exc}") from exc
+
+
+def write_text(path: Path, text: str, *, label: str) -> None:
+    """Write a command artifact, creating its parent directory when needed."""
+
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(text, encoding="utf-8")
+    except OSError as exc:
+        raise GrudgeError(f"could not write {label} file {path}: {exc}") from exc
+
+
+def model_json(model: Any) -> dict[str, Any]:
+    """Return a Pydantic model as JSON-compatible data."""
+
+    return cast(dict[str, Any], model.model_dump(mode="json"))
+
+
+__all__ = [
+    "command",
+    "fail",
+    "finish",
+    "load_callable",
+    "model_json",
+    "read_text",
+    "require_file",
+    "write_text",
+]

grudge/cli/admit.py

@@ -0,0 +1,93 @@
+"""Build and prove a Scar from a cassette using the admission court."""
+
+# Typer's declarative Option defaults are intentional here.
+# ruff: noqa: B008
+
+import hashlib
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+import typer
+
+from grudge.admission import admit
+from grudge.cli._common import command, finish, load_callable, read_text
+from grudge.errors import GrudgeError
+from grudge.predicates.dsl import parse
+from grudge.predicates.fallback_rules import demo_rule
+from grudge.schema.cassette import Cassette
+from grudge.schema.scar import Scar
+from grudge.schema.trace import Trace
+
+
+def admit_cassette(
+    cassette: Path,
+    bad: str = typer.Option(..., "--bad", help="Failing agent module."),
+    good: str = typer.Option(..., "--good", help="Fixed agent module."),
+    out: Path = typer.Option(..., "--out", help="Directory for admitted Scars."),
+    scar_id: str | None = typer.Option(None, "--id", help="Scar filename/id."),
+    predicate: str = typer.Option(
+        "demo",
+        "--predicate",
+        help="Predicate DSL, or 'demo' for the hand-verified rule.",
+    ),
+    controls: Path = typer.Option(
+        Path("examples/leaky_agent/controls"),
+        "--controls",
+        help="Directory containing benign control JSONL traces.",
+    ),
+    runs: int = typer.Option(3, "--runs", min=1, help="Proof runs per agent."),
+    json_output: bool = typer.Option(False, "--json", help="Emit JSON output."),
+    debug: bool = typer.Option(False, "--debug", help="Show a traceback on errors."),
+) -> None:
+    """Prove a cassette red on the bad agent and green on the good agent."""
+
+    def run() -> dict[str, Any]:
+        payload = read_text(cassette, label="cassette")
+        try:
+            candidate_cassette = Cassette.model_validate_json(payload)
+        except Exception as exc:
+            raise GrudgeError(f"invalid cassette file {cassette}: {exc}") from exc
+        rule = demo_rule() if predicate.strip().lower() == "demo" else parse(predicate)
+        identifier = scar_id or candidate_cassette.content_hash[:12]
+        fingerprint = hashlib.sha256(
+            (candidate_cassette.content_hash + rule.model_dump_json()).encode("utf-8")
+        ).hexdigest()
+        candidate = Scar(
+            id=identifier,
+            cassette=candidate_cassette,
+            predicate=rule,
+            seed_predicate=rule.model_copy(deep=True),
+            source_trace_hash=candidate_cassette.content_hash,
+            agent_version_failed=bad,
+            agent_version_fixed=good,
+            created_at=datetime.now(UTC),
+            fingerprint=fingerprint,
+        )
+        control_paths = sorted(controls.glob("*.jsonl"))
+        if not control_paths:
+            raise GrudgeError(f"no control JSONL files found in {controls}")
+        control_traces = [
+            Trace.from_jsonl(read_text(path, label="control")) for path in control_paths
+        ]
+        verdict = admit(
+            candidate,
+            load_callable(bad),
+            load_callable(good),
+            runs=runs,
+            controls=control_traces,
+            output_dir=out,
+        )
+        result = {
+            "command": "admit",
+            "admitted": verdict.admitted,
+            "scar_id": candidate.id,
+            "saved_path": str(verdict.saved_path) if verdict.saved_path else None,
+            "reasons": verdict.reasons,
+        }
+        if not verdict.admitted:
+            finish(result, json_output=json_output)
+            raise typer.Exit(code=1)
+        return result
+
+    finish(command(run, json_output=json_output, debug=debug), json_output=json_output)

grudge/cli/capture.py

@@ -0,0 +1,39 @@
+"""Capture an agent module's typed trace as canonical JSONL."""
+
+# Typer's declarative Option defaults are intentional here.
+# ruff: noqa: B008
+
+from pathlib import Path
+from typing import Any
+
+import typer
+
+from grudge.cli._common import command, finish, load_callable, write_text
+from grudge.errors import GrudgeError
+from grudge.schema.trace import Trace
+
+
+def capture_agent(
+    agent_module: str,
+    out: Path = typer.Option(..., "--out", help="Destination JSONL trace."),
+    json_output: bool = typer.Option(False, "--json", help="Emit JSON output."),
+    debug: bool = typer.Option(False, "--debug", help="Show a traceback on errors."),
+) -> None:
+    """Capture an agent run into a trace."""
+
+    def run() -> dict[str, Any]:
+        trace = load_callable(agent_module)()
+        if not isinstance(trace, Trace):
+            raise GrudgeError(
+                f"agent {agent_module!r} returned {type(trace).__name__}, not a Trace"
+            )
+        write_text(out, trace.to_jsonl(), label="trace")
+        return {
+            "command": "capture",
+            "trace": str(out),
+            "run_id": trace.run_id,
+            "steps": len(trace.steps),
+            "outcome": trace.outcome,
+        }
+
+    finish(command(run, json_output=json_output, debug=debug), json_output=json_output)

grudge/cli/compile.py

@@ -0,0 +1,37 @@
+"""CLI command for compiling admitted Scars for the local guard runtime."""
+
+# Typer requires option factories in function defaults; this is the project's
+# established command-module pattern.
+# ruff: noqa: B008
+
+from pathlib import Path
+from typing import Any
+
+import typer
+
+from grudge.config import SCARS_DIR
+from grudge.guard.compile import compile_scars, load_scars, write_compiled
+
+
+def compile_command(
+    scars: Path = typer.Option(SCARS_DIR, "--scars", help="Scar directory."),
+    out: Path = typer.Option(
+        Path(".grudge/compiled.json"), "--out", help="Compiled rule output."
+    ),
+    json_output: bool = typer.Option(False, "--json", help="Emit JSON output."),
+) -> None:
+    """Compile admitted Scar files into the flat hook contract."""
+
+    payload = compile_scars(load_scars(scars), created_from=str(scars))
+    target = write_compiled(payload, out)
+    result: dict[str, Any] = {
+        "command": "compile",
+        "scars": str(scars),
+        "out": str(target),
+        "rules": len(payload["rules"]),
+        "version": payload["version"],
+    }
+    if json_output:
+        typer.echo(__import__("json").dumps(result, sort_keys=True))
+    else:
+        typer.echo(f"compiled {result['rules']} rule(s) to {target}")