grudge-cli 0.1.0__py3-none-any.whl

grudge/cli/demo.py

@@ -0,0 +1,136 @@
+"""Run the deterministic, offline five-beat demonstration."""
+
+# Typer's declarative Option defaults are intentional here.
+
+import json
+import os
+import shutil
+import subprocess
+import webbrowser
+from pathlib import Path
+from threading import Event
+from typing import Any, cast
+
+import typer
+
+from grudge.cli._common import command, finish
+from grudge.config import DEFAULTS, SCARS_DIR
+from grudge.demo.events import EventBus
+from grudge.demo.orchestrator import run_demo as orchestrate_demo
+from grudge.demo.server import DemoServer
+
+
+def run_demo(
+    live: bool = typer.Option(False, "--live", help="Reserved for a live provider."),
+    port: int = typer.Option(8765, "--port", help="Local viewer server port."),
+    no_browser: bool = typer.Option(
+        False, "--no-browser", help="Do not open a browser."
+    ),
+    runs: int = typer.Option(DEFAULTS.n_runs, "--runs", min=1),
+    fast: bool = typer.Option(False, "--fast", help="Disable presenter pacing."),
+    guard: bool = typer.Option(
+        False, "--guard", help="Run the offline two-session hook proof."
+    ),
+    json_output: bool = typer.Option(False, "--json", help="Emit JSON output."),
+    debug: bool = typer.Option(False, "--debug", help="Show a traceback on errors."),
+) -> None:
+    """Play the offline proof against the fixed and vulnerable demo agents."""
+
+    def run() -> dict[str, Any]:
+        if live:
+            raise RuntimeError(
+                "live demo mode is reserved for a future provider integration"
+            )
+        if guard:
+            node = shutil.which("node")
+            if node is None:
+                raise RuntimeError("--guard requires Node.js >= 18 on PATH")
+            script = Path(__file__).resolve().parents[2] / "scripts" / "e2e_guard.mjs"
+            completed = subprocess.run(
+                [node, str(script)],
+                cwd=script.parent.parent,
+                check=False,
+                capture_output=True,
+                text=True,
+                env={**os.environ, "GRUDGE_DISABLE": ""},
+            )
+            if completed.returncode != 0:
+                raise RuntimeError(completed.stderr.strip() or "guard demo failed")
+            payload = cast(dict[str, Any], json.loads(completed.stdout))
+            payload["command"] = "demo --guard"
+            if json_output:
+                return payload
+            typer.echo("Session 1: leak captured, proposed, and accepted")
+            typer.echo("Session 2: leak blocked before the HTTP call")
+            typer.echo(payload["reason"])
+            return payload
+        if not fast and not json_output:
+            bus = EventBus()
+            finished = Event()
+            failure: list[BaseException] = []
+
+            def run_viewer_demo() -> None:
+                try:
+                    orchestrate_demo(bus=bus, runs=runs)
+                except BaseException as exc:
+                    failure.append(exc)
+                finally:
+                    finished.set()
+
+            server = DemoServer(
+                port=port,
+                bus=bus,
+                runner=run_viewer_demo,
+            ).start(run_demo=True)
+            url = f"http://{server.address[0]}:{server.address[1]}/"
+            if not no_browser:
+                webbrowser.open(url)
+            typer.echo(f"Grudge viewer: {url}")
+            finished.wait()
+            server.close()
+            if failure:
+                raise RuntimeError("viewer demo failed") from failure[0]
+            return {"command": "demo", "mode": "viewer", "url": url}
+        events = orchestrate_demo(fast=fast, runs=runs)
+        event_payloads = [event.model_dump(mode="json") for event in events]
+        gates = [event for event in event_payloads if event["type"] == "gate.result"]
+        vulnerable = next(
+            event
+            for event in gates
+            if event["agent"] == "vulnerable" and not event["passed"]
+        )
+        fixed = next(event for event in gates if event["agent"] == "fixed")
+        return {
+            "command": "demo",
+            "mode": "offline/deterministic",
+            "scar": str(SCARS_DIR / "sample_leak.scar.json"),
+            "events": event_payloads,
+            "fixed": {
+                "passed": fixed["passed"],
+                "successes": fixed["passes"],
+                "runs": fixed["n"],
+                "lower_bound": fixed["wilson"],
+            },
+            "vulnerable": {
+                "passed": vulnerable["passed"],
+                "successes": vulnerable["passes"],
+                "runs": vulnerable["n"],
+                "lower_bound": vulnerable["wilson"],
+            },
+        }
+
+    result = command(run, json_output=json_output, debug=debug)
+    if result.get("mode") == "viewer":
+        return
+    if guard:
+        if json_output:
+            finish(result, json_output=True)
+        return
+    if json_output:
+        finish(result, json_output=True)
+    else:
+        for event in result["events"]:
+            typer.echo(
+                f"{event['type']} "
+                f"{json.dumps(event, ensure_ascii=False, sort_keys=True)}"
+            )

grudge/cli/main.py

@@ -0,0 +1,54 @@
+"""Typer command surface for Grudge's deterministic pipeline."""
+
+from typing import Any, NoReturn
+
+import typer
+
+from grudge._stubs import roadmap
+from grudge.cli._common import command
+from grudge.cli.admit import admit_cassette
+from grudge.cli.capture import capture_agent
+from grudge.cli.compile import compile_command
+from grudge.cli.demo import run_demo
+from grudge.cli.run import run_scars
+from grudge.cli.scars import scars_app
+from grudge.cli.shrink import shrink_trace
+from grudge.errors import GrudgeError
+
+app = typer.Typer(
+    name="grudge",
+    help="Compile agent failures into deterministic regression scars.",
+    no_args_is_help=True,
+)
+
+
+def _future(command: str) -> NoReturn:
+    roadmap(f"cli.{command}")
+
+
+app.command("capture")(capture_agent)
+app.command("compile")(compile_command)
+app.command("shrink")(shrink_trace)
+app.command("admit")(admit_cassette)
+app.command("run")(run_scars)
+app.command("demo")(run_demo)
+
+
+@app.command()
+def viewer(
+    json_output: bool = typer.Option(False, "--json", help="Emit JSON output."),
+    debug: bool = typer.Option(False, "--debug", help="Show a traceback on errors."),
+) -> None:
+    """Start the local viewer server."""
+
+    def run() -> dict[str, Any]:
+        try:
+            _future("viewer")
+        except NotImplementedError as exc:
+            raise GrudgeError(str(exc)) from exc
+        raise AssertionError("unreachable")
+
+    command(run, json_output=json_output, debug=debug)
+
+
+app.add_typer(scars_app, name="scars")

grudge/cli/run.py

@@ -0,0 +1,60 @@
+"""Run admitted Scars through the deterministic statistical gate."""
+
+# Typer's declarative Option defaults are intentional here.
+# ruff: noqa: B008
+
+from pathlib import Path
+from typing import Any
+
+import typer
+
+from grudge.cli._common import command, finish, load_callable, read_text
+from grudge.config import DEFAULTS
+from grudge.gate.report import render_terminal
+from grudge.gate.runner import run_gate
+from grudge.schema.scar import Scar
+
+
+def run_scars(
+    scars: Path = typer.Option(..., "--scars", help="Scar file or directory."),
+    agent: str = typer.Option(..., "--agent", help="Agent module to gate."),
+    runs: int = typer.Option(DEFAULTS.n_runs, "--runs", min=1),
+    threshold: float = typer.Option(DEFAULTS.gate_threshold, "--threshold"),
+    json_output: bool = typer.Option(False, "--json", help="Emit JSON output."),
+    debug: bool = typer.Option(False, "--debug", help="Show a traceback on errors."),
+) -> None:
+    """Run all admitted Scars in a directory, or one Scar file."""
+
+    def run() -> dict[str, Any]:
+        paths = [scars] if scars.is_file() else sorted(scars.glob("*.scar.json"))
+        if not paths:
+            raise FileNotFoundError(f"no Scar files found at {scars}")
+        agent_fn = load_callable(agent)
+        results: list[dict[str, Any]] = []
+        for path in paths:
+            scar = Scar.model_validate_json(read_text(path, label="Scar"))
+            result = run_gate(scar, agent_fn, runs=runs, threshold=threshold)
+            if not json_output:
+                typer.echo(render_terminal(result))
+            results.append(
+                {
+                    "scar_id": result.scar_id,
+                    "passed": result.passed,
+                    "exit_code": result.exit_code,
+                    "successes": result.successes,
+                    "runs": result.runs,
+                    "pass_rate": result.pass_rate,
+                    "lower_bound": result.lower_bound,
+                    "threshold": result.threshold,
+                    "reasons": result.reasons,
+                    "observations": result.observations,
+                }
+            )
+        payload = {"command": "run", "results": results}
+        return payload
+
+    payload = command(run, json_output=json_output, debug=debug)
+    if json_output:
+        finish(payload, json_output=True)
+    if any(not item["passed"] for item in payload["results"]):
+        raise typer.Exit(code=1)

grudge/cli/scars.py

@@ -0,0 +1,91 @@
+"""List and inspect saved Scars; retirement remains a roadmap stub."""
+
+# Typer's declarative Option defaults are intentional here.
+# ruff: noqa: B008
+
+from pathlib import Path
+from typing import Any
+
+import typer
+
+from grudge._stubs import roadmap
+from grudge.cli._common import command, finish, read_text
+from grudge.config import SCARS_DIR
+from grudge.errors import GrudgeError
+from grudge.schema.scar import Scar
+
+scars_app = typer.Typer(help="Browse saved regression Scars.")
+
+
+@scars_app.command("list")
+def list_scars(
+    scars_dir: Path = typer.Option(SCARS_DIR, "--scars", "--scars-dir"),
+    json_output: bool = typer.Option(False, "--json"),
+    debug: bool = typer.Option(False, "--debug"),
+) -> None:
+    """List saved Scar summaries."""
+
+    def run() -> dict[str, Any]:
+        paths = sorted(scars_dir.glob("*.scar.json"))
+        entries = []
+        for path in paths:
+            scar = Scar.model_validate_json(read_text(path, label="Scar"))
+            entries.append(
+                {
+                    "id": scar.id,
+                    "path": str(path),
+                    "admitted": bool(scar.admission and scar.admission.admitted),
+                    "fingerprint": scar.fingerprint,
+                }
+            )
+        payload = {"command": "scars list", "scars": entries}
+        if not json_output:
+            for item in entries:
+                status = "admitted" if item["admitted"] else "unadmitted"
+                typer.echo(f"{item['id']}  {status}")
+        return payload
+
+    finish(command(run, json_output=json_output, debug=debug), json_output=json_output)
+
+
+@scars_app.command("show")
+def show_scar(
+    scar: str,
+    scars_dir: Path = typer.Option(SCARS_DIR, "--scars", "--scars-dir"),
+    json_output: bool = typer.Option(False, "--json"),
+    debug: bool = typer.Option(False, "--debug"),
+) -> None:
+    """Show one Scar by path or id."""
+
+    def run() -> dict[str, Any]:
+        path = Path(scar)
+        if not path.is_file():
+            filename = scar if scar.endswith(".scar.json") else f"{scar}.scar.json"
+            path = scars_dir / filename
+        loaded = Scar.model_validate_json(read_text(path, label="Scar"))
+        return {"command": "scars show", "scar": loaded.model_dump(mode="json")}
+
+    result = command(run, json_output=json_output, debug=debug)
+    if not json_output:
+        typer.echo(Scar.model_validate(result["scar"]).model_dump_json(indent=2))
+    else:
+        finish(result, json_output=True)
+
+
+@scars_app.command("retire")
+def retire_scar(
+    scar: str,
+    json_output: bool = typer.Option(False, "--json"),
+    debug: bool = typer.Option(False, "--debug"),
+) -> None:
+    """Retire a Scar (registry support is a later roadmap item)."""
+
+    del scar
+
+    def run() -> None:
+        try:
+            roadmap("cli.scars.retire")
+        except NotImplementedError as exc:
+            raise GrudgeError(str(exc)) from exc
+
+    command(run, json_output=json_output, debug=debug)

grudge/cli/shrink.py

@@ -0,0 +1,108 @@
+"""Shrink a captured trace into a minimal replay cassette."""
+
+# Typer's declarative Option defaults are intentional here.
+# ruff: noqa: B008
+
+from pathlib import Path
+from typing import Any
+
+import typer
+
+from grudge.cli._common import command, finish, load_callable, read_text, write_text
+from grudge.predicates.compiler import compile_predicate
+from grudge.predicates.dsl import parse
+from grudge.predicates.fallback_rules import demo_rule
+from grudge.replay.sandbox import world_from_trace
+from grudge.replay.scripted_llm import ScriptedLLM
+from grudge.schema.trace import ToolResult, Trace
+from grudge.shrinker.cassette_builder import build_minimal_cassette
+from grudge.shrinker.oracle import OracleResult, PredicateOracle
+
+
+class _ReplayCompleteOracle:
+    """Keep CLI cassettes admissible by rejecting incomplete replays."""
+
+    def __init__(self, inner: PredicateOracle, seed: Any) -> None:
+        self.inner = inner
+        self.checker = compile_predicate(seed)
+        self.calls = 0
+        self.cache_hits = 0
+
+    def reproduces(self, cassette: Any) -> OracleResult:
+        self.calls = self.inner.calls
+        self.cache_hits = self.inner.cache_hits
+        result = self.inner.reproduces(cassette)
+        self.calls = self.inner.calls
+        self.cache_hits = self.inner.cache_hits
+        if not result.reproduced or result.trace is None:
+            return result
+        errors = [
+            step.error
+            for step in result.trace.steps
+            if isinstance(step, ToolResult) and step.error is not None
+        ]
+        if result.trace.outcome != "ok" or errors:
+            reason = errors[0] if errors else "failed or truncated trace"
+            return OracleResult(False, result.trace, f"replay failed: {reason}")
+        checked = self.checker.check(result.trace)
+        if not checked.violated:
+            return OracleResult(False, result.trace, "seed predicate did not reproduce")
+        return result
+
+
+def shrink_trace(
+    trace: Path,
+    failure: str = typer.Option(
+        ..., "--failure", help="Seed failure rule in Grudge DSL."
+    ),
+    out: Path = typer.Option(..., "--out", help="Destination cassette JSON."),
+    agent: str = typer.Option(
+        "examples.leaky_agent.agent",
+        "--agent",
+        help="Agent module used to evaluate candidate cassettes.",
+    ),
+    json_output: bool = typer.Option(False, "--json", help="Emit JSON output."),
+    debug: bool = typer.Option(False, "--debug", help="Show a traceback on errors."),
+) -> None:
+    """Shrink a failing trace while retaining the supplied seed failure."""
+
+    def run() -> dict[str, Any]:
+        source = Trace.from_jsonl(read_text(trace, label="trace"))
+        cassette = world_from_trace(source)
+        seed = demo_rule() if failure.strip().lower() == "demo" else parse(failure)
+        result = build_minimal_cassette(
+            cassette,
+            _ReplayCompleteOracle(
+                PredicateOracle(
+                    load_callable(agent),
+                    seed,
+                    llm=ScriptedLLM(),
+                    max_oracle_calls=5000,
+                ),
+                seed,
+            ),
+        )
+        write_text(
+            out,
+            result.minimal.model_dump_json(indent=2) + "\n",
+            label="cassette",
+        )
+        if not json_output:
+            for event in result.history:
+                typer.echo(
+                    f"shrink: {event['reducer']} "
+                    f"{event['size_before']} -> {event['size_after']} "
+                    f"({'kept' if event['kept'] else 'discarded'})"
+                )
+        return {
+            "command": "shrink",
+            "cassette": str(out),
+            "original_entries": result.stats["original_entries"],
+            "minimal_entries": result.stats["minimal_entries"],
+            "original_bytes": result.stats["original_bytes"],
+            "minimal_bytes": result.stats["minimal_bytes"],
+            "oracle_calls": result.stats["oracle_calls"],
+            "history_events": result.stats["history_events"],
+        }
+
+    finish(command(run, json_output=json_output, debug=debug), json_output=json_output)

grudge/config.py

@@ -0,0 +1,35 @@
+"""Bootstrap configuration shared by every pipeline stage.
+
+Phase 0 keeps defaults deterministic and offline-first; later phases may add
+validated environment loading without changing these stable names.
+"""
+
+from dataclasses import dataclass
+from pathlib import Path
+
+N_RUNS: int = 20
+WILSON_Z: float = 1.96
+GATE_THRESHOLD: float = 0.80
+OFFLINE: bool = True
+DEFAULT_MODEL: str = "offline-scripted"
+
+PACKAGE_DIR: Path = Path(__file__).resolve().parent
+PROJECT_ROOT: Path = PACKAGE_DIR.parent
+CACHE_DIR: Path = PROJECT_ROOT / ".grudge" / "cache"
+SCARS_DIR: Path = PROJECT_ROOT / "scars"
+
+
+@dataclass(frozen=True)
+class Settings:
+    """Immutable application defaults used by the Phase 0 command surface."""
+
+    n_runs: int = N_RUNS
+    wilson_z: float = WILSON_Z
+    gate_threshold: float = GATE_THRESHOLD
+    offline: bool = OFFLINE
+    model: str = DEFAULT_MODEL
+    cache_dir: Path = CACHE_DIR
+    scars_dir: Path = SCARS_DIR
+
+
+DEFAULTS = Settings()

grudge/demo/__init__.py

@@ -0,0 +1 @@
+"""Demo package; the five-beat offline story is implemented in a later phase."""