agentprobe-testing 0.5.0__py3-none-any.whl

agentprobe/cli.py

@@ -0,0 +1,684 @@
+"""CLI entrypoint: python -m agentprobe.cli run --mode robustness|recovery"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+from dotenv import load_dotenv
+
+from agentprobe.report import InjectionRecord, Report
+from agentprobe.runner import run_recovery, run_robustness_pair
+from agentprobe.scenarios.registry import ALL_SCENARIOS
+from agentprobe.termui import Spinner, green, pass_fail, red, yellow
+
+
+def _make_injector(
+    name: str,
+    model: str,
+    hardcoded_offsets: list[int],
+    hardcoded_tool: str,
+    kind_policy: str,
+    scenario_id: str | None = None,
+    replay_data: dict | None = None,
+    playbook=None,
+    scenario_shape=None,
+    domain=None,
+):
+    if name == "none":
+        from agentprobe.injector import NullInjector
+
+        return NullInjector()
+    if name == "hardcoded":
+        from agentprobe.injector import HardcodedToolErrorInjector
+
+        return HardcodedToolErrorInjector(offsets=hardcoded_offsets, tool_name=hardcoded_tool)
+    if name == "model":
+        from agentprobe.domain import TICKET_DOMAIN
+
+        domain = domain or TICKET_DOMAIN
+        if domain.injector_system_prompt is None:
+            # This Domain hasn't been given its own Injector system prompt
+            # (see domain.py's injector_system_prompt field) -- pointing
+            # ModelInjector at it anyway would send a prompt written for a
+            # different domain's vocabulary. hardcoded/none work fine under
+            # any domain in the meantime (see _make_injector's other
+            # branches -- neither references domain vocabulary at all).
+            raise ValueError(
+                "--injector model isn't wired up for this --domain yet (no injector_system_prompt "
+                "set on its Domain). Use --injector hardcoded or none instead."
+            )
+        from agentprobe.injector import ModelInjector
+
+        return ModelInjector(
+            model=model,
+            policy=kind_policy,
+            playbook=playbook,
+            scenario_shape=scenario_shape,
+            system_prompt=domain.injector_system_prompt,
+            all_tools=domain.all_tools,
+            commit_tools=domain.commit_tools,
+            entity_types=domain.injector_entity_types,
+        )
+    if name == "replay":
+        from agentprobe.injector import NullInjector, ReplayInjector, deserialize_armed_injection
+
+        recorded = (replay_data or {}).get(scenario_id)
+        if recorded is None:
+            print(
+                f"no recorded injections for scenario {scenario_id!r} in replay file -- using NullInjector",
+                file=sys.stderr,
+            )
+            return NullInjector()
+        return ReplayInjector([deserialize_armed_injection(d) for d in recorded])
+    raise ValueError(f"unknown injector {name!r}")
+
+
+def _load_json_file(path: str) -> dict | None:
+    """Loads a JSON file for a CLI flag/argument, printing a clean
+    one-line message and returning None on a bad path or malformed JSON
+    instead of letting FileNotFoundError/JSONDecodeError surface as a raw
+    traceback -- both are ordinary user typos (wrong path, half-written
+    file), not harness bugs."""
+    try:
+        with open(path) as f:
+            return json.load(f)
+    except FileNotFoundError:
+        print(f"no such file: {path}", file=sys.stderr)
+        return None
+    except json.JSONDecodeError as e:
+        print(f"{path} is not valid JSON: {e}", file=sys.stderr)
+        return None
+
+
+def _injector_cost_usd(injector) -> float:
+    """ModelInjector is the only injector type with API cost (none/
+    hardcoded/replay never call a model). Unwraps RecordingInjector's
+    wrapping too, since --record-injections swaps in a wrapper that
+    doesn't have its own total_cost_usd."""
+    inner = getattr(injector, "_inner", injector)
+    return getattr(inner, "total_cost_usd", 0.0)
+
+
+def _target_factory(target: str, model: str, domain_name: str = "ticket"):
+    """Target is an Agent (agentprobe/agents/base.py) -- the runner only
+    ever calls start()/next_action()/observe(), so any implementation is
+    swappable here. "claude" wraps the real Claude tool-calling loop for
+    whichever domain is selected; "scripted" is the dummy replay agent
+    used to exercise the world/tools/runner without live API calls (see
+    agents/scripted.py) -- it always immediately gives a final answer with
+    no script, since a useful script is scenario-specific and has no
+    general CLI form, and it's domain-agnostic (no tool-name knowledge
+    baked in), so it works under any --domain. "rule_based" is a real (if
+    arbitrary) deterministic decision tree -- zero API cost like scripted,
+    but with actual branching/retry/recovery logic for the harness to
+    chaos-test against -- each domain has its own (see agents/
+    rule_based.py for ticket, domains/access_control/rule_based_agent.py
+    for access_control).
+    """
+    if target == "scripted":
+        from agentprobe.agents.scripted import ScriptedAgent
+
+        return lambda: ScriptedAgent(script=[])
+    if domain_name == "access_control":
+        if target == "claude":
+            from agentprobe.domains.access_control.agent import AccessControlAgent
+
+            return lambda: AccessControlAgent(model=model)
+        if target == "rule_based":
+            from agentprobe.domains.access_control.rule_based_agent import AccessControlRuleBasedAgent
+
+            return lambda: AccessControlRuleBasedAgent()
+        raise ValueError(f"target {target!r} is not available under --domain access_control (try 'claude', 'scripted', or 'rule_based')")
+    if target == "claude":
+        from agentprobe.agents.target_agent import TargetAgent
+
+        return lambda: TargetAgent(model=model)
+    if target == "rule_based":
+        from agentprobe.agents.rule_based import RuleBasedAgent
+
+        return lambda: RuleBasedAgent()
+    raise ValueError(f"unknown target {target!r}")
+
+
+def _serialize_trajectory(t) -> dict:
+    return {
+        "scenario_id": t.scenario_id,
+        "variant": t.variant,
+        "passed": t.passed,
+        "final_status": t.final_status.value,
+        "breaking_step": t.breaking_step,
+        "wasted_steps": t.wasted_steps,
+        "halted_reason": t.halted_reason,
+        "final_answer": t.final_answer,
+        "discarded": t.discarded,
+        "injector_errors": t.injector_errors,
+        "steps": [
+            {
+                "index": s.index,
+                "tool_name": s.tool_name,
+                "tool_args": s.tool_args,
+                "ok": s.ok,
+                "result": s.result,
+                "reachability": s.reachability.status.value,
+                "reason": s.reachability.reason,
+                "injected_error": s.injected_error,
+            }
+            for s in t.steps
+        ],
+        "injections": [
+            {
+                "armed_at_step": a.armed_at_step,
+                "fired_at_step": a.fired_at_step,
+                "kind": a.injection.kind.value,
+                "payload": a.injection.payload,
+                "intent": a.injection.intent,
+                "expected_signature": a.injection.expected_signature.description,
+                "rationale": a.injection.rationale,
+                "effect": a.effect,
+                "valid": a.valid,
+            }
+            for a in t.injections
+        ],
+        "expired_injections": [
+            {
+                "armed_at_step": e.armed_at_step,
+                "kind": e.injection.kind.value,
+                "payload": e.injection.payload,
+                "trigger": e.trigger.describe(),
+                "rationale": e.injection.rationale,
+            }
+            for e in t.expired_injections
+        ],
+    }
+
+
+def main(argv: list[str] | None = None) -> int:
+    from agentprobe import __version__
+
+    parser = argparse.ArgumentParser(prog="agentprobe")
+    parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    run_p = sub.add_parser("run", help="Run scenarios in robustness or recovery mode.")
+    run_p.add_argument("--mode", default="robustness", choices=["robustness", "recovery"])
+    run_p.add_argument("--injector", default="hardcoded", choices=["none", "hardcoded", "model", "replay"])
+    run_p.add_argument("--injector-model", default="claude-sonnet-5")
+    run_p.add_argument(
+        "--replay-file",
+        default=None,
+        help="Required with --injector replay: a JSON file (from --record-injections) of "
+        "recorded armed injections keyed by scenario id. Lets you re-run the identical chaos "
+        "sequence against a different --target/--target-model with zero Injector-side API cost.",
+    )
+    run_p.add_argument(
+        "--record-injections",
+        default=None,
+        help="Write every armed injection decided during this run to this JSON file, keyed by "
+        "scenario id, for later replay via --injector replay --replay-file. Most useful with "
+        "--injector model: pay for the Injector's decisions once, replay them against as many "
+        "Target models as you want for free.",
+    )
+    run_p.add_argument(
+        "--kind-policy",
+        default="round_robin",
+        choices=["free", "round_robin", "forced_coverage"],
+        help="Only applies to --injector model. free: no restriction, matches whichever kind is "
+        "easiest to justify (starves TOOL_ERROR/AMBIGUITY). round_robin (default): offers only "
+        "the next uncovered kind, forces once budget is tight. forced_coverage: forces an "
+        "injection every step while any kind remains uncovered, no wait allowed.",
+    )
+    run_p.add_argument(
+        "--domain",
+        default="ticket",
+        choices=["ticket", "access_control"],
+        help="Which business domain's world/tools/scenarios to run against. ticket (default): the "
+        "customer-support domain this harness shipped with. access_control: employees requesting "
+        "access to internal systems -- a structurally different domain built on the same generic "
+        "chaos-injection engine, with its own Injector prompt too (see domain.py).",
+    )
+    run_p.add_argument(
+        "--target",
+        default="claude",
+        choices=["claude", "scripted", "rule_based"],
+        help="claude: real Claude tool-calling loop (--target-model selects which model). "
+        "scripted: dummy replay Agent, no live calls -- for exercising the harness itself. "
+        "rule_based: deterministic decision-tree Agent, no live calls -- a real (if arbitrary) "
+        "policy to chaos-test for free -- each --domain has its own (see agents/rule_based.py).",
+    )
+    run_p.add_argument("--target-model", default="claude-haiku-4-5-20251001")
+    run_p.add_argument(
+        "--hardcoded-offsets",
+        default="0",
+        help="Comma-separated offsets (from baseline end, not absolute step index) at which the hardcoded injector fires, e.g. '0,2'.",
+    )
+    run_p.add_argument(
+        "--hardcoded-tool",
+        default=None,
+        help="Defaults to the chosen --domain's own default_hardcoded_tool (e.g. issue_refund "
+        "for ticket, grant_access for access_control) -- a tool name from the wrong domain can "
+        "never match a real call, so the injector would silently arm every step and never fire.",
+    )
+    run_p.add_argument("--scenario", default=None, help="Run a single scenario id.")
+    run_p.add_argument(
+        "--list-scenarios",
+        action="store_true",
+        help="Print every scenario in --domain as JSON (id, scenario_class, baseline_until, "
+        "max_steps, required_commits_count) and exit without running anything -- for building a "
+        "CI test matrix or picking scenario ids programmatically.",
+    )
+    run_p.add_argument(
+        "--classify", action="store_true", help="Run the response classifier on every valid injection."
+    )
+    run_p.add_argument(
+        "--max-total-cost-usd",
+        type=float,
+        default=None,
+        help="Stop starting new scenarios once combined Target + Injector spend so far reaches this "
+        "amount -- a whole-invocation safety net, separate from each scenario's own (much smaller) "
+        "max_cost_usd. Only applies to --target claude and/or --injector model; checked between "
+        "scenarios, not mid-scenario, so a single scenario can still slightly overshoot it. Does not "
+        "cover --classify's cost, which runs after every scenario in this invocation has finished.",
+    )
+    run_p.add_argument("--classifier-model", default="claude-sonnet-5")
+    run_p.add_argument("--json-out", default=None, help="Write full trajectory JSON here (mandatory per spec section 13 in spirit -- never analyze from memory only).")
+    run_p.add_argument("--html-out", default=None, help="Write a self-contained HTML report here -- for handing to a non-engineer stakeholder, not just reading in a terminal. Always includes the scenario-by-scenario step detail (collapsed by default).")
+    run_p.add_argument(
+        "--detailed",
+        action="store_true",
+        help="Also print a scenario-by-scenario, step-by-step breakdown of every tool call in both "
+        "the clean and chaos runs -- what render() summarizes into pass/fail counts, in full. Off by "
+        "default since it's a lot of output; the --html-out report always includes it either way "
+        "(collapsed, one click per scenario).",
+    )
+    run_p.add_argument(
+        "--export-for-labeling",
+        default=None,
+        help="Requires --classify. Write a sample of classified injections to this JSON file for "
+        "hand-labeling (fill in human_classification per entry), then run "
+        "`agentprobe score-agreement` on it -- spec section 7's mandate: classifier agreement "
+        "below ~80%% means downstream results mean nothing.",
+    )
+    run_p.add_argument("--labeling-sample-size", type=int, default=30, help="Sample size for --export-for-labeling.")
+    run_p.add_argument(
+        "--use-playbook",
+        action="store_true",
+        help="Only applies to --injector model. Surface an advisory hint in the Injector's prompt "
+        "naming whichever trigger has historically fired most often for scenarios shaped like this "
+        "one, and record this run's outcomes back into the playbook for future runs. Off by "
+        "default -- opt in once you have runs worth learning from.",
+    )
+    run_p.add_argument("--playbook-file", default="~/.agentprobe/playbook.json", help="Path for --use-playbook.")
+    run_p.add_argument(
+        "--export-for-feedback",
+        default=None,
+        help="Requires --use-playbook. Write a summary of this run's fired injections plus one "
+        "blank free-text 'feedback' field for the whole run (what was good, what should change, "
+        "what was missing) to this JSON file, then run `agentprobe apply-feedback` on it -- feeds "
+        "human judgment of injection quality into the playbook, a different axis from the "
+        "automated fire-rate/handled tracking --use-playbook already does on its own.",
+    )
+
+    score_p = sub.add_parser("score-agreement", help="Compute classifier agreement from a hand-labeled --export-for-labeling file.")
+    score_p.add_argument("labeled_file", help="A file from --export-for-labeling with human_classification filled in.")
+
+    feedback_p = sub.add_parser("apply-feedback", help="Fold a hand-written --export-for-feedback file back into the playbook.")
+    feedback_p.add_argument("feedback_file", help="A file from --export-for-feedback with the run-level 'feedback' field filled in.")
+    feedback_p.add_argument("--playbook-file", default="~/.agentprobe/playbook.json", help="Playbook file to update.")
+
+    diff_p = sub.add_parser("diff", help="Compare two --json-out dumps and report pass/fail regressions.")
+    diff_p.add_argument("before", help="Path to an earlier --json-out dump.")
+    diff_p.add_argument("after", help="Path to a later --json-out dump to compare against it.")
+    diff_p.add_argument(
+        "--fail-on-regression",
+        action="store_true",
+        help="Exit 1 if any scenario that passed in `before` fails in `after` -- for a CI gate.",
+    )
+
+    args = parser.parse_args(argv)
+    load_dotenv()
+
+    if args.command == "diff":
+        from agentprobe.diff import diff_runs, render_diff
+
+        before_payload = _load_json_file(args.before)
+        if before_payload is None:
+            return 1
+        after_payload = _load_json_file(args.after)
+        if after_payload is None:
+            return 1
+        result = diff_runs(before_payload, after_payload)
+        print(render_diff(result))
+        return 1 if (args.fail_on_regression and result.has_regressions) else 0
+
+    if args.command == "score-agreement":
+        from agentprobe.agreement import score_agreement
+
+        labeled = _load_json_file(args.labeled_file)
+        if labeled is None:
+            return 1
+        try:
+            agreed, total = score_agreement(labeled)
+        except ValueError as e:
+            print(str(e), file=sys.stderr)
+            return 1
+        if total == 0:
+            print("no labeled entries found (human_classification is empty for all of them)")
+            return 1
+        pct = 100 * agreed / total
+        pct_color = green if pct >= 80 else red
+        print(f"classifier agreement with human labels: {agreed}/{total} ({pct_color(f'{pct:.0f}%')})")
+        if pct < 80:
+            print(red("below the ~80% bar spec section 7 sets -- downstream results from this classifier configuration should not be trusted yet"))
+        return 0
+
+    if args.command == "apply-feedback":
+        from agentprobe.feedback import apply_feedback
+        from agentprobe.playbook import Playbook
+
+        payload = _load_json_file(args.feedback_file)
+        if payload is None:
+            return 1
+        playbook = Playbook(path=args.playbook_file)
+        applied = apply_feedback(payload, playbook)
+        if applied == 0:
+            print("no feedback found (the 'feedback' field was left blank)")
+            return 1
+        playbook.save()
+        print(f"applied feedback to {applied} kind/trigger/shape combination{'s' if applied != 1 else ''} -> {args.playbook_file}")
+        return 0
+
+    if args.command != "run":
+        return 1
+
+    if args.injector == "replay" and not args.replay_file:
+        print("--injector replay requires --replay-file", file=sys.stderr)
+        return 1
+
+    if args.export_for_labeling and not args.classify:
+        print("--export-for-labeling requires --classify", file=sys.stderr)
+        return 1
+
+    if args.export_for_feedback and not args.use_playbook:
+        print("--export-for-feedback requires --use-playbook", file=sys.stderr)
+        return 1
+
+    try:
+        hardcoded_offsets = [int(x) for x in args.hardcoded_offsets.split(",")]
+    except ValueError:
+        print(
+            f"--hardcoded-offsets must be a comma-separated list of integers, got {args.hardcoded_offsets!r}",
+            file=sys.stderr,
+        )
+        return 1
+
+    replay_data = None
+    if args.replay_file:
+        replay_data = _load_json_file(args.replay_file)
+        if replay_data is None:
+            return 1
+
+    if args.domain == "access_control":
+        from agentprobe.domains.access_control.domain import ACCESS_CONTROL_DOMAIN
+        from agentprobe.domains.access_control.scenarios import SCENARIOS as ALL_DOMAIN_SCENARIOS
+
+        domain_obj = ACCESS_CONTROL_DOMAIN
+    else:
+        from agentprobe.domain import TICKET_DOMAIN
+
+        ALL_DOMAIN_SCENARIOS = ALL_SCENARIOS
+        domain_obj = TICKET_DOMAIN
+
+    if args.hardcoded_tool is None:
+        args.hardcoded_tool = domain_obj.default_hardcoded_tool
+
+    if args.list_scenarios:
+        print(
+            json.dumps(
+                [
+                    {
+                        "id": s.id,
+                        "domain": domain_obj.name,
+                        "scenario_class": s.scenario_class,
+                        "baseline_until": s.baseline_until,
+                        "max_steps": s.max_steps,
+                        "required_commits_count": len(s.goal.required_commits),
+                    }
+                    for s in ALL_DOMAIN_SCENARIOS
+                ],
+                indent=2,
+            )
+        )
+        return 0
+
+    scenarios = ALL_DOMAIN_SCENARIOS
+    if args.scenario:
+        scenarios = [s for s in scenarios if s.id == args.scenario]
+        if not scenarios:
+            print(f"no such scenario: {args.scenario}", file=sys.stderr)
+            return 1
+
+    try:
+        target_factory = _target_factory(args.target, args.target_model, domain_name=args.domain)
+    except ValueError as e:
+        print(str(e), file=sys.stderr)
+        return 1
+
+    playbook = None
+    if args.use_playbook:
+        from agentprobe.playbook import Playbook
+
+        playbook = Playbook(path=args.playbook_file)
+
+    recorded_by_scenario: dict[str, list[dict]] = {}
+    shapes_by_scenario: dict[str, object] = {}
+    injector_cost_usd = 0.0
+    clean_trajectories = []
+    chaos_trajectories = []
+    total_scenarios = len(scenarios)
+    for scenario_index, s in enumerate(scenarios, start=1):
+        progress = f"[{scenario_index}/{total_scenarios}] " if total_scenarios > 1 else ""
+        with Spinner(f"{progress}running {s.id}") as spin:
+            current_shape = None
+            if playbook is not None:
+                from agentprobe.playbook import scenario_shape
+
+                current_shape = scenario_shape(s, domain_name=domain_obj.name)
+                shapes_by_scenario[s.id] = current_shape
+            # a fresh injector instance per scenario -- hardcoded/model injectors
+            # carry per-run state (e.g. "have I fired yet").
+            try:
+                injector = _make_injector(
+                    args.injector,
+                    args.injector_model,
+                    hardcoded_offsets,
+                    args.hardcoded_tool,
+                    args.kind_policy,
+                    scenario_id=s.id,
+                    replay_data=replay_data,
+                    playbook=playbook,
+                    scenario_shape=current_shape,
+                    domain=domain_obj,
+                )
+            except ValueError as e:
+                print(str(e), file=sys.stderr)
+                return 1
+            recorder = None
+            if args.record_injections:
+                from agentprobe.injector import RecordingInjector
+
+                recorder = RecordingInjector(injector)
+                injector = recorder
+            if args.mode == "robustness":
+                clean, chaos = run_robustness_pair(s, target_factory, injector, domain=domain_obj)
+                clean_trajectories.append(clean)
+                chaos_trajectories.append(chaos)
+                suffix = f"clean: {pass_fail(clean.passed)}   chaos: {pass_fail(chaos.passed)}"
+                if chaos.discarded:
+                    suffix += f"  {yellow('[DISCARDED: invalid injection]')}"
+                spin.finish(ok=clean.passed and chaos.passed, suffix=suffix)
+            else:
+                chaos = run_recovery(s, target_factory, injector, domain=domain_obj)
+                chaos_trajectories.append(chaos)
+                suffix = pass_fail(chaos.passed)
+                if chaos.discarded:
+                    suffix += f"  {yellow('[DISCARDED: invalid injection]')}"
+                spin.finish(ok=chaos.passed, suffix=suffix)
+            if recorder is not None:
+                from agentprobe.injector import serialize_armed_injection
+
+                recorded_by_scenario[s.id] = [serialize_armed_injection(a) for a in recorder.recorded]
+            injector_cost_usd += _injector_cost_usd(injector)
+
+        if args.max_total_cost_usd is not None:
+            target_cost_so_far = sum(t.total_cost_usd for t in clean_trajectories) + sum(
+                t.total_cost_usd for t in chaos_trajectories
+            )
+            spent_so_far = target_cost_so_far + injector_cost_usd
+            if spent_so_far >= args.max_total_cost_usd:
+                remaining = total_scenarios - scenario_index
+                print(
+                    f"\n--max-total-cost-usd reached (${spent_so_far:.4f} >= ${args.max_total_cost_usd:.2f}) "
+                    f"after {s.id} -- stopping before {remaining} remaining scenario(s). "
+                    "Report below covers only what actually ran.",
+                    file=sys.stderr,
+                )
+                break
+
+    if args.record_injections:
+        with open(args.record_injections, "w") as f:
+            json.dump(recorded_by_scenario, f, indent=2)
+        print(f"recorded injections for {len(recorded_by_scenario)} scenario(s) -> {args.record_injections}", file=sys.stderr)
+
+    from agentprobe.injection import injection_was_triggered
+
+    injection_records = [
+        InjectionRecord(
+            scenario_id=t.scenario_id, applied=a, triggered=injection_was_triggered(a, t.steps)
+        )
+        for t in chaos_trajectories
+        for a in t.injections
+    ]
+
+    if args.classify:
+        from agentprobe.classifier import classify_response
+
+        for i, record in enumerate(injection_records):
+            if not record.applied.valid or not record.triggered:
+                continue
+            traj = next(t for t in chaos_trajectories if t.scenario_id == record.scenario_id)
+            print(f"classifying {record.scenario_id} step {record.applied.fired_at_step} ...", file=sys.stderr)
+            classification = classify_response(
+                record.applied, traj.steps, traj.final_answer, model=args.classifier_model
+            )
+            injection_records[i] = InjectionRecord(
+                scenario_id=record.scenario_id,
+                applied=record.applied,
+                classification=classification,
+                triggered=record.triggered,
+            )
+
+    if playbook is not None:
+        from agentprobe.playbook import OutcomeRecord
+
+        for record in injection_records:
+            shape = shapes_by_scenario.get(record.scenario_id)
+            if shape is None:
+                continue
+            handled = None
+            if record.classification is not None:
+                handled = record.classification.classification.value == "HANDLED"
+            playbook.record(
+                OutcomeRecord(
+                    kind=record.applied.injection.kind.value,
+                    trigger_kind=record.applied.trigger_kind,
+                    scenario_shape=shape,
+                    fired=True,
+                    valid=record.applied.valid,
+                    handled=handled,
+                )
+            )
+        for t in chaos_trajectories:
+            shape = shapes_by_scenario.get(t.scenario_id)
+            if shape is None:
+                continue
+            for e in t.expired_injections:
+                playbook.record(
+                    OutcomeRecord(
+                        kind=e.injection.kind.value,
+                        trigger_kind=e.trigger.kind.value,
+                        scenario_shape=shape,
+                        fired=False,
+                    )
+                )
+        playbook.save()
+        print(f"playbook updated -> {args.playbook_file} ({len(playbook.records)} total records)", file=sys.stderr)
+
+    if args.export_for_labeling:
+        # already validated earlier that --export-for-labeling requires --classify
+        from agentprobe.agreement import export_for_labeling
+
+        sample = export_for_labeling(injection_records, n=args.labeling_sample_size)
+        with open(args.export_for_labeling, "w") as f:
+            json.dump(sample, f, indent=2)
+        print(
+            f"exported {len(sample)} classification(s) for hand-labeling -> {args.export_for_labeling} "
+            "(fill in human_classification per entry, then run `agentprobe score-agreement`)",
+            file=sys.stderr,
+        )
+
+    if args.export_for_feedback:
+        # already validated earlier that --export-for-feedback requires --use-playbook
+        from agentprobe.feedback import export_for_feedback
+
+        summary = export_for_feedback(injection_records, shapes_by_scenario)
+        with open(args.export_for_feedback, "w") as f:
+            json.dump(summary, f, indent=2)
+        print(
+            f"exported {len(summary['summary'])} injection(s) for feedback -> {args.export_for_feedback} "
+            "(fill in the 'feedback' field, then run `agentprobe apply-feedback`)",
+            file=sys.stderr,
+        )
+
+    injector_labels = {
+        "none": "none",
+        "hardcoded": f"hardcoded({args.hardcoded_tool}@{args.hardcoded_offsets})",
+        "model": args.injector_model,
+        "replay": f"replay({args.replay_file})",
+    }
+    report = Report(
+        mode=args.mode,
+        injector_model=injector_labels[args.injector],
+        target_model=args.target_model if args.target == "claude" else args.target,
+        clean_trajectories=clean_trajectories,
+        chaos_trajectories=chaos_trajectories,
+        injection_records=injection_records,
+        injector_cost_usd=injector_cost_usd,
+    )
+    print()
+    print(report.render())
+
+    if args.detailed:
+        print()
+        print(report.render_scenario_detail())
+
+    if args.json_out:
+        payload = {
+            "mode": args.mode,
+            "clean_trajectories": [_serialize_trajectory(t) for t in clean_trajectories],
+            "chaos_trajectories": [_serialize_trajectory(t) for t in chaos_trajectories],
+        }
+        with open(args.json_out, "w") as f:
+            json.dump(payload, f, indent=2)
+
+    if args.html_out:
+        with open(args.html_out, "w") as f:
+            f.write(report.render_html())
+        print(f"HTML report -> {args.html_out}", file=sys.stderr)
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())