agentprobe-testing 0.5.0__py3-none-any.whl

agentprobe/llm.py

@@ -0,0 +1,123 @@
+"""Shared helper for calling messages.create with best-effort determinism.
+
+The anthropic SDK (1.0.0, latest on PyPI) has no typed `temperature`
+parameter at all, but the raw API still validates and honors it for some
+models -- reachable via `extra_body` (confirmed: an out-of-range value gets
+a 400 with "temperature: range: 0..1", and temperature=0 measurably
+converges completions vs the default). Newer models reject it outright
+(observed: claude-sonnet-5 -> 400 "`temperature` is deprecated for this
+model"). This probes once per model and falls back cleanly, instead of
+either hardcoding a guess per call site or crashing every call for models
+where it's unavailable.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+_TEMPERATURE_SUPPORTED: dict[str, bool] = {}
+
+
+def create_deterministic(client: Any, **kwargs: Any) -> Any:
+    model = kwargs["model"]
+    if _TEMPERATURE_SUPPORTED.get(model, True):
+        try:
+            return client.messages.create(**kwargs, extra_body={"temperature": 0})
+        except Exception as e:
+            if "temperature" in str(e).lower():
+                _TEMPERATURE_SUPPORTED[model] = False
+            else:
+                raise
+    return client.messages.create(**kwargs)
+
+
+_PRICE_PER_TOKEN: dict[str, tuple[float, float]] = {
+    # (input, output) USD per token, from the pricing table this was
+    # checked against (2026-08). Keyed by the exact model ids this
+    # codebase actually passes around.
+    "claude-haiku-4-5-20251001": (1e-6, 5e-6),
+    "claude-sonnet-5": (2e-6, 10e-6),
+    "claude-opus-5": (5e-6, 25e-6),
+}
+_PRICE_FALLBACK_BY_FAMILY: list[tuple[str, tuple[float, float]]] = [
+    # substring match against the model id, for a version not listed
+    # above (e.g. a dated snapshot) -- still prices at that family's
+    # current rate instead of silently defaulting to one fixed rate
+    # regardless of which model actually ran, which is the bug this
+    # replaces (cost tracking previously hardcoded Sonnet-era pricing for
+    # every model, over-costing Haiku runs ~5x and under-costing Opus
+    # runs ~5x on input).
+    ("haiku", (1e-6, 5e-6)),
+    ("sonnet", (2e-6, 10e-6)),
+    ("opus", (5e-6, 25e-6)),
+]
+
+
+def cost_per_token(model: str) -> tuple[float, float]:
+    """(input, output) USD-per-token for `model`."""
+    if model in _PRICE_PER_TOKEN:
+        return _PRICE_PER_TOKEN[model]
+    lowered = model.lower()
+    for family, price in _PRICE_FALLBACK_BY_FAMILY:
+        if family in lowered:
+            return price
+    return _PRICE_PER_TOKEN["claude-sonnet-5"]  # last-resort default
+
+
+_CACHE_WRITE_MULTIPLIER = 1.25  # Anthropic prompt caching, 5-minute (ephemeral) TTL
+_CACHE_READ_MULTIPLIER = 0.1  # -- fixed multiples of the model's own base input price,
+# not separate absolute rates, and stable across model families; see
+# https://docs.anthropic.com/en/docs/build-with-claude/prompt-caching
+
+
+def compute_cost_usd(
+    model: str,
+    input_tokens: int,
+    output_tokens: int,
+    cache_creation_input_tokens: int = 0,
+    cache_read_input_tokens: int = 0,
+) -> float:
+    """Every live call site in this codebase sends its system prompt
+    through cacheable_system(), so a real response.usage nearly always has
+    non-zero cache_creation_input_tokens (first call in a ~5-min window)
+    or cache_read_input_tokens (every call after) -- neither is included
+    in input_tokens by the API, so omitting them here silently undercounts
+    the real bill on every single live run, not just an edge case."""
+    in_price, out_price = cost_per_token(model)
+    return (
+        input_tokens * in_price
+        + output_tokens * out_price
+        + cache_creation_input_tokens * in_price * _CACHE_WRITE_MULTIPLIER
+        + cache_read_input_tokens * in_price * _CACHE_READ_MULTIPLIER
+    )
+
+
+def response_cost_usd(model: str, response: Any) -> float:
+    """compute_cost_usd from a raw messages.create response's usage --
+    the single call site every caller should use instead of reading
+    response.usage.input_tokens/output_tokens directly, so a mocked test
+    Usage (no cache_* attributes) and a real one (cache_* present, 0 when
+    caching wasn't triggered) both price correctly."""
+    usage = response.usage
+    return compute_cost_usd(
+        model,
+        usage.input_tokens,
+        usage.output_tokens,
+        getattr(usage, "cache_creation_input_tokens", 0) or 0,
+        getattr(usage, "cache_read_input_tokens", 0) or 0,
+    )
+
+
+def cacheable_system(text: str) -> list[dict[str, Any]]:
+    """Wraps a plain-string system prompt into the block form prompt
+    caching requires, with cache_control on the (only) block. The
+    Injector/Target/Classifier system prompts are large, fixed, and
+    resent on nearly every call within a run -- a natural cache-hit
+    target. A cache hit is priced far below base input tokens (e.g.
+    Sonnet 5: $0.20/MTok vs $2/MTok base, per the pricing table this was
+    built against). No-op if the prompt is under the API's minimum
+    cacheable length for the model -- the API just serves it uncached,
+    same as before this existed. Cache writes cost slightly more than a
+    plain uncached call on the FIRST use within the ~5-minute TTL; the
+    savings come from every call after that within the window."""
+    return [{"type": "text", "text": text, "cache_control": {"type": "ephemeral"}}]

agentprobe/playbook.py

@@ -0,0 +1,211 @@
+"""Cross-run accumulated data on which trigger choices actually work,
+keyed by structural features only (injection kind + scenario shape) --
+never actual scenario/customer content (ticket text, customer names,
+task descriptions). That's deliberate: aggregating structural outcome
+statistics doesn't touch what a BYOK customer's data isolation promise
+is actually about (their trajectories, their tickets), so it doesn't
+undercut that pitch the way pooling raw trajectory content would.
+
+v1: a local JSON file plus an advisory hint surfaced into the Injector's
+prompt. This does NOT change the model in any way -- no fine-tuning, no
+persistent state inside the model, every API call is still stateless.
+The compounding advantage lives entirely in this file, which is then
+pasted into the prompt as extra context each time. Whether the model
+actually follows the hint is not guaranteed -- same as any other prompt
+instruction -- this is a stronger prior than reasoning from scratch every
+run, not a hard override of the model's judgment.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+import tempfile
+from collections import defaultdict
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import Optional
+
+from agentprobe.scenario import Scenario
+
+
+@dataclass(frozen=True)
+class ScenarioShape:
+    """Structural features that generalize across customers/scenarios --
+    deliberately excludes anything content-specific."""
+
+    baseline_kind: str  # "first_commit" | "fixed"
+    required_commits_count: int
+    domain: str = "ticket"
+    """Which Domain (see domain.py) this shape belongs to, e.g. "ticket"
+    or "access_control". Two domains' scenarios can easily land on the
+    same (baseline_kind, required_commits_count) -- without this, their
+    trigger-recommendation data would silently blend together in
+    Playbook.recommend(), giving the Injector a prior computed across
+    domains with potentially very different dynamics. Defaults to
+    "ticket" so a playbook.json written before this field existed (when
+    this project only had one domain) still loads and matches correctly."""
+
+    def key(self) -> str:
+        return f"{self.domain}:{self.baseline_kind}:{self.required_commits_count}"
+
+
+def scenario_shape(scenario: Scenario, domain_name: str = "ticket") -> ScenarioShape:
+    baseline_kind = "first_commit" if scenario.baseline_until == "first_commit" else "fixed"
+    return ScenarioShape(
+        baseline_kind=baseline_kind,
+        required_commits_count=len(scenario.goal.required_commits),
+        domain=domain_name,
+    )
+
+
+@dataclass(frozen=True)
+class OutcomeRecord:
+    kind: str  # InjectionKind value
+    trigger_kind: str  # TriggerKind value
+    scenario_shape: ScenarioShape
+    fired: bool
+    valid: Optional[bool] = None  # only meaningful if fired
+    handled: Optional[bool] = None  # only meaningful if fired, valid, and classified
+    # Human judgment, from agentprobe/feedback.py's export/apply round trip --
+    # distinct axis from `handled`. `handled` is the classifier's mechanical
+    # verdict on whether the Target's response addressed the injection.
+    # `feedback` is free text: a customer's own words on what was good,
+    # what should change, and what was missing about this injection. None
+    # until someone writes it. Deliberately not a numeric rating -- forcing
+    # a customer's judgment into yes/no loses exactly the detail (WHY it
+    # was or wasn't realistic) that's useful to feed back to the Injector.
+    feedback: Optional[str] = None
+
+
+class Playbook:
+    """Accumulates OutcomeRecords across runs into a local JSON file, and
+    answers "which trigger has historically worked best for this (kind,
+    shape)?" Read/write are both cheap, small-file operations -- this is
+    not meant to scale past thousands of records without moving to a real
+    database, which is a fine v2 problem to have.
+    """
+
+    def __init__(self, path: str = "~/.agentprobe/playbook.json"):
+        self._path = Path(path).expanduser()
+        self.records: list[OutcomeRecord] = self._load()
+
+    def _load(self) -> list[OutcomeRecord]:
+        if not self._path.exists():
+            return []
+        try:
+            with open(self._path) as f:
+                raw = json.load(f)
+            return [
+                OutcomeRecord(
+                    kind=r["kind"],
+                    trigger_kind=r["trigger_kind"],
+                    scenario_shape=ScenarioShape(**r["scenario_shape"]),
+                    fired=r["fired"],
+                    valid=r.get("valid"),
+                    handled=r.get("handled"),
+                    feedback=r.get("feedback"),
+                )
+                for r in raw
+            ]
+        except (json.JSONDecodeError, KeyError, TypeError) as e:
+            # A crash or interruption mid-write (see save()'s atomic-rename
+            # comment for why this used to be possible) or hand-editing
+            # gone wrong can leave this file unparsable. Losing accumulated
+            # playbook history is bad, but it's strictly better than every
+            # future --use-playbook run crashing with a traceback until
+            # someone finds and deletes the file by hand -- preserve the
+            # bad file for inspection and start fresh instead.
+            backup = self._path.with_suffix(self._path.suffix + ".corrupted")
+            try:
+                self._path.replace(backup)
+                note = f"moved to {backup}"
+            except OSError:
+                note = "could not be backed up"
+            print(
+                f"agentprobe: {self._path} was corrupted ({e}) -- {note}, starting a fresh playbook",
+                file=sys.stderr,
+            )
+            return []
+
+    def save(self) -> None:
+        self._path.parent.mkdir(parents=True, exist_ok=True)
+        payload = [
+            {
+                "kind": r.kind,
+                "trigger_kind": r.trigger_kind,
+                "scenario_shape": asdict(r.scenario_shape),
+                "fired": r.fired,
+                "valid": r.valid,
+                "handled": r.handled,
+                "feedback": r.feedback,
+            }
+            for r in self.records
+        ]
+        # Write to a temp file in the same directory, then atomically
+        # rename over the real path -- os.replace is atomic on both POSIX
+        # and Windows, so a crash or interruption mid-write can never leave
+        # a half-written, corrupted playbook.json behind; the old file
+        # stays intact (or the new one lands whole), never a partial mix.
+        fd, tmp_path = tempfile.mkstemp(dir=self._path.parent, prefix=".playbook-", suffix=".tmp")
+        try:
+            with os.fdopen(fd, "w") as f:
+                json.dump(payload, f, indent=2)
+            os.replace(tmp_path, self._path)
+        except BaseException:
+            try:
+                os.unlink(tmp_path)
+            except OSError:
+                pass
+            raise
+
+    def record(self, outcome: OutcomeRecord) -> None:
+        self.records.append(outcome)
+
+    def recommend(self, kind: str, shape: ScenarioShape, min_n: int = 10) -> Optional[dict]:
+        """Best-performing trigger_kind (by fire rate) for this (kind,
+        shape) combination, or None if there's fewer than min_n armed
+        attempts total -- callers should fall back to the existing
+        hard-coded default in that case, not act on noise from a handful
+        of runs. Fire rate, not handled rate: a trigger that never fires
+        can't be handled at all, and this is specifically about the
+        expiry problem this session spent most of its time on, not yet
+        about which trigger produces the best Target behavior once fired.
+        """
+        relevant = [r for r in self.records if r.kind == kind and r.scenario_shape == shape]
+        if len(relevant) < min_n:
+            return None
+
+        by_trigger: dict[str, list[OutcomeRecord]] = defaultdict(list)
+        for r in relevant:
+            by_trigger[r.trigger_kind].append(r)
+
+        best_trigger, best_rate, best_n = None, -1.0, 0
+        for trigger_kind, records in by_trigger.items():
+            n = len(records)
+            fire_rate = sum(1 for r in records if r.fired) / n
+            if fire_rate > best_rate:
+                best_trigger, best_rate, best_n = trigger_kind, fire_rate, n
+
+        if best_trigger is None:
+            return None
+        return {"trigger_kind": best_trigger, "fire_rate": best_rate, "n": best_n}
+
+    def recent_feedback(self, kind: str, shape: ScenarioShape, limit: int = 3) -> list[str]:
+        """Free-text customer notes for this (kind, shape), most recent
+        first -- from records a customer actually wrote via
+        agentprobe/feedback.py. Unlike recommend(), there's no rate to
+        average here (text doesn't reduce to a number), so this returns raw
+        notes for the Injector's prompt to read directly, capped at `limit`
+        so a long feedback history doesn't dominate the prompt. "Recent"
+        means "most recently appended to self.records" -- records don't
+        carry a timestamp, and append order is a good enough proxy since
+        feedback is always applied after the outcomes it comments on.
+        """
+        notes = [
+            r.feedback
+            for r in self.records
+            if r.kind == kind and r.scenario_shape == shape and r.feedback
+        ]
+        return notes[::-1][:limit]

agentprobe/quickstart.py

@@ -0,0 +1,295 @@
+"""The lowest-friction way into this library: wrap_agent() turns a plain
+function into a valid Target (no Agent subclass, no manual state), and
+quick_test() collapses run_robustness_pair()+Report() into one call. Both
+are pure convenience over the same public API in __init__.py -- nothing
+here does anything the longer form couldn't already do, this just removes
+the boilerplate for the common case of "I have a function that decides
+what to do next, test it against one scenario."
+"""
+
+from __future__ import annotations
+
+from typing import Any, Callable, Optional
+
+from agentprobe.agents.base import Agent, AgentAction
+from agentprobe.domain import Domain, TICKET_DOMAIN
+from agentprobe.injection import injection_was_triggered
+from agentprobe.injector import Injector, NullInjector, RecordingInjector, serialize_armed_injection
+from agentprobe.report import InjectionRecord, Report
+from agentprobe.runner import run_recovery, run_robustness_pair
+from agentprobe.scenario import Scenario
+
+DecideFn = Callable[[str, list[dict[str, Any]]], AgentAction]
+
+
+class _FunctionAgent(Agent):
+    """Adapts a plain decide_fn(task, history) -> AgentAction function to
+    the Agent interface. One instance per run (see wrap_agent), so state
+    never leaks between the clean and chaos trajectories of a robustness
+    pair -- the caller's function never has to think about that."""
+
+    def __init__(self, decide_fn: DecideFn):
+        self._decide_fn = decide_fn
+        self._task = ""
+        self._history: list[dict[str, Any]] = []
+
+    def start(self, task: str, tool_schemas: list[dict]) -> None:
+        self._task = task
+        self._history = []
+
+    def next_action(self) -> AgentAction:
+        return self._decide_fn(self._task, list(self._history))
+
+    def observe(self, tool_name: str, tool_args: dict, result: Any, ok: bool) -> None:
+        self._history.append({"tool_name": tool_name, "tool_args": tool_args, "result": result, "ok": ok})
+
+
+def wrap_agent(decide_fn: DecideFn) -> Callable[[], Agent]:
+    """Wrap a plain function into a target_factory for run_robustness_pair/
+    run_recovery/quick_test. decide_fn receives (task, history) each turn --
+    history is everything observed so far this run, as a list of
+    {"tool_name", "tool_args", "result", "ok"} dicts -- and returns an
+    AgentAction (AgentAction(kind="tool_call", tool_name=..., tool_args=...)
+    or AgentAction(kind="final_answer", text=...)). No class, no manual
+    start()/observe() bookkeeping -- call whatever your real agent already
+    is from inside decide_fn and translate its answer into an AgentAction.
+    """
+    return lambda: _FunctionAgent(decide_fn)
+
+
+def quick_test(
+    scenario: Scenario,
+    agent: Callable[[], Agent] | DecideFn,
+    injector: Optional[Injector] = None,
+    mode: str = "robustness",
+    target_model: str = "custom",
+    injector_model: Optional[str] = None,
+    domain: Domain = TICKET_DOMAIN,
+    upload_api_key: Optional[str] = None,
+    domain_key: Optional[str] = None,
+    record_injections: bool = False,
+) -> Report:
+    """Run one scenario and get back a ready Report -- the one-call version
+    of run_robustness_pair()/run_recovery() + Report(). `agent` can be a
+    target_factory (a zero-arg callable returning an Agent, e.g. RuleBasedAgent)
+    or a plain decide_fn(task, history) -> AgentAction, which gets wrap_agent()
+    applied automatically -- either way, nothing else here changes.
+
+    Pass `upload_api_key` (a personal key from /dashboard) to also upload the
+    finished report to your AgentProbe account, so it shows up under Your
+    Runs -- see registry.upload_run for what this does and its failure
+    behavior. Pass `domain_key` too if `domain` came from fetch_domain(), so
+    the uploaded run links back to it. Omit upload_api_key (the default) and
+    nothing is uploaded -- quick_test() stays fully local and offline.
+
+    Pass `record_injections=True` to wrap `injector` in a RecordingInjector
+    and, when uploading, include the recorded sequence so this exact run can
+    be replayed later via registry.replay_run() -- see its docstring. Has no
+    effect without upload_api_key (there'd be nowhere to store the recording).
+    """
+    injector = injector if injector is not None else NullInjector()
+    target_factory = agent if _looks_like_factory(agent) else wrap_agent(agent)
+    resolved_injector_model = injector_model if injector_model is not None else type(injector).__name__
+
+    recording: Optional[RecordingInjector] = None
+    if record_injections:
+        recording = RecordingInjector(injector)
+        injector = recording
+
+    if mode == "robustness":
+        clean, chaos = run_robustness_pair(scenario, target_factory, injector, domain=domain)
+        clean_trajectories = [clean]
+    else:
+        chaos = run_recovery(scenario, target_factory, injector, domain=domain)
+        clean_trajectories = []
+
+    # Report's "injections fired" stats come from injection_records, not
+    # from chaos_trajectories directly (see report.py's _compute) -- skip
+    # this and a quick_test() report would always show 0 fired regardless
+    # of what actually happened, same mistake the CLI has to avoid too.
+    injection_records = [
+        InjectionRecord(scenario_id=chaos.scenario_id, applied=a, triggered=injection_was_triggered(a, chaos.steps))
+        for a in chaos.injections
+    ]
+
+    report = Report(
+        mode=mode,
+        injector_model=resolved_injector_model,
+        target_model=target_model,
+        clean_trajectories=clean_trajectories,
+        chaos_trajectories=[chaos],
+        injection_records=injection_records,
+        injector_cost_usd=_injector_cost_usd(injector),
+    )
+    if recording is not None:
+        # Stashed here (not a Report field -- Report is a general-purpose
+        # shape other callers build by hand too) so a caller who skips
+        # upload_api_key here to call check_regression() first, then
+        # upload_run() manually, still gets the recording -- upload_run()
+        # auto-detects this attribute when its own recorded_injections
+        # param is omitted. See registry.upload_run.
+        report._recorded_injections = [serialize_armed_injection(a) for a in recording.recorded]
+
+    if upload_api_key is not None:
+        from agentprobe.registry import upload_run
+
+        upload_run(report, api_key=upload_api_key, domain_key=domain_key)
+
+    return report
+
+
+def quick_test_all(
+    scenarios: dict[str, Scenario] | list[Scenario],
+    agent: Callable[[], Agent] | DecideFn,
+    injector_factory: Optional[Callable[[], Injector]] = None,
+    mode: str = "robustness",
+    target_model: str = "custom",
+    injector_model: Optional[str] = None,
+    domain: Domain = TICKET_DOMAIN,
+    upload_api_key: Optional[str] = None,
+    domain_key: Optional[str] = None,
+    record_injections: bool = False,
+) -> Report:
+    """Run every scenario in `scenarios` (a dict[str, Scenario] -- e.g.
+    TICKET_SCENARIOS or a FetchedDomain's .scenarios -- or a plain list of
+    Scenario) against the same agent, and combine every result into ONE
+    Report -- the batch version of calling quick_test() once per scenario
+    and merging the results by hand.
+
+    `injector_factory` is a zero-arg callable returning a FRESH Injector
+    instance per scenario (same shape as wrap_agent's target_factory), e.g.
+    `injector_factory=lambda: ModelInjector()`. This must be a factory, not
+    an already-constructed Injector -- several injectors (ModelInjector's
+    kind rotation, HardcodedToolErrorInjector's fired-offset tracking) carry
+    state across a run that must not leak between unrelated scenarios.
+    Omit it (the default) to run every scenario clean, with no chaos at all.
+
+    If `upload_api_key` is given, uploads EACH scenario's result as its own
+    run (so Your Runs shows them individually, the same granularity as
+    calling quick_test() per scenario and uploading each) -- the Report
+    this function returns is still the combined summary across all of them.
+    """
+    items = list(scenarios.items()) if isinstance(scenarios, dict) else [(s.id, s) for s in scenarios]
+    if not items:
+        raise ValueError("quick_test_all requires at least one scenario")
+
+    target_factory = agent if _looks_like_factory(agent) else wrap_agent(agent)
+    make_injector = injector_factory if injector_factory is not None else (lambda: NullInjector())
+    if not callable(make_injector):
+        raise ValueError(
+            "injector_factory must be a zero-arg callable returning a fresh Injector each time "
+            "(e.g. lambda: ModelInjector()), not an already-constructed Injector instance"
+        )
+
+    all_clean: list = []
+    all_chaos: list = []
+    all_records: list[InjectionRecord] = []
+    total_injector_cost = 0.0
+    combined_injector_model: Optional[str] = None
+
+    for _scenario_id, scenario in items:
+        injector = make_injector()
+        resolved_injector_model = injector_model if injector_model is not None else type(injector).__name__
+        if combined_injector_model is None:
+            combined_injector_model = resolved_injector_model
+
+        recording: Optional[RecordingInjector] = None
+        if record_injections:
+            recording = RecordingInjector(injector)
+            injector = recording
+
+        if mode == "robustness":
+            clean, chaos = run_robustness_pair(scenario, target_factory, injector, domain=domain)
+            all_clean.append(clean)
+        else:
+            clean = None
+            chaos = run_recovery(scenario, target_factory, injector, domain=domain)
+        all_chaos.append(chaos)
+
+        records = [
+            InjectionRecord(scenario_id=chaos.scenario_id, applied=a, triggered=injection_was_triggered(a, chaos.steps))
+            for a in chaos.injections
+        ]
+        all_records.extend(records)
+        injector_cost = _injector_cost_usd(injector)
+        total_injector_cost += injector_cost
+
+        if upload_api_key is not None:
+            from agentprobe.registry import upload_run
+
+            scenario_report = Report(
+                mode=mode,
+                injector_model=resolved_injector_model,
+                target_model=target_model,
+                clean_trajectories=[clean] if clean is not None else [],
+                chaos_trajectories=[chaos],
+                injection_records=records,
+                injector_cost_usd=injector_cost,
+            )
+            recorded = [serialize_armed_injection(a) for a in recording.recorded] if recording is not None else None
+            upload_run(scenario_report, api_key=upload_api_key, domain_key=domain_key, recorded_injections=recorded)
+
+    return Report(
+        mode=mode,
+        injector_model=combined_injector_model or "none",
+        target_model=target_model,
+        clean_trajectories=all_clean,
+        chaos_trajectories=all_chaos,
+        injection_records=all_records,
+        injector_cost_usd=total_injector_cost,
+    )
+
+
+def assert_passes(report: Report, save_html_on_failure: Optional[str] = None) -> None:
+    """Assert every scenario in `report` passed -- chaos, and clean too when
+    this was a robustness-mode run -- the one-line pytest integration: call
+    this at the end of a test instead of hand-picking which Report fields
+    to check.
+
+    Raises AssertionError with a summary of what failed if not, so this
+    reads naturally as the last line of a pytest test function:
+
+        report = quick_test(scenario, my_agent, injector=ModelInjector())
+        assert_passes(report)
+
+    Pass `save_html_on_failure="path.html"` to also write the full
+    render_html() report there when the assertion fails, so a failed CI run
+    leaves something to open instead of just a pass/fail count.
+    """
+    c = report._compute()
+    failures = []
+    if c["clean_passed"] is not None and c["clean_passed"] != c["clean_total"]:
+        failures.append(f"clean pass rate {c['clean_passed']}/{c['clean_total']}")
+    if c["chaos_passed"] != c["n"]:
+        failures.append(f"chaos pass rate {c['chaos_passed']}/{c['n']}")
+    if not failures:
+        return
+    if save_html_on_failure:
+        with open(save_html_on_failure, "w") as f:
+            f.write(report.render_html())
+        failures.append(f"full report written to {save_html_on_failure}")
+    raise AssertionError("agentprobe: " + "; ".join(failures))
+
+
+def _injector_cost_usd(injector: Injector) -> float:
+    """ModelInjector is the only injector type with real API cost (none/
+    hardcoded/replay never call a model) -- same logic as cli.py's version,
+    duplicated rather than imported to keep quickstart.py's only dependency
+    on cli.py at zero. Unwraps RecordingInjector's wrapping too, since it
+    doesn't have its own total_cost_usd."""
+    inner = getattr(injector, "_inner", injector)
+    return getattr(inner, "total_cost_usd", 0.0)
+
+
+def _looks_like_factory(agent: Callable[[], Agent] | DecideFn) -> bool:
+    """A target_factory takes zero arguments; a decide_fn takes two
+    (task, history). Distinguish by arity rather than asking the caller to
+    say which one they passed."""
+    import inspect
+
+    try:
+        params = inspect.signature(agent).parameters
+    except (TypeError, ValueError):
+        return True
+    required = [p for p in params.values() if p.default is p.empty and p.kind in (p.POSITIONAL_ONLY, p.POSITIONAL_OR_KEYWORD)]
+    return len(required) == 0