agentprobe-testing 0.5.0__py3-none-any.whl

agentprobe/report.py

@@ -0,0 +1,666 @@
+"""Aggregates robustness-mode runs into the report format from spec
+section 9.
+
+Reporting rules (repair sheet): report n alongside every percentage;
+suppress a per-kind rate where n < 5 and print the raw count instead
+("LATE_INFO 0/1 (0%)" reads like a finding and is one event); never pool
+classifier agreement across runs with different injector/prompt versions.
+"""
+
+from __future__ import annotations
+
+import math
+from collections import Counter
+from dataclasses import dataclass, field
+from typing import Optional
+
+from agentprobe.classifier import ClassifiedResponse
+from agentprobe.injection import AppliedInjection, InjectionKind
+from agentprobe.termui import blue, bold, cyan, dim, green, pass_fail, purple, red, yellow
+from agentprobe.trajectory import Trajectory
+
+_MAX_REPR_LEN = 160
+
+
+def _truncate(s: str, limit: int = _MAX_REPR_LEN) -> str:
+    return s if len(s) <= limit else s[: limit - 1] + "…"
+
+
+_REPORT_CSS = """
+  :root {
+    color-scheme: light dark;
+    --bg: #ffffff; --bg-alt: #f7f8fa; --bg-card: #ffffff; --border: #e3e5ea;
+    --text: #14161c; --text-dim: #666b7a;
+    --purple: #7c3aed; --blue: #2563eb; --green: #16a34a; --red: #dc2626; --amber: #b45309;
+    --purple-bg: rgba(124,58,237,.08); --blue-bg: rgba(37,99,235,.08);
+    --green-bg: rgba(22,163,74,.10); --red-bg: rgba(220,38,38,.10); --amber-bg: rgba(180,83,9,.10);
+  }
+  @media (prefers-color-scheme: dark) {
+    :root {
+      --bg: #0a0b0f; --bg-alt: #14161d; --bg-card: #14161d; --border: #262a35;
+      --text: #e7e9ee; --text-dim: #8b90a3;
+      --purple: #a78bfa; --blue: #60a5fa; --green: #4ade80; --red: #f87171; --amber: #fbbf24;
+      --purple-bg: rgba(167,139,250,.14); --blue-bg: rgba(96,165,250,.14);
+      --green-bg: rgba(74,222,128,.14); --red-bg: rgba(248,113,113,.14); --amber-bg: rgba(251,191,36,.14);
+    }
+  }
+  * { box-sizing: border-box; }
+  body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
+         background: var(--bg); color: var(--text); margin: 0; line-height: 1.5; }
+  .page { max-width: 900px; margin: 0 auto; padding: 2.5rem 1.5rem 4rem; }
+  .title-row { display: flex; align-items: center; gap: 0.55rem; }
+  .health-dot { width: 10px; height: 10px; border-radius: 50%; flex-shrink: 0; }
+  .health-dot--pass { background: var(--green); box-shadow: 0 0 0 4px var(--green-bg); }
+  .health-dot--warn { background: var(--amber); box-shadow: 0 0 0 4px var(--amber-bg); }
+  .health-dot--fail { background: var(--red); box-shadow: 0 0 0 4px var(--red-bg); }
+  h1 { font-size: 1.5rem; margin: 0; font-weight: 700; letter-spacing: -0.01em; }
+  .subtitle { color: var(--text-dim); margin: 0.35rem 0 1.75rem; font-size: 0.875rem; }
+  .stats { display: grid; grid-template-columns: repeat(auto-fit, minmax(158px, 1fr)); gap: 0.75rem; margin-bottom: 2rem; }
+  .stat { border: 1px solid var(--border); border-radius: 10px; padding: 0.85rem 1rem;
+          background: var(--bg-card); border-top: 3px solid var(--border); }
+  .stat--blue { border-top-color: var(--blue); }
+  .stat--purple { border-top-color: var(--purple); }
+  .stat--amber { border-top-color: var(--amber); }
+  .stat--green { border-top-color: var(--green); }
+  .stat-label { font-size: 0.7rem; color: var(--text-dim); text-transform: uppercase;
+                letter-spacing: 0.04em; font-weight: 600; }
+  .stat-value { font-size: 1.55rem; font-weight: 700; margin-top: 0.15rem; letter-spacing: -0.01em; }
+  .stat-value .frac { font-size: 1rem; font-weight: 500; color: var(--text-dim); }
+  .stat-sub { font-size: 0.78rem; color: var(--text-dim); margin-top: 0.2rem; }
+  .stat-bar { height: 5px; border-radius: 3px; background: var(--border); margin-top: 0.6rem; overflow: hidden; }
+  .stat-bar--warn { background: var(--amber-bg); }
+  .stat-bar--fail { background: var(--red-bg); }
+  .stat-bar-fill { height: 100%; border-radius: 3px; }
+  .stat-bar-fill--pass { background: var(--green); }
+  .stat-bar-fill--warn { background: var(--amber); }
+  .stat-bar-fill--fail { background: var(--red); }
+  .muted { color: var(--text-dim); font-size: 0.85rem; }
+  .empty-state { padding: 0.9rem 0; }
+  section { margin-bottom: 2rem; }
+  h2 { font-size: 0.8rem; text-transform: uppercase; letter-spacing: 0.04em; color: var(--text-dim);
+       font-weight: 700; display: flex; align-items: center; gap: 0.5rem; margin: 0 0 0.75rem; }
+  h2::before { content: ""; width: 3px; height: 0.85rem; border-radius: 2px; background: var(--purple); }
+  .card { border: 1px solid var(--border); border-radius: 10px; overflow: hidden; background: var(--bg-card); }
+  table { border-collapse: collapse; width: 100%; }
+  .card table { margin: 0; }
+  th, td { text-align: left; padding: 0.5rem 0.7rem; border-bottom: 1px solid var(--border); }
+  tr:last-child td { border-bottom: none; }
+  th { font-size: 0.7rem; color: var(--text-dim); text-transform: uppercase; letter-spacing: 0.03em; font-weight: 600; }
+  td.num { font-variant-numeric: tabular-nums; }
+  .kind-tag { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: 0.82em;
+              background: var(--purple-bg); color: var(--purple); padding: 0.1em 0.45em; border-radius: 4px; }
+  .bar-track { position: relative; height: 6px; background: var(--border); border-radius: 3px; margin-top: 5px; }
+  .bar-ci { position: absolute; top: 0; height: 100%; background: var(--blue-bg); border-radius: 3px; }
+  .bar-point { position: absolute; top: -3px; width: 2px; height: 12px; background: var(--blue); border-radius: 1px; }
+  details { border: 1px solid var(--border); border-left: 3px solid var(--border); border-radius: 8px;
+            margin-bottom: 0.55rem; background: var(--bg-card); overflow: hidden; }
+  details:has(.pill--fail) { border-left-color: var(--red); }
+  details:has(.pill--warn):not(:has(.pill--fail)) { border-left-color: var(--amber); }
+  details:has(.pill--pass):not(:has(.pill--fail)):not(:has(.pill--warn)) { border-left-color: var(--green); }
+  summary { cursor: pointer; font-weight: 600; padding: 0.65rem 0.9rem; display: flex; align-items: center;
+            justify-content: space-between; gap: 0.75rem; list-style: none; }
+  summary::-webkit-details-marker { display: none; }
+  summary::before { content: "\\25B8"; color: var(--purple); font-size: 0.95em; margin-right: 0.6rem; }
+  details[open] > summary::before { content: "\\25BE"; }
+  .tag-group { display: flex; gap: 0.35rem; flex-wrap: wrap; flex-shrink: 0; }
+  .pill { font-size: 0.68rem; font-weight: 700; padding: 0.16rem 0.5rem; border-radius: 999px;
+          text-transform: uppercase; letter-spacing: 0.02em; white-space: nowrap; }
+  .pill--pass { color: var(--green); background: var(--green-bg); }
+  .pill--fail { color: var(--red); background: var(--red-bg); }
+  .pill--warn { color: var(--amber); background: var(--amber-bg); }
+  .scenario-body { padding: 0 0.9rem 0.9rem; border-top: 1px solid var(--border); }
+  .run-label { font-size: 0.7rem; color: var(--text-dim); text-transform: uppercase; letter-spacing: 0.04em;
+               font-weight: 700; margin: 0.9rem 0 0.4rem; }
+  .step-table { font-size: 0.83rem; }
+  .step-table td, .step-table th { padding: 0.3rem 0.55rem; }
+  code { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: 0.85em;
+         background: var(--bg-alt); padding: 0.1em 0.3em; border-radius: 4px; }
+  .pass { color: var(--green); font-weight: 600; }
+  .fail { color: var(--red); font-weight: 600; }
+  .warn { color: var(--amber); font-weight: 600; }
+  .injection-note { font-size: 0.83rem; margin-top: 0.5rem; padding: 0.55rem 0.7rem; border-radius: 6px;
+                     border-left: 3px solid var(--amber); background: var(--amber-bg); }
+  .injection-note--invalid { border-left-color: var(--red); background: var(--red-bg); }
+  .injection-note--expired { border-left-color: var(--border); background: var(--bg-alt); }
+  footer { margin-top: 3rem; padding-top: 1.25rem; border-top: 1px solid var(--border);
+           font-size: 0.78rem; color: var(--text-dim); }
+  @media print {
+    .page { max-width: none; }
+    details { break-inside: avoid; }
+    summary::before { display: none; }
+  }
+"""
+
+
+def _format_args(args: dict) -> str:
+    return ", ".join(f"{k}={v!r}" for k, v in args.items())
+
+
+def _notable_steps(steps: list) -> list:
+    """The steps worth showing in a human-facing report: irreversible
+    actions and anything that went wrong. A successful read (get_ticket,
+    get_order, ...) is the agent looking something up -- noise in a report
+    meant to answer "what did it actually do," not a trace of its
+    reasoning. Commits, errors, and injected failures are the actual
+    story."""
+    return [s for s in steps if s.is_commit or not s.ok or s.injected_error]
+
+
+def _format_step_line(step) -> str:
+    call = f"{step.tool_name}({_format_args(step.tool_args)})"
+    tag = " [COMMIT]" if step.is_commit else ""
+    if step.injected_error:
+        status = red("injected error")
+        detail = f"  {dim(_truncate(str(step.result)))}"
+    elif step.ok:
+        status = green("ok")
+        detail = ""
+    else:
+        status = red("error")
+        detail = f"  {dim(_truncate(str(step.result)))}"
+    return f"  step {step.index:<2d} {call}{tag}  {status}{detail}"
+
+_MIN_N_FOR_RATE = 5
+_Z_95 = 1.96
+
+
+def wilson_interval(successes: int, n: int, z: float = _Z_95) -> tuple[float, float]:
+    """95% Wilson score interval for a binomial proportion, as (lower,
+    upper) fractions in [0, 1]. Used instead of the naive
+    successes/n +/- z*sqrt(p(1-p)/n) (Wald) interval, which badly
+    misbehaves near 0%/100% and at the small n (5-20) this harness's
+    per-kind handled-rates typically have -- exactly where an honest
+    interval matters most. A bare percentage at n=5 ("80%") reads as far
+    more precise than it is; reporting it without a range overstates
+    confidence a company would reasonably act on.
+    """
+    if n == 0:
+        return (0.0, 0.0)
+    phat = successes / n
+    z2 = z * z
+    denom = 1 + z2 / n
+    center = phat + z2 / (2 * n)
+    margin = z * math.sqrt(phat * (1 - phat) / n + z2 / (4 * n * n))
+    lower = (center - margin) / denom
+    upper = (center + margin) / denom
+    return (max(0.0, lower), min(1.0, upper))
+
+
+@dataclass(frozen=True)
+class InjectionRecord:
+    scenario_id: str
+    applied: AppliedInjection
+    classification: Optional[ClassifiedResponse] = None
+    triggered: bool = True
+    """False when a fired injection's effect was never observable by the
+    Target (zero tool calls after it fired) -- there's no response to
+    classify, so these are excluded from classification stats rather than
+    fed to the classifier to confabulate a verdict for. Always True for
+    TOOL_ERROR now: firing IS interception, by construction (see
+    injection_was_triggered)."""
+
+
+@dataclass
+class Report:
+    mode: str
+    injector_model: str
+    target_model: str
+    clean_trajectories: list[Trajectory]
+    chaos_trajectories: list[Trajectory]
+    injection_records: list[InjectionRecord] = field(default_factory=list)
+    classifier_agreement: Optional[tuple[int, int]] = None
+    """(agreed, total) against hand labels, spec section 7's mandated
+    check. Only meaningful for THIS injector/classifier configuration --
+    never pool across runs with different injector or prompt versions."""
+    injector_cost_usd: float = 0.0
+    """Sum of every ModelInjector instance's total_cost_usd across the
+    run -- the caller (cli.py) has to supply this since a Report only
+    holds Trajectories/InjectionRecords, not the Injector instances
+    themselves. 0.0 (and omitted from output) for none/hardcoded/replay
+    injectors, which never call a model at all."""
+
+    def _compute(self) -> dict:
+        """Shared computation behind both render() and render_html() --
+        kept in one place so the two views can't silently drift apart on
+        what a number actually means."""
+        n = len(self.chaos_trajectories)
+        chaos_passed = sum(1 for t in self.chaos_trajectories if t.passed)
+        clean_passed = sum(1 for t in self.clean_trajectories if t.passed) if self.clean_trajectories else None
+
+        total = len(self.injection_records)
+        invalid = sum(1 for r in self.injection_records if not r.applied.valid)
+        valid = total - invalid
+        not_observed = sum(1 for r in self.injection_records if r.applied.valid and not r.triggered)
+        expired_total = sum(len(t.expired_injections) for t in self.chaos_trajectories)
+
+        valid_records = [r for r in self.injection_records if r.applied.valid and r.triggered]
+        classified = [r for r in valid_records if r.classification is not None]
+        classification_counts = Counter(r.classification.classification.value for r in classified)
+
+        target_cost_usd = sum(t.total_cost_usd for t in self.clean_trajectories) + sum(
+            t.total_cost_usd for t in self.chaos_trajectories
+        )
+        classifier_cost_usd = sum(r.classification.cost_usd for r in classified)
+
+        per_kind = []
+        for kind in InjectionKind:
+            kind_records = [r for r in classified if r.applied.injection.kind == kind]
+            if not kind_records:
+                continue
+            handled = sum(1 for r in kind_records if r.classification.classification.value == "HANDLED")
+            k = len(kind_records)
+            interval = wilson_interval(handled, k) if k >= _MIN_N_FOR_RATE else None
+            per_kind.append({"kind": kind.value, "handled": handled, "n": k, "interval": interval})
+
+        return {
+            "n": n,
+            "chaos_passed": chaos_passed,
+            "clean_passed": clean_passed,
+            "clean_total": len(self.clean_trajectories),
+            "total": total,
+            "invalid": invalid,
+            "valid": valid,
+            "not_observed": not_observed,
+            "expired_total": expired_total,
+            "classified_n": len(classified),
+            "classification_counts": classification_counts,
+            "per_kind": per_kind,
+            "target_cost_usd": target_cost_usd,
+            "classifier_cost_usd": classifier_cost_usd,
+            "injector_cost_usd": self.injector_cost_usd,
+        }
+
+    def render(self) -> str:
+        c = self._compute()
+        lines = [
+            blue(
+                f"scenarios: {c['n']}   mode: {self.mode}   injector: {self.injector_model}   "
+                f"target: {self.target_model}"
+            ),
+        ]
+        if c["clean_passed"] is not None:
+            lines.append(f"{purple('clean pass rate:')}  {c['clean_passed']:2d}/{c['clean_total']}")
+        else:
+            lines.append(f"{purple('clean pass rate:')}  n/a (no control run in recovery mode -- see spec section 12)")
+        lines.append(f"{purple('chaos pass rate:')}  {c['chaos_passed']:2d}/{c['n']}")
+        if c["target_cost_usd"] > 0:
+            cost = cyan(f"${c['target_cost_usd']:.4f}")
+            lines.append(f"{dim('target API cost:')}     {cost}")
+        if c["injector_cost_usd"] > 0:
+            cost = cyan(f"${c['injector_cost_usd']:.4f}")
+            lines.append(f"{dim('injector API cost:')}   {cost}")
+        if c["classifier_cost_usd"] > 0:
+            cost = cyan(f"${c['classifier_cost_usd']:.4f}")
+            lines.append(f"{dim('classifier API cost:')} {cost}")
+        total_cost = c["target_cost_usd"] + c["injector_cost_usd"] + c["classifier_cost_usd"]
+        if total_cost > 0:
+            cost = cyan(f"${total_cost:.4f}")
+            lines.append(f"{dim('total API cost:')}      {cost}")
+        lines.append("")
+
+        invalid_pct = f"{100 * c['invalid'] / c['total']:.0f}%" if c["total"] else "0%"
+        lines.append(
+            f"injections fired: {c['total']}   invalid: {c['invalid']} ({invalid_pct}, n={c['total']})   "
+            f"valid: {c['valid']}"
+        )
+
+        if c["not_observed"]:
+            lines.append(
+                f"  of which unobservable: {c['not_observed']} (fired, but zero Target actions "
+                "afterward -- excluded below, not classified)"
+            )
+
+        if c["expired_total"]:
+            lines.append(
+                yellow(
+                    f"expired (armed, trigger never matched): {c['expired_total']} -- a bad prediction "
+                    "by the Injector, not an unfair one (that's invalid)"
+                )
+            )
+        lines.append("")
+
+        if c["classified_n"]:
+            lines.append(bold(purple(f"response classification (valid injections, n={c['classified_n']}):")))
+            for label in ["HANDLED", "IGNORED", "MISREAD", "OVERREACTED", "LOOPED"]:
+                if c["classification_counts"].get(label):
+                    lines.append(f"  {label:<12} {c['classification_counts'][label]}")
+            lines.append("")
+
+            lines.append(bold(purple("by injection kind (handled rate):")))
+            for row in c["per_kind"]:
+                kind_label = blue(f"{row['kind']:<15}")
+                if row["interval"] is None:
+                    lines.append(f"  {kind_label} {row['handled']}/{row['n']}  (n={row['n']}, too few to rate)")
+                else:
+                    lo, hi = row["interval"]
+                    pct = f"{100 * row['handled'] / row['n']:.0f}%"
+                    lines.append(
+                        f"  {kind_label} {row['handled']}/{row['n']}  "
+                        f"({pct}, 95% CI {100*lo:.0f}-{100*hi:.0f}%, n={row['n']})"
+                    )
+            lines.append("")
+
+        if self.classifier_agreement is not None:
+            agreed, checked = self.classifier_agreement
+            lines.append(f"classifier agreement with human labels (this configuration only): {agreed}/{checked}")
+
+        return "\n".join(lines).rstrip()
+
+    def render_scenario_detail(self) -> str:
+        """Scenario-by-scenario, step-by-step breakdown -- every tool call
+        the Target made in both the clean and chaos run, in order, plus
+        exactly where an injection fired and what it did. render()'s
+        aggregate numbers answer "how did this configuration do overall";
+        this answers "what actually happened in scenario X, step by step"
+        -- the thing you actually want to read when a specific scenario
+        failed and you need to know why, not just that it did.
+        """
+        clean_by_id = {t.scenario_id: t for t in self.clean_trajectories}
+        chaos_by_id = {t.scenario_id: t for t in self.chaos_trajectories}
+        scenario_ids = list(dict.fromkeys([*clean_by_id, *chaos_by_id]))  # stable order, no dupes
+
+        lines = [bold(purple("scenario detail")), ""]
+        for scenario_id in scenario_ids:
+            clean = clean_by_id.get(scenario_id)
+            chaos = chaos_by_id.get(scenario_id)
+            header = blue(f"=== {scenario_id} ===")
+            if clean is not None:
+                header += f"  clean: {pass_fail(clean.passed)}"
+            if chaos is not None:
+                header += f"  chaos: {pass_fail(chaos.passed)}"
+                if chaos.discarded:
+                    header += f"  {yellow('[DISCARDED: invalid injection]')}"
+            lines.append(bold(header))
+
+            if clean is not None:
+                lines.append(f"  {dim('-- clean run --')}")
+                notable = _notable_steps(clean.steps)
+                for step in notable:
+                    lines.append(_format_step_line(step))
+                skipped = len(clean.steps) - len(notable)
+                if skipped:
+                    plural = "s" if skipped != 1 else ""
+                    lines.append(f"  {dim(f'({skipped} read-only call{plural} omitted)')}")
+                lines.append(f"  final answer: {_truncate(str(clean.final_answer))}")
+                lines.append("")
+
+            if chaos is not None:
+                lines.append(f"  {dim('-- chaos run --')}")
+                notable = _notable_steps(chaos.steps)
+                for step in notable:
+                    lines.append(_format_step_line(step))
+                skipped = len(chaos.steps) - len(notable)
+                if skipped:
+                    plural = "s" if skipped != 1 else ""
+                    lines.append(f"  {dim(f'({skipped} read-only call{plural} omitted)')}")
+                lines.append(f"  final answer: {_truncate(str(chaos.final_answer))}")
+                if chaos.injections:
+                    for a in chaos.injections:
+                        validity = "" if a.valid else f" {red('INVALID')}"
+                        lines.append(
+                            f"  {purple('->')} injection fired: {purple(a.injection.kind.value)} @ step {a.fired_at_step} "
+                            f"(trigger: {a.trigger_kind}){validity} -- {a.injection.intent}"
+                        )
+                if chaos.expired_injections:
+                    for e in chaos.expired_injections:
+                        lines.append(
+                            f"  {dim('->')} injection armed but never fired: {e.injection.kind.value} "
+                            f"(trigger: {e.trigger.describe()})"
+                        )
+                lines.append("")
+
+        return "\n".join(lines).rstrip()
+
+    def render_html(self) -> str:
+        """A self-contained HTML report -- no external CSS/JS/fonts --
+        for handing to a stakeholder who isn't going to read a terminal
+        dump. Same numbers as render(), same _compute(), just a different
+        presentation.
+
+        No <script> anywhere, by design (see test_render_html_is_well_
+        formed_and_self_contained) -- every bit of interactivity here
+        (collapsible scenarios, status-colored accents) is native <details>
+        plus CSS :has(), not JS. That also means <details> and <summary>
+        must stay exactly as bare/plain as they were before this method
+        grew styling (see test_render_html_includes_a_collapsible_
+        scenario_detail_section, which matches the literal substrings
+        "<details>" and "<summary>a") -- status coloring is applied via
+        :has(.pill--fail) etc. on a *wrapper* class instead of attributes
+        on those two tags.
+        """
+        import html as _html
+
+        c = self._compute()
+
+        def esc(s: object) -> str:
+            return _html.escape(str(s))
+
+        def rate_tier(pct: float) -> str:
+            if pct >= 80:
+                return "pass"
+            if pct >= 40:
+                return "warn"
+            return "fail"
+
+        def stat_card(label: str, value_html: str, sub_html: str = "", accent: str = "blue", bar_pct=None) -> str:
+            bar = ""
+            if bar_pct is not None:
+                tier = rate_tier(bar_pct)
+                bar = (
+                    f"<div class='stat-bar stat-bar--{tier}'><div class='stat-bar-fill stat-bar-fill--{tier}' "
+                    f"style='width:{bar_pct:.0f}%'></div></div>"
+                )
+            sub = f"<div class='stat-sub'>{sub_html}</div>" if sub_html else ""
+            return (
+                f"<div class='stat stat--{accent}'><div class='stat-label'>{esc(label)}</div>"
+                f"<div class='stat-value'>{value_html}</div>{sub}{bar}</div>"
+            )
+
+        stats_html = []
+        if c["clean_passed"] is not None:
+            clean_pct = 100 * c["clean_passed"] / c["clean_total"] if c["clean_total"] else 0
+            stats_html.append(
+                stat_card(
+                    "Clean pass rate",
+                    f"{c['clean_passed']}<span class='frac'>/{c['clean_total']}</span>",
+                    accent="blue", bar_pct=clean_pct,
+                )
+            )
+        else:
+            stats_html.append(stat_card("Clean pass rate", "n/a (recovery mode)", accent="blue"))
+
+        chaos_pct = 100 * c["chaos_passed"] / c["n"] if c["n"] else 0
+        stats_html.append(
+            stat_card(
+                "Chaos pass rate",
+                f"{c['chaos_passed']}<span class='frac'>/{c['n']}</span>",
+                f"{chaos_pct:.0f}%" if c["n"] else "n/a",
+                accent="blue", bar_pct=chaos_pct if c["n"] else None,
+            )
+        )
+
+        invalid_pct = 100 * c["invalid"] / c["total"] if c["total"] else 0
+        stats_html.append(
+            stat_card(
+                "Injections fired", str(c["total"]),
+                f"invalid: {c['invalid']} ({invalid_pct:.0f}%) &middot; valid: {c['valid']}",
+                accent="purple",
+            )
+        )
+        stats_html.append(stat_card("Expired (never fired)", str(c["expired_total"]), accent="amber"))
+
+        if c["target_cost_usd"] > 0:
+            stats_html.append(stat_card("Target API cost", f"${c['target_cost_usd']:.4f}", accent="green"))
+        if c["injector_cost_usd"] > 0:
+            stats_html.append(stat_card("Injector API cost", f"${c['injector_cost_usd']:.4f}", accent="green"))
+        if c["classifier_cost_usd"] > 0:
+            stats_html.append(stat_card("Classifier API cost", f"${c['classifier_cost_usd']:.4f}", accent="green"))
+        total_cost = c["target_cost_usd"] + c["injector_cost_usd"] + c["classifier_cost_usd"]
+        if total_cost > 0:
+            stats_html.append(stat_card("Total API cost", f"${total_cost:.4f}", accent="green"))
+
+        if self.classifier_agreement is not None:
+            agreed, checked = self.classifier_agreement
+            stats_html.append(
+                stat_card("Classifier agreement", f"{agreed}/{checked}", "this configuration only", accent="blue")
+            )
+
+        health = rate_tier(chaos_pct) if c["n"] else "warn"
+
+        classification_rows = "".join(
+            f"<tr><td>{esc(label)}</td><td class='num'>{c['classification_counts'][label]}</td></tr>"
+            for label in ["HANDLED", "IGNORED", "MISREAD", "OVERREACTED", "LOOPED"]
+            if c["classification_counts"].get(label)
+        )
+        classification_html = (
+            f"<div class='card'><table><tbody>{classification_rows}</tbody></table></div>"
+            if classification_rows
+            else "<p class='muted empty-state'>No classified responses yet.</p>"
+        )
+
+        kind_rows = []
+        for row in c["per_kind"]:
+            if row["interval"] is None:
+                kind_rows.append(
+                    f"<tr><td><span class='kind-tag'>{esc(row['kind'])}</span></td>"
+                    f"<td class='num'>{row['handled']}/{row['n']}</td>"
+                    f"<td class='muted'>n={row['n']}, too few to rate</td></tr>"
+                )
+            else:
+                lo, hi = row["interval"]
+                pct = 100 * row["handled"] / row["n"]
+                kind_rows.append(
+                    f"<tr><td><span class='kind-tag'>{esc(row['kind'])}</span></td>"
+                    f"<td class='num'>{row['handled']}/{row['n']}</td>"
+                    f"<td>{pct:.0f}% <span class='muted'>(95% CI {100*lo:.0f}-{100*hi:.0f}%, n={row['n']})</span>"
+                    f"<div class='bar-track'><div class='bar-ci' style='left:{100*lo:.1f}%;width:{100*(hi-lo):.1f}%'></div>"
+                    f"<div class='bar-point' style='left:{pct:.1f}%'></div></div></td></tr>"
+                )
+        kind_table = (
+            f"<div class='card'><table><thead><tr><th>Injection kind</th><th>Handled</th><th>Rate</th></tr></thead>"
+            f"<tbody>{''.join(kind_rows)}</tbody></table></div>"
+            if kind_rows
+            else "<p class='muted empty-state'>No classified injections yet.</p>"
+        )
+
+        scenario_detail_html = self._scenario_detail_html(esc)
+
+        return f"""<!doctype html>
+<html><head><meta charset="utf-8"><title>AgentProbe report</title>
+<style>
+{_REPORT_CSS}
+</style></head>
+<body>
+<div class="page">
+  <header>
+    <div class="title-row"><span class="health-dot health-dot--{health}"></span><h1>AgentProbe report</h1></div>
+    <div class="subtitle">mode: {esc(self.mode)} &middot; injector: {esc(self.injector_model)} &middot; target: {esc(self.target_model)} &middot; {esc(c['n'])} scenarios</div>
+  </header>
+  <div class="stats">
+    {"".join(stats_html)}
+  </div>
+  <section>
+    <h2>Response classification (n={c['classified_n']})</h2>
+    {classification_html}
+  </section>
+  <section>
+    <h2>By injection kind &mdash; handled rate</h2>
+    {kind_table}
+  </section>
+  <section>
+    <h2>Scenario detail &mdash; step by step</h2>
+    {scenario_detail_html}
+  </section>
+  <footer>Generated by AgentProbe &mdash; adaptive chaos-testing for LLM agents.</footer>
+</div>
+</body></html>"""
+
+    def _scenario_detail_html(self, esc) -> str:
+        """One collapsible <details> block per scenario, each with a full
+        step-by-step table for the clean and chaos runs -- the HTML
+        counterpart to render_scenario_detail()."""
+
+        def step_rows(steps) -> str:
+            rows = []
+            for s in steps:
+                if s.injected_error:
+                    status = "<span class='fail'>injected error</span>"
+                elif s.ok:
+                    status = "<span class='pass'>ok</span>"
+                else:
+                    status = "<span class='fail'>error</span>"
+                tag = " <span class='muted'>[commit]</span>" if s.is_commit else ""
+                args = ", ".join(f"{k}={v!r}" for k, v in s.tool_args.items())
+                # only show the result payload for something that went
+                # wrong -- a successful call's full JSON dump is noise, not
+                # signal, in a report meant to show what happened
+                result = "" if s.ok and not s.injected_error else esc(_truncate(str(s.result), 200))
+                rows.append(
+                    f"<tr><td class='num'>{s.index}</td>"
+                    f"<td><code>{esc(s.tool_name)}({esc(args)})</code>{tag}</td>"
+                    f"<td>{status}</td><td class='muted'>{result}</td></tr>"
+                )
+            return "".join(rows)
+
+        def run_section(label: str, traj) -> str:
+            if traj is None:
+                return ""
+            notable = _notable_steps(traj.steps)
+            skipped = len(traj.steps) - len(notable)
+            skipped_note = (
+                f"<p class='muted'>({skipped} read-only call{'s' if skipped != 1 else ''} omitted)</p>"
+                if skipped
+                else ""
+            )
+            table = (
+                "<table class='step-table'><thead><tr><th>#</th><th>Call</th><th>Status</th>"
+                f"<th>Result</th></tr></thead><tbody>{step_rows(notable)}</tbody></table>{skipped_note}"
+                if traj.steps
+                else "<p class='muted'>No steps.</p>"
+            )
+            notes = ""
+            for a in traj.injections:
+                note_class = "injection-note" if a.valid else "injection-note injection-note--invalid"
+                validity = "" if a.valid else " &mdash; <strong>INVALID</strong>"
+                notes += (
+                    f"<div class='{note_class}'>&#9889; injection fired: <strong>{esc(a.injection.kind.value)}</strong> "
+                    f"@ step {a.fired_at_step} (trigger: <code>{esc(a.trigger_kind)}</code>){validity}"
+                    f"<br><span class='muted'>{esc(a.injection.intent)}</span></div>"
+                )
+            for e in traj.expired_injections:
+                notes += (
+                    f"<div class='injection-note injection-note--expired'>armed but never fired: "
+                    f"<strong>{esc(e.injection.kind.value)}</strong> (trigger: <code>{esc(e.trigger.describe())}</code>)</div>"
+                )
+            return (
+                f"<div class='run-label'>{esc(label)}</div>{table}"
+                f"<p class='muted'>final answer: {esc(_truncate(str(traj.final_answer), 200))}</p>{notes}"
+            )
+
+        clean_by_id = {t.scenario_id: t for t in self.clean_trajectories}
+        chaos_by_id = {t.scenario_id: t for t in self.chaos_trajectories}
+        scenario_ids = list(dict.fromkeys([*clean_by_id, *chaos_by_id]))
+
+        if not scenario_ids:
+            return "<p class='muted empty-state'>No scenarios run.</p>"
+
+        blocks = []
+        for scenario_id in scenario_ids:
+            clean = clean_by_id.get(scenario_id)
+            chaos = chaos_by_id.get(scenario_id)
+            pills = []
+            if clean is not None:
+                pills.append(
+                    f"<span class='pill pill--{'pass' if clean.passed else 'fail'}'>clean: {'PASS' if clean.passed else 'FAIL'}</span>"
+                )
+            if chaos is not None:
+                pills.append(
+                    f"<span class='pill pill--{'pass' if chaos.passed else 'fail'}'>chaos: {'PASS' if chaos.passed else 'FAIL'}</span>"
+                )
+                if chaos.discarded:
+                    pills.append("<span class='pill pill--warn'>DISCARDED</span>")
+            blocks.append(
+                f"<details><summary>{esc(scenario_id)}<span class='tag-group'>{''.join(pills)}</span></summary>"
+                f"<div class='scenario-body'>{run_section('Clean run', clean)}{run_section('Chaos run', chaos)}</div></details>"
+            )
+        return "".join(blocks)