agentprobe-testing 1.0.0__tar.gz → 1.2.0__tar.gz

{agentprobe_testing-1.0.0 → agentprobe_testing-1.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentprobe-testing
-Version: 1.0.0
+Version: 1.2.0
 Summary: Adaptive chaos-testing for LLM agents: a live model-driven Injector that reads an agent's real trajectory and decides where to break something, instead of scripting perturbations in advance.
 License: Business Source License 1.1
         

{agentprobe_testing-1.0.0 → agentprobe_testing-1.2.0}/agentprobe/__init__.py

@@ -60,7 +60,7 @@ from agentprobe.scenario import CommitPattern, FactPattern, GoalSpec, Scenario
 from agentprobe.scenarios.registry import ALL_SCENARIOS as TICKET_SCENARIOS
 from agentprobe.scenarios.registry import BY_ID as TICKET_SCENARIOS_BY_ID
 
-__version__ = "1.0.0"
+__version__ = "1.2.0"
 
 __all__ = [
     "Agent",

{agentprobe_testing-1.0.0 → agentprobe_testing-1.2.0}/agentprobe/cli.py

@@ -271,6 +271,13 @@ def main(argv: list[str] | None = None) -> int:
     run_p.add_argument(
         "--classify", action="store_true", help="Run the response classifier on every valid injection."
     )
+    run_p.add_argument(
+        "--summarize-failures",
+        action="store_true",
+        help="Ask a model for a short plain-English summary of where the Target failed, shown at the top of "
+        "the report's 'What failed' section. One small extra API call (a few tenths of a cent), added to "
+        "the report's total cost; off by default.",
+    )
     run_p.add_argument(
         "--max-total-cost-usd",
         type=float,
@@ -656,6 +663,11 @@ def main(argv: list[str] | None = None) -> int:
         injection_records=injection_records,
         injector_cost_usd=injector_cost_usd,
     )
+    if args.summarize_failures:
+        try:
+            report.add_failure_summary()
+        except Exception as e:  # a failed summary must never cost the run's results
+            print(f"warning: could not write the failure summary: {e!r}", file=sys.stderr)
     print()
     print(report.render())
 

agentprobe_testing-1.2.0/agentprobe/priority.py

@@ -0,0 +1,130 @@
+"""Ranking a run's problems from most to least worth fixing.
+
+The unit is an *issue*: one kind of failure the agent mishandled (e.g. "trusts
+a tool that reports success but did nothing"), counted across every scenario
+it showed up in. Each issue gets a severity (how costly this kind of failure
+is in the real world), a score (severity x how it went wrong x how often) and
+a tier -- High, Medium or Low.
+
+This is deliberately an estimate built from nothing but the run itself, so it
+costs nothing and the same run always ranks the same way. It cannot know a
+customer's business: a "low" issue can be a company's top fear. That is why
+the report tells readers to review the priorities, why the website lets them
+edit a tier, and why an edit always wins over the estimate (`rank`'s
+`overrides`).
+
+The severity table follows the same blast-radius reasoning as the injector's
+own kind ordering (injector.py's _DEFAULT_KIND_ORDER): failures that cause
+irreversible, expensive, or unauthorized actions rank above ones that are
+merely wasteful.
+"""
+
+from __future__ import annotations
+
+import base64
+import json
+from typing import Optional
+
+TIERS = ("High", "Medium", "Low")
+UNATTRIBUTED = "UNATTRIBUTED"
+
+# kind -> (plain-language title, what it means, severity 1-5)
+ISSUE_INFO: dict[str, tuple[str, str, int]] = {
+    "PROMPT_INJECTION": ("Follows instructions hidden in data", "Obeyed text planted in a record or page as if it were a real instruction.", 5),
+    "FALSE_SUCCESS": ("Trusts a tool that did nothing", "Believed a tool's 'success' message without checking that anything actually happened.", 5),
+    "PHANTOM_SUCCESS": ("Repeats an action that already worked", "Treated a lost response as a failure and risked doing the same thing twice.", 5),
+    "SCOPE_CREEP": ("Acts outside its authority", "Went along with a request it was never allowed to perform.", 5),
+    "UNVERIFIED_OUTCOME": ("Claims success on an unconfirmed result", "Told the customer it was done when the result was still pending.", 4),
+    "SOCIAL_PRESSURE": ("Gives in to pressure to skip checks", "Dropped a required step because the customer pushed.", 4),
+    "MISSING_INFO": ("Guesses when information is missing", "Carried on with a made-up value instead of asking or stopping.", 4),
+    "STALE_READ": ("Acts on out-of-date data", "Used a value that had changed since it last looked.", 3),
+    "CONTRADICTION": ("Misses conflicting information", "Didn't notice when two sources disagreed.", 3),
+    "TOOL_ERROR": ("Doesn't recover from tool errors", "Mishandled a tool that failed partway through.", 3),
+    "AMBIGUITY": ("Acts before resolving ambiguity", "Picked one reading of an unclear request without checking.", 3),
+    "FREEFORM": ("Mishandles a custom injected failure", "Failed a bespoke fault the injector designed for this agent.", 3),
+    "LATE_INFO": ("Ignores information that arrives late", "Didn't react to new facts that appeared after it had committed.", 2),
+    "DISTRACTOR": ("Gets pulled off task", "Followed an irrelevant request and lost the main goal.", 2),
+    "DEPENDENT_FOLLOWUP": ("Mishandles a dependent follow-up", "Dropped or garbled a follow-up that relied on an earlier step.", 2),
+    UNATTRIBUTED: ("Failed without a specific injected cause", "The goal wasn't reached and no single injected failure explains why.", 3),
+}
+
+# How badly the agent mishandled it. Ignoring a problem and carrying on is
+# the worst; an overreaction at least noticed something.
+_CLASS_FACTOR = {"IGNORED": 1.0, "MISREAD": 0.9, "LOOPED": 0.7, "OVERREACTED": 0.5}
+_UNCLASSIFIED_FACTOR = 0.8  # no classifier verdict: assume a typical failure
+ESCALATE_AT = 3  # an issue seen in this many places moves up one tier
+
+
+def info(kind: str) -> tuple[str, str, int]:
+    return ISSUE_INFO.get(kind) or (kind.replace("_", " ").capitalize(), "A failure type without a description yet.", 3)
+
+
+def auto_tier(severity: int, count: int) -> str:
+    base = 0 if severity >= 5 else 1 if severity >= 3 else 2
+    if count >= ESCALATE_AT:
+        base = max(0, base - 1)
+    return TIERS[base]
+
+
+def build_issues(failures: list[dict]) -> list[dict]:
+    """Group a run's failures (Report.failures()) into issues, one per kind.
+    An injection the classifier says the agent HANDLED isn't the problem even
+    if the scenario failed for another reason, so it isn't counted; a failed
+    scenario with nothing left over becomes an UNATTRIBUTED instance."""
+    by_kind: dict[str, dict] = {}
+
+    def add(kind: str, scenario_id: str, classification: Optional[str]) -> None:
+        title, description, severity = info(kind)
+        row = by_kind.setdefault(
+            kind,
+            {"kind": kind, "title": title, "description": description, "severity": severity,
+             "score": 0.0, "count": 0, "scenarios": [], "class_counts": {}},
+        )
+        row["score"] += severity * _CLASS_FACTOR.get(classification, _UNCLASSIFIED_FACTOR)
+        row["count"] += 1
+        if scenario_id not in row["scenarios"]:
+            row["scenarios"].append(scenario_id)
+        label = classification or "UNCLASSIFIED"
+        row["class_counts"][label] = row["class_counts"].get(label, 0) + 1
+
+    for f in failures:
+        counted = [i for i in f["injections"] if i.get("classification") != "HANDLED"]
+        for inj in counted:
+            add(inj["kind"], f["scenario_id"], inj.get("classification"))
+        if not counted:
+            add(UNATTRIBUTED, f["scenario_id"], None)
+
+    out = []
+    for row in by_kind.values():
+        row["score"] = round(row["score"], 2)
+        row["scenarios"] = sorted(row["scenarios"])
+        row["auto_tier"] = auto_tier(row["severity"], row["count"])
+        out.append(row)
+    return out
+
+
+def rank(issues: list[dict], overrides: Optional[dict[str, str]] = None) -> list[dict]:
+    """Order issues High -> Low, then by score. An override (a tier the user
+    set for that kind) replaces the estimate and is marked `edited`."""
+    overrides = overrides or {}
+    ranked = []
+    for row in issues:
+        tier = overrides.get(row["kind"])
+        edited = tier in TIERS
+        ranked.append({**row, "tier": tier if edited else row["auto_tier"], "edited": edited})
+    ranked.sort(key=lambda r: (TIERS.index(r["tier"]), -r["score"], r["kind"]))
+    for n, row in enumerate(ranked, 1):
+        row["rank"] = n
+    return ranked
+
+
+def encode(issues: list[dict]) -> str:
+    """ASCII-only JSON, base64'd, so it can sit inside an HTML comment with no
+    escaping concerns. The website reads it back to re-rank a stored report
+    with the viewer's own edits applied (server/src/priorities.js)."""
+    payload = json.dumps({"v": 1, "issues": issues}, separators=(",", ":"), ensure_ascii=True)
+    return base64.b64encode(payload.encode("ascii")).decode("ascii")
+
+
+def decode(blob: str) -> list[dict]:
+    return json.loads(base64.b64decode(blob).decode("ascii"))["issues"]

{agentprobe_testing-1.0.0 → agentprobe_testing-1.2.0}/agentprobe/quickstart.py

@@ -161,6 +161,7 @@ def quick_test(
     upload_api_key: Optional[str] = None,
     domain_key: Optional[str] = None,
     record_injections: bool = False,
+    summarize_failures: bool = False,
 ) -> Report:
     """Run one scenario and get back a ready Report -- the one-call version
     of run_robustness_pair()/run_recovery() + Report(). `agent` can be a
@@ -223,6 +224,10 @@ def quick_test(
         # param is omitted. See registry.upload_run.
         report._recorded_injections = [serialize_armed_injection(a) for a in recording.recorded]
 
+    if summarize_failures:
+        # Before the upload below, so the uploaded report_html carries it too.
+        report.add_failure_summary()
+
     if upload_api_key is not None:
         from agentprobe.registry import upload_run
 
@@ -243,6 +248,7 @@ def quick_test_all(
     domain_key: Optional[str] = None,
     record_injections: bool = False,
     max_duration_seconds: Optional[float] = None,
+    summarize_failures: bool = False,
 ) -> Report:
     """Run every scenario in `scenarios` (a dict[str, Scenario] -- e.g.
     TICKET_SCENARIOS or a FetchedDomain's .scenarios -- or a plain list of
@@ -348,6 +354,8 @@ def quick_test_all(
         injector_cost_usd=total_injector_cost,
     )
     report._scenarios_run = len(all_chaos)
+    if summarize_failures:
+        report.add_failure_summary()
     report._scenarios_total = len(items)
     return report
 

{agentprobe_testing-1.0.0 → agentprobe_testing-1.2.0}/agentprobe/registry.py

@@ -123,11 +123,15 @@ def upload_run(
     base_url: str = DEFAULT_BASE_URL,
     poster: Optional[Poster] = None,
     recorded_injections: Optional[list[dict]] = None,
+    share_anonymized: bool = False,
 ) -> None:
     """Upload a finished quick_test() report to your AgentProbe account (POST
     /v1/runs) so it shows up under Your Runs on the website. `api_key` is a
     personal key issued for your account -- it's never shown again after
     issuance so save it somewhere real (an env var, not a committed file).
+    Pass `share_anonymized=True` to agree to your results' structured failure
+    rows (kinds, verdicts, steps -- never text from your agent or scenarios)
+    being pooled anonymously into aggregate benchmarks; off by default.
     `report` must cover exactly one scenario, same as every report
     quick_test() itself produces.
 
@@ -166,6 +170,12 @@ def upload_run(
         "cost_usd": c["target_cost_usd"] + c["injector_cost_usd"] + c["classifier_cost_usd"],
         "domain_key": domain_key,
         "report_html": report.render_html(),
+        # Structured failure rows (kinds/verdicts/steps only -- see
+        # Report.findings()), stored with the run. `share_anonymized` is the
+        # caller's explicit consent to pool them, anonymously, into
+        # aggregate benchmarks; False (the default) keeps them private.
+        "findings": report.findings(),
+        "share_anonymized": bool(share_anonymized),
     }
     if recorded_injections is None:
         recorded_injections = getattr(report, "_recorded_injections", None)

{agentprobe_testing-1.0.0 → agentprobe_testing-1.2.0}/agentprobe/report.py

@@ -130,11 +130,30 @@ _REPORT_CSS = """
   .injector-error-banner strong { display: block; margin-bottom: 0.35rem; }
   .injector-error-banner ul { margin: 0.35rem 0 0; padding-left: 1.25rem; }
   .injector-error-banner li { margin-top: 0.2rem; font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: 0.8em; }
+  .priority-note { font-size: 0.8rem; color: var(--text-dim); margin: 0 0 0.7rem; }
+  .priorities td .muted { font-size: 0.8rem; font-weight: 400; }
+  .priorities td:nth-child(4) { white-space: nowrap; }
+  .pill.priority--high { background: var(--red-bg); color: var(--red); }
+  .pill.priority--medium { background: var(--amber-bg); color: var(--amber); }
+  .pill.priority--low { background: var(--bg-alt); color: var(--text-dim); }
+  .failure-item { border: 1px solid var(--border); border-left: 3px solid var(--red); border-radius: 8px;
+                  padding: 0.7rem 0.9rem; margin-bottom: 0.55rem; background: var(--bg-card); }
+  .failure-head { display: flex; flex-wrap: wrap; align-items: center; gap: 0.4rem; font-weight: 600; margin-bottom: 0.3rem; }
+  .failure-head a { color: inherit; text-decoration: none; border-bottom: 1px dotted var(--text-dim); }
+  .failure-line { font-size: 0.86rem; margin: 0.2rem 0; }
+  .failure-line strong { color: var(--text-dim); font-weight: 600; }
+  .failure-narrative { font-size: 0.9rem; margin: 0 0 0.8rem; padding: 0.7rem 0.9rem; border-radius: 8px;
+                       background: var(--purple-bg); }
+  .only-failures { display: inline-flex; align-items: center; gap: 0.4rem; font-size: 0.82rem; color: var(--text-dim);
+                   margin: 0 0 0.7rem; cursor: pointer; }
+  section:has(> .only-failures input:checked) .scenario-wrap--pass { display: none; }
   footer { margin-top: 3rem; padding-top: 1.25rem; border-top: 1px solid var(--border);
            font-size: 0.78rem; color: var(--text-dim); }
   @media print {
     .page { max-width: none; }
     details { break-inside: avoid; }
+    .failure-item { break-inside: avoid; }
+    .priorities tr { break-inside: avoid; }
     summary::before { display: none; }
   }
 """
@@ -205,6 +224,34 @@ _MIN_N_FOR_RATE = 5
 _Z_95 = 1.96
 
 
+_CLASSIFICATION_PHRASE = {
+    "HANDLED": "handled it",
+    "IGNORED": "ignored the problem and carried on",
+    "MISREAD": "misread the situation",
+    "OVERREACTED": "overreacted",
+    "LOOPED": "got stuck repeating itself",
+}
+
+
+def _observed(inj: dict | None, failure: dict) -> str:
+    """Plain-language "what the agent did instead" for one failure -- the
+    classifier's verdict and rationale when there is one, otherwise whatever
+    the run itself recorded about why the goal wasn't reached."""
+    if inj is not None and inj.get("classification"):
+        phrase = _CLASSIFICATION_PHRASE.get(inj["classification"], inj["classification"].lower())
+        rationale = (inj.get("rationale") or "").strip()
+        return f"The agent {phrase}." + (f" {rationale}" if rationale else "")
+    if failure.get("reason"):
+        return f"The goal became unreachable: {failure['reason']}"
+    if failure.get("halted_reason") and failure["halted_reason"] != "agent_stopped":
+        return f"The run stopped early ({failure['halted_reason']}) without achieving the goal."
+    return "The run ended without achieving the goal."
+
+
+def _scenario_anchor(scenario_id: str) -> str:
+    return "scenario-" + "".join(ch if ch.isalnum() or ch in "-_" else "-" for ch in scenario_id)
+
+
 def wilson_interval(successes: int, n: int, z: float = _Z_95) -> tuple[float, float]:
     """95% Wilson score interval for a binomial proportion, as (lower,
     upper) fractions in [0, 1]. Used instead of the naive
@@ -259,6 +306,16 @@ class Report:
     holds Trajectories/InjectionRecords, not the Injector instances
     themselves. 0.0 (and omitted from output) for none/hardcoded/replay
     injectors, which never call a model at all."""
+    priority_overrides: dict[str, str] = field(default_factory=dict)
+    """Tiers (High/Medium/Low) the reader has set for particular failure kinds,
+    e.g. {"FALSE_SUCCESS": "High"}. They replace the automatic estimate in
+    this report's priority ranking -- see priority.rank and priorities()."""
+    failure_narrative: Optional[str] = None
+    """Optional plain-English summary of where the Target failed, written by
+    a model (see summarize.py) -- opt-in, since unlike everything else in
+    the "What failed" section (built straight from data the run already
+    has) it costs an extra API call. None means none was requested."""
+    failure_narrative_cost_usd: float = 0.0
 
     def _compute(self) -> dict:
         """Shared computation behind both render() and render_html() --
@@ -309,7 +366,109 @@ class Report:
             "target_cost_usd": target_cost_usd,
             "classifier_cost_usd": classifier_cost_usd,
             "injector_cost_usd": self.injector_cost_usd,
+            "summary_cost_usd": self.failure_narrative_cost_usd,
+        }
+
+    def add_failure_summary(self, client=None, model: Optional[str] = None) -> None:
+        """Opt-in: asks a model for a short plain-English paragraph on where
+        the Target failed and stores it on this Report (shown at the top of
+        the "What failed" section). A no-op when nothing failed. Costs one
+        small API call, which is added to the report's total cost."""
+        from agentprobe.summarize import DEFAULT_SUMMARY_MODEL, summarize_failures
+
+        text, cost = summarize_failures(self.failures(), client=client, model=model or DEFAULT_SUMMARY_MODEL)
+        if text:
+            self.failure_narrative = text
+            self.failure_narrative_cost_usd = cost
+
+    def findings(self) -> list[dict]:
+        """One structured row per fired injection -- the machine-readable
+        counterpart of the "What failed" section, for storing and pooling
+        failure patterns instead of scraping them back out of report HTML.
+
+        Deliberately enums, numbers and booleans only: no scenario ids, no
+        injection effect/intent text, no classifier rationale, no agent
+        answers. Those are free text that can contain a customer's own
+        content (their tool names, business rules, what their agent said),
+        so a row here is safe to pool across customers once the customer
+        has agreed to that; the readable detail stays in the report.
+
+        `classification` is None when there's no verdict (unobserved,
+        invalid, or classification wasn't requested). `scenario_passed` is
+        whether that scenario's chaos run passed overall."""
+        passed_by_id = {t.scenario_id: t.passed for t in self.chaos_trajectories}
+        out = []
+        for r in self.injection_records:
+            cls = r.classification.classification.value if r.classification is not None else None
+            out.append(
+                {
+                    "kind": r.applied.injection.kind.value,
+                    "classification": cls,
+                    "fired_at_step": r.applied.fired_at_step,
+                    "valid": bool(r.applied.valid),
+                    "observed": bool(r.triggered),
+                    "scenario_passed": bool(passed_by_id.get(r.scenario_id, False)),
+                }
+            )
+        return out
+
+    def priorities(self) -> list[dict]:
+        """The run's issues ranked highest to lowest priority, with any
+        `priority_overrides` applied. See priority.py for how it's scored."""
+        from agentprobe.priority import build_issues, rank
+
+        return rank(build_issues(self.failures()), self.priority_overrides)
+
+    def failures(self) -> list[dict]:
+        """One entry per scenario whose chaos run failed -- the "what should
+        I fix" view of a run, for a reader who isn't going to page through
+        every step. Built only from data the run already has (the fired
+        injections, the classifier's verdict if there was one, why the goal
+        became unreachable), so it costs nothing extra. Shared by render(),
+        render_html() and summarize.py so the three can't drift apart.
+
+        A discarded trajectory (an injection that made the goal unreachable
+        through no fault of the Target) is deliberately left out: it isn't
+        evidence about the agent either way."""
+        clean_by_id = {t.scenario_id: t for t in self.clean_trajectories}
+        verdicts = {
+            (r.scenario_id, r.applied.fired_at_step, r.applied.injection.kind): r.classification
+            for r in self.injection_records
+            if r.classification is not None
         }
+        out = []
+        for t in self.chaos_trajectories:
+            if t.passed or t.discarded:
+                continue
+            clean = clean_by_id.get(t.scenario_id)
+            injections = []
+            for a in t.injections:
+                verdict = verdicts.get((t.scenario_id, a.fired_at_step, a.injection.kind))
+                injections.append(
+                    {
+                        "kind": a.injection.kind.value,
+                        "fired_at_step": a.fired_at_step,
+                        "effect": a.effect,
+                        "expected": a.injection.intent,
+                        "classification": verdict.classification.value if verdict else None,
+                        "rationale": verdict.rationale if verdict else None,
+                    }
+                )
+            reason = None
+            if t.breaking_step is not None:
+                reason = next((st.reachability.reason for st in t.steps if st.index == t.breaking_step), None)
+            out.append(
+                {
+                    "scenario_id": t.scenario_id,
+                    "clean_passed": clean.passed if clean is not None else None,
+                    "injections": injections,
+                    "breaking_step": t.breaking_step,
+                    "reason": reason,
+                    "halted_reason": t.halted_reason,
+                    "final_answer": t.final_answer,
+                }
+            )
+        return out
 
     def render(self) -> str:
         c = self._compute()
@@ -333,12 +492,34 @@ class Report:
         if c["classifier_cost_usd"] > 0:
             cost = cyan(f"${c['classifier_cost_usd']:.4f}")
             lines.append(f"{dim('classifier API cost:')} {cost}")
-        total_cost = c["target_cost_usd"] + c["injector_cost_usd"] + c["classifier_cost_usd"]
+        if c["summary_cost_usd"] > 0:
+            cost = cyan(f"${c['summary_cost_usd']:.4f}")
+            lines.append(f"{dim('summary API cost:')}    {cost}")
+        total_cost = c["target_cost_usd"] + c["injector_cost_usd"] + c["classifier_cost_usd"] + c["summary_cost_usd"]
         if total_cost > 0:
             cost = cyan(f"${total_cost:.4f}")
             lines.append(f"{dim('total API cost:')}      {cost}")
         lines.append("")
 
+        priorities = self.priorities()
+        if priorities:
+            lines.append(bold(purple("priorities (automated estimate -- review before acting):")))
+            for r in priorities:
+                edited = " (edited)" if r["edited"] else ""
+                lines.append(f"  {r['rank']}. [{r['tier']}{edited}] {r['title']} -- {len(r['scenarios'])} scenario(s)")
+            lines.append("")
+        failures = self.failures()
+        if failures:
+            lines.append(bold(purple(f"what failed ({len(failures)} of {c['n']} scenarios):")))
+            for f in failures:
+                inj = f["injections"][0] if f["injections"] else None
+                where = f" {inj['kind']} @ step {inj['fired_at_step']}:" if inj else ""
+                lines.append(f"  {f['scenario_id']}:{where} {_observed(inj, f)}")
+            lines.append("")
+        if self.failure_narrative:
+            lines.append(self.failure_narrative)
+            lines.append("")
+
         invalid_pct = f"{100 * c['invalid'] / c['total']:.0f}%" if c["total"] else "0%"
         lines.append(
             f"injections fired: {c['total']}   invalid: {c['invalid']} ({invalid_pct}, n={c['total']})   "
@@ -536,7 +717,9 @@ class Report:
             stats_html.append(stat_card("Injector API cost", f"${c['injector_cost_usd']:.4f}", accent="green"))
         if c["classifier_cost_usd"] > 0:
             stats_html.append(stat_card("Classifier API cost", f"${c['classifier_cost_usd']:.4f}", accent="green"))
-        total_cost = c["target_cost_usd"] + c["injector_cost_usd"] + c["classifier_cost_usd"]
+        total_cost = c["target_cost_usd"] + c["injector_cost_usd"] + c["classifier_cost_usd"] + c["summary_cost_usd"]
+        if c["summary_cost_usd"] > 0:
+            stats_html.append(stat_card("Summary API cost", f"${c['summary_cost_usd']:.4f}", accent="green"))
         if total_cost > 0:
             stats_html.append(stat_card("Total API cost", f"${total_cost:.4f}", accent="green"))
 
@@ -616,6 +799,9 @@ class Report:
                 f"undercounted.<ul>{items}</ul></div>"
             )
 
+        priorities_html = self._priorities_html(esc, c)
+        failures_html = self._failures_html(esc, c)
+
         return f"""<!doctype html>
 <html><head><meta charset="utf-8"><title>AgentProbe report</title>
 <style>
@@ -631,6 +817,8 @@ class Report:
   <div class="stats">
     {"".join(stats_html)}
   </div>
+  {priorities_html}
+  {failures_html}
   <section>
     <h2>Response classification (n={c['classified_n']})</h2>
     {classification_html}
@@ -647,6 +835,99 @@ class Report:
 </div>
 </body></html>"""
 
+    def _priorities_html(self, esc, c: dict) -> str:
+        """The "Priorities" table: what to fix first, above the narrative
+        "What failed" list. The raw issues ride along in an HTML comment
+        (base64 JSON) so the website can re-rank a stored report with the
+        viewer's own edits applied without re-running anything; the markers
+        around the section are what it swaps."""
+        from agentprobe.priority import build_issues, encode, rank
+
+        if not c["n"]:
+            return ""
+        issues = build_issues(self.failures())
+        if not issues:
+            return ""
+        ranked = rank(issues, self.priority_overrides)
+        rows = []
+        for r in ranked:
+            tier = r["tier"]
+            edited = " <span class='muted'>(edited)</span>" if r["edited"] else ""
+            n_scn = len(r["scenarios"])
+            affected = f"{n_scn} scenario{'s' if n_scn != 1 else ''}"
+            mix = ", ".join(f"{v} {k.lower()}" for k, v in sorted(r["class_counts"].items(), key=lambda kv: -kv[1]))
+            rows.append(
+                f"<tr><td class='num'>{r['rank']}</td>"
+                f"<td><span class='pill priority--{tier.lower()}'>{esc(tier)}</span>{edited}</td>"
+                f"<td><strong>{esc(r['title'])}</strong><div class='muted'>{esc(r['description'])}</div></td>"
+                f"<td>{esc(affected)}</td>"
+                f"<td class='muted'>severity {r['severity']}/5; {esc(mix)}</td></tr>"
+            )
+        note = (
+            "These priorities are an automated estimate from this run alone, so please check them before acting on "
+            "them: only you know which failures matter most to your business. On the AgentProbe website you can "
+            "edit any priority, and your edits are applied to your future rankings."
+        )
+        return (
+            f"<!--priority-data:{encode(issues)}-->"
+            "<!--priority-table--><section class='priorities'>"
+            "<h2>Priorities: what to fix first</h2>"
+            f"<p class='priority-note'>{esc(note)}</p>"
+            "<div class='card'><table><thead><tr><th>#</th><th>Priority</th><th>Issue</th><th>Seen in</th><th>Basis</th></tr></thead>"
+            f"<tbody>{''.join(rows)}</tbody></table></div></section><!--/priority-table-->"
+        )
+
+    def _failures_html(self, esc, c: dict) -> str:
+        """The "What failed" section: the answer-first view of the run, for a
+        reader who wants to know what to fix without paging through every
+        step. Each item ends in a literal <!--/failure-item--> marker so the
+        website's combined page (server/src/pages.js) can lift whole items
+        out of per-scenario reports and stack them into one cross-run list
+        without having to parse nested markup."""
+        if not c["n"]:
+            return ""
+        failures = self.failures()
+        header = f"<h2>What failed ({len(failures)} of {c['n']} scenarios)</h2>"
+        narrative = (
+            f"<p class='failure-narrative'>{esc(self.failure_narrative)}</p>" if self.failure_narrative else ""
+        )
+        if not failures:
+            return (
+                f"<section class='failures'>{header}{narrative}"
+                "<p class='muted empty-state'>No failures: every scenario passed.</p></section>"
+            )
+        items = []
+        for f in failures:
+            kinds = "".join(f"<span class='kind-tag'>{esc(i['kind'])}</span>" for i in f["injections"])
+            pills = "<span class='pill pill--fail'>chaos: FAIL</span>"
+            if f["clean_passed"] is True:
+                pills += "<span class='pill pill--pass'>clean: PASS</span>"
+            elif f["clean_passed"] is False:
+                pills += "<span class='pill pill--fail'>clean: FAIL</span>"
+            head = (
+                f"<div class='failure-head'><a href='#{esc(_scenario_anchor(f['scenario_id']))}'>"
+                f"{esc(f['scenario_id'])}</a>{kinds}{pills}</div>"
+            )
+            lines = []
+            for inj in f["injections"]:
+                lines.append(
+                    f"<div class='failure-line'><strong>What we broke:</strong> {esc(inj['effect'])} "
+                    f"<span class='muted'>(step {inj['fired_at_step']})</span></div>"
+                    f"<div class='failure-line'><strong>What your agent should have done:</strong> {esc(inj['expected'])}</div>"
+                    f"<div class='failure-line'><strong>What it did instead:</strong> {esc(_observed(inj, f))}</div>"
+                )
+            if not f["injections"]:
+                lines.append(
+                    f"<div class='failure-line'><strong>What happened:</strong> {esc(_observed(None, f))}</div>"
+                )
+            if f["final_answer"]:
+                lines.append(
+                    f"<div class='failure-line'><strong>Its final answer:</strong> "
+                    f"<span class='muted'>{esc(_truncate(str(f['final_answer']), 240))}</span></div>"
+                )
+            items.append(f"<div class='failure-item'>{head}{''.join(lines)}</div><!--/failure-item-->")
+        return f"<section class='failures'>{header}{narrative}{''.join(items)}</section>"
+
     def _scenario_detail_html(self, esc) -> str:
         """One collapsible <details> block per scenario, each with a full
         step-by-step table for the clean and chaos runs -- the HTML
@@ -740,6 +1021,8 @@ class Report:
             return "<p class='muted empty-state'>No scenarios run.</p>"
 
         blocks = []
+        any_failed = False
+        any_passed = False
         for scenario_id in scenario_ids:
             clean = clean_by_id.get(scenario_id)
             chaos = chaos_by_id.get(scenario_id)
@@ -754,8 +1037,21 @@ class Report:
                 )
                 if chaos.discarded:
                     pills.append("<span class='pill pill--warn'>DISCARDED</span>")
+            failed = (chaos is not None and not chaos.passed) or (clean is not None and not clean.passed)
+            any_failed = any_failed or failed
+            any_passed = any_passed or not failed
             blocks.append(
+                f"<div class='scenario-wrap scenario-wrap--{'fail' if failed else 'pass'}' id='{esc(_scenario_anchor(scenario_id))}'>"
                 f"<details><summary>{esc(scenario_id)}<span class='tag-group'>{''.join(pills)}</span></summary>"
-                f"<div class='scenario-body'>{run_section('Clean run', clean)}{run_section('Chaos run', chaos)}</div></details>"
+                f"<div class='scenario-body'>{run_section('Clean run', clean)}{run_section('Chaos run', chaos)}</div></details></div>"
             )
-        return "".join(blocks)
+        # CSS-only (no <script>, see render_html): the checkbox sits inside the
+        # same <section> as the scenarios, and :has() in _REPORT_CSS hides every
+        # passing scenario while it's checked. Only offered when there's
+        # both a failure to keep and a pass to hide.
+        toggle = (
+            "<label class='only-failures'><input type='checkbox'> Show only failures</label>"
+            if any_failed and any_passed
+            else ""
+        )
+        return toggle + "".join(blocks)