agentprobe-testing 1.8.0__tar.gz → 1.9.0__tar.gz

{agentprobe_testing-1.8.0 → agentprobe_testing-1.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentprobe-testing
-Version: 1.8.0
+Version: 1.9.0
 Summary: Adaptive chaos-testing for LLM agents: a live model-driven Injector that reads an agent's real trajectory and decides where to break something, instead of scripting perturbations in advance.
 License: Business Source License 1.1
         

{agentprobe_testing-1.8.0 → agentprobe_testing-1.9.0}/agentprobe/__init__.py

@@ -56,6 +56,7 @@ from agentprobe.registry import (
 )
 from agentprobe.report import InjectionRecord, Report
 from agentprobe.runner import run_recovery, run_robustness_pair
+from agentprobe.focus import FOCUS_KINDS, FOCUS_LABELS, focus_kind_order, normalize_focus
 from agentprobe.savings import FixedFaultInjector, estimate_cost, fixed_fault_for, model_injector_for, retest_failures
 from agentprobe.connect import MODES as CONNECTION_MODES, connect
 from agentprobe.sandbox import DriverAgent, ToolBox, ToolCallError
@@ -65,7 +66,7 @@ from agentprobe.scenario import CommitPattern, FactPattern, GoalSpec, Scenario
 from agentprobe.scenarios.registry import ALL_SCENARIOS as TICKET_SCENARIOS
 from agentprobe.scenarios.registry import BY_ID as TICKET_SCENARIOS_BY_ID
 
-__version__ = "1.8.0"
+__version__ = "1.9.0"
 
 __all__ = [
     "CONNECTION_MODES",
@@ -79,6 +80,10 @@ __all__ = [
     "estimate_cost",
     "fixed_fault_for",
     "model_injector_for",
+    "FOCUS_KINDS",
+    "FOCUS_LABELS",
+    "focus_kind_order",
+    "normalize_focus",
     "retest_failures",
     "Agent",
     "AgentAction",

agentprobe_testing-1.9.0/agentprobe/focus.py

@@ -0,0 +1,77 @@
+"""Tell the test which failures cost you the most, so those are tried first.
+
+`focus` is a list of failure kind names (InjectionKind values), most important first. The adaptive
+injector works through its failure kinds in order, so putting yours first means they are the ones
+still covered if a run stops early (for example on a spending budget). The report then says, for each
+kind you chose, whether it was actually tested and how the agent did, and says so plainly when a
+chosen failure could not be tried in a scenario.
+
+    injector = ap.model_injector_for(domain, focus=["FALSE_SUCCESS", "PROMPT_INJECTION"])
+    report = ap.quick_test_all(scenarios, agent, injector_factory=lambda: injector_for(...), focus=[...])
+"""
+from __future__ import annotations
+
+from typing import Iterable, Optional
+
+from agentprobe.injection import InjectionKind
+from agentprobe.injector import _DEFAULT_KIND_ORDER
+
+# Plain-language names for each kind, in the words a customer would use. Also what the website shows.
+FOCUS_LABELS: dict[str, str] = {
+    "PHANTOM_SUCCESS": "A system says it did something, but nothing happened",
+    "FALSE_SUCCESS": "A system reports a success that isn't true",
+    "UNVERIFIED_OUTCOME": "A result is pending or unconfirmed, and the agent calls it done",
+    "TOOL_ERROR": "A tool call fails",
+    "PROMPT_INJECTION": "Hidden instructions in ordinary-looking data",
+    "SOCIAL_PRESSURE": "A customer pushes with emotion or authority",
+    "SCOPE_CREEP": "A request that goes beyond what the agent should do",
+    "STALE_READ": "A record changes after the agent already read it",
+    "CONTRADICTION": "A new fact contradicts an earlier one",
+    "LATE_INFO": "Something important arrives only after the agent acted",
+    "MISSING_INFO": "A field the agent needs is missing",
+    "AMBIGUITY": "A second, equally valid record appears",
+    "DEPENDENT_FOLLOWUP": "A note arrives that follows from the agent's own action",
+    "DISTRACTOR": "An irrelevant note appears",
+    "FREEFORM": "A failure the injector invents for the situation",
+}
+
+FOCUS_KINDS: tuple[str, ...] = tuple(FOCUS_LABELS)
+
+
+def normalize_focus(values: Optional[Iterable[str]]) -> list[str]:
+    """Validated, de-duplicated focus list in the order given. Raises ValueError on an unknown kind."""
+    out: list[str] = []
+    for v in values or []:
+        name = str(v).strip().upper()
+        if not name:
+            continue
+        if name not in FOCUS_LABELS:
+            raise ValueError(f"unknown failure kind {v!r}; choose from: {', '.join(FOCUS_KINDS)}")
+        if name not in out:
+            out.append(name)
+    return out
+
+
+def focus_kind_order(focus: Optional[Iterable[str]]) -> list[InjectionKind]:
+    """The injector's kind order: your kinds first (in your order), then the usual order."""
+    chosen = [InjectionKind(n) for n in normalize_focus(focus)]
+    rest = [k for k in _DEFAULT_KIND_ORDER if k not in chosen]
+    return chosen + rest
+
+
+def focus_coverage(per_kind: list[dict], focus: Iterable[str]) -> list[dict]:
+    """For each chosen kind: was it tested, and how did the agent do. `per_kind` is Report._compute()["per_kind"]."""
+    rows = {r["kind"]: r for r in per_kind}
+    out = []
+    for name in normalize_focus(focus):
+        r = rows.get(name)
+        out.append(
+            {
+                "kind": name,
+                "label": FOCUS_LABELS[name],
+                "tested": r is not None and r["n"] > 0,
+                "n": r["n"] if r else 0,
+                "handled": r["handled"] if r else 0,
+            }
+        )
+    return out

{agentprobe_testing-1.8.0 → agentprobe_testing-1.9.0}/agentprobe/quickstart.py

@@ -14,6 +14,7 @@ import time
 import urllib.request
 from typing import Any, Callable, Optional
 
+from agentprobe.focus import normalize_focus as _normalize_focus
 from agentprobe.agents.base import Agent, AgentAction
 from agentprobe.domain import Domain, TICKET_DOMAIN
 from agentprobe.injection import injection_was_triggered
@@ -165,6 +166,7 @@ def quick_test(
     record_injections: bool = False,
     summarize_failures: bool = False,
     priority_overrides: Optional[dict] = None,
+    focus: Optional[list[str]] = None,
 ) -> Report:
     """Run one scenario and get back a ready Report -- the one-call version
     of run_robustness_pair()/run_recovery() + Report(). `agent` can be a
@@ -221,6 +223,7 @@ def quick_test(
         priority_overrides=clean_overrides(priority_overrides),
         injection_mode=injection_mode,
         tool_expectations={"known": sorted(domain.all_tools), "required": {scenario.id: sorted(required_tools(scenario))}},
+        focus_kinds=_normalize_focus(focus),
     )
     if recording is not None:
         # Stashed here (not a Report field -- Report is a general-purpose
@@ -262,6 +265,7 @@ def quick_test_all(
     stop_when_achieved: bool = False,
     preflight: bool = False,
     agent_tools: Optional[list[str]] = None,
+    focus: Optional[list[str]] = None,
 ) -> Report:
     """Run every scenario in `scenarios` (a dict[str, Scenario] -- e.g.
     TICKET_SCENARIOS or a FetchedDomain's .scenarios -- or a plain list of
@@ -269,6 +273,9 @@ def quick_test_all(
     Report -- the batch version of calling quick_test() once per scenario
     and merging the results by hand.
 
+    `focus` is a list of failure kinds that cost you the most (see focus.py): the report says whether each
+    was actually tested. To make the injector try them first, build it with `model_injector_for(..., focus=...)`.
+
     `injector_factory` is a zero-arg callable returning a FRESH Injector
     instance per scenario (same shape as wrap_agent's target_factory), e.g.
     `injector_factory=lambda: ModelInjector()`. This must be a factory, not
@@ -338,6 +345,10 @@ def quick_test_all(
             )
 
     target_factory = agent if _looks_like_factory(agent) else wrap_agent(agent)
+    from agentprobe.focus import normalize_focus
+
+    focus_kinds = normalize_focus(focus)  # refuse an unknown kind before spending anything
+
     raw_make_injector = injector_factory if injector_factory is not None else (lambda: NullInjector())
     if not callable(raw_make_injector):
         raise ValueError(
@@ -446,6 +457,7 @@ def quick_test_all(
             "required": {sc.id: sorted(required_tools(sc)) for _id, sc in items},
         },
     )
+    report.focus_kinds = focus_kinds
     report._scenarios_run = len(all_chaos)
     report._skipped = skipped
     report._budget_stopped = budget_stopped

{agentprobe_testing-1.8.0 → agentprobe_testing-1.9.0}/agentprobe/report.py

@@ -327,6 +327,9 @@ class Report:
     the "What failed" section (built straight from data the run already
     has) it costs an extra API call. None means none was requested."""
     failure_narrative_cost_usd: float = 0.0
+    focus_kinds: list[str] = field(default_factory=list)
+    """Failure kinds the reader said cost them the most, most important first (see focus.py). The
+    report says, for each, whether it was actually tested."""
 
     def _compute(self) -> dict:
         """Shared computation behind both render() and render_html() --
@@ -871,7 +874,7 @@ class Report:
             )
 
         priorities_html = self._priorities_html(esc, c) + self._prompt_fixes_html(esc, c)
-        trust_html = self._landing_html(esc) + self._tool_check_html(esc)
+        trust_html = self._focus_html(esc) + self._landing_html(esc) + self._tool_check_html(esc)
         failures_html = self._failures_html(esc, c)
 
         return f"""<!doctype html>
@@ -950,6 +953,27 @@ class Report:
             f"<tbody>{''.join(rows)}</tbody></table></div></section><!--/priority-table-->"
         )
 
+    def focus_coverage(self) -> list[dict]:
+        """For each failure kind the reader asked to be tested first: tested or not, and how it went."""
+        from agentprobe.focus import focus_coverage
+
+        return focus_coverage(self._compute()["per_kind"], self.focus_kinds)
+
+    def _focus_html(self, esc) -> str:
+        if not self.focus_kinds:
+            return ""
+        rows = []
+        for r in self.focus_coverage():
+            if r["tested"]:
+                status = f"tested {r['n']} time{'s' if r['n'] != 1 else ''}: the agent handled {r['handled']} of {r['n']}"
+            else:
+                status = "<strong>not tested</strong>: it could not be tried in these scenarios (or the run ended first)"
+            rows.append(f"<li><strong>{esc(r['label'])}</strong>: {status}</li>")
+        return (
+            "<div class='injector-error-banner trust-banner focus-banner'><strong>You asked us to test these first</strong>"
+            f"<ul>{''.join(rows)}</ul></div>"
+        )
+
     def _landing_html(self, esc) -> str:
         """A banner when failures were expected but did not (all) land: a pass in such a scenario
         is not evidence of anything."""

{agentprobe_testing-1.8.0 → agentprobe_testing-1.9.0}/agentprobe/savings.py

@@ -159,8 +159,13 @@ def fixed_fault_for(scenario: Scenario, kind: InjectionKind = InjectionKind.FALS
 CHEAP_INJECTOR_MODEL = "claude-haiku-4-5-20251001"
 
 
-def model_injector_for(domain: Domain = TICKET_DOMAIN, model: str = CHEAP_INJECTOR_MODEL, **kwargs: Any) -> ModelInjector:
-    """The adaptive injector on a chosen (by default cheaper) model, wired to `domain`."""
+def model_injector_for(domain: Domain = TICKET_DOMAIN, model: str = CHEAP_INJECTOR_MODEL, focus: Optional[list[str]] = None, **kwargs: Any) -> ModelInjector:
+    """The adaptive injector on a chosen (by default cheaper) model, wired to `domain`.
+    `focus` lists the failure kinds that cost you the most; the injector tries those first."""
+    if focus:
+        from agentprobe.focus import focus_kind_order
+
+        kwargs["kind_order"] = focus_kind_order(focus)
     if domain.injector_system_prompt is None:
         return ModelInjector(model=model, **kwargs)
     return ModelInjector(

{agentprobe_testing-1.8.0 → agentprobe_testing-1.9.0}/agentprobe_testing.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentprobe-testing
-Version: 1.8.0
+Version: 1.9.0
 Summary: Adaptive chaos-testing for LLM agents: a live model-driven Injector that reads an agent's real trajectory and decides where to break something, instead of scripting perturbations in advance.
 License: Business Source License 1.1
         

{agentprobe_testing-1.8.0 → agentprobe_testing-1.9.0}/agentprobe_testing.egg-info/SOURCES.txt

@@ -10,6 +10,7 @@ agentprobe/diff.py
 agentprobe/domain.py
 agentprobe/feedback.py
 agentprobe/fixes.py
+agentprobe/focus.py
 agentprobe/generic_world.py
 agentprobe/injection.py
 agentprobe/injector.py
@@ -104,6 +105,7 @@ tests/test_domain.py
 tests/test_failures.py
 tests/test_feedback.py
 tests/test_fixes.py
+tests/test_focus.py
 tests/test_generic_world.py
 tests/test_injection.py
 tests/test_injection_mode.py

{agentprobe_testing-1.8.0 → agentprobe_testing-1.9.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "agentprobe-testing"
-version = "1.8.0"
+version = "1.9.0"
 description = "Adaptive chaos-testing for LLM agents: a live model-driven Injector that reads an agent's real trajectory and decides where to break something, instead of scripting perturbations in advance."
 requires-python = ">=3.11"
 dependencies = ["anthropic>=1.0.0", "python-dotenv>=1.0.0"]

agentprobe_testing-1.9.0/tests/test_focus.py

@@ -0,0 +1,87 @@
+import pytest
+
+from agentprobe.agents.base import AgentAction
+from agentprobe.focus import FOCUS_KINDS, FOCUS_LABELS, focus_coverage, focus_kind_order, normalize_focus
+from agentprobe.injection import InjectionKind
+from agentprobe.injector import _DEFAULT_KIND_ORDER
+from agentprobe.quickstart import quick_test_all
+from agentprobe.savings import fixed_fault_for, model_injector_for
+from agentprobe.scenarios.registry import BY_ID
+
+S1 = BY_ID["clean-1"]
+
+
+def _agent(task, history):
+    calls = [h["tool_name"] for h in history]
+    if "get_ticket" not in calls:
+        return AgentAction(kind="tool_call", tool_name="get_ticket", tool_args={"id": "T-1001"})
+    if "issue_refund" not in calls:
+        return AgentAction(kind="tool_call", tool_name="issue_refund", tool_args={"ticket_id": "T-1001", "amount": 49.99})
+    if "close_ticket" not in calls:
+        return AgentAction(kind="tool_call", tool_name="close_ticket", tool_args={"ticket_id": "T-1001", "resolution": "refunded"})
+    return AgentAction(kind="final_answer", text="done")
+
+
+def test_every_injection_kind_has_a_plain_label():
+    assert set(FOCUS_KINDS) == {k.value for k in InjectionKind}
+    assert all(FOCUS_LABELS[k] for k in FOCUS_KINDS)
+
+
+def test_normalize_validates_dedupes_and_keeps_order():
+    assert normalize_focus(["false_success", "TOOL_ERROR", "FALSE_SUCCESS", " "]) == ["FALSE_SUCCESS", "TOOL_ERROR"]
+    assert normalize_focus(None) == []
+    with pytest.raises(ValueError, match="unknown failure kind"):
+        normalize_focus(["NOT_A_KIND"])
+
+
+def test_kind_order_puts_the_chosen_kinds_first_then_the_usual_order():
+    order = focus_kind_order(["SCOPE_CREEP", "FALSE_SUCCESS"])
+    assert order[:2] == [InjectionKind.SCOPE_CREEP, InjectionKind.FALSE_SUCCESS]
+    assert [k for k in order[2:]] == [k for k in _DEFAULT_KIND_ORDER if k not in (InjectionKind.SCOPE_CREEP, InjectionKind.FALSE_SUCCESS)]
+    assert focus_kind_order(None) == list(_DEFAULT_KIND_ORDER)
+
+
+def test_model_injector_for_uses_the_focus_order():
+    inj = model_injector_for(focus=["PROMPT_INJECTION", "LATE_INFO"])
+    assert inj._kind_order[:2] == [InjectionKind.PROMPT_INJECTION, InjectionKind.LATE_INFO]
+
+
+def test_coverage_marks_tested_and_untested_kinds():
+    rows = focus_coverage([{"kind": "TOOL_ERROR", "handled": 2, "n": 3}], ["TOOL_ERROR", "SCOPE_CREEP"])
+    assert rows[0] | {"label": ""} == {"kind": "TOOL_ERROR", "label": "", "tested": True, "n": 3, "handled": 2}
+    assert rows[1]["tested"] is False and rows[1]["n"] == 0
+
+
+def test_report_says_which_chosen_failures_were_tested_and_which_were_not():
+    report = quick_test_all(
+        [S1], _agent, injector_factory=lambda sc: fixed_fault_for(sc, InjectionKind.TOOL_ERROR), focus=["TOOL_ERROR", "SCOPE_CREEP"]
+    )
+    assert report.focus_kinds == ["TOOL_ERROR", "SCOPE_CREEP"]
+    html = report.render_html()
+    assert "You asked us to test these first" in html
+    assert FOCUS_LABELS["TOOL_ERROR"] in html and FOCUS_LABELS["SCOPE_CREEP"] in html
+    cov = {r["kind"]: r for r in report.focus_coverage()}
+    assert cov["SCOPE_CREEP"]["tested"] is False
+    assert "not tested" in html
+
+
+def test_no_focus_means_no_focus_section():
+    report = quick_test_all([S1], _agent)
+    assert report.focus_kinds == []
+    assert "You asked us to test these first" not in report.render_html()
+
+
+def test_bad_focus_is_refused_up_front():
+    with pytest.raises(ValueError):
+        quick_test_all([S1], _agent, focus=["NOPE"])
+
+
+def test_the_websites_list_of_failure_types_matches_the_library():
+    import re
+    from pathlib import Path
+
+    js = (Path(__file__).resolve().parent.parent / "server" / "src" / "focus.js").read_text()
+    site = re.findall(r'\["([A-Z_]+)", "', js)
+    assert sorted(site) == sorted(FOCUS_KINDS)
+    labels = dict(re.findall(r'\["([A-Z_]+)", "([^"]+)"\]', js))
+    assert labels == FOCUS_LABELS