agentprobe-testing 1.2.0__tar.gz → 1.4.0__tar.gz

{agentprobe_testing-1.2.0 → agentprobe_testing-1.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentprobe-testing
-Version: 1.2.0
+Version: 1.4.0
 Summary: Adaptive chaos-testing for LLM agents: a live model-driven Injector that reads an agent's real trajectory and decides where to break something, instead of scripting perturbations in advance.
 License: Business Source License 1.1
         

{agentprobe_testing-1.2.0 → agentprobe_testing-1.4.0}/agentprobe/__init__.py

@@ -60,7 +60,7 @@ from agentprobe.scenario import CommitPattern, FactPattern, GoalSpec, Scenario
 from agentprobe.scenarios.registry import ALL_SCENARIOS as TICKET_SCENARIOS
 from agentprobe.scenarios.registry import BY_ID as TICKET_SCENARIOS_BY_ID
 
-__version__ = "1.2.0"
+__version__ = "1.4.0"
 
 __all__ = [
     "Agent",

{agentprobe_testing-1.2.0 → agentprobe_testing-1.4.0}/agentprobe/priority.py

@@ -118,6 +118,20 @@ def rank(issues: list[dict], overrides: Optional[dict[str, str]] = None) -> list
     return ranked
 
 
+def clean_overrides(raw: object) -> dict[str, str]:
+    """Keep only well-formed {FAILURE_KIND: "High"|"Medium"|"Low"} entries from
+    whatever a caller hands in (the hosted runner gets this from the website's
+    saved edits, over the wire), so a malformed or hostile value can never
+    reach a report."""
+    if not isinstance(raw, dict):
+        return {}
+    return {
+        k: v
+        for k, v in raw.items()
+        if isinstance(k, str) and isinstance(v, str) and v in TIERS and 1 <= len(k) <= 40 and k.replace("_", "").isalnum() and k.upper() == k
+    }
+
+
 def encode(issues: list[dict]) -> str:
     """ASCII-only JSON, base64'd, so it can sit inside an HTML comment with no
     escaping concerns. The website reads it back to re-rank a stored report

{agentprobe_testing-1.2.0 → agentprobe_testing-1.4.0}/agentprobe/quickstart.py

@@ -17,7 +17,8 @@ from typing import Any, Callable, Optional
 from agentprobe.agents.base import Agent, AgentAction
 from agentprobe.domain import Domain, TICKET_DOMAIN
 from agentprobe.injection import injection_was_triggered
-from agentprobe.injector import Injector, NullInjector, RecordingInjector, serialize_armed_injection
+from agentprobe.injector import Injector, ModelInjector, NullInjector, RecordingInjector, ReplayInjector, serialize_armed_injection
+from agentprobe.priority import clean_overrides
 from agentprobe.report import InjectionRecord, Report
 from agentprobe.runner import run_recovery, run_robustness_pair
 from agentprobe.scenario import Scenario
@@ -162,6 +163,7 @@ def quick_test(
     domain_key: Optional[str] = None,
     record_injections: bool = False,
     summarize_failures: bool = False,
+    priority_overrides: Optional[dict] = None,
 ) -> Report:
     """Run one scenario and get back a ready Report -- the one-call version
     of run_robustness_pair()/run_recovery() + Report(). `agent` can be a
@@ -184,6 +186,7 @@ def quick_test(
     injector = injector if injector is not None else NullInjector()
     target_factory = agent if _looks_like_factory(agent) else wrap_agent(agent)
     resolved_injector_model = injector_model if injector_model is not None else type(injector).__name__
+    injection_mode = _injection_mode(injector)
 
     recording: Optional[RecordingInjector] = None
     if record_injections:
@@ -214,6 +217,8 @@ def quick_test(
         chaos_trajectories=[chaos],
         injection_records=injection_records,
         injector_cost_usd=_injector_cost_usd(injector),
+        priority_overrides=clean_overrides(priority_overrides),
+        injection_mode=injection_mode,
     )
     if recording is not None:
         # Stashed here (not a Report field -- Report is a general-purpose
@@ -249,6 +254,7 @@ def quick_test_all(
     record_injections: bool = False,
     max_duration_seconds: Optional[float] = None,
     summarize_failures: bool = False,
+    priority_overrides: Optional[dict] = None,
 ) -> Report:
     """Run every scenario in `scenarios` (a dict[str, Scenario] -- e.g.
     TICKET_SCENARIOS or a FetchedDomain's .scenarios -- or a plain list of
@@ -296,6 +302,7 @@ def quick_test_all(
     all_records: list[InjectionRecord] = []
     total_injector_cost = 0.0
     combined_injector_model: Optional[str] = None
+    combined_injection_mode: Optional[str] = None
     start = time.monotonic()
     scenarios_run = 0
 
@@ -307,6 +314,7 @@ def quick_test_all(
         resolved_injector_model = injector_model if injector_model is not None else type(injector).__name__
         if combined_injector_model is None:
             combined_injector_model = resolved_injector_model
+            combined_injection_mode = _injection_mode(injector)
 
         recording: Optional[RecordingInjector] = None
         if record_injections:
@@ -352,6 +360,8 @@ def quick_test_all(
         chaos_trajectories=all_chaos,
         injection_records=all_records,
         injector_cost_usd=total_injector_cost,
+        priority_overrides=clean_overrides(priority_overrides),
+        injection_mode=combined_injection_mode,
     )
     report._scenarios_run = len(all_chaos)
     if summarize_failures:
@@ -391,6 +401,21 @@ def assert_passes(report: Report, save_html_on_failure: Optional[str] = None) ->
     raise AssertionError("agentprobe: " + "; ".join(failures))
 
 
+def _injection_mode(injector: Injector) -> Optional[str]:
+    """How this run's failures were chosen, for labeling its report: an
+    adaptive ModelInjector decides afresh each time ("exploration" -- finds new
+    weaknesses, but two runs aren't directly comparable), a ReplayInjector
+    re-fires a recorded sequence ("pinned" -- the same failures every time, so
+    scores compare). Anything else (none, hardcoded) isn't labeled. Looks
+    through a RecordingInjector's wrapping."""
+    inner = getattr(injector, "_inner", injector)
+    if isinstance(inner, ReplayInjector):
+        return "pinned"
+    if isinstance(inner, ModelInjector):
+        return "exploration"
+    return None
+
+
 def _injector_cost_usd(injector: Injector) -> float:
     """ModelInjector is the only injector type with real API cost (none/
     hardcoded/replay never call a model) -- same logic as cli.py's version,

{agentprobe_testing-1.2.0 → agentprobe_testing-1.4.0}/agentprobe/registry.py

@@ -175,6 +175,7 @@ def upload_run(
         # caller's explicit consent to pool them, anonymously, into
         # aggregate benchmarks; False (the default) keeps them private.
         "findings": report.findings(),
+        "injection_mode": report.injection_mode,
         "share_anonymized": bool(share_anonymized),
     }
     if recorded_injections is None:

{agentprobe_testing-1.2.0 → agentprobe_testing-1.4.0}/agentprobe/report.py

@@ -306,6 +306,11 @@ class Report:
     holds Trajectories/InjectionRecords, not the Injector instances
     themselves. 0.0 (and omitted from output) for none/hardcoded/replay
     injectors, which never call a model at all."""
+    injection_mode: Optional[str] = None
+    """How this run's failures were chosen: "exploration" (an adaptive injector
+    decided them -- finds new weaknesses, but two runs aren't directly
+    comparable) or "pinned" (a recorded sequence was replayed -- the same
+    failures every time, so scores compare). None when unknown/unlabeled."""
     priority_overrides: dict[str, str] = field(default_factory=dict)
     """Tiers (High/Medium/Low) the reader has set for particular failure kinds,
     e.g. {"FALSE_SUCCESS": "High"}. They replace the automatic estimate in
@@ -412,6 +417,13 @@ class Report:
             )
         return out
 
+    def injection_mode_label(self) -> Optional[str]:
+        """Plain-language form of injection_mode for report headers."""
+        return {
+            "exploration": "adaptive failures (exploration: not directly comparable with other runs)",
+            "pinned": "replayed failures (pinned: comparable with other replays of the same recording)",
+        }.get(self.injection_mode or "")
+
     def priorities(self) -> list[dict]:
         """The run's issues ranked highest to lowest priority, with any
         `priority_overrides` applied. See priority.py for how it's scored."""
@@ -475,7 +487,7 @@ class Report:
         lines = [
             blue(
                 f"scenarios: {c['n']}   mode: {self.mode}   injector: {self.injector_model}   "
-                f"target: {self.target_model}"
+                f"target: {self.target_model}" + (f"   failures: {self.injection_mode_label()}" if self.injection_mode_label() else "")
             ),
         ]
         if c["clean_passed"] is not None:
@@ -811,7 +823,7 @@ class Report:
 <div class="page">
   <header>
     <div class="title-row"><span class="health-dot health-dot--{health}"></span><h1>AgentProbe report</h1></div>
-    <div class="subtitle">mode: {esc(self.mode)} &middot; injector: {esc(self.injector_model)} &middot; target: {esc(self.target_model)} &middot; {esc(c['n'])} scenarios</div>
+    <div class="subtitle">mode: {esc(self.mode)} &middot; injector: {esc(self.injector_model)} &middot; target: {esc(self.target_model)} &middot; {esc(c['n'])} scenarios{(' &middot; ' + esc(self.injection_mode_label())) if self.injection_mode_label() else ''}</div>
   </header>
   {injector_error_banner}
   <div class="stats">

{agentprobe_testing-1.2.0 → agentprobe_testing-1.4.0}/agentprobe_testing.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentprobe-testing
-Version: 1.2.0
+Version: 1.4.0
 Summary: Adaptive chaos-testing for LLM agents: a live model-driven Injector that reads an agent's real trajectory and decides where to break something, instead of scripting perturbations in advance.
 License: Business Source License 1.1
         

{agentprobe_testing-1.2.0 → agentprobe_testing-1.4.0}/agentprobe_testing.egg-info/SOURCES.txt

@@ -85,6 +85,7 @@ tests/test_failures.py
 tests/test_feedback.py
 tests/test_generic_world.py
 tests/test_injection.py
+tests/test_injection_mode.py
 tests/test_injector.py
 tests/test_llm.py
 tests/test_package_api.py

{agentprobe_testing-1.2.0 → agentprobe_testing-1.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "agentprobe-testing"
-version = "1.2.0"
+version = "1.4.0"
 description = "Adaptive chaos-testing for LLM agents: a live model-driven Injector that reads an agent's real trajectory and decides where to break something, instead of scripting perturbations in advance."
 requires-python = ">=3.11"
 dependencies = ["anthropic>=1.0.0", "python-dotenv>=1.0.0"]

agentprobe_testing-1.4.0/tests/test_injection_mode.py

@@ -0,0 +1,80 @@
+"""Labeling a run exploration (adaptive injector) or pinned (replayed recording)."""
+
+from agentprobe.agents.scripted import ScriptedAgent
+from agentprobe.injection import ArmedInjection, Injection, InjectionKind, ResponseSignature, Trigger
+from agentprobe.injector import HardcodedToolErrorInjector, ModelInjector, NullInjector, RecordingInjector, ReplayInjector, serialize_armed_injection, deserialize_armed_injection
+from agentprobe.quickstart import _injection_mode, quick_test, quick_test_all
+from agentprobe.registry import upload_run
+from agentprobe.scenarios.registry import BY_ID
+
+AGENT = lambda: ScriptedAgent(script=[("get_ticket", {"id": "T-1001"})])
+
+
+def _armed(step=1):
+    return ArmedInjection(
+        injection=Injection(kind=InjectionKind.TOOL_ERROR, payload={"tool_name": "close_ticket", "count": 1}, intent="i",
+                            expected_signature=ResponseSignature("s"), rationale="r"),
+        trigger=Trigger.on_next_action(),
+        armed_at_step=step,
+    )
+
+
+def test_injection_mode_reads_the_injector_type_through_a_recording_wrapper():
+    adaptive = ModelInjector.__new__(ModelInjector)  # only its type matters here; no client needed
+    replay = ReplayInjector([_armed()])
+    assert _injection_mode(adaptive) == "exploration"
+    assert _injection_mode(replay) == "pinned"
+    assert _injection_mode(RecordingInjector(adaptive)) == "exploration"
+    assert _injection_mode(RecordingInjector(replay)) == "pinned"
+    assert _injection_mode(NullInjector()) is None
+    assert _injection_mode(HardcodedToolErrorInjector()) is None
+
+
+def test_a_replayed_run_is_labeled_pinned_in_the_report():
+    report = quick_test(BY_ID["clean-1"], AGENT, injector=ReplayInjector([]), mode="recovery")
+    assert report.injection_mode == "pinned"
+    assert "replayed failures (pinned: comparable with other replays of the same recording)" in report.render_html()
+    assert "failures: replayed failures (pinned" in report.render()
+
+
+def test_recording_a_replay_still_labels_it_pinned():
+    report = quick_test(BY_ID["clean-1"], AGENT, injector=ReplayInjector([]), mode="recovery", record_injections=True)
+    assert report.injection_mode == "pinned"
+
+
+def test_an_unlabeled_injector_adds_no_mode_text():
+    report = quick_test(BY_ID["clean-1"], AGENT, injector=NullInjector(), mode="recovery")
+    assert report.injection_mode is None and report.injection_mode_label() is None
+    assert "pinned" not in report.render_html() and "exploration" not in report.render_html()
+
+
+def test_the_exploration_label_says_runs_are_not_directly_comparable():
+    from agentprobe.report import Report
+
+    r = Report("robustness", "m", "t", [], [], injection_mode="exploration")
+    assert "adaptive failures (exploration: not directly comparable with other runs)" in r.render_html()
+
+
+def test_quick_test_all_labels_the_combined_report_from_its_injectors():
+    scenarios = {k: BY_ID[k] for k in ("clean-1", "clean-2")}
+    report = quick_test_all(scenarios, AGENT, injector_factory=lambda: ReplayInjector([]), mode="recovery")
+    assert report.injection_mode == "pinned"
+
+
+def test_upload_run_sends_the_mode_so_the_website_can_label_it():
+    sent = {}
+    report = quick_test(BY_ID["clean-1"], AGENT, injector=ReplayInjector([]), mode="recovery")
+    upload_run(report, api_key="k", poster=lambda url, body, key: sent.update(body))
+    assert sent["injection_mode"] == "pinned"
+    report2 = quick_test(BY_ID["clean-1"], AGENT, injector=NullInjector(), mode="recovery")
+    upload_run(report2, api_key="k", poster=lambda url, body, key: sent.update(body))
+    assert sent["injection_mode"] is None
+
+
+def test_a_serialized_recording_survives_the_round_trip_the_website_relies_on():
+    # The website stores serialize_armed_injection() dicts and the runner rebuilds them with
+    # deserialize_armed_injection() before replaying -- this is the contract between the two.
+    original = [_armed(2), _armed(5)]
+    rebuilt = ReplayInjector([deserialize_armed_injection(serialize_armed_injection(a)) for a in original])
+    assert sorted(rebuilt._by_step) == [2, 5]
+    assert rebuilt._by_step[2].injection.kind == InjectionKind.TOOL_ERROR

{agentprobe_testing-1.2.0 → agentprobe_testing-1.4.0}/tests/test_priority.py

@@ -139,3 +139,52 @@ def test_html_stays_script_free_and_the_text_report_lists_priorities():
     out = r.render()
     assert "priorities (automated estimate -- review before acting):" in out
     assert "1. [High] Trusts a tool that did nothing -- 1 scenario(s)" in out
+
+
+# --- overrides flowing in from outside (the hosted runner / Temporal worker) ---
+
+def test_clean_overrides_keeps_only_well_formed_entries():
+    from agentprobe.priority import clean_overrides
+
+    assert clean_overrides({"FALSE_SUCCESS": "High", "STALE_READ": "Low"}) == {"FALSE_SUCCESS": "High", "STALE_READ": "Low"}
+    dirty = {"false_success": "High", "STALE READ": "Low", "TOOL_ERROR": "Urgent", "DISTRACTOR": 3, 5: "High", "X" * 41: "High", "": "High", "OK_KIND": "Medium"}
+    assert clean_overrides(dirty) == {"OK_KIND": "Medium"}
+    for junk in (None, "High", ["High"], 7):
+        assert clean_overrides(junk) == {}
+
+
+def test_quick_test_ranks_with_the_overrides_it_is_given():
+    from agentprobe.agents.scripted import ScriptedAgent
+    from agentprobe.injector import NullInjector
+    from agentprobe.quickstart import quick_test
+    from agentprobe.scenarios.registry import BY_ID
+
+    agent = lambda: ScriptedAgent(script=[("get_ticket", {"id": "T-1001"})])  # never finishes -> the chaos run fails
+    report = quick_test(BY_ID["clean-1"], agent, injector=NullInjector(), mode="recovery", priority_overrides={"UNATTRIBUTED": "High", "BAD": "nope", "x": "Low"})
+    assert report.priority_overrides == {"UNATTRIBUTED": "High"}  # junk dropped
+    ranked = report.priorities()
+    assert [(r["kind"], r["tier"], r["edited"]) for r in ranked] == [("UNATTRIBUTED", "High", True)]
+    assert "priority--high'>High</span> <span class='muted'>(edited)" in report.render_html()
+
+
+def test_quick_test_without_overrides_ranks_automatically():
+    from agentprobe.agents.scripted import ScriptedAgent
+    from agentprobe.injector import NullInjector
+    from agentprobe.quickstart import quick_test
+    from agentprobe.scenarios.registry import BY_ID
+
+    report = quick_test(BY_ID["clean-1"], lambda: ScriptedAgent(script=[("get_ticket", {"id": "T-1001"})]), injector=NullInjector(), mode="recovery")
+    assert report.priority_overrides == {}
+    assert [(r["tier"], r["edited"]) for r in report.priorities()] == [("Medium", False)]
+
+
+def test_quick_test_all_carries_the_overrides_onto_the_combined_report():
+    from agentprobe.agents.scripted import ScriptedAgent
+    from agentprobe.injector import NullInjector
+    from agentprobe.quickstart import quick_test_all
+    from agentprobe.scenarios.registry import BY_ID
+
+    scenarios = {k: BY_ID[k] for k in ("clean-1", "clean-2")}
+    report = quick_test_all(scenarios, lambda: ScriptedAgent(script=[("get_ticket", {"id": "T-1001"})]), injector_factory=NullInjector, mode="recovery", priority_overrides={"UNATTRIBUTED": "Low"})
+    assert report.priority_overrides == {"UNATTRIBUTED": "Low"}
+    assert all(r["tier"] == "Low" and r["edited"] for r in report.priorities())