agentprobe-testing 1.3.0__tar.gz → 1.4.0__tar.gz

{agentprobe_testing-1.3.0 → agentprobe_testing-1.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentprobe-testing
-Version: 1.3.0
+Version: 1.4.0
 Summary: Adaptive chaos-testing for LLM agents: a live model-driven Injector that reads an agent's real trajectory and decides where to break something, instead of scripting perturbations in advance.
 License: Business Source License 1.1
         

{agentprobe_testing-1.3.0 → agentprobe_testing-1.4.0}/agentprobe/__init__.py

@@ -60,7 +60,7 @@ from agentprobe.scenario import CommitPattern, FactPattern, GoalSpec, Scenario
 from agentprobe.scenarios.registry import ALL_SCENARIOS as TICKET_SCENARIOS
 from agentprobe.scenarios.registry import BY_ID as TICKET_SCENARIOS_BY_ID
 
-__version__ = "1.3.0"
+__version__ = "1.4.0"
 
 __all__ = [
     "Agent",

{agentprobe_testing-1.3.0 → agentprobe_testing-1.4.0}/agentprobe/quickstart.py

@@ -17,7 +17,7 @@ from typing import Any, Callable, Optional
 from agentprobe.agents.base import Agent, AgentAction
 from agentprobe.domain import Domain, TICKET_DOMAIN
 from agentprobe.injection import injection_was_triggered
-from agentprobe.injector import Injector, NullInjector, RecordingInjector, serialize_armed_injection
+from agentprobe.injector import Injector, ModelInjector, NullInjector, RecordingInjector, ReplayInjector, serialize_armed_injection
 from agentprobe.priority import clean_overrides
 from agentprobe.report import InjectionRecord, Report
 from agentprobe.runner import run_recovery, run_robustness_pair
@@ -186,6 +186,7 @@ def quick_test(
     injector = injector if injector is not None else NullInjector()
     target_factory = agent if _looks_like_factory(agent) else wrap_agent(agent)
     resolved_injector_model = injector_model if injector_model is not None else type(injector).__name__
+    injection_mode = _injection_mode(injector)
 
     recording: Optional[RecordingInjector] = None
     if record_injections:
@@ -217,6 +218,7 @@ def quick_test(
         injection_records=injection_records,
         injector_cost_usd=_injector_cost_usd(injector),
         priority_overrides=clean_overrides(priority_overrides),
+        injection_mode=injection_mode,
     )
     if recording is not None:
         # Stashed here (not a Report field -- Report is a general-purpose
@@ -300,6 +302,7 @@ def quick_test_all(
     all_records: list[InjectionRecord] = []
     total_injector_cost = 0.0
     combined_injector_model: Optional[str] = None
+    combined_injection_mode: Optional[str] = None
     start = time.monotonic()
     scenarios_run = 0
 
@@ -311,6 +314,7 @@ def quick_test_all(
         resolved_injector_model = injector_model if injector_model is not None else type(injector).__name__
         if combined_injector_model is None:
             combined_injector_model = resolved_injector_model
+            combined_injection_mode = _injection_mode(injector)
 
         recording: Optional[RecordingInjector] = None
         if record_injections:
@@ -357,6 +361,7 @@ def quick_test_all(
         injection_records=all_records,
         injector_cost_usd=total_injector_cost,
         priority_overrides=clean_overrides(priority_overrides),
+        injection_mode=combined_injection_mode,
     )
     report._scenarios_run = len(all_chaos)
     if summarize_failures:
@@ -396,6 +401,21 @@ def assert_passes(report: Report, save_html_on_failure: Optional[str] = None) ->
     raise AssertionError("agentprobe: " + "; ".join(failures))
 
 
+def _injection_mode(injector: Injector) -> Optional[str]:
+    """How this run's failures were chosen, for labeling its report: an
+    adaptive ModelInjector decides afresh each time ("exploration" -- finds new
+    weaknesses, but two runs aren't directly comparable), a ReplayInjector
+    re-fires a recorded sequence ("pinned" -- the same failures every time, so
+    scores compare). Anything else (none, hardcoded) isn't labeled. Looks
+    through a RecordingInjector's wrapping."""
+    inner = getattr(injector, "_inner", injector)
+    if isinstance(inner, ReplayInjector):
+        return "pinned"
+    if isinstance(inner, ModelInjector):
+        return "exploration"
+    return None
+
+
 def _injector_cost_usd(injector: Injector) -> float:
     """ModelInjector is the only injector type with real API cost (none/
     hardcoded/replay never call a model) -- same logic as cli.py's version,

{agentprobe_testing-1.3.0 → agentprobe_testing-1.4.0}/agentprobe/registry.py

@@ -175,6 +175,7 @@ def upload_run(
         # caller's explicit consent to pool them, anonymously, into
         # aggregate benchmarks; False (the default) keeps them private.
         "findings": report.findings(),
+        "injection_mode": report.injection_mode,
         "share_anonymized": bool(share_anonymized),
     }
     if recorded_injections is None:

{agentprobe_testing-1.3.0 → agentprobe_testing-1.4.0}/agentprobe/report.py

@@ -306,6 +306,11 @@ class Report:
     holds Trajectories/InjectionRecords, not the Injector instances
     themselves. 0.0 (and omitted from output) for none/hardcoded/replay
     injectors, which never call a model at all."""
+    injection_mode: Optional[str] = None
+    """How this run's failures were chosen: "exploration" (an adaptive injector
+    decided them -- finds new weaknesses, but two runs aren't directly
+    comparable) or "pinned" (a recorded sequence was replayed -- the same
+    failures every time, so scores compare). None when unknown/unlabeled."""
     priority_overrides: dict[str, str] = field(default_factory=dict)
     """Tiers (High/Medium/Low) the reader has set for particular failure kinds,
     e.g. {"FALSE_SUCCESS": "High"}. They replace the automatic estimate in
@@ -412,6 +417,13 @@ class Report:
             )
         return out
 
+    def injection_mode_label(self) -> Optional[str]:
+        """Plain-language form of injection_mode for report headers."""
+        return {
+            "exploration": "adaptive failures (exploration: not directly comparable with other runs)",
+            "pinned": "replayed failures (pinned: comparable with other replays of the same recording)",
+        }.get(self.injection_mode or "")
+
     def priorities(self) -> list[dict]:
         """The run's issues ranked highest to lowest priority, with any
         `priority_overrides` applied. See priority.py for how it's scored."""
@@ -475,7 +487,7 @@ class Report:
         lines = [
             blue(
                 f"scenarios: {c['n']}   mode: {self.mode}   injector: {self.injector_model}   "
-                f"target: {self.target_model}"
+                f"target: {self.target_model}" + (f"   failures: {self.injection_mode_label()}" if self.injection_mode_label() else "")
             ),
         ]
         if c["clean_passed"] is not None:
@@ -811,7 +823,7 @@ class Report:
 <div class="page">
   <header>
     <div class="title-row"><span class="health-dot health-dot--{health}"></span><h1>AgentProbe report</h1></div>
-    <div class="subtitle">mode: {esc(self.mode)} &middot; injector: {esc(self.injector_model)} &middot; target: {esc(self.target_model)} &middot; {esc(c['n'])} scenarios</div>
+    <div class="subtitle">mode: {esc(self.mode)} &middot; injector: {esc(self.injector_model)} &middot; target: {esc(self.target_model)} &middot; {esc(c['n'])} scenarios{(' &middot; ' + esc(self.injection_mode_label())) if self.injection_mode_label() else ''}</div>
   </header>
   {injector_error_banner}
   <div class="stats">

{agentprobe_testing-1.3.0 → agentprobe_testing-1.4.0}/agentprobe_testing.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentprobe-testing
-Version: 1.3.0
+Version: 1.4.0
 Summary: Adaptive chaos-testing for LLM agents: a live model-driven Injector that reads an agent's real trajectory and decides where to break something, instead of scripting perturbations in advance.
 License: Business Source License 1.1
         

{agentprobe_testing-1.3.0 → agentprobe_testing-1.4.0}/agentprobe_testing.egg-info/SOURCES.txt

@@ -85,6 +85,7 @@ tests/test_failures.py
 tests/test_feedback.py
 tests/test_generic_world.py
 tests/test_injection.py
+tests/test_injection_mode.py
 tests/test_injector.py
 tests/test_llm.py
 tests/test_package_api.py

{agentprobe_testing-1.3.0 → agentprobe_testing-1.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "agentprobe-testing"
-version = "1.3.0"
+version = "1.4.0"
 description = "Adaptive chaos-testing for LLM agents: a live model-driven Injector that reads an agent's real trajectory and decides where to break something, instead of scripting perturbations in advance."
 requires-python = ">=3.11"
 dependencies = ["anthropic>=1.0.0", "python-dotenv>=1.0.0"]

agentprobe_testing-1.4.0/tests/test_injection_mode.py

@@ -0,0 +1,80 @@
+"""Labeling a run exploration (adaptive injector) or pinned (replayed recording)."""
+
+from agentprobe.agents.scripted import ScriptedAgent
+from agentprobe.injection import ArmedInjection, Injection, InjectionKind, ResponseSignature, Trigger
+from agentprobe.injector import HardcodedToolErrorInjector, ModelInjector, NullInjector, RecordingInjector, ReplayInjector, serialize_armed_injection, deserialize_armed_injection
+from agentprobe.quickstart import _injection_mode, quick_test, quick_test_all
+from agentprobe.registry import upload_run
+from agentprobe.scenarios.registry import BY_ID
+
+AGENT = lambda: ScriptedAgent(script=[("get_ticket", {"id": "T-1001"})])
+
+
+def _armed(step=1):
+    return ArmedInjection(
+        injection=Injection(kind=InjectionKind.TOOL_ERROR, payload={"tool_name": "close_ticket", "count": 1}, intent="i",
+                            expected_signature=ResponseSignature("s"), rationale="r"),
+        trigger=Trigger.on_next_action(),
+        armed_at_step=step,
+    )
+
+
+def test_injection_mode_reads_the_injector_type_through_a_recording_wrapper():
+    adaptive = ModelInjector.__new__(ModelInjector)  # only its type matters here; no client needed
+    replay = ReplayInjector([_armed()])
+    assert _injection_mode(adaptive) == "exploration"
+    assert _injection_mode(replay) == "pinned"
+    assert _injection_mode(RecordingInjector(adaptive)) == "exploration"
+    assert _injection_mode(RecordingInjector(replay)) == "pinned"
+    assert _injection_mode(NullInjector()) is None
+    assert _injection_mode(HardcodedToolErrorInjector()) is None
+
+
+def test_a_replayed_run_is_labeled_pinned_in_the_report():
+    report = quick_test(BY_ID["clean-1"], AGENT, injector=ReplayInjector([]), mode="recovery")
+    assert report.injection_mode == "pinned"
+    assert "replayed failures (pinned: comparable with other replays of the same recording)" in report.render_html()
+    assert "failures: replayed failures (pinned" in report.render()
+
+
+def test_recording_a_replay_still_labels_it_pinned():
+    report = quick_test(BY_ID["clean-1"], AGENT, injector=ReplayInjector([]), mode="recovery", record_injections=True)
+    assert report.injection_mode == "pinned"
+
+
+def test_an_unlabeled_injector_adds_no_mode_text():
+    report = quick_test(BY_ID["clean-1"], AGENT, injector=NullInjector(), mode="recovery")
+    assert report.injection_mode is None and report.injection_mode_label() is None
+    assert "pinned" not in report.render_html() and "exploration" not in report.render_html()
+
+
+def test_the_exploration_label_says_runs_are_not_directly_comparable():
+    from agentprobe.report import Report
+
+    r = Report("robustness", "m", "t", [], [], injection_mode="exploration")
+    assert "adaptive failures (exploration: not directly comparable with other runs)" in r.render_html()
+
+
+def test_quick_test_all_labels_the_combined_report_from_its_injectors():
+    scenarios = {k: BY_ID[k] for k in ("clean-1", "clean-2")}
+    report = quick_test_all(scenarios, AGENT, injector_factory=lambda: ReplayInjector([]), mode="recovery")
+    assert report.injection_mode == "pinned"
+
+
+def test_upload_run_sends_the_mode_so_the_website_can_label_it():
+    sent = {}
+    report = quick_test(BY_ID["clean-1"], AGENT, injector=ReplayInjector([]), mode="recovery")
+    upload_run(report, api_key="k", poster=lambda url, body, key: sent.update(body))
+    assert sent["injection_mode"] == "pinned"
+    report2 = quick_test(BY_ID["clean-1"], AGENT, injector=NullInjector(), mode="recovery")
+    upload_run(report2, api_key="k", poster=lambda url, body, key: sent.update(body))
+    assert sent["injection_mode"] is None
+
+
+def test_a_serialized_recording_survives_the_round_trip_the_website_relies_on():
+    # The website stores serialize_armed_injection() dicts and the runner rebuilds them with
+    # deserialize_armed_injection() before replaying -- this is the contract between the two.
+    original = [_armed(2), _armed(5)]
+    rebuilt = ReplayInjector([deserialize_armed_injection(serialize_armed_injection(a)) for a in original])
+    assert sorted(rebuilt._by_step) == [2, 5]
+    assert rebuilt._by_step[2].injection.kind == InjectionKind.TOOL_ERROR