agentprobe-testing 0.8.1__tar.gz → 0.8.2__tar.gz

{agentprobe_testing-0.8.1 → agentprobe_testing-0.8.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentprobe-testing
-Version: 0.8.1
+Version: 0.8.2
 Summary: Adaptive chaos-testing for LLM agents: a live model-driven Injector that reads an agent's real trajectory and decides where to break something, instead of scripting perturbations in advance.
 License: Business Source License 1.1
         

{agentprobe_testing-0.8.1 → agentprobe_testing-0.8.2}/agentprobe/__init__.py

@@ -60,7 +60,7 @@ from agentprobe.scenario import CommitPattern, FactPattern, GoalSpec, Scenario
 from agentprobe.scenarios.registry import ALL_SCENARIOS as TICKET_SCENARIOS
 from agentprobe.scenarios.registry import BY_ID as TICKET_SCENARIOS_BY_ID
 
-__version__ = "0.8.1"
+__version__ = "0.8.2"
 
 __all__ = [
     "Agent",

{agentprobe_testing-0.8.1 → agentprobe_testing-0.8.2}/agentprobe/report.py

@@ -118,10 +118,18 @@ _REPORT_CSS = """
   .pass { color: var(--green); font-weight: 600; }
   .fail { color: var(--red); font-weight: 600; }
   .warn { color: var(--amber); font-weight: 600; }
+  .injected { color: var(--amber); font-weight: 600; }
+  tr.step-injected { background: var(--amber-bg); }
+  tr.step-injected td { border-bottom-color: var(--amber); }
   .injection-note { font-size: 0.83rem; margin-top: 0.5rem; padding: 0.55rem 0.7rem; border-radius: 6px;
                      border-left: 3px solid var(--amber); background: var(--amber-bg); }
   .injection-note--invalid { border-left-color: var(--red); background: var(--red-bg); }
   .injection-note--expired { border-left-color: var(--border); background: var(--bg-alt); }
+  .injector-error-banner { margin: 1rem 0 1.5rem; padding: 0.85rem 1rem; border-radius: 10px;
+                            border: 1px solid var(--amber); background: var(--amber-bg); font-size: 0.85rem; }
+  .injector-error-banner strong { display: block; margin-bottom: 0.35rem; }
+  .injector-error-banner ul { margin: 0.35rem 0 0; padding-left: 1.25rem; }
+  .injector-error-banner li { margin-top: 0.2rem; font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: 0.8em; }
   footer { margin-top: 3rem; padding-top: 1.25rem; border-top: 1px solid var(--border);
            font-size: 0.78rem; color: var(--text-dim); }
   @media print {
@@ -136,20 +144,41 @@ def _format_args(args: dict) -> str:
     return ", ".join(f"{k}={v!r}" for k, v in args.items())
 
 
+def _is_injected_outcome(step) -> bool:
+    """True for any of the four ways a step's perceived outcome can be
+    chaos-injected -- not just injected_error. unverified_outcome in
+    particular leaves `ok=True` (an ambiguous, non-committal response, not
+    a clean failure), so a check that only looked at `not s.ok` would
+    silently drop it from a report entirely."""
+    return step.injected_error or step.phantom_success or step.unverified_outcome or step.freeform_override
+
+
 def _notable_steps(steps: list) -> list:
     """The steps worth showing in a human-facing report: irreversible
     actions and anything that went wrong. A successful read (get_ticket,
     get_order, ...) is the agent looking something up -- noise in a report
     meant to answer "what did it actually do," not a trace of its
-    reasoning. Commits, errors, and injected failures are the actual
+    reasoning. Commits, errors, and injected outcomes are the actual
     story."""
-    return [s for s in steps if s.is_commit or not s.ok or s.injected_error]
+    return [s for s in steps if s.is_commit or not s.ok or _is_injected_outcome(s)]
 
 
 def _format_step_line(step) -> str:
     call = f"{step.tool_name}({_format_args(step.tool_args)})"
     tag = " [COMMIT]" if step.is_commit else ""
-    if step.injected_error:
+    if step.phantom_success:
+        # The call genuinely succeeded -- the world already changed -- but
+        # the Target was shown a fabricated failure. Labeling this "error"
+        # would itself misreport what actually happened.
+        status = yellow("secretly succeeded (told it failed)")
+        detail = f"  {dim(_truncate(str(step.result)))}"
+    elif step.unverified_outcome:
+        status = yellow("unverified (ambiguous response)")
+        detail = f"  {dim(_truncate(str(step.result)))}"
+    elif step.freeform_override:
+        status = yellow("injected (freeform)")
+        detail = f"  {dim(_truncate(str(step.result)))}"
+    elif step.injected_error:
         status = red("injected error")
         detail = f"  {dim(_truncate(str(step.result)))}"
     elif step.ok:
@@ -545,6 +574,36 @@ class Report:
 
         scenario_detail_html = self._scenario_detail_html(esc)
 
+        # injector.decide() raising (a bad API key, a rate limit, anything)
+        # is caught by runner.py and silently treated as "no injection this
+        # step" -- indistinguishable, from the stats above alone, from the
+        # Injector genuinely choosing not to intervene. Surface it loudly
+        # instead of leaving a failing Injector looking identical to a quiet
+        # one (confirmed live: an auto-revoked BYOK key produced exactly
+        # this silent-looking "0 fired, 0 expired" result across an entire
+        # run before this was traced back to the real cause).
+        injector_error_lines = [
+            f"{esc(t.scenario_id)}: {esc(err)}"
+            for t in self.chaos_trajectories
+            for err in t.injector_errors
+        ]
+        injector_error_banner = ""
+        if injector_error_lines:
+            shown = injector_error_lines[:10]
+            more = len(injector_error_lines) - len(shown)
+            items = "".join(f"<li>{line}</li>" for line in shown)
+            if more > 0:
+                items += f"<li>&hellip; and {more} more</li>"
+            injector_error_banner = (
+                "<div class='injector-error-banner'>"
+                f"<strong>The Injector failed on {len(injector_error_lines)} step"
+                f"{'s' if len(injector_error_lines) != 1 else ''} during this run</strong>"
+                "Each failure (a bad API key, a rate limit, anything raised by injector.decide()) was silently "
+                "treated as “no injection this step” -- the stats above cannot tell that apart from the "
+                "Injector genuinely choosing not to intervene. Injections fired/expired counts below are likely "
+                f"undercounted.<ul>{items}</ul></div>"
+            )
+
         return f"""<!doctype html>
 <html><head><meta charset="utf-8"><title>AgentProbe report</title>
 <style>
@@ -556,6 +615,7 @@ class Report:
     <div class="title-row"><span class="health-dot health-dot--{health}"></span><h1>AgentProbe report</h1></div>
     <div class="subtitle">mode: {esc(self.mode)} &middot; injector: {esc(self.injector_model)} &middot; target: {esc(self.target_model)} &middot; {esc(c['n'])} scenarios</div>
   </header>
+  {injector_error_banner}
   <div class="stats">
     {"".join(stats_html)}
   </div>
@@ -583,8 +643,24 @@ class Report:
         def step_rows(steps) -> str:
             rows = []
             for s in steps:
-                if s.injected_error:
-                    status = "<span class='fail'>injected error</span>"
+                # Four distinct ways a step's PERCEIVED outcome can be
+                # chaos-injected (see trajectory.py's Step) -- rendering
+                # them all as plain "ok"/"error" would hide exactly the
+                # steps a chaos report exists to show. phantom_success in
+                # particular actually SUCCEEDED (the world changed) despite
+                # ok=False here, so labeling it "error" would itself
+                # misreport what happened -- distinct from injected_error,
+                # where nothing really executed.
+                is_injected = s.injected_error or s.phantom_success or s.unverified_outcome or s.freeform_override
+                row_class = " class='step-injected'" if is_injected else ""
+                if s.phantom_success:
+                    status = "<span class='injected'>&#9889; secretly succeeded (told it failed)</span>"
+                elif s.unverified_outcome:
+                    status = "<span class='injected'>&#9889; unverified (ambiguous response)</span>"
+                elif s.freeform_override:
+                    status = "<span class='injected'>&#9889; injected (freeform)</span>"
+                elif s.injected_error:
+                    status = "<span class='injected'>&#9889; injected error</span>"
                 elif s.ok:
                     status = "<span class='pass'>ok</span>"
                 else:
@@ -592,11 +668,12 @@ class Report:
                 tag = " <span class='muted'>[commit]</span>" if s.is_commit else ""
                 args = ", ".join(f"{k}={v!r}" for k, v in s.tool_args.items())
                 # only show the result payload for something that went
-                # wrong -- a successful call's full JSON dump is noise, not
-                # signal, in a report meant to show what happened
-                result = "" if s.ok and not s.injected_error else esc(_truncate(str(s.result), 200))
+                # wrong or was injected -- a successful call's full JSON
+                # dump is noise, not signal, in a report meant to show what
+                # happened
+                result = "" if s.ok and not is_injected else esc(_truncate(str(s.result), 200))
                 rows.append(
-                    f"<tr><td class='num'>{s.index}</td>"
+                    f"<tr{row_class}><td class='num'>{s.index}</td>"
                     f"<td><code>{esc(s.tool_name)}({esc(args)})</code>{tag}</td>"
                     f"<td>{status}</td><td class='muted'>{result}</td></tr>"
                 )

{agentprobe_testing-0.8.1 → agentprobe_testing-0.8.2}/agentprobe_testing.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentprobe-testing
-Version: 0.8.1
+Version: 0.8.2
 Summary: Adaptive chaos-testing for LLM agents: a live model-driven Injector that reads an agent's real trajectory and decides where to break something, instead of scripting perturbations in advance.
 License: Business Source License 1.1
         

{agentprobe_testing-0.8.1 → agentprobe_testing-0.8.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "agentprobe-testing"
-version = "0.8.1"
+version = "0.8.2"
 description = "Adaptive chaos-testing for LLM agents: a live model-driven Injector that reads an agent's real trajectory and decides where to break something, instead of scripting perturbations in advance."
 requires-python = ">=3.11"
 dependencies = ["anthropic>=1.0.0", "python-dotenv>=1.0.0"]

{agentprobe_testing-0.8.1 → agentprobe_testing-0.8.2}/tests/test_report.py

@@ -553,6 +553,73 @@ def test_report_text_shows_injected_error_and_plain_error_step_statuses():
     assert "close_ticket" in text and "error" in text
 
 
+def test_render_html_marks_a_phantom_success_step_distinctly_from_a_plain_error():
+    # phantom_success means the call genuinely succeeded (the world
+    # changed) despite ok=False here -- rendering it as a plain "error"
+    # would misreport what actually happened, and it must not be silently
+    # dropped from the notable-steps list either.
+    step = Step(
+        index=0, tool_name="issue_refund", tool_args={"amount": 75}, is_commit=True,
+        result={"error": "HTTP 500"}, ok=False, world_hash="h",
+        reachability=Reachability(status=ReachabilityStatus.ACHIEVED), progress=0,
+        phantom_success=True,
+    )
+    chaos = [Trajectory(scenario_id="a", steps=[step], final_status=ReachabilityStatus.ACHIEVED)]
+    report = Report(mode="robustness", injector_model="m", target_model="m", clean_trajectories=[], chaos_trajectories=chaos)
+    out = report.render_html()
+
+    assert "class='step-injected'" in out
+    assert "secretly succeeded" in out
+    assert "HTTP 500" in out  # the fabricated result the Target actually saw
+
+
+def test_render_html_marks_an_unverified_outcome_step_despite_ok_being_true():
+    # unverified_outcome is the one flag that leaves ok=True (an ambiguous,
+    # non-committal response, not a clean failure) -- a check that only
+    # looked at `not ok` would silently drop this step from the report.
+    step = Step(
+        index=0, tool_name="issue_refund", tool_args={"amount": 75}, is_commit=True,
+        result={"status": "pending confirmation"}, ok=True, world_hash="h",
+        reachability=Reachability(status=ReachabilityStatus.ACHIEVED), progress=0,
+        unverified_outcome=True,
+    )
+    chaos = [Trajectory(scenario_id="a", steps=[step], final_status=ReachabilityStatus.ACHIEVED)]
+    report = Report(mode="robustness", injector_model="m", target_model="m", clean_trajectories=[], chaos_trajectories=chaos)
+    out = report.render_html()
+
+    assert "class='step-injected'" in out
+    assert "unverified (ambiguous response)" in out
+    assert "pending confirmation" in out
+
+
+def test_render_html_marks_a_freeform_override_step():
+    step = Step(
+        index=0, tool_name="get_policy", tool_args={"name": "refund"}, is_commit=False,
+        result={"text": "fabricated policy text"}, ok=True, world_hash="h",
+        reachability=Reachability(status=ReachabilityStatus.ACHIEVED), progress=0,
+        freeform_override=True,
+    )
+    chaos = [Trajectory(scenario_id="a", steps=[step], final_status=ReachabilityStatus.ACHIEVED)]
+    report = Report(mode="robustness", injector_model="m", target_model="m", clean_trajectories=[], chaos_trajectories=chaos)
+    out = report.render_html()
+
+    assert "class='step-injected'" in out
+    assert "injected (freeform)" in out
+
+
+def test_report_text_marks_a_phantom_success_step_distinctly():
+    step = Step(
+        index=0, tool_name="issue_refund", tool_args={}, is_commit=True, result={"error": "HTTP 500"},
+        ok=False, world_hash="h", reachability=Reachability(status=ReachabilityStatus.ACHIEVED),
+        progress=0, phantom_success=True,
+    )
+    chaos = [Trajectory(scenario_id="a", steps=[step], final_status=ReachabilityStatus.ACHIEVED)]
+    report = Report(mode="robustness", injector_model="m", target_model="m", clean_trajectories=[], chaos_trajectories=chaos)
+    text = report.render_scenario_detail()
+
+    assert "secretly succeeded" in text
+
+
 def test_report_text_shows_a_discarded_note_for_an_invalid_injection():
     applied = make_applied(InjectionKind.TOOL_ERROR, valid=False)
     chaos = [make_trajectory_with_injection("a", "chaos", True, applied=applied)]
@@ -602,3 +669,42 @@ def test_report_text_notes_unobservable_injections():
     )
     text = report.render()
     assert "of which unobservable: 1" in text
+
+
+def test_render_html_shows_a_warning_banner_when_the_injector_raised():
+    # injector.decide() raising (a bad API key, a rate limit, ...) is caught
+    # by runner.py and silently treated as "no injection this step" -- from
+    # the stats alone that's indistinguishable from the Injector genuinely
+    # choosing not to intervene. The report must say so explicitly instead.
+    chaos = make_trajectory("a", "chaos", True)
+    chaos.injector_errors = ["step 3: injector.decide() raised AuthenticationError(...)"]
+    report = Report(
+        mode="robustness", injector_model="m", target_model="m",
+        clean_trajectories=[], chaos_trajectories=[chaos],
+    )
+    out = report.render_html()
+    assert "<div class='injector-error-banner'>" in out
+    assert "The Injector failed on 1 step" in out
+    assert "AuthenticationError" in out
+
+
+def test_render_html_omits_the_injector_error_banner_when_nothing_failed():
+    chaos = [make_trajectory("a", "chaos", True)]
+    report = Report(
+        mode="robustness", injector_model="m", target_model="m",
+        clean_trajectories=[], chaos_trajectories=chaos,
+    )
+    out = report.render_html()
+    assert "<div class='injector-error-banner'>" not in out
+
+
+def test_render_html_pluralizes_the_injector_error_banner_and_caps_the_listed_lines():
+    chaos = make_trajectory("a", "chaos", True)
+    chaos.injector_errors = [f"step {i}: boom" for i in range(12)]
+    report = Report(
+        mode="robustness", injector_model="m", target_model="m",
+        clean_trajectories=[], chaos_trajectories=[chaos],
+    )
+    out = report.render_html()
+    assert "The Injector failed on 12 steps" in out
+    assert "and 2 more" in out  # 12 total, only the first 10 are listed individually