agentprobe-testing 1.2.0__tar.gz → 1.3.0__tar.gz

{agentprobe_testing-1.2.0 → agentprobe_testing-1.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentprobe-testing
-Version: 1.2.0
+Version: 1.3.0
 Summary: Adaptive chaos-testing for LLM agents: a live model-driven Injector that reads an agent's real trajectory and decides where to break something, instead of scripting perturbations in advance.
 License: Business Source License 1.1
         

{agentprobe_testing-1.2.0 → agentprobe_testing-1.3.0}/agentprobe/__init__.py

@@ -60,7 +60,7 @@ from agentprobe.scenario import CommitPattern, FactPattern, GoalSpec, Scenario
 from agentprobe.scenarios.registry import ALL_SCENARIOS as TICKET_SCENARIOS
 from agentprobe.scenarios.registry import BY_ID as TICKET_SCENARIOS_BY_ID
 
-__version__ = "1.2.0"
+__version__ = "1.3.0"
 
 __all__ = [
     "Agent",

{agentprobe_testing-1.2.0 → agentprobe_testing-1.3.0}/agentprobe/priority.py

@@ -118,6 +118,20 @@ def rank(issues: list[dict], overrides: Optional[dict[str, str]] = None) -> list
     return ranked
 
 
+def clean_overrides(raw: object) -> dict[str, str]:
+    """Keep only well-formed {FAILURE_KIND: "High"|"Medium"|"Low"} entries from
+    whatever a caller hands in (the hosted runner gets this from the website's
+    saved edits, over the wire), so a malformed or hostile value can never
+    reach a report."""
+    if not isinstance(raw, dict):
+        return {}
+    return {
+        k: v
+        for k, v in raw.items()
+        if isinstance(k, str) and isinstance(v, str) and v in TIERS and 1 <= len(k) <= 40 and k.replace("_", "").isalnum() and k.upper() == k
+    }
+
+
 def encode(issues: list[dict]) -> str:
     """ASCII-only JSON, base64'd, so it can sit inside an HTML comment with no
     escaping concerns. The website reads it back to re-rank a stored report

{agentprobe_testing-1.2.0 → agentprobe_testing-1.3.0}/agentprobe/quickstart.py

@@ -18,6 +18,7 @@ from agentprobe.agents.base import Agent, AgentAction
 from agentprobe.domain import Domain, TICKET_DOMAIN
 from agentprobe.injection import injection_was_triggered
 from agentprobe.injector import Injector, NullInjector, RecordingInjector, serialize_armed_injection
+from agentprobe.priority import clean_overrides
 from agentprobe.report import InjectionRecord, Report
 from agentprobe.runner import run_recovery, run_robustness_pair
 from agentprobe.scenario import Scenario
@@ -162,6 +163,7 @@ def quick_test(
     domain_key: Optional[str] = None,
     record_injections: bool = False,
     summarize_failures: bool = False,
+    priority_overrides: Optional[dict] = None,
 ) -> Report:
     """Run one scenario and get back a ready Report -- the one-call version
     of run_robustness_pair()/run_recovery() + Report(). `agent` can be a
@@ -214,6 +216,7 @@ def quick_test(
         chaos_trajectories=[chaos],
         injection_records=injection_records,
         injector_cost_usd=_injector_cost_usd(injector),
+        priority_overrides=clean_overrides(priority_overrides),
     )
     if recording is not None:
         # Stashed here (not a Report field -- Report is a general-purpose
@@ -249,6 +252,7 @@ def quick_test_all(
     record_injections: bool = False,
     max_duration_seconds: Optional[float] = None,
     summarize_failures: bool = False,
+    priority_overrides: Optional[dict] = None,
 ) -> Report:
     """Run every scenario in `scenarios` (a dict[str, Scenario] -- e.g.
     TICKET_SCENARIOS or a FetchedDomain's .scenarios -- or a plain list of
@@ -352,6 +356,7 @@ def quick_test_all(
         chaos_trajectories=all_chaos,
         injection_records=all_records,
         injector_cost_usd=total_injector_cost,
+        priority_overrides=clean_overrides(priority_overrides),
     )
     report._scenarios_run = len(all_chaos)
     if summarize_failures:

{agentprobe_testing-1.2.0 → agentprobe_testing-1.3.0}/agentprobe_testing.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentprobe-testing
-Version: 1.2.0
+Version: 1.3.0
 Summary: Adaptive chaos-testing for LLM agents: a live model-driven Injector that reads an agent's real trajectory and decides where to break something, instead of scripting perturbations in advance.
 License: Business Source License 1.1
         

{agentprobe_testing-1.2.0 → agentprobe_testing-1.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "agentprobe-testing"
-version = "1.2.0"
+version = "1.3.0"
 description = "Adaptive chaos-testing for LLM agents: a live model-driven Injector that reads an agent's real trajectory and decides where to break something, instead of scripting perturbations in advance."
 requires-python = ">=3.11"
 dependencies = ["anthropic>=1.0.0", "python-dotenv>=1.0.0"]

{agentprobe_testing-1.2.0 → agentprobe_testing-1.3.0}/tests/test_priority.py

@@ -139,3 +139,52 @@ def test_html_stays_script_free_and_the_text_report_lists_priorities():
     out = r.render()
     assert "priorities (automated estimate -- review before acting):" in out
     assert "1. [High] Trusts a tool that did nothing -- 1 scenario(s)" in out
+
+
+# --- overrides flowing in from outside (the hosted runner / Temporal worker) ---
+
+def test_clean_overrides_keeps_only_well_formed_entries():
+    from agentprobe.priority import clean_overrides
+
+    assert clean_overrides({"FALSE_SUCCESS": "High", "STALE_READ": "Low"}) == {"FALSE_SUCCESS": "High", "STALE_READ": "Low"}
+    dirty = {"false_success": "High", "STALE READ": "Low", "TOOL_ERROR": "Urgent", "DISTRACTOR": 3, 5: "High", "X" * 41: "High", "": "High", "OK_KIND": "Medium"}
+    assert clean_overrides(dirty) == {"OK_KIND": "Medium"}
+    for junk in (None, "High", ["High"], 7):
+        assert clean_overrides(junk) == {}
+
+
+def test_quick_test_ranks_with_the_overrides_it_is_given():
+    from agentprobe.agents.scripted import ScriptedAgent
+    from agentprobe.injector import NullInjector
+    from agentprobe.quickstart import quick_test
+    from agentprobe.scenarios.registry import BY_ID
+
+    agent = lambda: ScriptedAgent(script=[("get_ticket", {"id": "T-1001"})])  # never finishes -> the chaos run fails
+    report = quick_test(BY_ID["clean-1"], agent, injector=NullInjector(), mode="recovery", priority_overrides={"UNATTRIBUTED": "High", "BAD": "nope", "x": "Low"})
+    assert report.priority_overrides == {"UNATTRIBUTED": "High"}  # junk dropped
+    ranked = report.priorities()
+    assert [(r["kind"], r["tier"], r["edited"]) for r in ranked] == [("UNATTRIBUTED", "High", True)]
+    assert "priority--high'>High</span> <span class='muted'>(edited)" in report.render_html()
+
+
+def test_quick_test_without_overrides_ranks_automatically():
+    from agentprobe.agents.scripted import ScriptedAgent
+    from agentprobe.injector import NullInjector
+    from agentprobe.quickstart import quick_test
+    from agentprobe.scenarios.registry import BY_ID
+
+    report = quick_test(BY_ID["clean-1"], lambda: ScriptedAgent(script=[("get_ticket", {"id": "T-1001"})]), injector=NullInjector(), mode="recovery")
+    assert report.priority_overrides == {}
+    assert [(r["tier"], r["edited"]) for r in report.priorities()] == [("Medium", False)]
+
+
+def test_quick_test_all_carries_the_overrides_onto_the_combined_report():
+    from agentprobe.agents.scripted import ScriptedAgent
+    from agentprobe.injector import NullInjector
+    from agentprobe.quickstart import quick_test_all
+    from agentprobe.scenarios.registry import BY_ID
+
+    scenarios = {k: BY_ID[k] for k in ("clean-1", "clean-2")}
+    report = quick_test_all(scenarios, lambda: ScriptedAgent(script=[("get_ticket", {"id": "T-1001"})]), injector_factory=NullInjector, mode="recovery", priority_overrides={"UNATTRIBUTED": "Low"})
+    assert report.priority_overrides == {"UNATTRIBUTED": "Low"}
+    assert all(r["tier"] == "Low" and r["edited"] for r in report.priorities())