agentprobe-testing 1.6.0__tar.gz → 1.7.0__tar.gz

{agentprobe_testing-1.6.0 → agentprobe_testing-1.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentprobe-testing
-Version: 1.6.0
+Version: 1.7.0
 Summary: Adaptive chaos-testing for LLM agents: a live model-driven Injector that reads an agent's real trajectory and decides where to break something, instead of scripting perturbations in advance.
 License: Business Source License 1.1
         

{agentprobe_testing-1.6.0 → agentprobe_testing-1.7.0}/agentprobe/__init__.py

@@ -57,13 +57,17 @@ from agentprobe.registry import (
 from agentprobe.report import InjectionRecord, Report
 from agentprobe.runner import run_recovery, run_robustness_pair
 from agentprobe.savings import FixedFaultInjector, estimate_cost, fixed_fault_for, model_injector_for, retest_failures
+from agentprobe.toolcheck import check_agent_tools
+from agentprobe.toolimport import normalize_tools
 from agentprobe.scenario import CommitPattern, FactPattern, GoalSpec, Scenario
 from agentprobe.scenarios.registry import ALL_SCENARIOS as TICKET_SCENARIOS
 from agentprobe.scenarios.registry import BY_ID as TICKET_SCENARIOS_BY_ID
 
-__version__ = "1.6.0"
+__version__ = "1.7.0"
 
 __all__ = [
+    "check_agent_tools",
+    "normalize_tools",
     "FixedFaultInjector",
     "estimate_cost",
     "fixed_fault_for",

{agentprobe_testing-1.6.0 → agentprobe_testing-1.7.0}/agentprobe/quickstart.py

@@ -22,6 +22,7 @@ from agentprobe.priority import clean_overrides
 from agentprobe.report import InjectionRecord, Report
 from agentprobe.runner import run_recovery, run_robustness_pair, run_target
 from agentprobe.scenario import Scenario
+from agentprobe.toolcheck import check_agent_tools, required_tools
 
 DecideFn = Callable[[str, list[dict[str, Any]]], AgentAction]
 
@@ -219,6 +220,7 @@ def quick_test(
         injector_cost_usd=_injector_cost_usd(injector),
         priority_overrides=clean_overrides(priority_overrides),
         injection_mode=injection_mode,
+        tool_expectations={"known": sorted(domain.all_tools), "required": {scenario.id: sorted(required_tools(scenario))}},
     )
     if recording is not None:
         # Stashed here (not a Report field -- Report is a general-purpose
@@ -259,6 +261,7 @@ def quick_test_all(
     skip_injection_if_clean_fails: bool = False,
     stop_when_achieved: bool = False,
     preflight: bool = False,
+    agent_tools: Optional[list[str]] = None,
 ) -> Report:
     """Run every scenario in `scenarios` (a dict[str, Scenario] -- e.g.
     TICKET_SCENARIOS or a FetchedDomain's .scenarios -- or a plain list of
@@ -305,6 +308,9 @@ def quick_test_all(
         agent's final claim) is not tested.
       preflight -- validate every scenario first (free, no model calls) and
         raise ValueError if any is BROKEN, before spending anything.
+      agent_tools -- the tool names your agent really has. If given, a scenario that
+        needs a tool outside this list raises ValueError before anything is spent: a failure
+        aimed at a tool the agent doesn't have can never fire, and the run would look clean.
 
     `injector_factory` may take the Scenario as its one argument, e.g.
     `lambda sc: fixed_fault_for(sc)`, to aim an injector per scenario.
@@ -324,6 +330,13 @@ def quick_test_all(
         if broken:
             raise ValueError("preflight found broken scenarios (nothing was run, nothing was spent):\n  " + "\n  ".join(broken))
 
+    if agent_tools is not None:
+        problems = check_agent_tools([sc for _id, sc in items], agent_tools)
+        if problems:
+            raise ValueError(
+                "the tests need tools your agent does not have (nothing was run, nothing was spent):\n  " + "\n  ".join(problems)
+            )
+
     target_factory = agent if _looks_like_factory(agent) else wrap_agent(agent)
     raw_make_injector = injector_factory if injector_factory is not None else (lambda: NullInjector())
     if not callable(raw_make_injector):
@@ -428,6 +441,10 @@ def quick_test_all(
         injector_cost_usd=total_injector_cost,
         priority_overrides=clean_overrides(priority_overrides),
         injection_mode=combined_injection_mode,
+        tool_expectations={
+            "known": sorted(domain.all_tools),
+            "required": {sc.id: sorted(required_tools(sc)) for _id, sc in items},
+        },
     )
     report._scenarios_run = len(all_chaos)
     report._skipped = skipped
@@ -441,7 +458,7 @@ def quick_test_all(
     return report
 
 
-def assert_passes(report: Report, save_html_on_failure: Optional[str] = None) -> None:
+def assert_passes(report: Report, save_html_on_failure: Optional[str] = None, allow_inconclusive: bool = False) -> None:
     """Assert every scenario in `report` passed -- chaos, and clean too when
     this was a robustness-mode run -- the one-line pytest integration: call
     this at the end of a test instead of hand-picking which Report fields
@@ -456,9 +473,15 @@ def assert_passes(report: Report, save_html_on_failure: Optional[str] = None) ->
     Pass `save_html_on_failure="path.html"` to also write the full
     render_html() report there when the assertion fails, so a failed CI run
     leaves something to open instead of just a pass/fail count.
+
+    A run where failures were expected but none fired is "inconclusive": every scenario
+    "passes" only because nothing was tested. That is not a pass, so it raises too, unless
+    you pass `allow_inconclusive=True`.
     """
     c = report._compute()
     failures = []
+    if report.inconclusive and not allow_inconclusive:
+        failures.append("inconclusive: no failure was introduced in any scenario, so the passes prove nothing")
     if c["clean_passed"] is not None and c["clean_passed"] != c["clean_total"]:
         failures.append(f"clean pass rate {c['clean_passed']}/{c['clean_total']}")
     if c["chaos_passed"] != c["n"]:

{agentprobe_testing-1.6.0 → agentprobe_testing-1.7.0}/agentprobe/report.py

@@ -310,6 +310,9 @@ class Report:
     themselves. 0.0 (and omitted from output) for none/hardcoded/replay
     injectors, which never call a model at all."""
     injection_mode: Optional[str] = None
+    tool_expectations: Optional[dict] = None
+    """{"known": [tool names the test defines], "required": {scenario_id: [tools its goal needs]}}, set
+    by quick_test/quick_test_all so the report can say when the agent and the test disagree about tools."""
     """How this run's failures were chosen: "exploration" (an adaptive injector
     decided them -- finds new weaknesses, but two runs aren't directly
     comparable) or "pinned" (a recorded sequence was replayed -- the same
@@ -424,9 +427,35 @@ class Report:
         """Plain-language form of injection_mode for report headers."""
         return {
             "exploration": "adaptive failures (exploration: not directly comparable with other runs)",
-            "pinned": "replayed failures (pinned: comparable with other replays of the same recording)",
+            "pinned": "pinned failures (a fixed or replayed set: comparable with other runs that use the same failures)",
         }.get(self.injection_mode or "")
 
+    def fault_landing(self) -> dict:
+        """Did the failures this run was meant to introduce actually fire? See toolcheck.py."""
+        from agentprobe.toolcheck import fault_landing, faults_expected_for
+
+        return fault_landing(self.chaos_trajectories, faults_expected_for(self.injector_model))
+
+    @property
+    def inconclusive(self) -> bool:
+        """True when failures were expected but none fired in any scenario: every pass below is
+        meaningless as evidence of robustness."""
+        return self.fault_landing()["inconclusive"]
+
+    def tool_check(self) -> Optional[dict]:
+        """Where the agent and the test disagree about tools, or None when this report wasn't
+        built with that information (an older caller) or they agree."""
+        from agentprobe.toolcheck import required_never_called, unknown_tool_calls
+
+        exp = self.tool_expectations
+        if not exp:
+            return None
+        unknown = unknown_tool_calls([*self.clean_trajectories, *self.chaos_trajectories], exp.get("known", []))
+        never = required_never_called(exp.get("required", {}), self.clean_trajectories)
+        if not unknown and not never:
+            return None
+        return {"unknown_calls": unknown, "required_never_called": never}
+
     def priorities(self) -> list[dict]:
         """The run's issues ranked highest to lowest priority, with any
         `priority_overrides` applied. See priority.py for how it's scored."""
@@ -498,6 +527,21 @@ class Report:
         else:
             lines.append(f"{purple('clean pass rate:')}  n/a (no control run in recovery mode -- see spec section 12)")
         lines.append(f"{purple('chaos pass rate:')}  {c['chaos_passed']:2d}/{c['n']}")
+        landing = self.fault_landing()
+        if landing["expected"] and landing["not_landed"]:
+            if landing["inconclusive"]:
+                lines.append(bold(f"INCONCLUSIVE: no failure was introduced in any of the {landing['scenarios']} scenarios, so the pass rates above say nothing about robustness."))
+            else:
+                lines.append(
+                    f"{landing['landed']} of {landing['scenarios']} scenarios had a failure land; the rest tested nothing: "
+                    + ", ".join(landing["not_landed"])
+                )
+        tc = self.tool_check()
+        if tc:
+            if tc["unknown_calls"]:
+                lines.append("tool check: your agent called tools this test does not define: " + ", ".join(f"{k} (x{v})" for k, v in tc["unknown_calls"].items()))
+            for sid, tools in tc["required_never_called"].items():
+                lines.append(f"tool check: in {sid} your agent never tried {', '.join(tools)}, which the test expects")
         if c["target_cost_usd"] > 0:
             cost = cyan(f"${c['target_cost_usd']:.4f}")
             lines.append(f"{dim('target API cost:')}     {cost}")
@@ -827,6 +871,7 @@ class Report:
             )
 
         priorities_html = self._priorities_html(esc, c) + self._prompt_fixes_html(esc, c)
+        trust_html = self._landing_html(esc) + self._tool_check_html(esc)
         failures_html = self._failures_html(esc, c)
 
         return f"""<!doctype html>
@@ -841,6 +886,7 @@ class Report:
     <div class="subtitle">mode: {esc(self.mode)} &middot; injector: {esc(self.injector_model)} &middot; target: {esc(self.target_model)} &middot; {esc(c['n'])} scenarios{(' &middot; ' + esc(self.injection_mode_label())) if self.injection_mode_label() else ''}</div>
   </header>
   {injector_error_banner}
+  {trust_html}
   <div class="stats">
     {"".join(stats_html)}
   </div>
@@ -904,6 +950,44 @@ class Report:
             f"<tbody>{''.join(rows)}</tbody></table></div></section><!--/priority-table-->"
         )
 
+    def _landing_html(self, esc) -> str:
+        """A banner when failures were expected but did not (all) land: a pass in such a scenario
+        is not evidence of anything."""
+        landing = self.fault_landing()
+        if not landing["expected"] or not landing["not_landed"]:
+            return ""
+        if landing["inconclusive"]:
+            head = "Inconclusive: no failure was introduced"
+            body = (
+                f"None of the {landing['scenarios']} scenarios had a failure land, so the pass rates below say nothing about "
+                "how your agent handles failures. This usually means the test is aimed at tools or steps your agent never reaches. "
+                "Check the tool list and run again."
+            )
+        else:
+            head = f"{landing['landed']} of {landing['scenarios']} scenarios had a failure land"
+            body = (
+                "These scenarios tested nothing, so a pass in them is not evidence of robustness: "
+                + ", ".join(esc(x) for x in landing["not_landed"]) + "."
+            )
+        return f"<div class='injector-error-banner trust-banner'><strong>{esc(head)}</strong>{body}</div>"
+
+    def _tool_check_html(self, esc) -> str:
+        tc = self.tool_check()
+        if not tc:
+            return ""
+        items = []
+        if tc["unknown_calls"]:
+            calls = ", ".join(f"<code>{esc(k)}</code> (x{v})" for k, v in tc["unknown_calls"].items())
+            items.append(f"<li>Your agent called tools the test does not define: {calls}. The test has no answer for them.</li>")
+        for sid, tools in tc["required_never_called"].items():
+            names = ", ".join(f"<code>{esc(t)}</code>" for t in tools)
+            items.append(f"<li><code>{esc(sid)}</code>: your agent never tried {names}, which the test expects.</li>")
+        return (
+            "<div class='injector-error-banner trust-banner'><strong>Tool check: the test and your agent disagree about tools</strong>"
+            "A mismatch means failures aimed at those tools cannot land. Give the test your real tool definitions and run again."
+            f"<ul>{''.join(items)}</ul></div>"
+        )
+
     def _prompt_fixes_html(self, esc, c: dict) -> str:
         """"Add these to your system prompt": general rules for the failure kinds this run found,
         in priority order, right after the priorities table. The markers around it let the

agentprobe_testing-1.7.0/agentprobe/toolcheck.py

@@ -0,0 +1,86 @@
+"""Did the test and the agent agree about which tools exist, and did the failures land?
+
+Two quiet ways a run can look clean without proving anything:
+
+  * the test was built around tool names the agent doesn't have (so a failure aimed at
+    `cancel_order` never fires because the agent calls `cancelOrder`), and
+  * no failure was actually introduced in a scenario, so "passed" says nothing.
+
+This module holds the pure checks; Report uses them to say so, and quick_test_all can run
+the first one before spending anything.
+"""
+
+from __future__ import annotations
+
+from typing import Iterable, Optional
+
+
+def required_tools(scenario) -> set[str]:
+    """Every tool the scenario's goal depends on: required commits and required facts."""
+    goal = scenario.goal
+    return {p.tool for p in goal.required_commits} | {p.tool for p in goal.required_facts}
+
+
+def check_agent_tools(scenarios: Iterable, agent_tools: Iterable[str]) -> list[str]:
+    """Problems between what the scenarios need and the tool names the agent really has. Empty
+    when they agree. Call this BEFORE a run: a test aimed at a tool the agent doesn't have cannot
+    fire its failure, and the result would be a clean-looking pass."""
+    have = set(agent_tools)
+    problems = []
+    for sc in scenarios:
+        missing = sorted(required_tools(sc) - have)
+        if missing:
+            problems.append(f"{sc.id}: needs {', '.join(missing)}, which the agent does not have")
+    return problems
+
+
+def unknown_tool_calls(trajectories: Iterable, known_tools: Iterable[str]) -> dict[str, int]:
+    """Tool names the agent called that the test does not define, with how often. The agent
+    reached for something the test world has no answer for."""
+    known = set(known_tools)
+    out: dict[str, int] = {}
+    for t in trajectories:
+        for s in t.steps:
+            if s.tool_name not in known:
+                out[s.tool_name] = out.get(s.tool_name, 0) + 1
+    return dict(sorted(out.items(), key=lambda kv: (-kv[1], kv[0])))
+
+
+def required_never_called(scenario_ids_to_required: dict[str, list[str]], clean_trajectories: Iterable) -> dict[str, list[str]]:
+    """Per scenario, the required tools the agent never even tried in its clean run. Usually
+    this means the agent uses a different name for the same action."""
+    called: dict[str, set[str]] = {}
+    for t in clean_trajectories:
+        called.setdefault(t.scenario_id, set()).update(s.tool_name for s in t.steps)
+    out = {}
+    for sid, req in scenario_ids_to_required.items():
+        if sid not in called:
+            continue
+        missing = sorted(set(req) - called[sid])
+        if missing:
+            out[sid] = missing
+    return out
+
+
+def fault_landing(chaos_trajectories: Iterable, faults_expected: bool) -> dict:
+    """Whether the failures the run was meant to introduce actually fired. A scenario where none
+    fired was never tested for robustness, whatever its pass/fail says."""
+    trajs = list(chaos_trajectories)
+    not_landed = [t.scenario_id for t in trajs if not t.injections and not getattr(t, "discarded", False)]
+    n = len(trajs)
+    landed = n - len(not_landed)
+    return {
+        "expected": faults_expected,
+        "scenarios": n,
+        "landed": landed,
+        "not_landed": not_landed,
+        "inconclusive": bool(faults_expected and n > 0 and landed == 0),
+    }
+
+
+NO_FAULT_INJECTORS = ("NullInjector", "none")
+
+
+def faults_expected_for(injector_model: Optional[str]) -> bool:
+    """False for a run that deliberately injects nothing (a clean baseline)."""
+    return (injector_model or "") not in NO_FAULT_INJECTORS

agentprobe_testing-1.7.0/agentprobe/toolimport.py

@@ -0,0 +1,141 @@
+"""Turn the tool definitions you already have into the list AgentProbe tests against.
+
+Accepts, and auto-detects, any of:
+  * AgentProbe/Anthropic tools        [{"name", "description", "input_schema"}]
+  * OpenAI function calling           [{"type": "function", "function": {"name", "description", "parameters"}}]
+                                      or [{"name", "description", "parameters"}]
+  * an MCP server's tools/list reply  {"tools": [{"name", "description", "inputSchema"}]}
+                                      (or the whole JSON-RPC reply, {"result": {"tools": [...]}})
+  * an OpenAPI 3 / Swagger 2 document (one tool per operation)
+
+Real tool definitions matter: a test built around guessed tool names can aim a failure at a
+tool the agent doesn't have, so it never fires. server/src/toolimport.js is a port of this file
+for the website; tests/fixtures/tool_import is shared by both and a test on each side checks them.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Any
+
+MAX_TOOLS = 100
+_HTTP_METHODS = ("get", "post", "put", "patch", "delete")
+_NAME_OK = re.compile(r"^[A-Za-z0-9_-]{1,64}$")
+
+
+def _clean_name(raw: str) -> str:
+    name = re.sub(r"[^A-Za-z0-9_-]+", "_", str(raw)).strip("_")
+    return name[:64] or "tool"
+
+
+def _tool(name: Any, description: Any, schema: Any) -> dict:
+    if not isinstance(schema, dict) or not schema:
+        schema = {"type": "object", "properties": {}}
+    if "type" not in schema:
+        schema = {"type": "object", **schema}
+    return {"name": _clean_name(name), "description": str(description or "")[:1000], "input_schema": schema}
+
+
+def _from_list(items: list) -> list[dict]:
+    out = []
+    for it in items:
+        if not isinstance(it, dict):
+            raise ValueError("each tool must be an object")
+        fn = it.get("function") if it.get("type") == "function" and isinstance(it.get("function"), dict) else it
+        name = fn.get("name")
+        if not name:
+            raise ValueError("a tool has no name")
+        schema = fn.get("input_schema") or fn.get("inputSchema") or fn.get("parameters")
+        out.append(_tool(name, fn.get("description"), schema))
+    return out
+
+
+def _resolve(node: Any, spec: dict, depth: int = 0) -> Any:
+    """Inline local $refs (#/components/schemas/X, #/definitions/X), a few levels deep."""
+    if depth > 5:
+        return {}
+    if isinstance(node, dict):
+        ref = node.get("$ref")
+        if isinstance(ref, str) and ref.startswith("#/"):
+            target: Any = spec
+            for part in ref[2:].split("/"):
+                target = target.get(part, {}) if isinstance(target, dict) else {}
+            return _resolve(target, spec, depth + 1)
+        return {k: _resolve(v, spec, depth + 1) for k, v in node.items()}
+    if isinstance(node, list):
+        return [_resolve(v, spec, depth + 1) for v in node]
+    return node
+
+
+def _from_openapi(spec: dict) -> list[dict]:
+    out = []
+    for path, item in (spec.get("paths") or {}).items():
+        if not isinstance(item, dict):
+            continue
+        shared = item.get("parameters") or []
+        for method in _HTTP_METHODS:
+            op = item.get(method)
+            if not isinstance(op, dict):
+                continue
+            props: dict = {}
+            required: list = []
+            for p in [*shared, *(op.get("parameters") or [])]:
+                p = _resolve(p, spec)
+                if not isinstance(p, dict) or "name" not in p:
+                    continue
+                sch = dict(_resolve(p.get("schema") or {"type": p.get("type", "string")}, spec))
+                if p.get("description"):
+                    sch["description"] = p["description"]
+                props[p["name"]] = sch
+                if p.get("required") or p.get("in") == "path":
+                    required.append(p["name"])
+            body = (((op.get("requestBody") or {}).get("content") or {}).get("application/json") or {}).get("schema")
+            if body:
+                body = _resolve(body, spec)
+                if isinstance(body, dict) and isinstance(body.get("properties"), dict):
+                    props.update(body["properties"])
+                    required += [r for r in body.get("required", []) if r not in required]
+                else:
+                    props["body"] = body
+            schema = {"type": "object", "properties": props}
+            if required:
+                schema["required"] = sorted(set(required))
+            name = op.get("operationId") or f"{method}_{path}"
+            out.append(_tool(name, op.get("summary") or op.get("description"), schema))
+    return out
+
+
+def normalize_tools(source: Any) -> list[dict]:
+    """The tool list for `source` (a parsed object or a JSON string), in AgentProbe's shape.
+    Raises ValueError with a readable reason if it isn't a format above, or has no tools."""
+    if isinstance(source, (str, bytes)):
+        try:
+            source = json.loads(source)
+        except json.JSONDecodeError as e:
+            raise ValueError(f"not valid JSON: {e}") from None
+    if isinstance(source, dict) and isinstance(source.get("result"), dict) and "tools" in source["result"]:
+        source = source["result"]
+    if isinstance(source, list):
+        tools = _from_list(source)
+    elif isinstance(source, dict) and isinstance(source.get("tools"), list):
+        tools = _from_list(source["tools"])
+    elif isinstance(source, dict) and ("openapi" in source or "swagger" in source):
+        tools = _from_openapi(source)
+    else:
+        raise ValueError(
+            "unrecognized tool definitions: expected a list of tools, an MCP tools/list reply, or an OpenAPI document"
+        )
+    if not tools:
+        raise ValueError("no tools found in that file")
+    seen: dict[str, int] = {}
+    unique = []
+    for t in tools:
+        n = seen.get(t["name"], 0)
+        seen[t["name"]] = n + 1
+        if n:
+            t = {**t, "name": f"{t['name'][:60]}_{n + 1}"}
+        unique.append(t)
+    if len(unique) > MAX_TOOLS:
+        raise ValueError(f"too many tools ({len(unique)}); the limit is {MAX_TOOLS}. Send only the ones your agent uses")
+    return unique

{agentprobe_testing-1.6.0 → agentprobe_testing-1.7.0}/agentprobe_testing.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentprobe-testing
-Version: 1.6.0
+Version: 1.7.0
 Summary: Adaptive chaos-testing for LLM agents: a live model-driven Injector that reads an agent's real trajectory and decides where to break something, instead of scripting perturbations in advance.
 License: Business Source License 1.1
         

{agentprobe_testing-1.6.0 → agentprobe_testing-1.7.0}/agentprobe_testing.egg-info/SOURCES.txt

@@ -24,6 +24,8 @@ agentprobe/savings.py
 agentprobe/scenario.py
 agentprobe/summarize.py
 agentprobe/termui.py
+agentprobe/toolcheck.py
+agentprobe/toolimport.py
 agentprobe/tools.py
 agentprobe/trajectory.py
 agentprobe/triage.py
@@ -116,6 +118,8 @@ tests/test_scenarios.py
 tests/test_target_agent.py
 tests/test_tau_retail.py
 tests/test_termui.py
+tests/test_toolcheck.py
+tests/test_toolimport.py
 tests/test_tools.py
 tests/test_trajectory.py
 tests/test_triage.py

{agentprobe_testing-1.6.0 → agentprobe_testing-1.7.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "agentprobe-testing"
-version = "1.6.0"
+version = "1.7.0"
 description = "Adaptive chaos-testing for LLM agents: a live model-driven Injector that reads an agent's real trajectory and decides where to break something, instead of scripting perturbations in advance."
 requires-python = ">=3.11"
 dependencies = ["anthropic>=1.0.0", "python-dotenv>=1.0.0"]

{agentprobe_testing-1.6.0 → agentprobe_testing-1.7.0}/tests/test_injection_mode.py

@@ -33,8 +33,8 @@ def test_injection_mode_reads_the_injector_type_through_a_recording_wrapper():
 def test_a_replayed_run_is_labeled_pinned_in_the_report():
     report = quick_test(BY_ID["clean-1"], AGENT, injector=ReplayInjector([]), mode="recovery")
     assert report.injection_mode == "pinned"
-    assert "replayed failures (pinned: comparable with other replays of the same recording)" in report.render_html()
-    assert "failures: replayed failures (pinned" in report.render()
+    assert "pinned failures (a fixed or replayed set: comparable with other runs that use the same failures)" in report.render_html()
+    assert "failures: pinned failures (a fixed or replayed set" in report.render()
 
 
 def test_recording_a_replay_still_labels_it_pinned():

{agentprobe_testing-1.6.0 → agentprobe_testing-1.7.0}/tests/test_runner_timeouts.py

@@ -65,3 +65,9 @@ def test_the_requester_uses_the_step_timeout_helper_and_no_hardcoded_sixty_secon
     assert "_step_timeout(deadline, time.monotonic())" in SRC
     # every run builds its requester with a deadline
     assert "_make_httpx_requester(deadline=time.monotonic() + budget)" in SRC
+
+
+def test_the_runner_reports_whether_a_run_was_inconclusive():
+    # entry.py can't be imported here (it needs Cloudflare's `workers`), so check the source: the
+    # website relies on this field to show INCONCLUSIVE instead of a meaningless PASS.
+    assert '"inconclusive": report.inconclusive' in SRC

agentprobe_testing-1.7.0/tests/test_toolcheck.py

@@ -0,0 +1,114 @@
+import dataclasses
+import os
+import sys
+
+import pytest
+
+sys.path.insert(0, os.path.dirname(__file__))
+
+from agentprobe.agents.base import AgentAction  # noqa: E402
+from agentprobe.agents.scripted import ScriptedAgent  # noqa: E402
+from agentprobe.classifier import ResponseClassification as RC  # noqa: E402
+from agentprobe.injector import NullInjector  # noqa: E402
+from agentprobe.quickstart import assert_passes, quick_test, quick_test_all  # noqa: E402
+from agentprobe.savings import fixed_fault_for  # noqa: E402
+from agentprobe.scenarios.registry import BY_ID  # noqa: E402
+from agentprobe.toolcheck import check_agent_tools, fault_landing, required_tools  # noqa: E402
+from test_failures import _report  # noqa: E402
+
+S1 = BY_ID["clean-1"]
+REAL = ["get_ticket", "issue_refund", "close_ticket"]
+
+
+def _decide(task, history):
+    calls = [h["tool_name"] for h in history]
+    if "get_ticket" not in calls:
+        return AgentAction(kind="tool_call", tool_name="get_ticket", tool_args={"id": "T-1001"})
+    if "issue_refund" not in calls:
+        return AgentAction(kind="tool_call", tool_name="issue_refund", tool_args={"ticket_id": "T-1001", "amount": 49.99})
+    if "close_ticket" not in calls:
+        return AgentAction(kind="tool_call", tool_name="close_ticket", tool_args={"ticket_id": "T-1001", "resolution": "refunded"})
+    return AgentAction(kind="final_answer", text="done")
+
+
+# ---- #2 the preflight: does the agent have the tools the tests need? ----
+def test_required_tools_come_from_the_goal():
+    assert {"issue_refund"} <= required_tools(S1)
+
+
+def test_check_agent_tools_names_what_is_missing():
+    assert check_agent_tools([S1], REAL) == []
+    problems = check_agent_tools([S1], ["get_ticket", "refund"])
+    assert problems and "issue_refund" in problems[0] and S1.id in problems[0]
+
+
+def test_quick_test_all_refuses_to_spend_when_the_agent_lacks_a_required_tool():
+    ran = []
+
+    def spy(task, history):
+        ran.append(1)
+        return AgentAction(kind="final_answer", text="x")
+
+    with pytest.raises(ValueError, match="nothing was spent"):
+        quick_test_all([S1], spy, agent_tools=["get_ticket", "refund"])
+    assert ran == []
+    quick_test_all([S1], _decide, agent_tools=REAL)  # a matching list runs
+
+
+# ---- #5 the probe: the clean run reveals what the agent actually calls ----
+def test_the_report_says_when_the_agent_reaches_for_a_tool_the_test_does_not_define():
+    def calls_a_different_name(task, history):
+        if not history:
+            return AgentAction(kind="tool_call", tool_name="lookup_ticket", tool_args={"id": "T-1001"})
+        return AgentAction(kind="final_answer", text="done")
+
+    report = quick_test_all([S1], calls_a_different_name)
+    tc = report.tool_check()
+    assert tc["unknown_calls"] == {"lookup_ticket": 2}  # clean and chaos runs
+    assert S1.id in tc["required_never_called"] and "issue_refund" in tc["required_never_called"][S1.id]
+    html = report.render_html()
+    assert "the test and your agent disagree about tools" in html and "lookup_ticket" in html
+    assert "tool check:" in report.render()
+
+
+def test_no_tool_check_noise_when_they_agree():
+    report = quick_test_all([S1], _decide)
+    assert report.tool_check() is None
+    assert "disagree about tools" not in report.render_html()
+
+
+# ---- #3 did the failures land? ----
+def test_a_run_where_no_failure_fired_is_inconclusive_not_clean():
+    # Fault aimed at a tool the (scripted) agent never calls: nothing ever fires.
+    def never_commits(task, history):
+        return AgentAction(kind="final_answer", text="done")
+
+    report = quick_test_all([S1], never_commits, injector_factory=lambda sc: fixed_fault_for(sc))
+    landing = report.fault_landing()
+    assert landing["inconclusive"] and landing["not_landed"] == [S1.id] and landing["landed"] == 0
+    assert report.inconclusive
+    html = report.render_html()
+    assert "Inconclusive: no failure was introduced" in html
+    assert "INCONCLUSIVE" in report.render()
+    with pytest.raises(AssertionError, match="inconclusive"):
+        assert_passes(report)
+
+
+def test_a_run_where_the_failure_landed_is_not_inconclusive():
+    report = quick_test_all([S1], _decide, injector_factory=lambda sc: fixed_fault_for(sc))
+    assert report.fault_landing()["landed"] == 1 and not report.inconclusive
+    assert "Inconclusive" not in report.render_html()
+
+
+def test_a_deliberate_clean_run_is_not_called_inconclusive():
+    report = quick_test(S1, _decide, injector=NullInjector())
+    assert not report.inconclusive and not report.fault_landing()["expected"]
+    assert "Inconclusive" not in report.render_html()
+
+
+def test_partial_landing_is_called_out_by_scenario():
+    chaos_ok = _report([("a", False, RC.IGNORED)]).chaos_trajectories
+    quiet = _report([("b", True, RC.HANDLED)]).chaos_trajectories
+    quiet = [dataclasses.replace(t, injections=[]) for t in quiet]
+    landing = fault_landing([*chaos_ok, *quiet], True)
+    assert landing["landed"] == 1 and landing["not_landed"] == ["b"] and not landing["inconclusive"]

agentprobe_testing-1.7.0/tests/test_toolimport.py

@@ -0,0 +1,47 @@
+import json
+import os
+
+import pytest
+
+from agentprobe.toolimport import MAX_TOOLS, normalize_tools
+
+FIX = os.path.join(os.path.dirname(__file__), "fixtures", "tool_import")
+
+
+@pytest.mark.parametrize("name", ["openai_functions", "mcp_tools_list", "openapi_small"])
+def test_each_format_normalizes_to_the_expected_tools(name):
+    raw = open(os.path.join(FIX, f"{name}.json")).read()
+    expected = json.load(open(os.path.join(FIX, f"{name}.expected.json")))
+    assert normalize_tools(raw) == expected
+    assert normalize_tools(json.loads(raw)) == expected  # a parsed object works too
+
+
+def test_agentprobe_and_anthropic_shape_passes_through():
+    tools = [{"name": "get_ticket", "description": "d", "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}}}]
+    assert normalize_tools(tools) == tools
+
+
+def test_names_are_made_safe_and_unique():
+    out = normalize_tools([{"name": "a.b c"}, {"name": "a.b c"}, {"name": "!!!"}])
+    assert [t["name"] for t in out] == ["a_b_c", "a_b_c_2", "tool"]
+    assert all(t["input_schema"]["type"] == "object" for t in out)
+
+
+@pytest.mark.parametrize(
+    "bad,match",
+    [("{not json", "not valid JSON"), ({"hello": 1}, "unrecognized"), ([], "no tools"), ([{"description": "x"}], "no name"), (["x"], "object")],
+)
+def test_unusable_input_gets_a_readable_error(bad, match):
+    with pytest.raises(ValueError, match=match):
+        normalize_tools(bad)
+
+
+def test_too_many_tools_is_refused():
+    with pytest.raises(ValueError, match="too many"):
+        normalize_tools([{"name": f"t{i}"} for i in range(MAX_TOOLS + 1)])
+
+
+def test_self_referencing_openapi_schemas_do_not_loop():
+    spec = {"openapi": "3.0.0", "components": {"schemas": {"A": {"type": "object", "properties": {"a": {"$ref": "#/components/schemas/A"}}}}},
+            "paths": {"/x": {"post": {"operationId": "x", "requestBody": {"content": {"application/json": {"schema": {"$ref": "#/components/schemas/A"}}}}}}}}
+    assert normalize_tools(spec)[0]["name"] == "x"