agentprobe-testing 1.6.0__tar.gz → 1.8.0__tar.gz

{agentprobe_testing-1.6.0/agentprobe_testing.egg-info → agentprobe_testing-1.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentprobe-testing
-Version: 1.6.0
+Version: 1.8.0
 Summary: Adaptive chaos-testing for LLM agents: a live model-driven Injector that reads an agent's real trajectory and decides where to break something, instead of scripting perturbations in advance.
 License: Business Source License 1.1
         

{agentprobe_testing-1.6.0 → agentprobe_testing-1.8.0}/agentprobe/__init__.py

@@ -57,13 +57,24 @@ from agentprobe.registry import (
 from agentprobe.report import InjectionRecord, Report
 from agentprobe.runner import run_recovery, run_robustness_pair
 from agentprobe.savings import FixedFaultInjector, estimate_cost, fixed_fault_for, model_injector_for, retest_failures
+from agentprobe.connect import MODES as CONNECTION_MODES, connect
+from agentprobe.sandbox import DriverAgent, ToolBox, ToolCallError
+from agentprobe.toolcheck import check_agent_tools
+from agentprobe.toolimport import normalize_tools
 from agentprobe.scenario import CommitPattern, FactPattern, GoalSpec, Scenario
 from agentprobe.scenarios.registry import ALL_SCENARIOS as TICKET_SCENARIOS
 from agentprobe.scenarios.registry import BY_ID as TICKET_SCENARIOS_BY_ID
 
-__version__ = "1.6.0"
+__version__ = "1.8.0"
 
 __all__ = [
+    "CONNECTION_MODES",
+    "DriverAgent",
+    "ToolBox",
+    "ToolCallError",
+    "connect",
+    "check_agent_tools",
+    "normalize_tools",
     "FixedFaultInjector",
     "estimate_cost",
     "fixed_fault_for",

agentprobe_testing-1.8.0/agentprobe/agents/openai_chat.py

@@ -0,0 +1,92 @@
+"""A tool-calling agent over any OpenAI-compatible chat API (OpenAI, OpenRouter, Moonshot, vLLM, ...).
+
+The customer gives a system prompt and a model; the tools come from the test's domain. A plain
+text reply, with no tool call, is the agent's final answer.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import time
+import urllib.request
+from typing import Any, Callable, Optional
+
+from agentprobe.agents.base import Agent, AgentAction
+
+DEFAULT_BASE_URL = "https://api.openai.com/v1"
+
+
+class OpenAIChatAgent(Agent):
+    def __init__(
+        self,
+        model: str,
+        system_prompt: str,
+        api_key: Optional[str] = None,
+        base_url: str = DEFAULT_BASE_URL,
+        price_per_mtok: tuple[float, float] = (0.0, 0.0),
+        post: Optional[Callable[[str, dict, dict], dict]] = None,
+        max_tokens: int = 2048,
+        temperature: float = 0.0,
+    ):
+        self.model, self.system_prompt = model, system_prompt
+        self.url = base_url.rstrip("/") + "/chat/completions"
+        self.api_key = api_key or os.environ.get("OPENAI_API_KEY", "")
+        self.price = price_per_mtok
+        self.max_tokens, self.temperature = max_tokens, temperature
+        self._post = post or self._urllib_post
+        self.messages: list[dict] = []
+        self.tools: list[dict] = []
+        self._pending_call_id: Optional[str] = None
+
+    @staticmethod
+    def _urllib_post(url: str, body: dict, headers: dict) -> dict:
+        req = urllib.request.Request(url, data=json.dumps(body).encode(), method="POST", headers=headers)
+        with urllib.request.urlopen(req, timeout=180) as r:  # noqa: S310 -- https/http chosen by the caller
+            return json.load(r)
+
+    def start(self, task: str, tool_schemas: list[dict]) -> None:
+        self.tools = [
+            {"type": "function", "function": {"name": s["name"], "description": s.get("description", ""), "parameters": s["input_schema"]}}
+            for s in tool_schemas
+        ]
+        self.messages = [{"role": "system", "content": self.system_prompt}, {"role": "user", "content": task}]
+        self._pending_call_id = None
+
+    def next_action(self) -> AgentAction:
+        t0 = time.monotonic()
+        body = {"model": self.model, "messages": self.messages, "max_tokens": self.max_tokens, "temperature": self.temperature}
+        if self.tools:
+            body["tools"] = self.tools
+            body["parallel_tool_calls"] = False
+        headers = {"Authorization": f"Bearer {self.api_key}", "Content-Type": "application/json"}
+        data = self._post(self.url, body, headers)
+        msg = data["choices"][0]["message"]
+        usage = data.get("usage") or {}
+        cost = usage.get("cost")
+        if cost is None:
+            cost = (usage.get("prompt_tokens", 0) * self.price[0] + usage.get("completion_tokens", 0) * self.price[1]) / 1e6
+        latency = time.monotonic() - t0
+        calls = msg.get("tool_calls") or []
+        if calls:
+            call = calls[0]
+            self.messages.append({"role": "assistant", "content": msg.get("content"), "tool_calls": [call]})
+            self._pending_call_id = call["id"]
+            try:
+                args = json.loads(call["function"].get("arguments") or "{}")
+            except json.JSONDecodeError:
+                args = {}
+            return AgentAction(kind="tool_call", tool_name=call["function"]["name"], tool_args=args, latency_s=latency, cost_usd=cost)
+        text = msg.get("content") or ""
+        self.messages.append({"role": "assistant", "content": text})
+        return AgentAction(kind="final_answer", text=text, latency_s=latency, cost_usd=cost)
+
+    def observe(self, tool_name: str, tool_args: dict, result: Any, ok: bool) -> None:
+        text = result if isinstance(result, str) else json.dumps(result, default=str)
+        if not ok:
+            text = "Error: " + (result.get("error", text) if isinstance(result, dict) else text)
+        self.messages.append({"role": "tool", "tool_call_id": self._pending_call_id, "content": text})
+        self._pending_call_id = None
+
+    def receive_message(self, text: str) -> None:
+        self.messages.append({"role": "user", "content": text})

{agentprobe_testing-1.6.0 → agentprobe_testing-1.8.0}/agentprobe/cli.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 import argparse
 import json
+import os
 import sys
 
 from dotenv import load_dotenv
@@ -343,9 +344,44 @@ def main(argv: list[str] | None = None) -> int:
         help="Exit 1 if any scenario that passed in `before` fails in `after` -- for a CI gate.",
     )
 
+    def _local_opts(p, scenario_required: bool) -> None:
+        p.add_argument("--domain-key", help="A domain generated on sampact.com. Omit to use a built-in demo domain (--domain).")
+        p.add_argument("--domain", choices=["ticket", "access_control"], default="ticket", help="The built-in demo domain, when no --domain-key is given.")
+        p.add_argument("--api-key", default=os.environ.get("AGENTPROBE_API_KEY"), help="Your sampact.com API key (or set AGENTPROBE_API_KEY): fetches a private domain and uploads the report.")
+        p.add_argument("--base-url", default=None, help="Another AgentProbe server (default: https://sampact.com).")
+        p.add_argument("--fault", default="false_success", help="none, model (adaptive injector, needs ANTHROPIC_API_KEY), or one of: false_success, tool_error, phantom_success, unverified_outcome.")
+        p.add_argument("--no-upload", action="store_true", help="Don't upload the report to your account.")
+        p.add_argument("--out", default=None, help="Write the HTML report here.")
+        if scenario_required:
+            p.add_argument("--scenario", required=True, help="The scenario id to run.")
+
+    mcp_p = sub.add_parser("mcp", help="Test an MCP-capable agent: this process is its tool server (stdio).")
+    _local_opts(mcp_p, True)
+
+    rd_p = sub.add_parser("run-domain", help="Run a domain's scenarios from your machine against an agent on your network, then upload the report.")
+    _local_opts(rd_p, False)
+    rd_p.add_argument("--scenario", default=None, help="Comma-separated scenario ids (default: all).")
+    rd_p.add_argument("--connection", choices=["http", "model", "openai"], default="http", help="How your agent connects.")
+    rd_p.add_argument("--agent-url", help="--connection http: your agent's decide endpoint.")
+    rd_p.add_argument("--header", action="append", help="--connection http: an extra header, 'Name: value' (repeatable).")
+    rd_p.add_argument("--system-prompt", help="--connection model/openai: your system prompt, or a path to a file holding it.")
+    rd_p.add_argument("--model", help="--connection model/openai: the model name.")
+    rd_p.add_argument("--agent-api-key", help="--connection openai: the key for your OpenAI-compatible provider.")
+    rd_p.add_argument("--skip-if-clean-fails", action="store_true", help="Don't inject into scenarios your agent already fails without faults.")
+    rd_p.add_argument("--budget-usd", type=float, default=None, help="Stop starting scenarios once spend reaches this.")
+
     args = parser.parse_args(argv)
     load_dotenv()
 
+    if args.command in ("mcp", "run-domain"):
+        from agentprobe import local_runner
+
+        try:
+            return local_runner.cmd_mcp(args) if args.command == "mcp" else local_runner.cmd_run_domain(args)
+        except (ValueError, KeyError) as e:
+            print(f"agentprobe: {e}", file=sys.stderr)
+            return 1
+
     if args.command == "diff":
         from agentprobe.diff import diff_runs, render_diff
 

agentprobe_testing-1.8.0/agentprobe/connect.py

@@ -0,0 +1,62 @@
+"""One switch for every way of connecting an agent.
+
+    agent = ap.connect("http", url="https://my-agent.example.com/decide")
+    agent = ap.connect("function", fn=my_decide_fn)
+    agent = ap.connect("model", system_prompt="You are ...", model="claude-sonnet-5")
+    agent = ap.connect("openai", system_prompt="You are ...", model="gpt-4o", api_key="sk-...")
+    agent = ap.connect("tools", driver=my_driver)          # an agent that runs its own loop
+
+    report = ap.quick_test_all(scenarios, agent, ...)       # the same call whichever you picked
+
+MODES lists what's available and what each needs. (An MCP server, `agentprobe mcp`, is the other
+way in, for agents that are MCP clients; it is started from the command line, not from here.)
+"""
+
+from __future__ import annotations
+
+from typing import Any, Callable
+
+from agentprobe.agents.base import Agent
+
+MODES: dict[str, dict] = {
+    "http": {"needs": ["url"], "optional": ["headers", "timeout"], "what": "Your agent is a web endpoint that returns its next step."},
+    "function": {"needs": ["fn"], "optional": [], "what": "Your agent is a Python function (task, history) -> AgentAction."},
+    "model": {"needs": ["system_prompt"], "optional": ["model", "max_tokens"], "what": "AgentProbe runs a Claude tool loop with your system prompt (the tools come from the test)."},
+    "openai": {"needs": ["system_prompt", "model"], "optional": ["api_key", "base_url", "max_tokens"], "what": "The same, over any OpenAI-compatible API (OpenAI, OpenRouter, Moonshot, vLLM...)."},
+    "tools": {"needs": ["driver"], "optional": ["timeout"], "what": "Your agent runs its own loop, calling tools you take from the test (LangChain, OpenAI Agents SDK, anything)."},
+}
+
+
+def connect(mode: str, **options: Any) -> Callable[[], Agent]:
+    """A target factory for quick_test/quick_test_all, for the connection `mode` you choose."""
+    spec = MODES.get(mode)
+    if spec is None:
+        raise ValueError(f"unknown connection mode {mode!r}; choose one of {', '.join(MODES)}")
+    missing = [k for k in spec["needs"] if k not in options or options[k] in (None, "")]
+    if missing:
+        raise ValueError(f"connection mode {mode!r} needs: {', '.join(missing)}")
+    unknown = [k for k in options if k not in spec["needs"] and k not in spec["optional"]]
+    if unknown:
+        raise ValueError(f"connection mode {mode!r} does not take: {', '.join(unknown)}")
+
+    if mode == "http":
+        from agentprobe.quickstart import http_agent
+
+        return http_agent(options["url"], headers=options.get("headers"), **({"timeout": options["timeout"]} if "timeout" in options else {}))
+    if mode == "function":
+        from agentprobe.quickstart import wrap_agent
+
+        return wrap_agent(options["fn"])
+    if mode == "model":
+        from agentprobe.agents.target_agent import TargetAgent
+
+        kwargs = {k: options[k] for k in ("model", "max_tokens") if k in options}
+        return lambda: TargetAgent(system_prompt=options["system_prompt"], **kwargs)
+    if mode == "openai":
+        from agentprobe.agents.openai_chat import OpenAIChatAgent
+
+        kwargs = {k: options[k] for k in ("api_key", "base_url", "max_tokens") if k in options}
+        return lambda: OpenAIChatAgent(model=options["model"], system_prompt=options["system_prompt"], **kwargs)
+    from agentprobe.sandbox import DriverAgent
+
+    return DriverAgent.factory(options["driver"], **({"timeout": options["timeout"]} if "timeout" in options else {}))

agentprobe_testing-1.8.0/agentprobe/local_runner.py

@@ -0,0 +1,188 @@
+"""Run a test on YOUR machine, inside YOUR network, and send only the report to your account.
+
+This is the "customer-hosted runner": an agent that can't be reached from the internet (a staging
+service on a private network, a laptop, a browser agent) is tested locally, against a domain you
+generated on sampact.com, and the result shows up under Your Runs like any other run. Nothing
+about your agent has to be exposed.
+
+    agentprobe run-domain --domain-key K --api-key ap_live_... --connection http --agent-url http://localhost:8080/decide
+    agentprobe mcp --domain-key K --api-key ap_live_... --scenario refund-1      # an MCP client is the agent
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+from dataclasses import dataclass
+from typing import Any, Callable, Optional
+
+from agentprobe.connect import connect
+
+FAULTS = {
+    "false_success": "FALSE_SUCCESS",
+    "tool_error": "TOOL_ERROR",
+    "phantom_success": "PHANTOM_SUCCESS",
+    "unverified_outcome": "UNVERIFIED_OUTCOME",
+}
+
+
+@dataclass
+class ResolvedDomain:
+    domain: Any
+    scenarios: dict
+    key: Optional[str]
+
+
+def resolve_domain(domain_key: Optional[str], builtin: str = "ticket", api_key: Optional[str] = None, base_url: Optional[str] = None) -> ResolvedDomain:
+    """A generated domain from sampact.com (by key), or one of the built-in demo domains."""
+    if domain_key:
+        from agentprobe.registry import DEFAULT_BASE_URL, fetch_domain
+
+        fetched = fetch_domain(domain_key, base_url=base_url or DEFAULT_BASE_URL, api_key=api_key)
+        return ResolvedDomain(fetched.domain, fetched.scenarios, domain_key)
+    if builtin == "access_control":
+        from agentprobe.domains.access_control.domain import ACCESS_CONTROL_DOMAIN
+        from agentprobe.domains.access_control.scenarios import SCENARIOS
+
+        return ResolvedDomain(ACCESS_CONTROL_DOMAIN, dict(SCENARIOS), None)
+    from agentprobe.domain import TICKET_DOMAIN
+    from agentprobe.scenarios.registry import BY_ID
+
+    return ResolvedDomain(TICKET_DOMAIN, dict(BY_ID), None)
+
+
+def injector_factory_for(fault: str, domain, model: Optional[str] = None) -> Callable:
+    """`fault` is none, model (the adaptive injector), or one of FAULTS (a fixed fault on the
+    scenario's first required commit, no model call)."""
+    from agentprobe.injection import InjectionKind
+    from agentprobe.injector import NullInjector
+    from agentprobe.savings import fixed_fault_for, model_injector_for
+
+    if fault == "none":
+        return lambda sc: NullInjector()
+    if fault == "model":
+        return lambda sc: model_injector_for(domain, **({"model": model} if model else {}))
+    if fault in FAULTS:
+        kind = InjectionKind[FAULTS[fault]]
+        return lambda sc: fixed_fault_for(sc, kind)
+    raise ValueError(f"unknown fault {fault!r}; choose none, model, or one of {', '.join(FAULTS)}")
+
+
+def _read(path_or_text: Optional[str]) -> Optional[str]:
+    if path_or_text and os.path.isfile(path_or_text):
+        return open(path_or_text).read()
+    return path_or_text
+
+
+def build_connection(connection: str, opts: dict) -> Callable:
+    """Turn command-line style options into a connection (see agentprobe.connect)."""
+    if connection == "http":
+        headers = {}
+        for h in opts.get("header") or []:
+            k, _, v = h.partition(":")
+            headers[k.strip()] = v.strip()
+        return connect("http", url=opts["agent_url"], **({"headers": headers} if headers else {}))
+    if connection == "model":
+        return connect("model", system_prompt=_read(opts["system_prompt"]), **({"model": opts["model"]} if opts.get("model") else {}))
+    if connection == "openai":
+        extra = {k: opts[k] for k in ("base_url",) if opts.get(k)}
+        if opts.get("agent_api_key"):
+            extra["api_key"] = opts["agent_api_key"]
+        return connect("openai", system_prompt=_read(opts["system_prompt"]), model=opts["model"], **extra)
+    raise ValueError(f"--connection must be http, model or openai here (got {connection!r}); use `agentprobe mcp` for an MCP client")
+
+
+def run_domain(
+    resolved: ResolvedDomain,
+    agent_factory: Callable,
+    scenario_ids: Optional[list[str]] = None,
+    fault: str = "false_success",
+    api_key: Optional[str] = None,
+    base_url: Optional[str] = None,
+    upload: bool = True,
+    out_path: Optional[str] = None,
+    **quick_test_all_kwargs: Any,
+):
+    from agentprobe.quickstart import quick_test_all
+
+    chosen = {sid: resolved.scenarios[sid] for sid in (scenario_ids or resolved.scenarios)}
+    kwargs = dict(quick_test_all_kwargs)
+    if upload and api_key:
+        kwargs["upload_api_key"] = api_key
+        kwargs["domain_key"] = resolved.key
+    report = quick_test_all(
+        chosen,
+        agent_factory,
+        injector_factory=injector_factory_for(fault, resolved.domain),
+        domain=resolved.domain,
+        injector_model=None if fault == "model" else (f"fixed fault: {fault}" if fault != "none" else None),
+        record_injections=bool(upload and api_key),
+        **kwargs,
+    )
+    if out_path:
+        with open(out_path, "w") as f:
+            f.write(report.render_html())
+    return report
+
+
+def mcp_report(trajectory, scenario, domain, fault: str, target_label: str = "an MCP client"):
+    """A Report for the one run an MCP client just did (chaos only: there is no clean baseline)."""
+    from agentprobe.injection import injection_was_triggered
+    from agentprobe.report import InjectionRecord, Report
+    from agentprobe.toolcheck import required_tools
+
+    records = [InjectionRecord(scenario_id=trajectory.scenario_id, applied=a, triggered=injection_was_triggered(a, trajectory.steps)) for a in trajectory.injections]
+    return Report(
+        mode="recovery",
+        injector_model="none" if fault == "none" else f"fixed fault: {fault}" if fault in FAULTS else "model",
+        target_model=target_label,
+        clean_trajectories=[],
+        chaos_trajectories=[trajectory],
+        injection_records=records,
+        injection_mode="pinned" if fault in FAULTS else ("exploration" if fault == "model" else None),
+        tool_expectations={"known": sorted(domain.all_tools), "required": {scenario.id: sorted(required_tools(scenario))}},
+    )
+
+
+def cmd_mcp(args) -> int:
+    from agentprobe.mcp_server import serve
+    from agentprobe.registry import upload_run
+
+    resolved = resolve_domain(args.domain_key, args.domain, api_key=args.api_key, base_url=args.base_url)
+    if args.scenario not in resolved.scenarios:
+        print(f"agentprobe: no scenario {args.scenario!r}; choose one of {', '.join(sorted(resolved.scenarios))}", file=sys.stderr)
+        return 1
+    scenario = resolved.scenarios[args.scenario]
+    injector = injector_factory_for(args.fault, resolved.domain)(scenario)
+
+    def on_finish(traj):
+        report = mcp_report(traj, scenario, resolved.domain, args.fault)
+        out = args.out or f"agentprobe_mcp_{scenario.id}.html"
+        with open(out, "w") as f:
+            f.write(report.render_html())
+        print(report.render(), file=sys.stderr)
+        print(f"agentprobe: report written to {out}", file=sys.stderr)
+        if args.api_key and not args.no_upload:
+            upload_run(report, api_key=args.api_key, domain_key=resolved.key, **({"base_url": args.base_url} if args.base_url else {}))
+            print("agentprobe: uploaded to your account (Your Runs)", file=sys.stderr)
+
+    serve(scenario, resolved.domain, injector, on_finish=on_finish)
+    return 0
+
+
+def cmd_run_domain(args) -> int:
+    resolved = resolve_domain(args.domain_key, args.domain, api_key=args.api_key, base_url=args.base_url)
+    try:
+        agent = build_connection(args.connection, vars(args))
+    except (ValueError, KeyError) as e:
+        print(f"agentprobe: {e}", file=sys.stderr)
+        return 1
+    ids = [s.strip() for s in args.scenario.split(",")] if args.scenario else None
+    report = run_domain(
+        resolved, agent, ids, fault=args.fault, api_key=args.api_key, base_url=args.base_url,
+        upload=not args.no_upload, out_path=args.out,
+        **({"skip_injection_if_clean_fails": True} if args.skip_if_clean_fails else {}),
+        **({"budget_usd": args.budget_usd} if args.budget_usd is not None else {}),
+    )
+    print(report.render())
+    return 0

agentprobe_testing-1.8.0/agentprobe/mcp_server.py

@@ -0,0 +1,105 @@
+"""Test any MCP-capable agent (Claude Desktop, Cursor, an MCP client you wrote) by being its tools.
+
+Run `agentprobe mcp ...`, point the client at it as an MCP server (stdio), and give it the task.
+Every tool the client calls goes into the test world, where failures are injected; the answer goes
+back. When the agent is done it calls the `end_conversation` tool (or the client disconnects), and
+the report is written.
+
+JSON-RPC over stdin/stdout, one message per line, the subset of MCP a tool server needs:
+initialize, tools/list, tools/call (plus ping and the initialized notification).
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from typing import IO, Callable, Optional
+
+from agentprobe.sandbox import ExternalSession
+
+END_TOOL = "end_conversation"
+PROTOCOL = "2024-11-05"
+
+
+def _end_tool_schema() -> dict:
+    return {
+        "name": END_TOOL,
+        "description": "Call this when you are finished with the task. Put your final message to the customer in `message`.",
+        "inputSchema": {"type": "object", "properties": {"message": {"type": "string"}}, "required": ["message"]},
+    }
+
+
+def serve(
+    scenario,
+    domain,
+    injector,
+    stdin: IO[str] = None,
+    stdout: IO[str] = None,
+    on_finish: Optional[Callable] = None,
+) -> object:
+    """Run one test over MCP. Returns the finished Trajectory. `on_finish(trajectory)` is called
+    once, before the final reply, so a caller can write the report."""
+    stdin = stdin or sys.stdin
+    stdout = stdout or sys.stdout
+    session = ExternalSession(scenario, domain, injector, variant="recovery")
+    session.start()
+
+    def send(msg: dict) -> None:
+        stdout.write(json.dumps(msg) + "\n")
+        stdout.flush()
+
+    def result(i, payload: dict) -> None:
+        send({"jsonrpc": "2.0", "id": i, "result": payload})
+
+    instructions = (
+        f"You are working a task. Use ONLY the tools of this server. When you are done, call {END_TOOL} with your final message.\n\n"
+        f"TASK: {scenario.task}"
+    )
+    done = False
+
+    def finish(text: str) -> None:
+        nonlocal done
+        if done:
+            return
+        done = True
+        traj = session.finish(text)
+        if on_finish:
+            on_finish(traj)
+
+    for line in stdin:
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            req = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        method, i = req.get("method"), req.get("id")
+        if method == "initialize":
+            result(i, {
+                "protocolVersion": (req.get("params") or {}).get("protocolVersion", PROTOCOL),
+                "capabilities": {"tools": {}},
+                "serverInfo": {"name": "agentprobe", "version": "1"},
+                "instructions": instructions,
+            })
+        elif method == "tools/list":
+            tools = [{"name": s["name"], "description": s.get("description", ""), "inputSchema": s["input_schema"]} for s in session.tool_schemas]
+            result(i, {"tools": tools + [_end_tool_schema()]})
+        elif method == "tools/call":
+            params = req.get("params") or {}
+            name, args = params.get("name"), params.get("arguments") or {}
+            if name == END_TOOL:
+                finish(str(args.get("message", "")))
+                result(i, {"content": [{"type": "text", "text": "The test is over. Thank you."}], "isError": False})
+                break
+            res, ok = session.call(name, args)
+            text = res if isinstance(res, str) else json.dumps(res, default=str)
+            if not ok and isinstance(res, dict):
+                text = "Error: " + str(res.get("error", text))
+            result(i, {"content": [{"type": "text", "text": text}], "isError": not ok})
+        elif method == "ping":
+            result(i, {})
+        elif i is not None and method:
+            send({"jsonrpc": "2.0", "id": i, "error": {"code": -32601, "message": f"unknown method {method}"}})
+    finish("")  # the client disconnected without calling end_conversation
+    return session.trajectory

{agentprobe_testing-1.6.0 → agentprobe_testing-1.8.0}/agentprobe/quickstart.py

@@ -22,6 +22,7 @@ from agentprobe.priority import clean_overrides
 from agentprobe.report import InjectionRecord, Report
 from agentprobe.runner import run_recovery, run_robustness_pair, run_target
 from agentprobe.scenario import Scenario
+from agentprobe.toolcheck import check_agent_tools, required_tools
 
 DecideFn = Callable[[str, list[dict[str, Any]]], AgentAction]
 
@@ -219,6 +220,7 @@ def quick_test(
         injector_cost_usd=_injector_cost_usd(injector),
         priority_overrides=clean_overrides(priority_overrides),
         injection_mode=injection_mode,
+        tool_expectations={"known": sorted(domain.all_tools), "required": {scenario.id: sorted(required_tools(scenario))}},
     )
     if recording is not None:
         # Stashed here (not a Report field -- Report is a general-purpose
@@ -259,6 +261,7 @@ def quick_test_all(
     skip_injection_if_clean_fails: bool = False,
     stop_when_achieved: bool = False,
     preflight: bool = False,
+    agent_tools: Optional[list[str]] = None,
 ) -> Report:
     """Run every scenario in `scenarios` (a dict[str, Scenario] -- e.g.
     TICKET_SCENARIOS or a FetchedDomain's .scenarios -- or a plain list of
@@ -305,6 +308,9 @@ def quick_test_all(
         agent's final claim) is not tested.
       preflight -- validate every scenario first (free, no model calls) and
         raise ValueError if any is BROKEN, before spending anything.
+      agent_tools -- the tool names your agent really has. If given, a scenario that
+        needs a tool outside this list raises ValueError before anything is spent: a failure
+        aimed at a tool the agent doesn't have can never fire, and the run would look clean.
 
     `injector_factory` may take the Scenario as its one argument, e.g.
     `lambda sc: fixed_fault_for(sc)`, to aim an injector per scenario.
@@ -324,6 +330,13 @@ def quick_test_all(
         if broken:
             raise ValueError("preflight found broken scenarios (nothing was run, nothing was spent):\n  " + "\n  ".join(broken))
 
+    if agent_tools is not None:
+        problems = check_agent_tools([sc for _id, sc in items], agent_tools)
+        if problems:
+            raise ValueError(
+                "the tests need tools your agent does not have (nothing was run, nothing was spent):\n  " + "\n  ".join(problems)
+            )
+
     target_factory = agent if _looks_like_factory(agent) else wrap_agent(agent)
     raw_make_injector = injector_factory if injector_factory is not None else (lambda: NullInjector())
     if not callable(raw_make_injector):
@@ -428,6 +441,10 @@ def quick_test_all(
         injector_cost_usd=total_injector_cost,
         priority_overrides=clean_overrides(priority_overrides),
         injection_mode=combined_injection_mode,
+        tool_expectations={
+            "known": sorted(domain.all_tools),
+            "required": {sc.id: sorted(required_tools(sc)) for _id, sc in items},
+        },
     )
     report._scenarios_run = len(all_chaos)
     report._skipped = skipped
@@ -441,7 +458,7 @@ def quick_test_all(
     return report
 
 
-def assert_passes(report: Report, save_html_on_failure: Optional[str] = None) -> None:
+def assert_passes(report: Report, save_html_on_failure: Optional[str] = None, allow_inconclusive: bool = False) -> None:
     """Assert every scenario in `report` passed -- chaos, and clean too when
     this was a robustness-mode run -- the one-line pytest integration: call
     this at the end of a test instead of hand-picking which Report fields
@@ -456,9 +473,15 @@ def assert_passes(report: Report, save_html_on_failure: Optional[str] = None) ->
     Pass `save_html_on_failure="path.html"` to also write the full
     render_html() report there when the assertion fails, so a failed CI run
     leaves something to open instead of just a pass/fail count.
+
+    A run where failures were expected but none fired is "inconclusive": every scenario
+    "passes" only because nothing was tested. That is not a pass, so it raises too, unless
+    you pass `allow_inconclusive=True`.
     """
     c = report._compute()
     failures = []
+    if report.inconclusive and not allow_inconclusive:
+        failures.append("inconclusive: no failure was introduced in any scenario, so the passes prove nothing")
     if c["clean_passed"] is not None and c["clean_passed"] != c["clean_total"]:
         failures.append(f"clean pass rate {c['clean_passed']}/{c['clean_total']}")
     if c["chaos_passed"] != c["n"]: