agentprobe-testing 1.4.0__tar.gz → 1.5.0__tar.gz

{agentprobe_testing-1.4.0 → agentprobe_testing-1.5.0}/LICENSE

@@ -16,7 +16,7 @@ Additional Use Grant:  You may use, copy, modify, and make production use of
                        limitation, hosted domain generation, hosted
                        dashboards, or hosted test-run storage substantially
                        similar to those offered at
-                       https://agentprobe-api.agentprobe.workers.dev).
+                       https://sampact.com).
 Change Date:           2030-01-01
 Change License:        Apache License, Version 2.0
 

{agentprobe_testing-1.4.0 → agentprobe_testing-1.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentprobe-testing
-Version: 1.4.0
+Version: 1.5.0
 Summary: Adaptive chaos-testing for LLM agents: a live model-driven Injector that reads an agent's real trajectory and decides where to break something, instead of scripting perturbations in advance.
 License: Business Source License 1.1
         
@@ -20,7 +20,7 @@ License: Business Source License 1.1
                                limitation, hosted domain generation, hosted
                                dashboards, or hosted test-run storage substantially
                                similar to those offered at
-                               https://agentprobe-api.agentprobe.workers.dev).
+                               https://sampact.com).
         Change Date:           2030-01-01
         Change License:        Apache License, Version 2.0
         

{agentprobe_testing-1.4.0 → agentprobe_testing-1.5.0}/README.md

@@ -183,10 +183,10 @@ own world — the engine underneath is already generic.
 
 Writing `agentprobe/domain.py`'s `Domain` bundle by hand (entities, tools,
 business rules) is real, unavoidable work — see "Domains" above. If you'd
-rather not write it yourself, [agentprobe-api.agentprobe.workers.dev](https://agentprobe-api.agentprobe.workers.dev)
+rather not write it yourself, [sampact.com](https://sampact.com)
 can generate one from your existing tool schemas plus a plain-English
 description of your business rules, and hand you back a key — either via
-[/generate](https://agentprobe-api.agentprobe.workers.dev/generate) on the
+[/generate](https://sampact.com/generate) on the
 site (sign in, fill in a form, get the key and the generated source back
 right on the page — no `curl` needed) or the raw API:
 
@@ -215,7 +215,7 @@ rule as never installing a package from someone you don't trust.
 
 ## The website: sign in and run a real test in your browser
 
-[agentprobe-api.agentprobe.workers.dev](https://agentprobe-api.agentprobe.workers.dev)
+[sampact.com](https://sampact.com)
 is the real site, not just the API — sign in with GitHub and you get:
 
 - **`/run`** — one flow, describing to running: describe your business in
@@ -246,7 +246,7 @@ Attribute a generated domain to your account by passing the key as a
 bearer token:
 
 ```bash
-curl -X POST https://agentprobe-api.agentprobe.workers.dev/v1/generate \
+curl -X POST https://sampact.com/v1/generate \
   -H "Authorization: Bearer ap_live_..." \
   -H "content-type: application/json" \
   -d '{"name": "...", "tool_schemas": [...], "business_description": "..."}'

{agentprobe_testing-1.4.0 → agentprobe_testing-1.5.0}/agentprobe/__init__.py

@@ -56,13 +56,19 @@ from agentprobe.registry import (
 )
 from agentprobe.report import InjectionRecord, Report
 from agentprobe.runner import run_recovery, run_robustness_pair
+from agentprobe.savings import FixedFaultInjector, estimate_cost, fixed_fault_for, model_injector_for, retest_failures
 from agentprobe.scenario import CommitPattern, FactPattern, GoalSpec, Scenario
 from agentprobe.scenarios.registry import ALL_SCENARIOS as TICKET_SCENARIOS
 from agentprobe.scenarios.registry import BY_ID as TICKET_SCENARIOS_BY_ID
 
-__version__ = "1.4.0"
+__version__ = "1.5.0"
 
 __all__ = [
+    "FixedFaultInjector",
+    "estimate_cost",
+    "fixed_fault_for",
+    "model_injector_for",
+    "retest_failures",
     "Agent",
     "AgentAction",
     "AppliedInjection",

{agentprobe_testing-1.4.0 → agentprobe_testing-1.5.0}/agentprobe/domain.py

@@ -87,6 +87,12 @@ class Domain:
     pre_dispatch's ON_TOOL_CALL check, so the injector would silently arm
     on every step and never once fire (0 fired, 100% expired, no error)."""
 
+    uninjectable_tools: frozenset = frozenset()
+    """Tools that are a communication channel rather than a system call (e.g.
+    the tau-bench `respond` tool, which talks to the customer). The runner never
+    fires a tool-response injection (error, phantom/false success, unverified
+    outcome) on them; triggers stay armed until a different tool is called."""
+
     @property
     def all_tools(self) -> frozenset:
         """Derived from tool_schemas rather than stored separately -- the

agentprobe_testing-1.5.0/agentprobe/domains/tau_retail/__init__.py

@@ -0,0 +1,7 @@
+"""Sierra's tau-bench retail environment, run unmodified inside AgentProbe.
+
+tau-bench (https://github.com/sierra-research/tau-bench, MIT, (c) Sierra) is
+NOT vendored here. Point TAU_BENCH_DIR at a checkout and this package loads
+Sierra's own tool code, database JSON, task list and policy straight from it,
+so every result is measured against their ground truth, not a re-creation.
+"""

agentprobe_testing-1.5.0/agentprobe/domains/tau_retail/chat_agent.py

@@ -0,0 +1,38 @@
+"""tau-bench semantics for a plain assistant message: it IS the reply to the
+customer (their `respond` action), not the end of the run. The run ends when the
+customer says ###STOP### or the agent transfers to a human."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from agentprobe.agents.base import AgentAction
+from agentprobe.agents.target_agent import TargetAgent
+from agentprobe.domains.tau_retail.tools import STOP
+
+
+class TauChatAgent(TargetAgent):
+    def start(self, task: str, tool_schemas: list[dict]) -> None:
+        super().start(task, tool_schemas)
+        self._pending_text_reply = False
+        self._over = False
+
+    def next_action(self) -> AgentAction:
+        if self._over:
+            return AgentAction(kind="final_answer", text="")
+        action = super().next_action()
+        if action.kind == "final_answer" and action.text:
+            self._pending_text_reply = True
+            return AgentAction(kind="tool_call", tool_name="respond", tool_args={"content": action.text},
+                               latency_s=action.latency_s, cost_usd=action.cost_usd)
+        return action
+
+    def observe(self, tool_name: str, tool_args: dict, result: Any, ok: bool) -> None:
+        if STOP in str(result) or tool_name == "transfer_to_human_agents":
+            self._over = True
+        if self._pending_text_reply:
+            # The agent spoke in plain text: the customer's reply is the next user turn.
+            self._pending_text_reply = False
+            self._messages.append({"role": "user", "content": str(result)})
+            return
+        super().observe(tool_name, tool_args, result, ok)

agentprobe_testing-1.5.0/agentprobe/domains/tau_retail/domain.py

@@ -0,0 +1,110 @@
+"""Wires Sierra's retail env into AgentProbe: one Domain + Scenario per run
+(the customer simulator is per-run state, so the Domain closes over it)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Optional
+
+from agentprobe.domain import Domain
+from agentprobe.domains.tau_retail import loader, oracle
+from agentprobe.domains.tau_retail.injector_prompt import INJECTION_KINDS, TAU_RETAIL_INJECTOR_SYSTEM_PROMPT
+from agentprobe.domains.tau_retail.tools import TauToolkit, commit_tools, tool_schemas
+from agentprobe.domains.tau_retail.world import TauWorld, TauWorldSpec
+from agentprobe.runner import run_target
+from agentprobe.scenario import CommitPattern, GoalSpec, Scenario
+
+BASELINE_STEPS = 2
+"""Most tau tasks have a single commit, so the default "wait for the first commit" would
+wake the injector only after the one thing worth breaking has happened."""
+
+MAX_STEPS = 30
+"""tau-bench's default max_num_steps. Here a step is any tool call, including
+`respond`, which matches their count of agent actions."""
+
+
+def build_domain(user: Any, tau: Optional[loader.TauRetail] = None, injector_system_prompt: Optional[str] = None) -> Domain:
+    tau = tau or loader.load()
+    worlds: list[TauWorld] = []
+
+    def world_factory(spec: TauWorldSpec) -> TauWorld:
+        w = TauWorld(tau.fresh_data(), {"policy": tau.wiki})
+        worlds.append(w)
+        return w
+
+    domain = Domain(
+        name="tau_retail",
+        world_state_factory=world_factory,
+        toolkit_factory=lambda w: TauToolkit(w, user, tau),
+        tool_schemas=tool_schemas(tau),
+        commit_tools=commit_tools(tau),
+        precondition_checker=lambda tool, args, world: None,
+        entity_id_arg="order_id",
+        injector_system_prompt=injector_system_prompt,
+        injector_entity_types=("order", "user") if injector_system_prompt else None,
+        default_hardcoded_tool="cancel_pending_order",
+        uninjectable_tools=frozenset({"respond"}),
+    )
+    object.__setattr__(domain, "_worlds", worlds)
+    return domain
+
+
+def build_scenario(task_index: int, opening_message: str, tau: Optional[loader.TauRetail] = None,
+                   baseline_until=2) -> Scenario:
+    tau = tau or loader.load()
+    task = tau.tasks[task_index]
+    commits = commit_tools(tau)
+    required = [CommitPattern(a.name, dict(a.kwargs)) for a in task.actions if a.name in commits]
+    return Scenario(
+        id=f"tau-retail-{task_index}",
+        seed=task_index,
+        world=TauWorldSpec(task_index),
+        task=opening_message,
+        goal=GoalSpec(required_commits=required),
+        baseline_until=baseline_until,
+        max_steps=MAX_STEPS,
+        max_cost_usd=2.0,
+    )
+
+
+def build_model_injector(model: str = "claude-sonnet-5", tau: Optional[loader.TauRetail] = None):
+    """The adaptive Injector, limited to the database-safe kinds."""
+    from agentprobe.injector import ModelInjector
+
+    tau = tau or loader.load()
+    return ModelInjector(
+        model=model,
+        kind_order=list(INJECTION_KINDS),
+        system_prompt=TAU_RETAIL_INJECTOR_SYSTEM_PROMPT,
+        all_tools=frozenset(s["name"] for s in tool_schemas(tau)),
+        commit_tools=commit_tools(tau),
+        entity_types=("order", "user"),
+    )
+
+
+def build_target(model: str = "claude-sonnet-5-5", tau: Optional[loader.TauRetail] = None):
+    """tau-bench's own agent setup: a Claude tool-calling loop whose system prompt is Sierra's policy."""
+    from agentprobe.domains.tau_retail.chat_agent import TauChatAgent
+
+    tau = tau or loader.load()
+    return TauChatAgent(model=model, max_tokens=2048, system_prompt=tau.wiki)
+
+
+@dataclass
+class TauRun:
+    trajectory: Any
+    grade: oracle.Grade
+    world: TauWorld
+    user_cost_usd: float
+
+
+def run_task(task_index: int, agent, injector, user, tau: Optional[loader.TauRetail] = None,
+             baseline_until=2, injector_system_prompt: Optional[str] = None, variant: str = "chaos") -> TauRun:
+    """One tau-bench task end to end: customer opens, agent+injector run, Sierra's grader scores."""
+    tau = tau or loader.load()
+    opening = user.reset()
+    domain = build_domain(user, tau, injector_system_prompt)
+    scenario = build_scenario(task_index, opening, tau, baseline_until)
+    traj = run_target(scenario, agent, injector, variant=variant, domain=domain)
+    world = domain._worlds[-1]
+    return TauRun(traj, oracle.grade(world, tau.tasks[task_index], tau), world, getattr(user, "cost_usd", 0.0))

agentprobe_testing-1.5.0/agentprobe/domains/tau_retail/injector_prompt.py

@@ -0,0 +1,51 @@
+"""Injector prompt for the tau-bench retail domain.
+
+Built from the access-control prompt (the generic trigger semantics, kind
+definitions and rules are identical across domains) with its domain-specific
+opening -- the task description and the tool list -- replaced. The experiment
+offers the Injector only the four tool-response kinds (see INJECTION_KINDS),
+which never touch the database, so Sierra's ground-truth final state stays the
+correct answer under injection and the note/entity kinds' vocabulary is unused.
+"""
+
+from __future__ import annotations
+
+from agentprobe.domains.access_control.injector_prompt import ACCESS_CONTROL_INJECTOR_SYSTEM_PROMPT as _BASE
+from agentprobe.injection import InjectionKind
+
+INJECTION_KINDS = [
+    InjectionKind.TOOL_ERROR,
+    InjectionKind.PHANTOM_SUCCESS,
+    InjectionKind.UNVERIFIED_OUTCOME,
+    InjectionKind.FALSE_SUCCESS,
+]
+"""Kinds that change only what a tool call RETURNS (or whether it runs), never
+the stored data. Under these, "database equals ground truth" stays the right
+success test."""
+
+_OPEN = _BASE.index("You choose a TRIGGER")
+_HEAD = """You are the Injector in an adversarial testing harness. \
+A Target agent is a retail customer-service agent talking to a customer: it can look up \
+users, orders and products and change orders; you watch its trajectory and, once \
+past the baseline phase, may arm exactly one fragility probe per step (or none). \
+An armed injection does NOT fire immediately -- it fires later, whenever its \
+trigger condition actually occurs. You never talk to the Target and never see \
+its internal reasoning -- only its tool calls and their results.
+
+The real tools -- use these exact names, never invent one:
+  reads: find_user_id_by_email, find_user_id_by_name_zip, get_user_details, get_order_details, \
+get_product_details, list_all_product_types, calculate, think
+  talking to the customer: respond
+  commits: cancel_pending_order, modify_pending_order_address, modify_pending_order_items, \
+modify_pending_order_payment, modify_user_address, return_delivered_order_items, \
+exchange_delivered_order_items, transfer_to_human_agents
+Commit tools take an "order_id" (modify_user_address takes "user_id"). A trigger naming a tool \
+that doesn't exist can never fire, silently wasting the injection.
+
+You may only use the kinds you are offered. They change what a tool call returns, never the \
+stored data: the correct end state of the database is fixed, so an injection can make the task \
+harder but never change what the right answer is.
+
+"""
+
+TAU_RETAIL_INJECTOR_SYSTEM_PROMPT = _HEAD + _BASE[_OPEN:]

agentprobe_testing-1.5.0/agentprobe/domains/tau_retail/loader.py

@@ -0,0 +1,99 @@
+"""Loads Sierra's retail env from a tau-bench checkout without importing the
+`tau_bench` package proper (its __init__ pulls litellm/openai/etc.). Only the
+pure-Python tool, data and task modules are used; `tau_bench.types` is a
+dataclass stand-in with the same fields (their original is pydantic)."""
+
+from __future__ import annotations
+
+import copy
+import importlib
+import os
+import sys
+import types
+from dataclasses import dataclass, field
+from functools import lru_cache
+from pathlib import Path
+from typing import Any
+
+
+@dataclass
+class Action:
+    name: str
+    kwargs: dict[str, Any]
+
+
+@dataclass
+class Task:
+    user_id: str
+    actions: list[Action]
+    instruction: str
+    outputs: list[str]
+    annotator: str = ""
+
+
+class TauBenchNotFound(RuntimeError):
+    pass
+
+
+def tau_dir() -> Path:
+    raw = os.environ.get("TAU_BENCH_DIR")
+    if not raw:
+        raise TauBenchNotFound("set TAU_BENCH_DIR to a checkout of sierra-research/tau-bench")
+    root = Path(raw)
+    if not (root / "tau_bench" / "envs" / "retail" / "tools").is_dir():
+        raise TauBenchNotFound(f"{root} does not look like a tau-bench checkout")
+    return root
+
+
+def _package(name: str, path: Path) -> types.ModuleType:
+    mod = types.ModuleType(name)
+    mod.__path__ = [str(path)]
+    sys.modules[name] = mod
+    return mod
+
+
+@dataclass
+class TauRetail:
+    root: Path
+    tools: dict[str, Any]
+    tasks: list[Task]
+    wiki: str
+    tool_schemas: list[dict]
+    _load_data: Any = field(repr=False, default=None)
+
+    def fresh_data(self) -> dict:
+        return self._load_data()
+
+    def commit_tool_names(self) -> frozenset:
+        return frozenset(
+            {
+                "cancel_pending_order", "exchange_delivered_order_items", "modify_pending_order_address",
+                "modify_pending_order_items", "modify_pending_order_payment", "modify_user_address",
+                "return_delivered_order_items", "transfer_to_human_agents",
+            }
+        )
+
+
+@lru_cache(maxsize=1)
+def load(root: str | None = None) -> TauRetail:
+    base = Path(root) if root else tau_dir()
+    pkg = base / "tau_bench"
+    for name in [n for n in sys.modules if n == "tau_bench" or n.startswith("tau_bench.")]:
+        del sys.modules[name]
+    _package("tau_bench", pkg)
+    _package("tau_bench.envs", pkg / "envs")
+    _package("tau_bench.envs.retail", pkg / "envs" / "retail")
+    types_mod = types.ModuleType("tau_bench.types")
+    types_mod.Task, types_mod.Action = Task, Action
+    sys.modules["tau_bench.types"] = types_mod
+    importlib.invalidate_caches()
+    tools_mod = importlib.import_module("tau_bench.envs.retail.tools")
+    data_mod = importlib.import_module("tau_bench.envs.retail.data")
+    tasks_mod = importlib.import_module("tau_bench.envs.retail.tasks_test")
+    wiki = (pkg / "envs" / "retail" / "wiki.md").read_text()
+    tools = {t.get_info()["function"]["name"]: t for t in tools_mod.ALL_TOOLS}
+    schemas = []
+    for t in tools_mod.ALL_TOOLS:
+        fn = t.get_info()["function"]
+        schemas.append({"name": fn["name"], "description": fn["description"], "input_schema": copy.deepcopy(fn["parameters"])})
+    return TauRetail(base, tools, list(tasks_mod.TASKS_TEST), wiki, schemas, data_mod.load_data)

agentprobe_testing-1.5.0/agentprobe/domains/tau_retail/openai_compat_agent.py

@@ -0,0 +1,85 @@
+"""Tool-calling agent for any OpenAI-compatible chat API (OpenRouter, Moonshot, ...),
+with tau-bench chat semantics: a plain message is a reply to the customer."""
+
+from __future__ import annotations
+
+import json
+import os
+import time
+import urllib.request
+from typing import Any, Callable, Optional
+
+from agentprobe.agents.base import AgentAction
+from agentprobe.agents.base import Agent
+from agentprobe.domains.tau_retail.tools import STOP
+
+OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions"
+
+
+class OpenAICompatChatAgent(Agent):
+    def __init__(self, model: str, system_prompt: str, api_key: Optional[str] = None, url: str = OPENROUTER_URL,
+                 price_per_mtok: tuple[float, float] = (0.0, 0.0), post: Optional[Callable[[dict], dict]] = None,
+                 max_tokens: int = 2048):
+        self.model, self.system_prompt, self.url = model, system_prompt, url
+        self.api_key = api_key or os.environ.get("OPENROUTER_API_KEY", "")
+        self.price = price_per_mtok
+        self.max_tokens = max_tokens
+        self._post = post or self._http_post
+        self.messages: list[dict] = []
+        self.tools: list[dict] = []
+        self._pending_call_id: Optional[str] = None
+        self._over = False
+
+    def _http_post(self, body: dict) -> dict:
+        req = urllib.request.Request(self.url, data=json.dumps(body).encode(), method="POST",
+                                     headers={"Authorization": f"Bearer {self.api_key}", "Content-Type": "application/json"})
+        with urllib.request.urlopen(req, timeout=180) as r:
+            return json.load(r)
+
+    def start(self, task: str, tool_schemas: list[dict]) -> None:
+        self.tools = [{"type": "function", "function": {"name": s["name"], "description": s["description"],
+                                                        "parameters": s["input_schema"]}} for s in tool_schemas]
+        self.messages = [{"role": "system", "content": self.system_prompt}, {"role": "user", "content": task}]
+        self._pending_call_id, self._over = None, False
+
+    def next_action(self) -> AgentAction:
+        if self._over:
+            return AgentAction(kind="final_answer", text="")
+        t0 = time.monotonic()
+        data = self._post({"model": self.model, "messages": self.messages, "tools": self.tools,
+                           "parallel_tool_calls": False, "temperature": 0, "max_tokens": self.max_tokens})
+        msg = data["choices"][0]["message"]
+        usage = data.get("usage") or {}
+        cost = usage.get("cost")
+        if cost is None:
+            cost = (usage.get("prompt_tokens", 0) * self.price[0] + usage.get("completion_tokens", 0) * self.price[1]) / 1e6
+        lat = time.monotonic() - t0
+        calls = msg.get("tool_calls") or []
+        if calls:
+            call = calls[0]
+            self.messages.append({"role": "assistant", "content": msg.get("content"), "tool_calls": [call]})
+            self._pending_call_id = call["id"]
+            try:
+                args = json.loads(call["function"].get("arguments") or "{}")
+            except json.JSONDecodeError:
+                args = {}
+            return AgentAction(kind="tool_call", tool_name=call["function"]["name"], tool_args=args,
+                               latency_s=lat, cost_usd=cost)
+        text = msg.get("content") or ""
+        self.messages.append({"role": "assistant", "content": text})
+        self._pending_call_id = None
+        # tau semantics: plain text is a reply to the customer.
+        return AgentAction(kind="tool_call", tool_name="respond", tool_args={"content": text}, latency_s=lat, cost_usd=cost)
+
+    def observe(self, tool_name: str, tool_args: dict, result: Any, ok: bool) -> None:
+        text = str(result) if ok else f"Error: {result.get('error', result)}"
+        if STOP in text or tool_name == "transfer_to_human_agents":
+            self._over = True
+        if self._pending_call_id is None:      # plain-text reply -> customer's answer is a user turn
+            self.messages.append({"role": "user", "content": text})
+        else:
+            self.messages.append({"role": "tool", "tool_call_id": self._pending_call_id, "content": text})
+            self._pending_call_id = None
+
+    def receive_message(self, text: str) -> None:
+        self.messages.append({"role": "user", "content": text})

agentprobe_testing-1.5.0/agentprobe/domains/tau_retail/oracle.py

@@ -0,0 +1,75 @@
+"""Ground-truth grading, a port of tau-bench Env.calculate_reward: replay the
+task's annotated actions on a fresh DB, compare DB hashes, then check required
+output strings. Plus a row-level diff so a failure says WHAT differed.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from agentprobe.domains.tau_retail import loader
+from agentprobe.domains.tau_retail.world import TauWorld, data_hash
+
+_TERMINATE = {"transfer_to_human_agents"}
+
+
+@dataclass
+class Grade:
+    reward: float
+    db_match: bool
+    outputs_ok: bool
+    missing_outputs: list[str] = field(default_factory=list)
+    changed_rows: list[str] = field(default_factory=list)
+    """Rows (table/id) whose final state differs from ground truth, either way."""
+    unexpected_writes: list[str] = field(default_factory=list)
+    """Rows the agent changed that ground truth leaves untouched."""
+    missing_writes: list[str] = field(default_factory=list)
+    """Rows ground truth changes that the agent left untouched."""
+
+    def as_dict(self) -> dict:
+        return dict(self.__dict__)
+
+
+def ground_truth_data(task: loader.Task, tau: loader.TauRetail) -> dict:
+    data = tau.fresh_data()
+    for action in task.actions:
+        if action.name == "respond" or action.name in _TERMINATE:
+            continue
+        try:
+            tau.tools[action.name].invoke(data=data, **action.kwargs)
+        except Exception:
+            pass  # tau swallows these into an "Error:" observation too
+    return data
+
+
+def _row_diff(a: dict, b: dict) -> set[str]:
+    out = set()
+    for table in set(a) | set(b):
+        ra, rb = a.get(table, {}), b.get(table, {})
+        for k in set(ra) | set(rb):
+            if ra.get(k) != rb.get(k):
+                out.add(f"{table}/{k}")
+    return out
+
+
+def grade(world: TauWorld, task: loader.Task, tau: loader.TauRetail) -> Grade:
+    gt = ground_truth_data(task, tau)
+    db_match = data_hash(world.data) == data_hash(gt)
+    outputs_ok, missing = True, []
+    said = [m.lower().replace(",", "") for m in world.agent_messages]
+    for out in task.outputs:
+        if not any(out.lower() in m for m in said):
+            outputs_ok = False
+            missing.append(out)
+    changed = _row_diff(world.data, gt)
+    original = tau.fresh_data()
+    by_agent, by_gt = _row_diff(world.data, original), _row_diff(gt, original)
+    return Grade(
+        reward=1.0 if (db_match and outputs_ok) else 0.0,
+        db_match=db_match,
+        outputs_ok=outputs_ok,
+        missing_outputs=missing,
+        changed_rows=sorted(changed),
+        unexpected_writes=sorted(by_agent - by_gt),
+        missing_writes=sorted(by_gt - by_agent),
+    )

agentprobe_testing-1.5.0/agentprobe/domains/tau_retail/tools.py

@@ -0,0 +1,88 @@
+"""TauToolkit: Sierra's own tool code run against a TauWorld.
+
+Differences from tau-bench's agent loop, all deliberate and listed in the
+README: (1) talking to the customer is a `respond` tool (tau uses a plain
+assistant message) so AgentProbe's tool-call engine can drive the whole
+conversation; (2) a tool result that starts with "Error" is raised as ToolError
+so the engine records the call as failed, exactly like tau's own observation.
+The text the agent sees is unchanged.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any, Optional
+
+from agentprobe.domains.tau_retail import loader
+from agentprobe.domains.tau_retail.world import TauWorld
+from agentprobe.tools import ToolError
+
+STOP = "###STOP###"
+RESPOND_SCHEMA = {
+    "name": "respond",
+    "description": "Send a message to the customer and wait for their reply. This is the only way to talk to the customer.",
+    "input_schema": {
+        "type": "object",
+        "properties": {"content": {"type": "string", "description": "The message to the customer."}},
+        "required": ["content"],
+    },
+}
+_TERMINATE = {"transfer_to_human_agents"}
+_NOTE_KEYS = {"get_order_details": ("order", "order_id"), "get_user_details": ("user", "user_id")}
+
+
+def tool_schemas(tau: Optional[loader.TauRetail] = None) -> list[dict]:
+    tau = tau or loader.load()
+    return [RESPOND_SCHEMA] + tau.tool_schemas
+
+
+class TauToolkit:
+    def __init__(self, world: TauWorld, user: Any, tau: Optional[loader.TauRetail] = None):
+        self.world = world
+        self.user = user
+        self.tau = tau or loader.load()
+
+    def canonicalize_args(self, tool_name: str, args: dict) -> dict:
+        return args
+
+    def call(self, name: str, args: dict) -> Any:
+        if name == "respond":
+            return self._respond(args)
+        tool = self.tau.tools.get(name)
+        if tool is None:
+            raise ToolError(f"Unknown action {name}")
+        try:
+            out = tool.invoke(data=self.world.data, **args)
+        except Exception as e:  # tau's Env.step turns any exception into this string
+            out = f"Error: {e}"
+        if name in ("find_user_id_by_email", "find_user_id_by_name_zip") and isinstance(out, str) and out in self.world.data["users"]:
+            self.world.focus_user = out
+        if name in _TERMINATE:
+            self.world.conversation_over = True
+        if isinstance(out, str) and out.startswith("Error"):
+            raise ToolError(out[len("Error"):].lstrip(": ").strip() or "error")
+        return self._with_notes(name, args, out)
+
+    def _respond(self, args: dict) -> str:
+        content = args.get("content")
+        if not isinstance(content, str):
+            raise ToolError("respond requires a string `content`")
+        self.world.agent_messages.append(content)
+        reply = self.user.step(content)
+        if STOP in reply:
+            self.world.conversation_over = True
+        return reply
+
+    def _with_notes(self, name: str, args: dict, out: Any) -> Any:
+        spec = _NOTE_KEYS.get(name)
+        if not spec or not isinstance(out, str):
+            return out
+        etype, arg = spec
+        notes = self.world.notes.get((etype, str(args.get(arg))))
+        if not notes:
+            return out
+        return out + "".join(f"\n[note] {n}" for n in notes)
+
+
+def commit_tools(tau: Optional[loader.TauRetail] = None) -> frozenset:
+    return (tau or loader.load()).commit_tool_names()