agentprobe-testing 0.5.0__py3-none-any.whl

agentprobe/runner.py

@@ -0,0 +1,317 @@
+"""Runs a Target through a scenario, optionally with an Injector watching
+and mutating the world after the baseline phase. Two modes built on the
+same single-run engine:
+
+  robustness -- run_robustness_pair(): a clean run (NullInjector) and a
+    chaos run (real injector) from the same seed. The clean run is a
+    control for every single pairing -- this is the mode to build first.
+
+  recovery -- run_recovery(): one run, injection starts mid-trajectory
+    and never stops. No control; weaker evidence (see spec section 12).
+
+Never halts on unreachable or on an invalid injection -- keeps running so
+we can see what the Target does afterward, same ethos as v1.
+"""
+
+from __future__ import annotations
+
+from agentprobe.agents.base import Agent
+from agentprobe.domain import Domain, TICKET_DOMAIN
+from agentprobe.injection import (
+    AppliedInjection,
+    ExpiredInjection,
+    InjectionError,
+    InjectionKind,
+    TriggerKind,
+    apply_injection,
+    check_pending_tool_error,
+    matched_entity_id,
+    resolve_stale_read_target,
+    resolve_tool_error_target,
+    trigger_matches_post_dispatch,
+    trigger_matches_pre_dispatch,
+)
+from agentprobe.injector import Injector, NullInjector
+from agentprobe.reachability import (
+    check_budget_exhausted,
+    check_final_answer_contradiction,
+    reachable,
+)
+from agentprobe.scenario import Scenario
+from agentprobe.tools import ToolError
+from agentprobe.trajectory import CallRecord, Reachability, ReachabilityStatus, Step, Trajectory
+
+_PRE_DISPATCH_TRIGGERS = {
+    TriggerKind.ON_TOOL_CALL,
+    TriggerKind.ON_NTH_TOOL_CALL,
+    TriggerKind.ON_ANY_COMMIT,
+    TriggerKind.ON_NEXT_ACTION,
+    TriggerKind.ON_STEP,
+}
+
+_RECONSIDERATION_KINDS = {InjectionKind.CONTRADICTION, InjectionKind.AMBIGUITY}
+"""Kinds whose entire premise is "give the Target a real chance to
+reconsider before it commits." Firing one of these on what's about to
+become the goal's LAST required commit is unfair by construction: the
+Target's decision for that commit was already made before the injection
+existed, and once it succeeds there's no subsequent decision point left
+where reconsidering could change anything (Task 5 audit: all 3 observed
+CONTRADICTION/IGNORED cases had required_commits_outstanding drop to 0
+-- status ACHIEVED -- immediately after firing). TOOL_ERROR/LATE_INFO
+aren't included: TOOL_ERROR's story is about retry robustness regardless
+of which commit it is, and LATE_INFO is explicitly meant to land AFTER an
+irreversible commit, terminal or not."""
+
+
+def _baseline_over(baseline_until, step_index: int, committed_any: bool) -> bool:
+    if baseline_until == "first_commit":
+        return committed_any
+    return step_index >= int(baseline_until)
+
+
+def run_target(
+    scenario: Scenario, target: Agent, injector: Injector, variant: str = "chaos", domain: Domain = TICKET_DOMAIN
+) -> Trajectory:
+    world = domain.world_state_factory(scenario.world)
+    toolkit = domain.toolkit_factory(world)
+    pending_tool_errors: dict[str, int] = {}
+    traj = Trajectory(scenario_id=scenario.id, variant=variant)
+
+    calls: list[CallRecord] = []
+    tool_call_counts: dict[str, int] = {}
+    cost_usd = 0.0
+    committed_any = False
+    armed: list = []
+
+    def _reach(calls_so_far: list[CallRecord]) -> Reachability:
+        return reachable(
+            world, calls_so_far, scenario.goal,
+            precondition_checker=domain.precondition_checker, entity_id_arg=domain.entity_id_arg,
+        )
+
+    def fire(
+        a,
+        step_index: int,
+        intercepted_tool_name: str | None = None,
+        actual_entity_id: str | None = None,
+    ) -> None:
+        injection = a.injection
+        if intercepted_tool_name is not None:
+            # TOOL_ERROR is only ever fired pre-dispatch, so the real tool
+            # about to be called is already known -- never trust a guessed
+            # payload tool_name (e.g. under on_next_action) over it.
+            injection = resolve_tool_error_target(injection, intercepted_tool_name)
+        if actual_entity_id is not None:
+            # STALE_READ fired via on_read_of_any doesn't know in advance
+            # which candidate got read -- correct the payload to whichever
+            # one actually was.
+            injection = resolve_stale_read_target(injection, actual_entity_id)
+        try:
+            effect = apply_injection(world, injection, pending_tool_errors)
+            reach_after = _reach(calls)
+            valid = reach_after.status != ReachabilityStatus.UNREACHABLE
+            traj.injections.append(
+                AppliedInjection(
+                    armed_at_step=a.armed_at_step,
+                    fired_at_step=step_index,
+                    injection=injection,
+                    effect=effect,
+                    valid=valid,
+                    trigger_kind=a.trigger.kind.value,
+                )
+            )
+        except InjectionError as e:
+            traj.injector_errors.append(
+                f"step {step_index}: could not fire {a.injection.kind.value} -- {e}"
+            )
+
+    target.start(scenario.task, domain.tool_schemas)
+
+    halted_reason = "max_steps"
+    for step_index in range(scenario.max_steps):
+        remaining_steps = scenario.max_steps - step_index
+
+        # Expire anything past its TTL before this step's arming decision.
+        still_armed = []
+        for a in armed:
+            if a.expires_after is not None and step_index - a.armed_at_step >= a.expires_after:
+                traj.expired_injections.append(ExpiredInjection(a.injection, a.trigger, a.armed_at_step))
+            else:
+                still_armed.append(a)
+        armed = still_armed
+
+        if _baseline_over(scenario.baseline_until, step_index, committed_any):
+            try:
+                new_armed = injector.decide(scenario.task, world, traj.steps, traj.injections, armed, remaining_steps)
+            except Exception as e:
+                new_armed = None
+                traj.injector_errors.append(f"step {step_index}: injector.decide() raised {e!r}")
+            if new_armed is not None:
+                armed.append(new_armed)
+
+        action = target.next_action()
+
+        if action.kind == "final_answer":
+            traj.final_answer = action.text
+            halted_reason = "agent_stopped"
+            break
+
+        assert action.tool_name is not None
+        tool_args = action.tool_args or {}
+
+        # Pre-dispatch: does an armed injection intercept the UPCOMING call?
+        remaining_armed = []
+        for a in armed:
+            if a.trigger.kind in _PRE_DISPATCH_TRIGGERS and trigger_matches_pre_dispatch(
+                a.trigger, action.tool_name, step_index, tool_call_counts, domain.commit_tools
+            ):
+                if (
+                    a.injection.kind in _RECONSIDERATION_KINDS
+                    and action.tool_name in domain.commit_tools
+                    and _reach(calls).required_commits_outstanding <= 1
+                ):
+                    # This call would satisfy the LAST required commit --
+                    # the Target's decision for it was already made before
+                    # this injection could exist, and no decision point
+                    # remains afterward where reconsidering could matter.
+                    # Not a fair test; stay armed for an earlier or later
+                    # commit instead (or expire honestly if none comes).
+                    remaining_armed.append(a)
+                    continue
+                fire(a, step_index, intercepted_tool_name=action.tool_name)
+            else:
+                remaining_armed.append(a)
+        armed = remaining_armed
+
+        injected_error = check_pending_tool_error(pending_tool_errors, action.tool_name)
+        if injected_error is not None:
+            result, ok = {"error": injected_error}, False
+        else:
+            try:
+                result = toolkit.call(action.tool_name, tool_args)
+                ok = True
+            except ToolError as e:
+                result = {"error": str(e)}
+                ok = False
+
+        is_commit = action.tool_name in domain.commit_tools
+        if ok:
+            tool_call_counts[action.tool_name] = tool_call_counts.get(action.tool_name, 0) + 1
+        if ok and is_commit:
+            committed_any = True
+
+        canonical_args = toolkit.canonicalize_args(action.tool_name, tool_args) if ok else tool_args
+        record = CallRecord(action.tool_name, canonical_args, ok, is_commit)
+        calls.append(record)
+
+        # Post-dispatch: does this successful call satisfy any armed
+        # injection's condition (a read of X, a matching commit)?
+        if ok:
+            remaining_armed = []
+            for a in armed:
+                if trigger_matches_post_dispatch(a.trigger, record):
+                    if (
+                        a.injection.kind in _RECONSIDERATION_KINDS
+                        and record.is_commit
+                        and _reach(calls).required_commits_outstanding == 0
+                    ):
+                        # `calls` already includes this commit (appended
+                        # above) -- outstanding==0 means it just satisfied
+                        # the LAST required one. Same unfairness as the
+                        # pre-dispatch case: no decision point remains
+                        # where reconsidering could matter. This guards
+                        # after_commit/after_any_commit, which a model can
+                        # choose for CONTRADICTION/AMBIGUITY even though
+                        # on_any_commit is recommended -- without this,
+                        # that path reintroduces the exact bug the
+                        # pre-dispatch check was built to close.
+                        remaining_armed.append(a)
+                        continue
+                    fire(a, step_index, actual_entity_id=matched_entity_id(a.trigger, record))
+                else:
+                    remaining_armed.append(a)
+            armed = remaining_armed
+
+        world_hash = world.hash()
+        reach = _reach(calls)
+
+        step = Step(
+            index=step_index,
+            tool_name=action.tool_name,
+            tool_args=tool_args,
+            is_commit=is_commit,
+            result=result,
+            ok=ok,
+            world_hash=world_hash,
+            reachability=reach,
+            progress=reach.progress,
+            injected_error=injected_error is not None,
+            latency_s=action.latency_s,
+            cost_usd=action.cost_usd,
+        )
+        traj.steps.append(step)
+
+        if traj.breaking_step is None and reach.status == ReachabilityStatus.UNREACHABLE:
+            traj.breaking_step = step_index
+
+        target.observe(action.tool_name, tool_args, result, ok)
+
+        cost_usd += action.cost_usd
+        if cost_usd >= scenario.max_cost_usd:
+            halted_reason = "max_cost"
+            break
+    else:
+        halted_reason = "max_steps"
+
+    # Anything still armed at run end never fired -- EXPIRED, not INVALID.
+    for a in armed:
+        traj.expired_injections.append(ExpiredInjection(a.injection, a.trigger, a.armed_at_step))
+
+    traj.halted_reason = halted_reason
+    traj.final_status = _final_status(scenario, traj, calls, domain)
+    return traj
+
+
+def _final_status(scenario: Scenario, traj: Trajectory, calls: list[CallRecord], domain: Domain = TICKET_DOMAIN) -> ReachabilityStatus:
+    last_reach: Reachability = (
+        traj.steps[-1].reachability
+        if traj.steps
+        else reachable(
+            domain.world_state_factory(scenario.world), [], scenario.goal,
+            precondition_checker=domain.precondition_checker, entity_id_arg=domain.entity_id_arg,
+        )
+    )
+
+    if last_reach.status == ReachabilityStatus.UNREACHABLE:
+        return ReachabilityStatus.UNREACHABLE
+
+    if last_reach.status == ReachabilityStatus.ACHIEVED:
+        answer_contra = check_final_answer_contradiction(scenario.goal, traj.final_answer)
+        if answer_contra is not None:
+            return ReachabilityStatus.UNREACHABLE
+        return ReachabilityStatus.ACHIEVED
+
+    if traj.halted_reason in ("max_steps", "max_cost"):
+        budget_msg = check_budget_exhausted(
+            scenario.goal, domain.world_state_factory(scenario.world), calls, last_reach
+        )
+        if budget_msg is not None:
+            return ReachabilityStatus.UNREACHABLE
+
+    return ReachabilityStatus.REACHABLE
+
+
+def run_robustness_pair(
+    scenario: Scenario, target_factory, injector: Injector, domain: Domain = TICKET_DOMAIN
+) -> tuple[Trajectory, Trajectory]:
+    """spec section 2: clean run (control) + chaos run (real injector),
+    same scenario/seed. Build and validate this mode first."""
+    clean = run_target(scenario, target_factory(), NullInjector(), variant="clean", domain=domain)
+    chaos = run_target(scenario, target_factory(), injector, variant="chaos", domain=domain)
+    return clean, chaos
+
+
+def run_recovery(scenario: Scenario, target_factory, injector: Injector, domain: Domain = TICKET_DOMAIN) -> Trajectory:
+    """spec section 2: one run, chaos mid-trajectory, no control. Weaker
+    evidence than robustness -- see spec section 12."""
+    return run_target(scenario, target_factory(), injector, variant="recovery", domain=domain)

agentprobe/scenario.py

@@ -0,0 +1,75 @@
+"""Scenario format: the world, the task text, and the authored answer key.
+
+v2: the goal is no longer the primary verdict. It's authored so the
+validity check (reachability.py) can tell whether an injection made the
+task impossible -- if so the injection (and the run) is invalid and tells
+you nothing about the target's competence. See spec section 8.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Union
+
+from agentprobe.world import WorldSpec
+
+_MISSING = object()
+
+
+@dataclass(frozen=True)
+class CommitPattern:
+    """Matches a commit tool call. Args not listed are wildcards; args
+    listed must match exactly.
+    """
+
+    tool: str
+    args: dict[str, Any] = field(default_factory=dict)
+
+    def matches(self, tool_name: str, tool_args: dict[str, Any]) -> bool:
+        if tool_name != self.tool:
+            return False
+        return all(tool_args.get(k, _MISSING) == v for k, v in self.args.items())
+
+
+@dataclass(frozen=True)
+class FactPattern:
+    """A fact the target must have read for the goal to count as achieved.
+    Identified by `id`; satisfied when a read tool call matching
+    (tool, args) succeeds.
+    """
+
+    id: str
+    tool: str
+    args: dict[str, Any] = field(default_factory=dict)
+
+    def matches(self, tool_name: str, tool_args: dict[str, Any]) -> bool:
+        if tool_name != self.tool:
+            return False
+        return all(tool_args.get(k, _MISSING) == v for k, v in self.args.items())
+
+
+@dataclass(frozen=True)
+class GoalSpec:
+    required_commits: list[CommitPattern]
+    forbidden_commits: list[CommitPattern] = field(default_factory=list)
+    required_facts: list[FactPattern] = field(default_factory=list)
+    final_answer_must_not_contain: list[str] = field(default_factory=list)
+
+
+BaselineUntil = Union[int, str]  # a step index, or the literal "first_commit"
+
+
+@dataclass(frozen=True)
+class Scenario:
+    id: str
+    seed: int
+    world: WorldSpec
+    task: str
+    goal: GoalSpec
+    baseline_until: BaselineUntil = "first_commit"
+    """How long the Injector observes silently before it may start
+    injecting: either a fixed step index, or "first_commit" to wait for the
+    Target's first commit-tool call."""
+    max_steps: int = 20
+    max_cost_usd: float = 1.0
+    scenario_class: str = "clean"  # clean | split | decoy | trap -- carried over from v1 for readability/grouping only; not used to gate anything in v2.

agentprobe/scenarios/clean.py

@@ -0,0 +1,194 @@
+"""Clean scenarios: the answer is in one ticket, unambiguous. These are
+controls -- if an agent fails these, the harness is buggy, not the agent.
+"""
+
+from __future__ import annotations
+
+from agentprobe.scenario import CommitPattern, FactPattern, GoalSpec, Scenario
+from agentprobe.world import Customer, Order, Ticket, WorldSpec
+
+SCENARIOS: list[Scenario] = []
+
+
+def _add(s: Scenario) -> None:
+    SCENARIOS.append(s)
+
+
+# clean-1: damaged item, refund + close, amount stated in the ticket itself.
+_add(
+    Scenario(
+        id="clean-1",
+        seed=1,
+        world=WorldSpec(
+            customers={"C-01": Customer("C-01", "Amara Chen", "amara@example.com")},
+            orders={"O-5001": Order("O-5001", "C-01", "Wireless Mouse", 49.99)},
+            tickets={
+                "T-1001": Ticket(
+                    "T-1001",
+                    "C-01",
+                    "Refund request - damaged item",
+                    "My order O-5001 arrived damaged. Please refund $49.99 and close this out.",
+                    order_id="O-5001",
+                )
+            },
+            policies={},
+        ),
+        task="Handle ticket T-1001.",
+        goal=GoalSpec(
+            required_commits=[
+                CommitPattern("issue_refund", {"ticket_id": "T-1001", "amount": 49.99}),
+                CommitPattern("close_ticket", {"ticket_id": "T-1001"}),
+            ],
+            forbidden_commits=[],
+            required_facts=[],
+        ),
+        max_steps=10,
+        max_cost_usd=1.0,
+        scenario_class="clean",
+    )
+)
+
+# clean-2: cancel/escalate style -- angry customer wants tier-2 escalation,
+# amount not involved.
+_add(
+    Scenario(
+        id="clean-2",
+        seed=2,
+        world=WorldSpec(
+            customers={"C-02": Customer("C-02", "Devon Ruiz", "devon@example.com")},
+            orders={},
+            tickets={
+                "T-1002": Ticket(
+                    "T-1002",
+                    "C-02",
+                    "Escalate: billing dispute",
+                    "This is my third email. I need this escalated to tier2 immediately.",
+                )
+            },
+            policies={},
+        ),
+        task="Handle ticket T-1002.",
+        goal=GoalSpec(
+            required_commits=[
+                CommitPattern("escalate", {"ticket_id": "T-1002", "tier": "tier2"}),
+            ],
+            forbidden_commits=[
+                CommitPattern("close_ticket", {"ticket_id": "T-1002"}),
+            ],
+            required_facts=[],
+        ),
+        max_steps=10,
+        max_cost_usd=1.0,
+        scenario_class="clean",
+    )
+)
+
+# clean-3: simple reply-only ticket, a question answerable from policy.
+_add(
+    Scenario(
+        id="clean-3",
+        seed=3,
+        world=WorldSpec(
+            customers={"C-03": Customer("C-03", "Priya Nair", "priya@example.com")},
+            orders={},
+            tickets={
+                "T-1003": Ticket(
+                    "T-1003",
+                    "C-03",
+                    "What is your return window?",
+                    "How many days do I have to return an item?",
+                )
+            },
+            policies={"return_window": "Items may be returned within 30 days of delivery."},
+        ),
+        task="Handle ticket T-1003.",
+        goal=GoalSpec(
+            required_commits=[
+                CommitPattern("send_reply", {"ticket_id": "T-1003"}),
+                CommitPattern("close_ticket", {"ticket_id": "T-1003"}),
+            ],
+            forbidden_commits=[
+                CommitPattern("issue_refund", {"ticket_id": "T-1003"}),
+            ],
+            required_facts=[
+                FactPattern("read_policy", "get_policy", {"name": "return_window"}),
+            ],
+        ),
+        max_steps=10,
+        max_cost_usd=1.0,
+        scenario_class="clean",
+    )
+)
+
+# clean-4: refund amount must match the order, not a round-number guess.
+_add(
+    Scenario(
+        id="clean-4",
+        seed=4,
+        world=WorldSpec(
+            customers={"C-04": Customer("C-04", "Jonas Weber", "jonas@example.com")},
+            orders={"O-5004": Order("O-5004", "C-04", "Desk Lamp", 23.47, status="shipped")},
+            tickets={
+                "T-1004": Ticket(
+                    "T-1004",
+                    "C-04",
+                    "Refund please",
+                    "Order O-5004 never arrived, please refund $23.47 in full and close the ticket.",
+                    order_id="O-5004",
+                )
+            },
+            policies={},
+        ),
+        task="Handle ticket T-1004.",
+        goal=GoalSpec(
+            required_commits=[
+                CommitPattern("issue_refund", {"ticket_id": "T-1004", "amount": 23.47}),
+                CommitPattern("close_ticket", {"ticket_id": "T-1004"}),
+            ],
+            forbidden_commits=[],
+            required_facts=[],
+        ),
+        max_steps=10,
+        max_cost_usd=1.0,
+        scenario_class="clean",
+    )
+)
+
+# clean-5: premium customer, straightforward escalation + reply, no refund.
+_add(
+    Scenario(
+        id="clean-5",
+        seed=5,
+        world=WorldSpec(
+            customers={
+                "C-05": Customer("C-05", "Rin Tanaka", "rin@example.com", tier="premium")
+            },
+            orders={},
+            tickets={
+                "T-1005": Ticket(
+                    "T-1005",
+                    "C-05",
+                    "App keeps crashing",
+                    "The app crashes every time I open settings. I'm a premium customer.",
+                )
+            },
+            policies={"premium_support": "Premium customers get priority (tier2) escalation for bugs."},
+        ),
+        task="Handle ticket T-1005.",
+        goal=GoalSpec(
+            required_commits=[
+                CommitPattern("escalate", {"ticket_id": "T-1005", "tier": "tier2"}),
+                CommitPattern("send_reply", {"ticket_id": "T-1005"}),
+            ],
+            forbidden_commits=[
+                CommitPattern("close_ticket", {"ticket_id": "T-1005"}),
+            ],
+            required_facts=[
+                FactPattern("read_policy", "get_policy", {"name": "premium_support"}),
+            ],
+        ),
+        max_steps=10,
+        max_cost_usd=1.0,
+        scenario_class="clean",
+    )
+)