agentprobe-testing 0.5.0__py3-none-any.whl

agentprobe/trajectory.py

@@ -0,0 +1,107 @@
+"""Per-step and whole-run records produced by the runner."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any, NamedTuple, Optional
+
+from agentprobe.injection import AppliedInjection, ExpiredInjection
+
+
+class CallRecord(NamedTuple):
+    """Minimal record of one tool call, as seen by reachable(). Deliberately
+    decoupled from Step so the pure checker has no dependency on runner
+    bookkeeping (latency, cost, world_hash)."""
+
+    tool_name: str
+    args: dict[str, Any]
+    ok: bool
+    is_commit: bool
+
+
+class ReachabilityStatus(str, Enum):
+    ACHIEVED = "achieved"
+    REACHABLE = "reachable"
+    UNREACHABLE = "unreachable"
+
+
+@dataclass(frozen=True)
+class Reachability:
+    status: ReachabilityStatus
+    reason: Optional[str] = None
+    """Human-readable reason, set when status flips to UNREACHABLE."""
+    required_commits_outstanding: int = 0
+    required_facts_unknown: int = 0
+    forbidden_commits_hit: int = 0
+
+    @property
+    def progress(self) -> int:
+        return self.required_commits_outstanding + self.required_facts_unknown
+
+
+@dataclass
+class Step:
+    index: int
+    tool_name: str
+    tool_args: dict[str, Any]
+    is_commit: bool
+    result: Any  # tool return value, or {"error": ...} on ToolError/injected error
+    ok: bool
+    world_hash: str
+    reachability: Reachability
+    progress: int
+    injected_error: bool = False
+    """True if this step's tool call was intercepted by a TOOL_ERROR
+    injection rather than actually executed."""
+    latency_s: float = 0.0
+    cost_usd: float = 0.0
+
+
+@dataclass
+class Trajectory:
+    scenario_id: str
+    variant: str = "clean"  # "clean" | "chaos" | "recovery"
+    steps: list[Step] = field(default_factory=list)
+    final_answer: Optional[str] = None
+    halted_reason: str = ""  # "agent_stopped" | "max_steps" | "max_cost"
+    breaking_step: Optional[int] = None
+    """Index of the first step at which reachability became UNREACHABLE,
+    or None if the run never became unreachable."""
+    final_status: ReachabilityStatus = ReachabilityStatus.REACHABLE
+    injections: list[AppliedInjection] = field(default_factory=list)
+    """Injections that actually fired (their trigger matched)."""
+    expired_injections: list[ExpiredInjection] = field(default_factory=list)
+    """Injections armed but never fired -- a bad prediction by the
+    Injector, distinct from INVALID (which only applies to injections
+    that fired and then broke the goal)."""
+    injector_errors: list[str] = field(default_factory=list)
+    """Injector decisions that couldn't be applied (bad payload, API
+    error) -- never crashes the run, but worth reading (spec section 12:
+    injector gaming shows up here too)."""
+
+    @property
+    def passed(self) -> bool:
+        return self.final_status == ReachabilityStatus.ACHIEVED
+
+    @property
+    def total_cost_usd(self) -> float:
+        """Sum of the Target's own per-step API cost -- see Step.cost_usd.
+        Injector/classifier calls aren't included here (they aren't
+        per-trajectory in the same sense, and a real deployment's Target
+        may not even be a Claude call at all -- see agents/base.py)."""
+        return sum(s.cost_usd for s in self.steps)
+
+    @property
+    def wasted_steps(self) -> int:
+        """Steps taken after the run was already doomed."""
+        if self.breaking_step is None:
+            return 0
+        return len(self.steps) - (self.breaking_step + 1)
+
+    @property
+    def discarded(self) -> bool:
+        """True if any applied injection was found invalid -- the run
+        can't be attributed to the target's competence past that point
+        (spec section 8)."""
+        return any(not a.valid for a in self.injections)

agentprobe/triage.py

@@ -0,0 +1,153 @@
+"""Failure triage: for one failed Trajectory, render a diff block showing
+what the goal required, what the Target actually did, and whether the
+scenario's own answer key (per validate_scenarios.py) is implicated --
+distinguishing a scenario bug from a real Target failure at the point of
+failure, not just at the whole-scenario level.
+"""
+
+from __future__ import annotations
+
+from agentprobe.domain import Domain, TICKET_DOMAIN
+from agentprobe.reachability import _find_wrong_arg_commit
+from agentprobe.scenario import CommitPattern, Scenario
+from agentprobe.trajectory import CallRecord, ReachabilityStatus, Step, Trajectory
+from agentprobe.validate_scenarios import Severity, validate_scenario
+
+
+def _format_call(tool: str, args: dict) -> str:
+    return f"{tool}({', '.join(str(v) for v in args.values())})"
+
+
+def _calls_from_steps(steps: list[Step]) -> list[CallRecord]:
+    return [CallRecord(s.tool_name, s.tool_args, s.ok, s.is_commit) for s in steps]
+
+
+def _describe_diff(expected_args: dict, actual_args: dict, entity_id_arg: str = "ticket_id", entity_noun: str = "ticket") -> str:
+    if expected_args.get(entity_id_arg) != actual_args.get(entity_id_arg):
+        return f"wrong {entity_noun}"
+    diffs = [k for k in expected_args if k != entity_id_arg and expected_args.get(k) != actual_args.get(k)]
+    if diffs:
+        return f"correct {entity_noun}, wrong {' & '.join(diffs)}"
+    return f"correct {entity_noun}, extra/missing args"
+
+
+def _world_context(scenario: Scenario, tool: str, args: dict) -> str | None:
+    """Ground-truth value relevant to this call, for issue_refund: the
+    order's own amount, independent of what any commit pattern claims."""
+    if tool != "issue_refund":
+        return None
+    ticket = scenario.world.tickets.get(args.get("ticket_id"))
+    if ticket is None or ticket.order_id is None:
+        return None
+    order = scenario.world.orders.get(ticket.order_id)
+    if order is None:
+        return None
+    return f"{order.id}.amount = {order.amount}"
+
+
+def _goal_context(pattern: CommitPattern | None) -> str | None:
+    if pattern is not None and pattern.tool == "issue_refund" and "amount" in pattern.args:
+        return str(pattern.args["amount"])
+    return None
+
+
+def _matching_required_pattern(
+    patterns: list[CommitPattern], tool: str, entity_id: str | None, entity_id_arg: str = "ticket_id"
+) -> CommitPattern | None:
+    same_tool = [p for p in patterns if p.tool == tool]
+    same_entity = next((p for p in same_tool if p.args.get(entity_id_arg) == entity_id), None)
+    return same_entity or (same_tool[0] if same_tool else None)
+
+
+def triage_failure(scenario: Scenario, traj: Trajectory, domain: Domain = TICKET_DOMAIN) -> str:
+    """Renders the diff block for one failed trajectory. Assumes traj is
+    already known to have failed (traj.passed is False) -- callers should
+    filter to failures before calling this."""
+    lines = [f"FAIL {scenario.id}"]
+    entity_id_arg = domain.entity_id_arg
+    entity_noun = entity_id_arg.removesuffix("_id") or entity_id_arg
+
+    calls = _calls_from_steps(traj.steps)
+    committed_ok = [c for c in calls if c.ok and c.is_commit]
+    wrong_arg = _find_wrong_arg_commit(committed_ok, scenario.goal.required_commits, entity_id_arg)
+
+    if wrong_arg is not None:
+        expected_pattern = _matching_required_pattern(
+            scenario.goal.required_commits, wrong_arg.tool_name, wrong_arg.args.get(entity_id_arg), entity_id_arg
+        )
+        expected_str = _format_call(expected_pattern.tool, expected_pattern.args) if expected_pattern else "(no matching required pattern)"
+        actual_str = _format_call(wrong_arg.tool_name, wrong_arg.args)
+        diff = _describe_diff(expected_pattern.args if expected_pattern else {}, wrong_arg.args, entity_id_arg, entity_noun)
+        lines.append(f"  expected: {expected_str}")
+        lines.append(f"  actual:   {actual_str}")
+        lines.append(f"  diff:     {diff}")
+        world_line = _world_context(scenario, wrong_arg.tool_name, wrong_arg.args)
+        if world_line:
+            lines.append(f"  world:    {world_line}")
+        goal_line = _goal_context(expected_pattern)
+        if goal_line:
+            lines.append(f"  goal:     {goal_line}")
+    else:
+        forbidden_hit = next(
+            (c for c in committed_ok if any(p.matches(c.tool_name, c.args) for p in scenario.goal.forbidden_commits)),
+            None,
+        )
+        if forbidden_hit is not None:
+            lines.append(f"  actual:   {_format_call(forbidden_hit.tool_name, forbidden_hit.args)}")
+            lines.append("  diff:     forbidden action taken")
+        else:
+            satisfied_idx = {
+                i for i, p in enumerate(scenario.goal.required_commits) if any(p.matches(c.tool_name, c.args) for c in committed_ok)
+            }
+            outstanding = [p for i, p in enumerate(scenario.goal.required_commits) if i not in satisfied_idx]
+            if outstanding:
+                lines.append("  never attempted:")
+                for p in outstanding:
+                    lines.append(f"    - {_format_call(p.tool, p.args)}")
+            reason = None
+            if traj.breaking_step is not None and traj.breaking_step < len(traj.steps):
+                reason = traj.steps[traj.breaking_step].reachability.reason
+            lines.append(f"  reason:   {reason or f'run ended ({traj.halted_reason}) with the above never attempted'}")
+
+    # Task-1 check: is the scenario itself implicated, independent of what
+    # went wrong above? Only answer-key-correctness checks belong here --
+    # no_reconsideration_room is a chaos-mode injection-testability note
+    # (can CONTRADICTION/AMBIGUITY ever fire fairly here), completely
+    # unrelated to whether THIS trajectory's goal was reachable/achieved.
+    # It fires on the majority of scenarios (11/20 in this suite), so
+    # surfacing it here would misleadingly imply relevance to failures
+    # that have nothing to do with injections at all -- including plain
+    # clean-run failures, where there's no injector in the loop to begin
+    # with.
+    _NOT_A_FAILURE_EXPLANATION = {"no_reconsideration_room"}
+    report = validate_scenario(scenario, domain)
+    for issue in report.issues:
+        if issue.check in _NOT_A_FAILURE_EXPLANATION:
+            continue
+        if issue.severity == Severity.BROKEN:
+            lines.append(f"  ⚠ {issue.check}: {issue.message} -- SCENARIO BUG, not agent failure")
+        elif issue.severity == Severity.WARN:
+            lines.append(f"  ⚠ {issue.check}: {issue.message} -- possible scenario issue, verify by hand")
+
+    # last 5 steps before the point of failure
+    end = traj.breaking_step if traj.breaking_step is not None else len(traj.steps) - 1
+    start = max(0, end - 4)
+    if traj.steps:
+        lines.append("  last steps before failure:")
+        for s in traj.steps[start : end + 1]:
+            outcome = "ok" if s.ok else f"ERROR: {s.result}"
+            lines.append(f"    step {s.index}: {s.tool_name}({s.tool_args}) -> {outcome}")
+    else:
+        lines.append("  last steps before failure: (none -- zero tool calls made)")
+
+    return "\n".join(lines)
+
+
+def triage_all_failures(scenarios: list[Scenario], trajectories: list[Trajectory], domain: Domain = TICKET_DOMAIN) -> str:
+    by_id = {s.id: s for s in scenarios}
+    blocks = [
+        triage_failure(by_id[t.scenario_id], t, domain)
+        for t in trajectories
+        if not t.passed and t.scenario_id in by_id
+    ]
+    return "\n\n".join(blocks)