evalrx 0.1.2__py3-none-any.whl

evalrx/analyzers/agent/__init__.py

@@ -0,0 +1,25 @@
+"""Agent failure analyzers — operate on a FailureCase's Trajectory.
+
+Build order (cheapest/most reliable first): deterministic heuristics
+(loop_detect, ignored_obs) → LLM-judge (first_error_judge, trajectory_rubric)
+→ intervention probes that RE-RUN the agent (counterfactual = perturb a step,
+reliability_probe = pass@k over k reps, tool_shap = Shapley over tool subsets).
+"""
+
+from evalrx.analyzers.agent.counterfactual import CounterfactualReplay
+from evalrx.analyzers.agent.first_error_judge import FirstErrorJudge
+from evalrx.analyzers.agent.ignored_obs import IgnoredObservationDetector
+from evalrx.analyzers.agent.loop_detect import LoopDetector
+from evalrx.analyzers.agent.reliability import ReliabilityProbe
+from evalrx.analyzers.agent.tool_shap import ToolShap
+from evalrx.analyzers.agent.trajectory_rubric import TrajectoryRubricJudge
+
+__all__ = [
+    "LoopDetector",
+    "IgnoredObservationDetector",
+    "FirstErrorJudge",
+    "TrajectoryRubricJudge",
+    "CounterfactualReplay",
+    "ReliabilityProbe",
+    "ToolShap",
+]

evalrx/analyzers/agent/counterfactual.py

@@ -0,0 +1,84 @@
+"""Counterfactual replay — causal step attribution for agent trajectories.
+
+For each decision step, re-run the trajectory forked at that step and see whether
+the outcome flips: a high flip-rate ⇒ the step is causally influential.  The actual
+re-run is INJECTED as ``rerun_fn(trajectory, step_idx, seed) -> bool`` (it wraps a
+live Agent + verifier), keeping this analyzer decoupled and testable.
+
+Caveat (built into the design): with temperature > 0, run ``n_replays`` per step
+and read the flip-RATE — a single replay conflates "this step was causal" with
+model noise.  Prefer temperature=0 replay or a larger ``n_replays``.
+
+Causal framework:
+  "Causality: Models, Reasoning and Inference"
+  Pearl (2000/2009), Cambridge University Press — do-calculus, interventions
+
+Applied to neural NLP (causal mediation / interchange intervention):
+  "Causal Mediation Analysis for Interpreting Neural NLP: The Case of Gender Bias"
+  Vig et al., 2020 — https://arxiv.org/abs/2004.12265
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Callable
+
+from evalrx.core.analyzer import Analyzer
+from evalrx.core.capability import Capability
+from evalrx.core.case import Label
+from evalrx.core.registry import register_analyzer
+from evalrx.core.result import Result
+
+if TYPE_CHECKING:
+    from evalrx.core.case import CaseBatch
+    from evalrx.core.model import Model
+
+
+@register_analyzer("counterfactual")
+class CounterfactualReplay(Analyzer):
+    """Rank a trajectory's tool-call steps by how often re-running flips the outcome."""
+
+    name = "counterfactual"
+    requires = frozenset({Capability.TOOL_CALLS})
+    #: TOOL_CALLS says the model CAN call tools; this says the cases ARE agent
+    #: runs. A chat model whose template renders tools declares the former.
+    requires_trajectories = True
+    applies_to_modalities = frozenset({"text", "image"})  # trajectory analysis is modality-agnostic
+
+    def __init__(self, rerun_fn: Callable, n_replays: int = 3) -> None:
+        super().__init__(rerun_fn=rerun_fn, n_replays=n_replays)
+        self.rerun_fn = rerun_fn
+
+    def _run(self, model: "Model", cases: "CaseBatch") -> Result:
+        per_case = []
+        for case in cases:
+            traj = case.trajectory
+            if traj is None:
+                continue
+            original_success = traj.outcome == Label.PASS
+            candidates = [s for s in traj.steps if s.tool_call]
+            step_scores = []
+            for s in candidates:
+                flips = sum(
+                    1 for seed in range(self.n_replays)
+                    if bool(self.rerun_fn(traj, s.idx, seed)) != original_success
+                )
+                step_scores.append({
+                    "step": s.idx,
+                    "action": s.tool_call.get("name"),
+                    "flip_rate": round(flips / self.n_replays, 4) if self.n_replays else 0.0,
+                })
+            step_scores.sort(key=lambda d: -d["flip_rate"])
+            per_case.append({
+                "sample_id": traj.sample_id,
+                "original_success": original_success,
+                "most_influential_step": step_scores[0] if step_scores else None,
+                "steps": step_scores,
+            })
+        return Result(
+            analyzer=self.name, model=repr(model), cases=cases,
+            findings={
+                "n_trajectories": len(per_case),
+                "per_case": per_case,
+                "_caveat": "flip-rate ≈ causal influence; use temperature=0 or larger n_replays to separate from noise.",
+            },
+        )

evalrx/analyzers/agent/first_error_judge.py

@@ -0,0 +1,96 @@
+"""First-error attribution — LLM-as-judge (the Who&When 'all-at-once' protocol).
+
+Given a trajectory, ask a JUDGE model which step first introduced the error.
+The judge is a separate handle (judge != system-under-test, to avoid
+self-preference bias).  Writes ``is_first_error`` onto the implicated step.
+
+Reliability caveats (bake into any reporting): LLM judges show position/length
+bias and self-preference; calibrate against a human-gold subset and report
+step-accuracy/F1.  As reported by Zhang et al. (Who&When), even strong judges
+reach only ~14% step-level accuracy — treat outputs as weak signal, not truth.
+
+References:
+- Which Agent Causes Task Failures and When? Automated Failure Attribution (Who&When)
+  Zhang et al., ICML 2025 — arXiv:2505.00212
+- Failure-mode vocabulary — MAST, Cemri et al., 2025 — arXiv:2503.13657
+- LLM-as-judge biases — Judging LLM-as-a-Judge, Zheng et al., NeurIPS 2023 — arXiv:2306.05685
+"""
+
+from __future__ import annotations
+
+import re
+from typing import TYPE_CHECKING, Optional
+
+from evalrx.core.analyzer import Analyzer
+from evalrx.core.registry import register_analyzer
+from evalrx.core.result import Result
+
+if TYPE_CHECKING:
+    from evalrx.core.case import CaseBatch, Trajectory
+    from evalrx.core.model import Model
+
+_PROMPT = """You are debugging an AI agent trajectory. The agent failed to achieve the goal.
+Goal: {goal}
+
+Steps:
+{steps}
+
+Which step index FIRST introduced the error that led to failure?
+Answer with exactly one line: "STEP: <index>"  (use -1 if no step is at fault)."""
+
+
+def _render(traj: "Trajectory") -> str:
+    lines = []
+    for s in traj.steps:
+        piece = f"[{s.idx}] {s.role.value}"
+        if s.tool_call:
+            piece += f" calls {s.tool_call.get('name')}({s.tool_call.get('args')})"
+        if s.observation is not None:
+            piece += f" -> obs: {str(s.observation)[:200]}"
+        if s.content:
+            piece += f" : {str(s.content)[:200]}"
+        lines.append(piece)
+    return "\n".join(lines)
+
+
+@register_analyzer("first_error_judge")
+class FirstErrorJudge(Analyzer):
+    """Attribute the first-erroneous step via an LLM judge (all-at-once protocol)."""
+
+    name = "first_error_judge"
+    requires = frozenset()  # the analysed model is irrelevant; the judge does the work
+    #: Reads agent runs. The model is not what makes this applicable —
+    #: the DATA is, which is why `requires` stays empty (these run on
+    #: trajectories loaded from disk, with no model at all).
+    requires_trajectories = True
+    applies_to_modalities = frozenset({"text", "image"})  # trajectory analysis is modality-agnostic
+
+    def __init__(self, judge: Optional["Model"] = None) -> None:
+        super().__init__(judge=judge)
+
+    def _run(self, model: "Model", cases: "CaseBatch") -> Result:
+        judge = self.judge if self.judge is not None else model
+        if judge is None or not hasattr(judge, "generate"):
+            raise ValueError("FirstErrorJudge needs a judge model with generate(); pass judge=...")
+        per_case = []
+        for case in cases:
+            traj = case.trajectory
+            if traj is None:
+                continue
+            raw = judge.generate(_PROMPT.format(goal=traj.goal, steps=_render(traj)))
+            m = re.search(r"STEP:\s*(-?\d+)", str(raw))
+            idx = int(m.group(1)) if m else -1
+            if 0 <= idx < len(traj.steps):
+                traj.steps[idx].is_first_error = True
+            per_case.append({"sample_id": traj.sample_id, "first_error_step": idx, "judge_raw": str(raw)[:200]})
+        return Result(
+            analyzer=self.name,
+            model=repr(model),
+            cases=cases,
+            findings={
+                "n_trajectories": len(per_case),
+                "judge": repr(judge),
+                "per_case": per_case,
+                "_caveat": "LLM-judge: position/length/self-preference bias; calibrate vs human gold (Who&When).",
+            },
+        )

evalrx/analyzers/agent/ignored_obs.py

@@ -0,0 +1,81 @@
+"""Ignored-observation detection — deterministic trajectory heuristic.
+
+Flags the pattern: a tool returns an error/empty signal, yet the next action
+repeats the *same* tool call unchanged (the agent didn't adapt to the
+observation).  MAST "ignored tool output" / failure-to-recover mode.
+
+References:
+- Failure-mode taxonomy — Why Do Multi-Agent LLM Systems Fail? (MAST)
+  Cemri et al., 2025 — arXiv:2503.13657  (ignored-tool-output / fail-to-recover modes)
+"""
+
+from __future__ import annotations
+
+import json
+from typing import TYPE_CHECKING
+
+from evalrx.core.analyzer import Analyzer
+from evalrx.core.case import StepRole
+from evalrx.core.registry import register_analyzer
+from evalrx.core.result import Result
+
+if TYPE_CHECKING:
+    from evalrx.core.case import CaseBatch
+    from evalrx.core.model import Model
+
+
+@register_analyzer("ignored_obs")
+class IgnoredObservationDetector(Analyzer):
+    """Detect error observations that the agent then ignored (repeated the same call)."""
+
+    name = "ignored_obs"
+    requires = frozenset()
+    #: Reads agent runs. The model is not what makes this applicable —
+    #: the DATA is, which is why `requires` stays empty (these run on
+    #: trajectories loaded from disk, with no model at all).
+    requires_trajectories = True
+    applies_to_modalities = frozenset({"text", "image"})  # trajectory analysis is modality-agnostic
+
+    DEFAULT_MARKERS = ("error", "not found", "failed", "no result", "none", "invalid", "empty", "exception")
+
+    def __init__(self, markers: tuple[str, ...] | None = None) -> None:
+        super().__init__(markers=markers or self.DEFAULT_MARKERS)
+
+    def _is_error(self, observation) -> bool:
+        text = str(observation).lower()
+        return any(m in text for m in self.markers)
+
+    @staticmethod
+    def _sig(tool_call: dict | None) -> str | None:
+        if not tool_call:
+            return None
+        return json.dumps([tool_call.get("name"), tool_call.get("args", {})], sort_keys=True, default=str)
+
+    def _run(self, model: "Model", cases: "CaseBatch") -> Result:
+        per_case = []
+        for case in cases:
+            traj = case.trajectory
+            if traj is None:
+                continue
+            steps = traj.steps
+            flags = []
+            for i, s in enumerate(steps):
+                if s.role is not StepRole.TOOL or s.observation is None or not self._is_error(s.observation):
+                    continue
+                # the actor that produced this observation is the previous step;
+                # the next actor that repeats the same call ignored the error.
+                prev_sig = self._sig(steps[i - 1].tool_call) if i > 0 else None
+                nxt = next((x for x in steps[i + 1:] if x.role is StepRole.ACTOR), None)
+                if nxt is not None and prev_sig is not None and self._sig(nxt.tool_call) == prev_sig:
+                    flags.append({"error_obs_step": s.idx, "repeated_at_step": nxt.idx})
+            per_case.append({"sample_id": traj.sample_id, "n_ignored": len(flags), "ignored": flags})
+        return Result(
+            analyzer=self.name,
+            model=repr(model),
+            cases=cases,
+            findings={
+                "n_trajectories": len(per_case),
+                "n_with_ignored_obs": sum(1 for c in per_case if c["n_ignored"]),
+                "per_case": per_case,
+            },
+        )

evalrx/analyzers/agent/loop_detect.py

@@ -0,0 +1,79 @@
+"""Loop detection — a deterministic trajectory heuristic (no model, no LLM).
+
+Flags repeated ``(tool, normalised args)`` actions in an agent trajectory — the
+MAST "step repetition" failure mode.  Cheap, deterministic, and the highest-ROI
+agent signal to compute first.
+
+References:
+- Failure-mode taxonomy — Why Do Multi-Agent LLM Systems Fail? (MAST)
+  Cemri et al., 2025 — arXiv:2503.13657  (step-repetition mode)
+"""
+
+from __future__ import annotations
+
+import json
+from collections import Counter
+from typing import TYPE_CHECKING
+
+from evalrx.core.analyzer import Analyzer
+from evalrx.core.registry import register_analyzer
+from evalrx.core.result import Result
+
+if TYPE_CHECKING:
+    from evalrx.core.case import CaseBatch
+    from evalrx.core.model import Model
+
+
+def _sig(tool_call: dict) -> tuple[str, str]:
+    return (tool_call.get("name", ""), json.dumps(tool_call.get("args", {}), sort_keys=True, default=str))
+
+
+@register_analyzer("loop_detect")
+class LoopDetector(Analyzer):
+    """Detect repeated tool actions in a trajectory.
+
+    Hyper-parameters:
+        min_repeats: how many times an identical action must occur to count as a loop.
+    """
+
+    name = "loop_detect"
+    requires = frozenset()  # pure heuristic over Trajectory — no model needed
+    #: Reads agent runs. The model is not what makes this applicable —
+    #: the DATA is, which is why `requires` stays empty (these run on
+    #: trajectories loaded from disk, with no model at all).
+    requires_trajectories = True
+    applies_to_modalities = frozenset({"text", "image"})  # trajectory analysis is modality-agnostic
+
+    def __init__(self, min_repeats: int = 2) -> None:
+        super().__init__(min_repeats=min_repeats)
+
+    def _run(self, model: "Model", cases: "CaseBatch") -> Result:
+        per_case = []
+        for case in cases:
+            traj = case.trajectory
+            if traj is None:
+                continue
+            sigs = [_sig(s.tool_call) for s in traj.steps if s.tool_call]
+            counts = Counter(sigs)
+            loops = [
+                {"action": name, "args": args, "count": n}
+                for (name, args), n in counts.items()
+                if n >= self.min_repeats
+            ]
+            consecutive = any(sigs[i] == sigs[i + 1] for i in range(len(sigs) - 1))
+            per_case.append({
+                "sample_id": traj.sample_id,
+                "has_loop": bool(loops),
+                "consecutive_repeat": consecutive,
+                "loops": loops,
+            })
+        return Result(
+            analyzer=self.name,
+            model=repr(model),
+            cases=cases,
+            findings={
+                "n_trajectories": len(per_case),
+                "n_with_loops": sum(c["has_loop"] for c in per_case),
+                "per_case": per_case,
+            },
+        )

evalrx/analyzers/agent/reliability.py

@@ -0,0 +1,165 @@
+"""Reliability probe — pass@k / pass^k over repeated agent runs.
+
+A single run hides the capability/stability split: "reliably fails" and
+"flaky" are different diagnoses with different fixes (prompt/tool surgery vs
+decoding/consistency scaffolds).  This probe re-runs each case *k* times
+through an injected runner and reports, per case, the pass@k family plus
+answer- and trajectory-consistency measures.
+
+Reference: Pass@k (capability: at least one of k passes) vs Pass^k
+(reliability: all k pass) as popularised by agent-eval harnesses.
+
+The runner is INJECTED — the analyzer never builds an agent itself::
+
+    def runs_fn(case, k):
+        # e.g. a temperature>0 endpoint, or per-rep seeds; use run_batch for concurrency
+        return [Agent(handle, tools_factory(case), system=SYS).run(case) for _ in range(k)]
+
+    probe = ReliabilityProbe(runs_fn=runs_fn, k=5)
+    result = probe.run(model, cases)
+"""
+
+from __future__ import annotations
+
+import re
+import statistics
+from collections import Counter
+from typing import TYPE_CHECKING, Any, Callable, Optional
+
+from evalrx.core.analyzer import Analyzer
+from evalrx.core.registry import register_analyzer
+from evalrx.core.result import Result
+
+if TYPE_CHECKING:
+    from evalrx.core.case import CaseBatch, FailureCase, Trajectory
+    from evalrx.core.model import Model
+
+_YESNO = re.compile(r"\b(yes|no)\b")
+
+
+def default_grader(trajectory: "Trajectory", case: "FailureCase") -> Optional[bool]:
+    """Grade one run against ``case.expected`` (``None`` = ungradable).
+
+    yes/no expectations match the FIRST yes/no token in the answer; other
+    expectations match as a case-insensitive substring.  Inject a task-specific
+    grader for anything richer.
+    """
+    expected = case.expected
+    if expected is None or trajectory.final_answer is None:
+        return None
+    answer = str(trajectory.final_answer).strip().lower()
+    exp = str(expected).strip().lower()
+    if exp in ("yes", "no"):
+        m = _YESNO.search(answer)
+        return bool(m and m.group(1) == exp)
+    return exp in answer
+
+
+def _normalized_answer(trajectory: "Trajectory") -> str:
+    return " ".join(str(trajectory.final_answer or "").lower().split())[:200]
+
+
+def _tool_sequence(trajectory: "Trajectory") -> tuple:
+    return tuple(s.tool_call.get("name", "") for s in trajectory.steps if s.tool_call)
+
+
+@register_analyzer("reliability_probe")
+class ReliabilityProbe(Analyzer):
+    """Re-run each case k times and measure pass@k / pass^k / consistency.
+
+    Hyper-parameters:
+        runs_fn:   ``callable(case, k) -> list[Trajectory]`` — REQUIRED.  The
+                   caller owns sampling variation (temperature/seeds) and
+                   concurrency; deterministic runners make every rep identical
+                   and the probe degenerates (it will say so in the caveat).
+        k:         repetitions per case.
+        grader:    ``callable(trajectory, case) -> bool | None``; defaults to
+                   :func:`default_grader`.
+        max_cases: label-balanced cap on probed cases (k runs each is costly).
+    """
+
+    name = "reliability_probe"
+    requires = frozenset()  # the injected runner encapsulates the model
+    applies_to_modalities = frozenset({"text", "image"})
+
+    def __init__(
+        self,
+        runs_fn: Callable[["FailureCase", int], "list[Trajectory]"],
+        k: int = 5,
+        grader: Optional[Callable[["Trajectory", "FailureCase"], Optional[bool]]] = None,
+        max_cases: Optional[int] = None,
+    ) -> None:
+        super().__init__(k=k, max_cases=max_cases)
+        # stored under the ctor names so sklearn-style get_params() reflection works
+        self.runs_fn = runs_fn
+        self.grader = grader or default_grader
+
+    def _run(self, model: "Model", cases: "CaseBatch") -> Result:
+        selected = cases.stratified_head(self.max_cases) if self.max_cases else list(cases)
+        per_case: list[dict[str, Any]] = []
+        for case in selected:
+            trajectories = list(self.runs_fn(case, self.k) or [])
+            if not trajectories:
+                continue
+            n_runs = len(trajectories)
+            grades = [self.grader(t, case) for t in trajectories]
+            graded = [g for g in grades if g is not None]
+
+            answers = Counter(_normalized_answer(t) for t in trajectories)
+            agreement = answers.most_common(1)[0][1] / n_runs
+            sequences = Counter(_tool_sequence(t) for t in trajectories)
+            calls = [
+                int(t.metrics.get("n_tool_calls", sum(1 for s in t.steps if s.tool_call)))
+                for t in trajectories
+            ]
+
+            entry: dict[str, Any] = {
+                "sample_id": case.id,
+                "n_runs": n_runs,
+                "answer_agreement": round(agreement, 4),
+                "answer_consistent": 1 if len(answers) == 1 else 0,
+                "tool_seq_diversity": round(len(sequences) / n_runs, 4),
+                "n_tool_calls_mean": round(statistics.fmean(calls), 3) if calls else 0.0,
+                "n_tool_calls_std": round(statistics.pstdev(calls), 3) if len(calls) > 1 else 0.0,
+            }
+            if graded:
+                n_pass = sum(graded)
+                entry.update(
+                    {
+                        "n_graded": len(graded),
+                        "n_pass": n_pass,
+                        "success_rate": round(n_pass / len(graded), 4),
+                        "pass_at_k": 1 if n_pass > 0 else 0,
+                        "pass_all_k": 1 if n_pass == len(graded) else 0,
+                        "flaky": 1 if 0 < n_pass < len(graded) else 0,
+                    }
+                )
+            per_case.append(entry)
+
+        rates = [c["success_rate"] for c in per_case if "success_rate" in c]
+        all_consistent = per_case and all(c["answer_consistent"] == 1 for c in per_case)
+        findings: dict[str, Any] = {
+            "n_trajectories": len(per_case),
+            "k": self.k,
+            "n_graded_cases": len(rates),
+            "mean_success_rate": round(statistics.fmean(rates), 4) if rates else None,
+            "frac_flaky": (
+                round(sum(c.get("flaky", 0) for c in per_case) / len(rates), 4) if rates else None
+            ),
+            "per_case": per_case,
+            "_caveat": (
+                "Ungraded cases (no expected answer) report consistency only. "
+                + (
+                    "Every case produced k identical answers — the runner looks "
+                    "deterministic; use temperature>0 or per-rep seeds for a "
+                    "meaningful reliability read. "
+                    if all_consistent
+                    else ""
+                )
+                + "pass@k measures capability, pass^k reliability; the gap is the "
+                "instability mass. These are INTERVENTIONAL columns: held-out "
+                "verification must RE-RUN the k repetitions on the held-out "
+                "cases — never reuse exploration-set values."
+            ),
+        }
+        return Result(analyzer=self.name, model=repr(model), cases=cases, findings=findings)