evalrx 0.1.2__py3-none-any.whl

evalrx/eval_agent/agentic/actions.py

@@ -0,0 +1,216 @@
+"""Judge-decided actions for the agentic diagnosis loop.
+
+Each decision turn, the CLI judge returns one strict JSON object describing
+the next tool to call. ``parse_action`` turns that raw text into a validated
+:class:`Action`, using the shared :func:`~evalrx.agent_runtime.json_shape.validate_json_shape`
+(re-exported here for convenience — also used by M3's hypothesis parser and
+failure-mode cluster naming).
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+import time
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any
+
+from evalrx.agent_runtime.json_shape import validate_json_shape
+
+if TYPE_CHECKING:
+    from evalrx.eval_agent.agentic.board import EvidenceBoard
+    from evalrx.eval_agent.agentic.tools import ToolRegistry
+    from evalrx.eval_agent.run_logger_v2 import RunLoggerV2
+
+logger = logging.getLogger(__name__)
+
+ACTION_SCHEMA: dict[str, Any] = {
+    "type": "object",
+    "required": ["tool", "rationale"],
+    "properties": {
+        "tool": {"type": "string", "minLength": 1},
+        "params": {"type": "object"},
+        "rationale": {"type": "string", "minLength": 1},
+    },
+}
+
+
+class ActionParseError(Exception):
+    """Raised when the judge's raw output cannot be turned into a valid Action."""
+
+    def __init__(self, message: str, errors: list[str] | None = None) -> None:
+        super().__init__(message)
+        self.errors = errors or []
+
+
+@dataclass
+class Action:
+    """One judge-decided tool call."""
+
+    tool: str
+    params: dict[str, Any] = field(default_factory=dict)
+    rationale: str = ""
+    raw: str = ""
+    repair_attempts: int = 0
+    valid: bool = True
+
+
+def _strip_fences(text: str) -> str:
+    text = text.strip()
+    if text.startswith("```"):
+        text = re.sub(r"^```\w*\n?", "", text)
+        text = re.sub(r"\n?```\s*$", "", text)
+    return text.strip()
+
+
+def parse_action(raw: str, registry: "ToolRegistry") -> Action:
+    """Parse the judge's raw text into a validated :class:`Action`.
+
+    Raises :class:`ActionParseError` when the text is not JSON, fails the
+    action schema, or names a tool the registry doesn't know about.
+    """
+    text = _strip_fences(raw)
+    try:
+        data = json.loads(text)
+    except json.JSONDecodeError as exc:
+        raise ActionParseError(f"not valid JSON: {exc}") from exc
+
+    errors = validate_json_shape(data, ACTION_SCHEMA)
+    if errors:
+        raise ActionParseError("action schema validation failed", errors)
+
+    tool = str(data["tool"])
+    if tool not in registry.tool_names():
+        raise ActionParseError(
+            f"unknown tool {tool!r}", [f"must be one of {sorted(registry.tool_names())}"]
+        )
+
+    params = data.get("params") or {}
+    if not isinstance(params, dict):
+        raise ActionParseError("params must be an object", [f"$.params: got {type(params).__name__}"])
+
+    return Action(tool=tool, params=params, rationale=str(data["rationale"]), raw=raw)
+
+
+def _fallback_action(board: "EvidenceBoard") -> Action:
+    """Deterministic next-step heuristic: the first unmet M1->M4 stage.
+
+    Used when the judge's output is unusable even after one repair prompt, so
+    a stuck judge doesn't stall the run — the action is marked ``valid=False``
+    so the loop can track a consecutive-failure streak and give up cleanly.
+    """
+    if not board.probe_findings:
+        tool = "run_probe"
+    elif not board.stats_findings:
+        tool = "run_stats"
+    elif not board.hypotheses:
+        tool = "propose_hypotheses"
+    elif not board.has_supported_hypothesis():
+        tool = "test_hypothesis"
+    else:
+        tool = "stop"
+    params = {"resolved": True, "reason": "fallback heuristic"} if tool == "stop" else {}
+    return Action(
+        tool=tool, params=params,
+        rationale="fallback heuristic: judge output unusable", valid=False,
+    )
+
+
+def decide(
+    judge: Any,
+    board: "EvidenceBoard",
+    registry: "ToolRegistry",
+    *,
+    run_logger: "RunLoggerV2 | None" = None,
+    step: int = 0,
+    max_repairs: int = 1,
+) -> Action:
+    """Ask the judge for the next action, repairing/falling back as needed.
+
+    Always returns an :class:`Action` — never raises. When the judge's
+    response can't be salvaged after *max_repairs* repair prompts, returns the
+    deterministic :func:`_fallback_action` with ``valid=False``.
+    """
+    from evalrx.eval_agent.prompts.agentic import ACTION_REPAIR_PROMPT, DECISION_PROMPT
+
+    evidence = board.to_prompt(registry)
+    prompt = DECISION_PROMPT.format(evidence=evidence)
+    judge_calls: list[dict[str, Any]] = []
+    started = time.perf_counter()
+    try:
+        raw = judge.generate(prompt)
+    except Exception as exc:  # noqa: BLE001
+        judge_calls.append({
+            "role": "agent_decision", "operation": "generate", "inputs": prompt,
+            "output": None, "error": str(exc),
+            "duration_sec": time.perf_counter() - started,
+        })
+        raw = ""
+        action: Action | None = _fallback_action(board)
+    else:
+        judge_calls.append({
+            "role": "agent_decision", "operation": "generate", "inputs": prompt,
+            "output": str(raw), "error": None,
+            "duration_sec": time.perf_counter() - started,
+        })
+        try:
+            action = parse_action(raw, registry)
+        except ActionParseError as exc:
+            action = None
+            last_raw, last_exc = raw, exc
+    if action is None:
+        for attempt in range(1, max_repairs + 1):
+            repair_prompt = ACTION_REPAIR_PROMPT.format(
+                raw=last_raw,
+                errors="\n".join(last_exc.errors) or str(last_exc),
+                evidence=evidence,
+            )
+            started = time.perf_counter()
+            try:
+                last_raw = judge.generate(repair_prompt)
+            except Exception as exc:  # noqa: BLE001
+                judge_calls.append({
+                    "role": "agent_decision_repair", "operation": "generate",
+                    "inputs": repair_prompt, "output": None, "error": str(exc),
+                    "duration_sec": time.perf_counter() - started,
+                    "metadata": {"attempt": attempt},
+                })
+                break
+            judge_calls.append({
+                "role": "agent_decision_repair", "operation": "generate",
+                "inputs": repair_prompt, "output": str(last_raw), "error": None,
+                "duration_sec": time.perf_counter() - started,
+                "metadata": {"attempt": attempt},
+            })
+            try:
+                action = parse_action(last_raw, registry)
+                action.repair_attempts = attempt
+                break
+            except ActionParseError as exc2:
+                last_exc = exc2
+                action = None
+    if action is None:
+        action = _fallback_action(board)
+        action.raw = raw
+
+    if run_logger is not None:
+        try:
+            log_kwargs = dict(
+                step=step,
+                action=action.tool,
+                params=action.params,
+                rationale=action.rationale,
+                valid=action.valid,
+                repair_attempts=action.repair_attempts,
+                fallback_used=not action.valid,
+                judge_prompt=prompt,
+                judge_raw=raw,
+            )
+            if getattr(run_logger, "preserve_full_model_io", False):
+                log_kwargs["judge_calls"] = judge_calls
+            run_logger.log_agent_decision(**log_kwargs)
+        except Exception:  # noqa: BLE001 — logging must never break the loop
+            logger.warning("decide: log_agent_decision failed", exc_info=True)
+
+    return action

evalrx/eval_agent/agentic/board.py

@@ -0,0 +1,107 @@
+"""The evidence board: everything the judge sees when deciding the next action.
+
+The host reconstructs the full decision context from this board on every
+turn — there is no CLI session state carried between judge calls, only what
+is serialized here.
+"""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class BudgetState:
+    """Guardrails on how long an agentic run may keep deciding actions."""
+
+    max_actions: int = 12
+    token_budget: int = 0
+    time_budget_sec: float = 0.0
+    actions_taken: int = 0
+    tokens_used: int = 0
+    _started_monotonic: float = 0.0
+
+    def start(self) -> None:
+        self._started_monotonic = time.monotonic()
+
+    def exhausted(self) -> str | None:
+        """Return which budget tripped (for ``stopped_by``), or ``None``."""
+        if self.max_actions > 0 and self.actions_taken >= self.max_actions:
+            return "max_actions"
+        if self.token_budget > 0 and self.tokens_used >= self.token_budget:
+            return "budget"
+        if (
+            self.time_budget_sec > 0
+            and self._started_monotonic > 0
+            and (time.monotonic() - self._started_monotonic) >= self.time_budget_sec
+        ):
+            return "time_budget"
+        return None
+
+
+@dataclass
+class EvidenceBoard:
+    """Accumulated evidence the judge reasons over, one decision turn at a time."""
+
+    protocol_summary: str = ""
+    data_summary: dict[str, Any] = field(default_factory=dict)
+    probe_findings: list[dict[str, Any]] = field(default_factory=list)
+    stats_findings: list[dict[str, Any]] = field(default_factory=list)
+    stats_confirmatory: bool = False
+    explore_takeaways: list[dict[str, Any]] = field(default_factory=list)
+    failure_modes: list[dict[str, Any]] = field(default_factory=list)
+    probe_search_findings: list[dict[str, Any]] = field(default_factory=list)
+    hypotheses: list[dict[str, Any]] = field(default_factory=list)
+    action_log: list[dict[str, Any]] = field(default_factory=list)
+    budget: BudgetState = field(default_factory=BudgetState)
+
+    def has_supported_hypothesis(self) -> bool:
+        return any(
+            h.get("status") == "supported" and h.get("is_consistent_with_protocol", True)
+            for h in self.hypotheses
+        )
+
+    def calls_made(self, tool: str) -> int:
+        return sum(1 for a in self.action_log if a.get("tool") == tool)
+
+    def to_prompt(self, registry: "Any") -> str:
+        """Render the board as the decision prompt's evidence section."""
+        lines: list[str] = []
+        if self.protocol_summary:
+            lines.append(f"PROTOCOL:\n{self.protocol_summary}\n")
+        if self.data_summary:
+            lines.append(f"DATA: {self.data_summary}\n")
+
+        lines.append(f"PROBE FINDINGS (M1): {self.probe_findings or 'none yet'}")
+        lines.append(
+            f"STATS FINDINGS (M2, confirmatory={self.stats_confirmatory}): "
+            f"{self.stats_findings or 'none yet'}"
+        )
+        if self.explore_takeaways:
+            lines.append(f"EXPLORE TAKEAWAYS: {self.explore_takeaways}")
+        if self.failure_modes:
+            lines.append(f"FAILURE MODES (clustered): {self.failure_modes}")
+        if self.probe_search_findings:
+            lines.append(f"PROBE SEARCH (newly synthesized failures): {self.probe_search_findings}")
+        lines.append(f"HYPOTHESES: {self.hypotheses or 'none yet'}")
+
+        if self.action_log:
+            lines.append("\nACTIONS TAKEN SO FAR:")
+            for a in self.action_log:
+                lines.append(
+                    f"  [{a['step']}] {a['tool']}({a.get('params', {})}) "
+                    f"-> ok={a['ok']}: {a['summary']}"
+                )
+        else:
+            lines.append("\nACTIONS TAKEN SO FAR: none")
+
+        remaining = (
+            self.budget.max_actions - self.budget.actions_taken
+            if self.budget.max_actions > 0
+            else "unlimited"
+        )
+        lines.append(f"\nACTIONS REMAINING: {remaining}")
+        lines.append(f"\nAVAILABLE TOOLS:\n{registry.catalog_for_prompt(self)}")
+        return "\n".join(lines)

evalrx/eval_agent/agentic/loop.py

@@ -0,0 +1,190 @@
+"""AgenticDiagnoseLoop — an LLM-driven decision loop over the M1-M4 stages.
+
+``VLDiagnoseLoop`` runs M1->M2->M3->M4 in a hardcoded sequence every cycle.
+This loop keeps every stage (and the confirm-split / post-loop run_m5/run_fix
+discipline) exactly as-is, but replaces the fixed sequence with a CLI judge
+that decides, one tool call at a time, what to do next — probe, run stats,
+explore the raw data, propose hypotheses, test one, fix, or stop. The host,
+not the judge, enforces call caps, tool preconditions, and the pre-registered
+stopping discipline (no declaring success without a tested, supported,
+protocol-consistent hypothesis).
+
+``VLDiagnoseLoop`` itself is untouched — this is a new mode, not a
+replacement (see the module docstring in ``eval_agent/loop.py``).
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING, Any
+
+from evalrx.eval_agent.agentic.actions import decide
+from evalrx.eval_agent.agentic.board import BudgetState, EvidenceBoard
+from evalrx.eval_agent.agentic.tools import ToolRegistry, _RunState, build_default_registry
+from evalrx.eval_agent.loop import VLDiagnoseLoop
+from evalrx.eval_agent.loop_reports import VLDiagnoseReport
+from evalrx.eval_agent.run_metadata import _attach_run_logger, _data_provenance, _run_config
+
+if TYPE_CHECKING:
+    from evalrx.core.case import CaseBatch
+
+logger = logging.getLogger(__name__)
+
+_MAX_INVALID_STREAK = 3
+
+
+def _protocol_summary(protocol: "Any | None") -> str:
+    if protocol is None:
+        return ""
+    bits = [f"description: {getattr(protocol, 'description', '')}"]
+    domain = getattr(protocol, "task_domain", None)
+    if domain:
+        bits.append(f"task_domain: {domain}")
+    patterns = getattr(protocol, "failure_patterns", None)
+    if patterns:
+        bits.append(f"failure_patterns: {patterns}")
+    return "\n".join(bits)
+
+
+class AgenticDiagnoseLoop(VLDiagnoseLoop):
+    """M1-M4 investigation driven by judge decisions instead of a fixed cycle.
+
+    Reuses :class:`~evalrx.eval_agent.loop.VLDiagnoseLoop`'s constructor,
+    stage helpers (``_do_m1``..``_do_m4``), confirm-split, and post-loop
+    ``run_m5``/``run_fix`` — only :meth:`run` differs.
+
+    Args:
+        judge:            CLI-backed judge that decides the next action each
+                           turn (e.g. ``ClaudeModel()`` / ``AgyModel()``).
+                           Required — this is a genuinely new capability, not
+                           a Gemini-fallback stage judge.
+        explorer:         Optional ``ExploratoryAnalysisAgent`` — when given,
+                           registers the ``explore_data`` tool.
+        max_actions:      Hard cap on judge decision turns (default 12).
+        time_budget_sec:  Wall-clock cap for the whole run (0 = unlimited).
+        registry_factory: Advanced override — ``(loop, run_state) -> ToolRegistry``
+                          instead of :func:`build_default_registry`.
+        **kwargs:         Forwarded to ``VLDiagnoseLoop.__init__`` (model,
+                          protocol, probe_agent, stats_agent, hypothesis_tester,
+                          surgery_agent, fix_agent, store, token_budget,
+                          confirm_split, run_logger, ...).
+    """
+
+    def __init__(
+        self,
+        model: "Any",
+        protocol: "Any",
+        *,
+        judge: "Any",
+        explorer: "Any | None" = None,
+        max_actions: int = 12,
+        time_budget_sec: float = 0.0,
+        registry_factory: "Any | None" = None,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__(model, protocol, **kwargs)
+        if judge is None:
+            raise ValueError(
+                "AgenticDiagnoseLoop requires a decision judge, e.g. "
+                "judge=ClaudeModel() or judge=AgyModel()."
+            )
+        self.judge = judge
+        self.explorer = explorer
+        self.max_actions = max_actions
+        self.time_budget_sec = time_budget_sec
+        self._registry_factory = registry_factory
+
+    def run(self, data: "CaseBatch") -> VLDiagnoseReport:
+        """Drive the judge-decided action loop to a stop or a budget limit."""
+        self._tokens_used = 0
+        timings: dict[str, float] = {}
+
+        explore, confirm = self._split_explore_confirm(data)
+        original_data = data
+        if confirm is not None:
+            data = explore
+
+        _attach_run_logger(self.run_logger, self.probe_agent, self.stats_agent)
+        if self.run_logger is not None:
+            # _run_config records the M3 *stage* judge (loop.diagnosis_agent.judge),
+            # not the decision judge driving this loop's own tool-calling — add it
+            # explicitly so the dashboard header can show who is actually deciding.
+            cfg = _run_config(self, data, loop_name="AgenticDiagnoseLoop")
+            cfg["decision_judge"] = repr(self.judge)
+            cfg["max_actions"] = self.max_actions
+            self.run_logger.log_run_start(cfg)
+            self.run_logger.log_cases(data)
+
+        board = EvidenceBoard(
+            protocol_summary=_protocol_summary(self.protocol),
+            data_summary=_data_provenance(data),
+            budget=BudgetState(
+                max_actions=self.max_actions,
+                token_budget=self.token_budget,
+                time_budget_sec=self.time_budget_sec,
+            ),
+        )
+        board.budget.start()
+
+        state = _RunState(original_data=original_data, data=data, timings=timings)
+        registry: ToolRegistry = (
+            self._registry_factory(self, state)
+            if self._registry_factory is not None
+            else build_default_registry(self, state)
+        )
+
+        stopped_by = "max_actions"
+        invalid_streak = 0
+        step = 0
+        while True:
+            reason = board.budget.exhausted()
+            if reason:
+                stopped_by = reason
+                break
+
+            action = decide(self.judge, board, registry, run_logger=self.run_logger, step=step)
+            board.budget.actions_taken += 1
+            invalid_streak = 0 if action.valid else invalid_streak + 1
+
+            outcome = registry.dispatch(action, board)
+            board.action_log.append({
+                "step": step, "tool": action.tool, "params": action.params,
+                "ok": outcome.ok, "summary": outcome.summary,
+            })
+            if self.run_logger is not None:
+                self.run_logger.log_agent_tool(
+                    step, tool=action.tool, ok=outcome.ok,
+                    summary=outcome.summary, error=outcome.error,
+                )
+
+            if action.tool == "stop" and outcome.ok:
+                stopped_by = "agent_stop"
+                step += 1
+                break
+
+            if invalid_streak >= _MAX_INVALID_STREAK:
+                logger.warning(
+                    "AgenticDiagnoseLoop: %d consecutive invalid actions — stopping.",
+                    invalid_streak,
+                )
+                stopped_by = "invalid_actions"
+                step += 1
+                break
+
+            step += 1
+
+        verified = self.hypothesis_tester.best_hypotheses(state.all_test_results)
+        report = VLDiagnoseReport(
+            cycles=step,
+            resolved=bool(verified),
+            stopped_by=stopped_by,
+            verified_hypotheses=verified,
+            final_hypotheses=state.all_hypotheses,
+            all_test_results=state.all_test_results,
+            final_stats_report=state.stats_report,
+            store=self.store,
+            _run_id=self._run_id,
+        )
+        if self.run_logger:
+            self.run_logger.log_loop_end(report, tokens_used=self._tokens_used, timings=timings)
+        return report