evalrx 0.1.2__py3-none-any.whl

evalrx/eval_agent/run_metadata.py

@@ -0,0 +1,208 @@
+"""Shared run-provenance and logging-wiring helpers used by both
+``VLDiagnoseLoop`` (loop.py) and ``AutoDiagnoseLoop`` (legacy.py).
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+def _attach_run_logger(run_logger: Any, *agents: Any) -> None:
+    """Point each agent's ``run_logger`` at *run_logger* when it is unset.
+
+    Lets the probe / stats generators emit ``tool_codegen`` events without the
+    caller having to wire the logger into every agent by hand.  Respects an
+    explicitly-set logger (only fills in ``None``).
+    """
+    if run_logger is None:
+        return
+    for agent in agents:
+        if agent is not None and getattr(agent, "run_logger", None) is None:
+            try:
+                agent.run_logger = run_logger
+            except Exception:  # noqa: BLE001 - never let logging wiring break a run
+                pass
+
+
+def _data_provenance(data: Any) -> dict[str, Any]:
+    """Fingerprint the case batch + tally its labels for the ``run_start`` record.
+
+    Returns ``{"data_fingerprint": <12-hex>, "label_distribution": {...}}`` — the
+    fingerprint is a SHA-1 over each case's ``id`` (or its prompt when no id),
+    order-independent, so the same batch always hashes the same regardless of
+    iteration order, and a different batch (even same size) hashes differently.
+    Best-effort: returns ``{}`` if *data* isn't iterable.
+    """
+    import hashlib
+    from collections import Counter
+
+    try:
+        cases = list(data)
+    except Exception:  # noqa: BLE001
+        return {}
+
+    keys: list[str] = []
+    labels: Counter = Counter()
+    for c in cases:
+        cid = getattr(c, "id", None)
+        if not cid:
+            prompt = getattr(getattr(c, "inputs", None), "prompt", None)
+            cid = str(prompt) if prompt is not None else repr(c)
+        keys.append(str(cid))
+        label = getattr(c, "label", None)
+        labels[getattr(label, "name", "UNKNOWN")] += 1
+
+    out: dict[str, Any] = {}
+    if keys:
+        digest = hashlib.sha1("\n".join(sorted(keys)).encode("utf-8")).hexdigest()
+        out["data_fingerprint"] = digest[:12]
+    if labels:
+        out["label_distribution"] = dict(labels)
+    return out
+
+
+def _run_config(loop: Any, data: Any, *, loop_name: str) -> dict[str, Any]:
+    """Build the ``run_start`` provenance dict from a loop instance + its data.
+
+    Robust to both loop types and missing agents — every field is best-effort so
+    a partially-configured loop still produces a useful config record.
+    """
+    cfg: dict[str, Any] = {"loop": loop_name}
+    try:
+        cfg["model"] = repr(loop.model)
+    except Exception:  # noqa: BLE001
+        pass
+    cfg["max_cycles"] = getattr(loop, "max_cycles", None)
+    cfg["token_budget"] = getattr(loop, "token_budget", None)
+    cfg["analysis_only"] = getattr(loop, "analysis_only", None)
+    try:
+        cfg["n_cases"] = len(data)
+    except Exception:  # noqa: BLE001
+        pass
+    # Dataset provenance: a stable fingerprint over the cases plus their label
+    # breakdown.  n_cases alone says how many; this says *which* (so two runs
+    # can be confirmed to use the same batch) and the base failure rate the
+    # whole diagnosis is conditioned on — both essential to interpret the run.
+    _data_prov = _data_provenance(data)
+    if _data_prov:
+        cfg.update(_data_prov)
+    # How the batch was partitioned before anything ran. ``n_cases`` above is
+    # the explore partition (``data`` has already been narrowed to it); these
+    # say whether a confirm / test partition was withheld and at what fraction,
+    # so a report can tell a two-way run from a train/val/test one.
+    for attr in ("confirm_split", "test_split", "confirm_split_seed"):
+        value = getattr(loop, attr, None)
+        if isinstance(value, (int, float)) and not isinstance(value, bool):
+            cfg[attr] = value
+
+    protocol = getattr(loop, "protocol", None)
+    if protocol is not None:
+        cfg["protocol"] = {
+            "description": getattr(protocol, "description", ""),
+            "task_domain": getattr(protocol, "task_domain", None),
+        }
+
+    # Judge — read off the M3 agent (or its lazy default) when present.
+    diag_agent = getattr(loop, "diagnosis_agent", None)
+    judge = getattr(diag_agent, "judge", None) if diag_agent is not None else None
+    if judge is not None:
+        cfg["judge"] = repr(judge)
+
+    # Coder — the M5 surgery writer's CLI provider/model, when configured.
+    surgery = getattr(loop, "surgery_agent", None)
+    writer = getattr(surgery, "_writer", None) if surgery is not None else None
+    cli = getattr(getattr(writer, "_cfg", None), "cli_agent", None)
+    if cli is not None:
+        provider = getattr(cli, "provider", None)
+        model = getattr(cli, "model", "")
+        if provider:
+            cfg["coder"] = f"{provider}:{model}" if model else provider
+
+    # Whether tool synthesis (codegen) is enabled on either agent.
+    cfg["allow_codegen"] = bool(
+        getattr(getattr(loop, "probe_agent", None), "allow_codegen", False)
+        or getattr(getattr(loop, "stats_agent", None), "_allow_codegen", False)
+        or getattr(getattr(loop, "fix_agent", None), "_allow_codegen", False)
+    )
+    # Whether the in-cycle explore step (free-form EDA beside M2) is configured —
+    # so a run with explorer notes in M3 is distinguishable from one without.
+    explorer = getattr(loop, "explorer", None)
+    if explorer is not None:
+        cli = getattr(explorer, "_cli_config", None)
+        provider = getattr(cli, "provider", None)
+        model = getattr(cli, "model", "")
+        cfg["explorer"] = (
+            (f"{provider}:{model}" if model else provider) if provider
+            else type(explorer).__name__
+        )
+    return cfg
+
+
+def _log_generated_tools(run_logger: Any, cycle: int, module: str, agent: Any) -> None:
+    """Snapshot an agent's active generated-tool registry for *cycle*.
+
+    Reads ``_generated_probes`` (list of ``(generator, probe)`` tuples) for the
+    probe agent or ``_generated_tools`` (list of tools) for the stats agent and
+    forwards the bare tool objects to :meth:`RunLogger.log_tool_registry`.
+    Best-effort: silently does nothing for agents without those attributes.
+    """
+    if agent is None:
+        return
+    raw = getattr(agent, "_generated_probes", None)
+    if raw is None:
+        raw = getattr(agent, "_generated_tools", None)
+    if not raw:
+        return
+    tools = [t[1] if isinstance(t, tuple) else t for t in raw]
+    try:
+        run_logger.log_tool_registry(cycle, module, tools)
+    except Exception as exc:  # noqa: BLE001
+        logger.debug("log_tool_registry failed: %s", exc)
+
+
+def _coerce_explore_context(value: "Any | None") -> "Any | None":
+    """Normalise an explore_report argument into an ExploreContext or None.
+
+    Accepts an ``ExploreContext``, a report dict (e.g. ``fused_report.json``),
+    or ``None``. Never raises — a malformed value degrades to ``None``."""
+    if value is None:
+        return None
+    try:
+        from evalrx.eval_agent.stages.diagnosis import ExploreContext
+
+        if isinstance(value, ExploreContext):
+            return None if value.is_empty else value
+        if isinstance(value, dict):
+            return ExploreContext.from_report(value)
+    except Exception as exc:  # explore context is optional, never fatal
+        logger.warning("could not build ExploreContext from explore_report: %s", exc)
+    return None
+
+
+# ---------------------------------------------------------------------------
+# Internal adapter: convert HypothesisTestResult → InterventionResult-like
+# object for RunLogger.log_surgery (which expects an InterventionResult).
+# ---------------------------------------------------------------------------
+
+def _make_intervention_result_from_test(tr: "Any") -> Any:
+    """Wrap a HypothesisTestResult as a minimal InterventionResult."""
+    from evalrx.eval_agent.stages.surgery import InterventionResult
+
+    return InterventionResult(
+        hypothesis=tr.hypothesis,
+        status=tr.status,
+        fixed=False,
+        evidence={
+            "m4_test_name": tr.test_name,
+            "m4_effect_size": tr.effect_size,
+            "m4_confidence": tr.confidence,
+            "m4_protocol_consistent": tr.is_consistent_with_protocol,
+            "m4_verdict": tr.verdict,
+            "m4_evidence_grade": tr.evidence_grade,
+            **tr.evidence,
+        },
+        confidence_score=tr.confidence,
+    )

evalrx/eval_agent/stages/__init__.py

@@ -0,0 +1,56 @@
+"""Stage implementations for the M1–M4 diagnostic pipeline.
+
+Each module is a self-contained stage that can be injected into either loop:
+
+  AutoDiagnoseLoop  uses M1 → M2 → M3 → M5
+  VLDiagnoseLoop    uses M1 → M2 → M3 → M4 (inner), M5 called post-loop
+
+M1 — Probe (analyzer selection + execution)
+  probe.py        StrategyProbe   detect model kind; rank analyzers by failure-mode hints
+  probe_agent.py  ProbeAgent      execute ranked analyzers; route to Docker if needed;
+                                  protocol-guided via LLM analyzer selection
+  protocol.py     ExperimentProtocol   user's NL description of what to investigate;
+                  ProbingSchema        records which analyzers ran and why (M2/M4 trace)
+
+Data — Case discovery / labeling
+  case_discovery.py  CaseDiscoveryAgent  run candidate prompts through the model,
+                                         store observed outputs, and label PASS/FAIL
+
+M2 — Analysis (statistical interpretation)
+  Lives in ``evalrx.analysis`` (not this package) so it is usable standalone:
+  analysis_module.py  AnalysisModule       threshold rules + VLM derived metrics → AnalysisReport
+  stats_agent.py       StatsAnalysisAgent   protocol-aware extension of AnalysisModule;
+                                            adds LLM-written conclusion + evidence chain
+                       StatsAnalysisReport  AnalysisReport subclass — adds conclusion,
+                                            evidence_chain, qualitative_findings, protocol
+  stats_tool_agent.py  StatsToolAgent       selects/runs deterministic exploratory
+                                            stats tools and visualization specs
+  stats_tools.py       STATS_TOOL_CATALOG   signal/label association, McNemar+e-value,
+                                            Friedman, single-rate e-value, rank corr
+  stats_tool_generator.py StatsToolGenerator  LLM/CLI writes a new stats script when
+                                            no catalog tool fits
+
+M3 — Diagnosis (hypothesis generation)
+  diagnosis.py    DiagnosisAgent   LLM judge reads AnalysisReport → Hypothesis list;
+                                   falls back to threshold-derived hypotheses on NO_ISSUE
+
+M5 — Surgery (fix proposal + execution)
+  surgery.py           SurgeryAgent       four strategies: verify_fn → analyzer_params →
+                                          ExperimentWriter → label correlation
+  experiment_writer.py ExperimentWriter   multi-phase LLM/CLI agent writes + runs
+                                          diagnostic Python projects in a sandbox
+
+M4 — Hypothesis testing (statistical + protocol consistency gate)
+  hypothesis_tester.py HypothesisTester   per-case signal extraction → fail-rate comparison
+                                          → optional LLM protocol consistency check;
+                                          stopping_criteria_met() returns True when at
+                                          least one hypothesis is SUPPORTED + consistent
+
+Cross-stage shared types (top level, not in this package):
+  hypothesis.py   Hypothesis, HypothesisStatus   used by M3, M4, M5
+
+Shared CLI-agent runtime (evalrx.agent_runtime, not in this package):
+  codegen/        CodegenRunner   shared CLI code-generation orchestration
+  providers/      CLI coding-provider adapters (agy/codex/claude/etc.)
+  judges/         CLI-backed judge model wrappers
+"""

evalrx/eval_agent/stages/case_discovery.py

@@ -0,0 +1,293 @@
+"""Case discovery and labeling for automated VL diagnosis loops.
+
+This stage turns candidate prompts into labeled cases by running the target
+model, storing its observed answer, and scoring the result.  It is intentionally
+small: dataset loading and prompt generation can live outside this class, while
+M4 receives the PASS/FAIL labels it needs for statistical testing.
+"""
+
+import json
+import re
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any, Callable, Iterable
+
+from evalrx.core.case import CaseBatch, FailureCase, Label
+from evalrx.eval_agent.prompts.case_discovery import _JUDGE_PROMPT
+
+if TYPE_CHECKING:
+    from evalrx.core.model import Model
+    from evalrx.eval_agent.stages.protocol import ExperimentProtocol
+
+
+ScoreFn = Callable[[FailureCase, str], Label | bool | str]
+
+
+@dataclass
+class CaseDiscoveryReport:
+    """Summary of one discovery pass."""
+
+    cases: CaseBatch
+    n_pass: int = 0
+    n_fail: int = 0
+    n_unknown: int = 0
+    errors: list[str] = field(default_factory=list)
+
+    @property
+    def has_m4_groups(self) -> bool:
+        """True when M4 has both failure and control examples."""
+        return self.n_fail > 0 and self.n_pass > 0
+
+
+class CaseDiscoveryAgent:
+    """Run candidate cases through a model and label PASS/FAIL outcomes.
+
+    Args:
+        scorer:            Deterministic scoring function.  It receives the
+                           case and observed answer and returns ``Label``, bool,
+                           or a label string.
+        judge:             Optional LLM judge used when no scorer is supplied.
+        generation_kwargs: Extra kwargs forwarded to ``model.generate``.
+        include_unknown:   Keep UNKNOWN cases in the returned batch.
+        concurrency:       How many cases may be generated at once.  Only the
+                           ``model.generate`` calls fan out; scoring and
+                           labelling stay sequential and in input order, so the
+                           report is identical whatever this is set to.
+
+                           Values above 1 are honoured ONLY for an API-backed
+                           handle, following the same rule as
+                           :func:`evalrx.models.agent.run_batch`: HTTP is
+                           reentrant, a local backend sharing one GPU is not.
+                           Against a local ``vllm serve`` this is the
+                           difference between one request in flight and a
+                           batch -- the server can only batch what it has been
+                           given.
+    """
+
+    def __init__(
+        self,
+        scorer: ScoreFn | None = None,
+        judge: "Model | None" = None,
+        generation_kwargs: dict[str, Any] | None = None,
+        include_unknown: bool = True,
+        concurrency: int = 1,
+    ) -> None:
+        self.scorer = scorer
+        self.judge = judge
+        self.generation_kwargs = generation_kwargs or {}
+        self.include_unknown = include_unknown
+        self.concurrency = max(1, int(concurrency))
+
+    def discover(
+        self,
+        model: "Model",
+        candidates: Iterable[FailureCase] | CaseBatch,
+        *,
+        protocol: "ExperimentProtocol | None" = None,
+    ) -> CaseDiscoveryReport:
+        """Return a labeled batch created from candidate cases."""
+        labeled: list[FailureCase] = []
+        errors: list[str] = []
+        counts = {Label.PASS: 0, Label.FAIL: 0, Label.UNKNOWN: 0}
+        cases = list(candidates)
+
+        for case, observed, gen_error in self._generate(model, cases):
+            try:
+                if gen_error is not None:
+                    raise gen_error
+                case.observed = observed
+                label, reason = self._score_with_reason(case, str(observed), protocol)
+            except Exception as exc:  # noqa: BLE001 - discovery should continue
+                observed = ""
+                label = Label.UNKNOWN
+                reason = "model generation or scoring raised an exception"
+                errors.append(f"{case.id}: {exc}")
+                case.metadata["discovery_error"] = str(exc)
+
+            case.label = label
+            case.metadata["discovery_observed"] = str(observed)
+            case.metadata["discovery_label"] = label.value
+            case.metadata["discovery_reason"] = reason
+            counts[label] += 1
+
+            if label != Label.UNKNOWN or self.include_unknown:
+                labeled.append(case)
+
+        return CaseDiscoveryReport(
+            cases=CaseBatch(labeled),
+            n_pass=counts[Label.PASS],
+            n_fail=counts[Label.FAIL],
+            n_unknown=counts[Label.UNKNOWN],
+            errors=errors,
+        )
+
+    def _generate(
+        self, model: "Model", cases: list[FailureCase]
+    ) -> "list[tuple[FailureCase, Any, Exception | None]]":
+        """``(case, observed, error)`` per case, in input order.
+
+        The exception is carried rather than raised so the caller keeps its one
+        place that turns any failure into an UNKNOWN label -- a case whose
+        generation failed and a case whose scoring failed are the same
+        observation, and both must leave the batch intact.
+        """
+        workers = self._workers(model, len(cases))
+        if workers <= 1:
+            out = []
+            for case in cases:
+                try:
+                    out.append((case, model.generate(case.inputs, **self.generation_kwargs), None))
+                except Exception as exc:  # noqa: BLE001
+                    out.append((case, None, exc))
+            return out
+
+        from concurrent.futures import ThreadPoolExecutor
+
+        def _one(case: FailureCase):
+            try:
+                return case, model.generate(case.inputs, **self.generation_kwargs), None
+            except Exception as exc:  # noqa: BLE001
+                return case, None, exc
+
+        with ThreadPoolExecutor(max_workers=workers) as pool:
+            return list(pool.map(_one, cases))
+
+    def _workers(self, model: "Model", n_cases: int) -> int:
+        """Concurrency actually allowed for this handle.
+
+        A local backend is silently forced back to 1 rather than warned about:
+        unlike ``run_batch``, nothing here asked for parallelism per case --
+        the caller set one number for a whole run that may mix handles, and a
+        warning per discovery pass would be noise about a setting that is
+        behaving correctly.
+        """
+        if self.concurrency <= 1:
+            return 1
+        from evalrx.models.backends.api import APIModel
+
+        # A deployed-pipeline wrapper (fix_tools.SpecPipelineModel) is exactly
+        # as reentrant as the handle it drives — judge it by its inner model.
+        model = getattr(model, "inner_model", model)
+        # InstrumentedModel is also a transparent proxy; preserve the API
+        # handle's configured discovery concurrency while recording each call.
+        model = getattr(model, "_model", model)
+        if not isinstance(model, APIModel):
+            return 1
+        return min(self.concurrency, max(1, n_cases))
+
+    def _score_with_reason(
+        self,
+        case: FailureCase,
+        observed: str,
+        protocol: "ExperimentProtocol | None",
+    ) -> tuple[Label, str]:
+        if self.scorer is not None:
+            label = _coerce_label(self.scorer(case, observed))
+            return label, "scored by injected scorer"
+        if self.judge is not None:
+            judged, reason = self._judge_score(case, observed, protocol)
+            if judged != Label.UNKNOWN:
+                return judged, reason
+            fallback = _heuristic_score(case.expected, observed)
+            return fallback, f"judge returned UNKNOWN; heuristic fallback produced {fallback.value}"
+        label = _heuristic_score(case.expected, observed)
+        return label, "scored by expected-answer heuristic"
+
+    def _judge_score(
+        self,
+        case: FailureCase,
+        observed: str,
+        protocol: "ExperimentProtocol | None",
+    ) -> tuple[Label, str]:
+        protocol_text = protocol.description if protocol is not None else ""
+        success_criteria = protocol.success_criteria if protocol is not None else ""
+        prompt = _JUDGE_PROMPT.format(
+            protocol=protocol_text,
+            success_criteria=success_criteria,
+            prompt=case.inputs.prompt,
+            expected=case.expected,
+            observed=observed,
+        )
+        raw = self.judge.generate(prompt)  # type: ignore[union-attr]
+        text = str(raw).strip()
+        parsed = _parse_judge_json(text)
+        if parsed is not None:
+            return parsed
+        first_lines = "\n".join(text.splitlines()[:5]).upper()
+        if "PASS" in first_lines and "FAIL" not in first_lines:
+            return Label.PASS, "judge text contained PASS"
+        if "FAIL" in first_lines and "PASS" not in first_lines:
+            return Label.FAIL, "judge text contained FAIL"
+        first = text.splitlines()[0].strip().upper() if text else ""
+        if first.startswith("PASS"):
+            return Label.PASS, "judge first line was PASS"
+        if first.startswith("FAIL"):
+            return Label.FAIL, "judge first line was FAIL"
+        return Label.UNKNOWN, "judge output did not contain a parseable label"
+
+
+def _coerce_label(value: Label | bool | str) -> Label:
+    if isinstance(value, Label):
+        return value
+    if isinstance(value, bool):
+        return Label.PASS if value else Label.FAIL
+    text = str(value).strip().lower()
+    if text in {"pass", "passed", "true", "correct", "ok"}:
+        return Label.PASS
+    if text in {"fail", "failed", "false", "incorrect", "wrong"}:
+        return Label.FAIL
+    return Label.UNKNOWN
+
+
+def _parse_judge_json(text: str) -> tuple[Label, str] | None:
+    raw = text.strip()
+    if raw.startswith("```"):
+        raw = re.sub(r"^```\w*\n?", "", raw)
+        raw = re.sub(r"\n?```\s*$", "", raw)
+    match = re.search(r"\{.*\}", raw, flags=re.DOTALL)
+    if not match:
+        return None
+    try:
+        payload = json.loads(match.group())
+    except json.JSONDecodeError:
+        return None
+    label = _coerce_label(payload.get("label", "unknown"))
+    reason = str(payload.get("reason", "judge returned structured label"))
+    return label, reason
+
+
+def _heuristic_score(expected: Any, observed: str) -> Label:
+    """Score simple expected-answer rubrics without an LLM judge.
+
+    Supported expected formats:
+      - string: substring match after normalization
+      - list/tuple/set: every item must appear
+      - dict: ``all_of``, ``any_of``, and ``none_of`` substring constraints
+    """
+    if expected is None:
+        return Label.UNKNOWN
+
+    obs = _normalize(observed)
+    if isinstance(expected, str):
+        return Label.PASS if _normalize(expected) in obs else Label.FAIL
+
+    if isinstance(expected, (list, tuple, set)):
+        terms = [_normalize(x) for x in expected]
+        return Label.PASS if all(t in obs for t in terms if t) else Label.FAIL
+
+    if isinstance(expected, dict):
+        all_of = [_normalize(x) for x in expected.get("all_of", [])]
+        any_of = [_normalize(x) for x in expected.get("any_of", [])]
+        none_of = [_normalize(x) for x in expected.get("none_of", [])]
+        if any(t and t in obs for t in none_of):
+            return Label.FAIL
+        if all_of and not all(t in obs for t in all_of if t):
+            return Label.FAIL
+        if any_of and not any(t in obs for t in any_of if t):
+            return Label.FAIL
+        return Label.PASS
+
+    return Label.UNKNOWN
+
+
+def _normalize(text: Any) -> str:
+    return re.sub(r"\s+", " ", str(text).lower()).strip()