evalrx 0.1.2__py3-none-any.whl

evalrx/analysis/holdout.py

@@ -0,0 +1,300 @@
+"""Generic held-out verification for standalone explore runs.
+
+Generalizes the deco_hallu_explore example's ``test_hypotheses.py`` to ANY
+dataset the explorer can load (the upload workbench's mode 2, and
+``evalrx explore --holdout-frac F --holdout-confirm``):
+
+1. :func:`split_records` carves the loaded records into a deterministic,
+   outcome-stratified explore/holdout partition BEFORE exploration — the
+   explorer only ever sees the explore share.
+2. :func:`holdout_confirm` re-evaluates the explorer's FROZEN recipes verbatim
+   on the held-out rows (:func:`~evalrx.analysis.operationalize.compile_recipe`,
+   thresholds exactly as fitted — no re-fitting), rebuilds two-group sufficient
+   statistics, adjudicates with
+   :func:`~evalrx.analysis.adjudicate.adjudicate_signals`
+   (``split_label="held_out"`` — a REJECT here is a real held-out verdict), and
+   optionally has an LLM judge grade each M3 hypothesis against the held-out
+   table (supported / partial / refuted / not_testable + surgery routing).
+
+The returned dict is the ``confirm_report.json`` shape the dashboard's
+*Held-out Verdicts* tab renders. The judge is duck-typed (any object with
+``generate(str) -> str``); pass ``judge=None`` for signal verdicts only —
+hypotheses then carry ``verdict="not_judged"``.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Any
+
+from evalrx.analysis.adjudicate import adjudicate_signals
+from evalrx.analysis.explorer import CandidateSignal
+from evalrx.analysis.fused_pipeline import _split_records
+from evalrx.analysis.operationalize import RecipeError, SignalRecipe, compile_recipe
+from evalrx.analysis.prompts.holdout import JUDGE_PROMPT
+
+_FAIL_LIKE = ("fail", "failed", "failure", "error", "wrong", "incorrect", "no")
+
+
+def split_records(
+    records: "list[dict[str, Any]]",
+    holdout_frac: float,
+    *,
+    seed: int = 0,
+    outcome_col: str | None = None,
+) -> "tuple[list[dict[str, Any]], list[dict[str, Any]] | None]":
+    """Deterministic ``(explore, holdout)`` partition, stratified by outcome.
+
+    Wraps the fused pipeline's split so the standalone path and the loop's
+    fused path hold out rows the same way. Returns ``(records, None)`` when
+    the fraction or batch size makes a split meaningless — callers fall back
+    to a plain in-sample run with a caveat rather than a fake split.
+    """
+    return _split_records(
+        records, frac=holdout_frac, seed=seed, label_col=outcome_col or "label"
+    )
+
+
+def failure_indicator(
+    rows: "list[dict[str, Any]]", outcome_col: str
+) -> "tuple[list[int | None], Any, str]":
+    """Per-row binary failure indicator for an arbitrary outcome column.
+
+    Returns ``(indicators, positive_label, note)`` where ``indicators[i]`` is
+    1 (failure), 0 (non-failure) or None (missing outcome). The positive
+    ("failure") class is chosen by, in order: a fail-like string value
+    (fail/failed/error/wrong/...), the truthy pole of a boolean/0-1 column, or
+    — for any other two-valued column — the MINORITY value (failures are
+    normally the rare class); *note* says which rule fired so the report can
+    state it honestly.
+    """
+    values = [r.get(outcome_col) for r in rows]
+    present = [v for v in values if v is not None]
+    uniq = {str(v).strip().lower() for v in present}
+
+    positive: Any = None
+    note = ""
+    fail_hits = sorted(uniq & set(_FAIL_LIKE))
+    if fail_hits:
+        positive = fail_hits[0]
+        note = f"failure = {outcome_col!r} == {positive!r} (fail-like value)"
+    elif uniq <= {"0", "1", "true", "false", "0.0", "1.0"} and uniq:
+        positive = "truthy"
+        note = f"failure = truthy {outcome_col!r} (boolean/0-1 column)"
+    elif len(uniq) == 2:
+        counts: dict[str, int] = {}
+        for v in present:
+            key = str(v).strip().lower()
+            counts[key] = counts.get(key, 0) + 1
+        positive = min(counts, key=lambda k: counts[k])
+        note = (
+            f"failure = {outcome_col!r} == {positive!r} "
+            "(minority value of a two-valued column)"
+        )
+    else:
+        return [None] * len(rows), None, (
+            f"outcome column {outcome_col!r} is not binary "
+            f"({len(uniq)} distinct values) — held-out verdicts unavailable"
+        )
+
+    out: "list[int | None]" = []
+    for v in values:
+        if v is None:
+            out.append(None)
+            continue
+        key = str(v).strip().lower()
+        if positive == "truthy":
+            out.append(1 if key in {"1", "true", "1.0"} else 0)
+        else:
+            out.append(1 if key == positive else 0)
+    return out, positive, note
+
+
+def _holdout_sufficient(
+    values: "dict[str, float]",
+    id_to_fail: "dict[str, int]",
+) -> "dict[str, Any] | None":
+    """Two-group sufficient stats: failure indicators among signal-ABSENT (a)
+    vs signal-PRESENT (b) rows. Defined only for binary-flag recipes — a
+    continuous recipe has no frozen threshold to reuse."""
+    pairs = [(v, id_to_fail[cid]) for cid, v in values.items() if cid in id_to_fail]
+    if not pairs:
+        return None
+    if not {round(v, 6) for v, _ in pairs} <= {0.0, 1.0}:
+        return None
+    a = [f for v, f in pairs if v == 0.0]
+    b = [f for v, f in pairs if v == 1.0]
+    if not a or not b:
+        return None
+    return {"kind": "two_group", "a": a, "b": b}
+
+
+def _judge_json(text: str) -> "dict[str, Any] | None":
+    m = re.search(r"\{.*\}", text, re.S)
+    if not m:
+        return None
+    try:
+        parsed = json.loads(m.group(0))
+    except json.JSONDecodeError:
+        return None
+    return parsed if isinstance(parsed, dict) else None
+
+
+def holdout_confirm(
+    report: "dict[str, Any]",
+    holdout_rows: "list[dict[str, Any]]",
+    *,
+    outcome_col: str = "label",
+    alpha: float = 0.05,
+    judge: Any = None,
+    judge_meta: "dict[str, Any] | None" = None,
+) -> "dict[str, Any]":
+    """Re-test one exploratory report's recipes + hypotheses on held-out rows.
+
+    *report* is the ``exploratory_report.json`` dict (``to_dict()`` of a live
+    report works too). Returns the ``confirm_report.json`` payload; the caller
+    persists it next to the exploratory report.
+    """
+    rows = [dict(r) for r in holdout_rows]
+    indicators, positive_label, outcome_note = failure_indicator(rows, outcome_col)
+
+    # Stable ids: force the compile id_col so recipe values map back to rows
+    # regardless of what id-ish keys the upload carries.
+    id_to_fail: "dict[str, int]" = {}
+    for i, (row, ind) in enumerate(zip(rows, indicators)):
+        row["_holdout_id"] = f"h{i}"
+        if ind is not None:
+            id_to_fail[f"h{i}"] = ind
+    n_fail = sum(id_to_fail.values())
+
+    candidates_raw = [c for c in report.get("candidate_signals") or [] if isinstance(c, dict)]
+    hypotheses = [h for h in report.get("hypotheses") or [] if isinstance(h, dict)]
+
+    # ── 1. frozen-recipe re-evaluation on the held-out rows ─────────────────
+    signal_verdicts: "list[dict[str, Any]]" = []
+    candidates: "list[CandidateSignal]" = []
+    for c in candidates_raw:
+        entry: "dict[str, Any]" = {
+            "name": c.get("name"),
+            "display_name": c.get("display_name") or c.get("name"),
+            "recipe": c.get("recipe"),
+            "explore_effect": c.get("effect"),
+            "explore_reject_in_sample": c.get("reject"),
+        }
+        recipe_dict = c.get("recipe")
+        if not id_to_fail:
+            entry.update(status="skipped", reason=outcome_note or "no binary outcome")
+            signal_verdicts.append(entry)
+            continue
+        if not isinstance(recipe_dict, dict) or recipe_dict.get("kind") != "expr":
+            entry.update(status="skipped", reason="no expr recipe to re-evaluate")
+            signal_verdicts.append(entry)
+            continue
+        try:
+            values = compile_recipe(
+                SignalRecipe.from_dict(recipe_dict), rows, id_col="_holdout_id"
+            )
+        except (RecipeError, NotImplementedError) as exc:
+            entry.update(status="skipped", reason=f"recipe error: {exc}")
+            signal_verdicts.append(entry)
+            continue
+        sufficient = _holdout_sufficient(values, id_to_fail)
+        if sufficient is None:
+            entry.update(
+                status="skipped",
+                reason="recipe is not a binary flag on held-out rows "
+                       "(no frozen threshold to reuse)",
+            )
+            signal_verdicts.append(entry)
+            continue
+        cand = CandidateSignal(
+            name=str(c.get("name") or "signal"),
+            rationale=str(c.get("rationale") or ""),
+            recipe=recipe_dict,
+            sufficient=sufficient,
+        )
+        n_a, n_b = len(sufficient["a"]), len(sufficient["b"])
+        entry.update(
+            status="adjudicated",
+            n_holdout=n_a + n_b,
+            n_flagged=n_b,
+            fail_rate_flagged=round(sum(sufficient["b"]) / n_b, 4),
+            fail_rate_unflagged=round(sum(sufficient["a"]) / n_a, 4),
+        )
+        signal_verdicts.append(entry)
+        candidates.append(cand)
+
+    meta = adjudicate_signals(candidates, alpha=alpha, split_label="held_out")
+    by_name = {c.name: c for c in candidates}
+    for entry in signal_verdicts:
+        cand = by_name.get(str(entry.get("name")))
+        if entry.get("status") == "adjudicated" and cand is not None:
+            entry.update(
+                reject=bool(cand.reject),
+                effect=cand.effect,
+                ci=cand.ci,
+                fdr_corrected=bool(cand.fdr_corrected),
+            )
+
+    # ── 2. LLM judge per hypothesis, grounded in the held-out table ─────────
+    evidence_lines = [
+        f"- {e.get('name')}: flagged-group fail rate {e.get('fail_rate_flagged')} "
+        f"vs unflagged {e.get('fail_rate_unflagged')} "
+        f"(n={e.get('n_holdout')}, effect={e.get('effect')}, CI={e.get('ci')}, "
+        f"verdict={'REJECT H0' if e.get('reject') else 'not rejected'})"
+        for e in signal_verdicts
+        if e.get("status") == "adjudicated"
+    ] or ["- (no recipe could be re-evaluated on the held-out split)"]
+    evidence = "\n".join(evidence_lines)
+
+    hypothesis_verdicts: "list[dict[str, Any]]" = []
+    for h in hypotheses:
+        verdict = dict(h)
+        if judge is None:
+            verdict.update(verdict="not_judged", reasoning="", needs_surgery=True)
+            hypothesis_verdicts.append(verdict)
+            continue
+        prompt = JUDGE_PROMPT.format(
+            statement=h.get("statement", ""),
+            basis=h.get("basis", ""),
+            test_design=h.get("test_design", ""),
+            n_rows=len(rows),
+            evidence=evidence,
+        )
+        try:
+            raw = judge.generate(prompt) or ""
+        except Exception as exc:  # noqa: BLE001 — judge outage must not sink the verdicts
+            raw = ""
+            verdict["judge_error"] = str(exc)
+        parsed = _judge_json(raw)
+        if parsed and parsed.get("verdict") in {"supported", "partial", "refuted", "not_testable"}:
+            verdict.update(
+                verdict=parsed["verdict"],
+                reasoning=str(parsed.get("reasoning", "")),
+                needs_surgery=bool(parsed.get("needs_surgery", False)),
+            )
+        else:
+            verdict.update(
+                verdict="not_testable",
+                reasoning="judge output unparseable — defaulting to not_testable",
+                needs_surgery=True,
+            )
+        hypothesis_verdicts.append(verdict)
+
+    return {
+        "phase": "holdout_confirm",
+        "split": "held_out",
+        "n_validate_rows": len(rows),
+        "n_validate_fail": n_fail,
+        "alpha": alpha,
+        "outcome": {
+            "column": outcome_col,
+            "positive_label": positive_label,
+            "note": outcome_note,
+        },
+        "adjudication": meta,
+        "signal_verdicts": signal_verdicts,
+        "hypothesis_verdicts": hypothesis_verdicts,
+        "judge": judge_meta if judge is not None else None,
+    }

evalrx/analysis/hypothesis_agent.py

@@ -0,0 +1,230 @@
+"""M3 — HypothesisAgent: propose falsifiable hypotheses from an exploratory
+analysis report (an ``ExploratoryAnalysisAgent`` / M2 output).
+
+This is a lightweight, standalone counterpart to the diagnosis loop's
+``DiagnosisAgent`` (``evalrx.eval_agent.stages.diagnosis``): that one is
+hard-bound to the loop's own ``AnalysisReport`` type and only accepts a judge
+``Model`` (Gemini by default) — it cannot read an ``ExploratoryAnalysisReport``
+without raising or silently producing nothing (there is no adapter between the
+two report shapes). This module reads the standalone explorer's own report
+directly and supports the same ``judge``/``cli_config`` backend flexibility as
+``ExploratoryAnalysisAgent``, so it can run with whatever ``--backend`` the
+``evalrx explore`` CLI was given.
+
+Proposal only — no validation. A hypothesis here is a candidate explanation to
+investigate further, not a conclusion; there is no confirm/test phase wired up
+for it (see ``StatsAnalysisAgent``/``HypothesisTester`` in the diagnosis loop
+for that, a separate, currently out-of-scope system).
+"""
+
+from __future__ import annotations
+
+import logging
+import tempfile
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from evalrx.analysis.plain_language import jargon_violation
+from evalrx.analysis.prompts.hypothesis_agent import (
+    AGENT_COLUMN_MARKERS as _AGENT_COLUMN_MARKERS,
+)
+from evalrx.analysis.prompts.hypothesis_agent import (
+    AGENT_TRAJECTORY_HINT as _AGENT_TRAJECTORY_HINT,
+)
+from evalrx.analysis.prompts.hypothesis_agent import PLAIN_REPAIR_PROMPT as _PLAIN_REPAIR_PROMPT
+from evalrx.analysis.prompts.hypothesis_agent import PROPOSE_PROMPT as _PROPOSE_PROMPT
+
+
+def _is_agent_report(report: dict) -> bool:
+    """True when the exploratory report shows agent-trajectory column families.
+
+    Detection scans the question, takeaway titles/analysis, and candidate
+    signal names/rationales for the marker substrings — the same places the
+    propose prompt quotes, so the hint fires exactly when the model will see
+    trajectory columns.
+    """
+    pieces: list[str] = [str(report.get("question") or "")]
+    for t in report.get("takeaways") or []:
+        if isinstance(t, dict):
+            pieces.append(str(t.get("title", "")))
+            pieces.append(str(t.get("analysis", "")))
+    for s in report.get("candidate_signals") or []:
+        if isinstance(s, dict):
+            pieces.append(str(s.get("name", "")))
+            pieces.append(str(s.get("display_name", "")))
+            pieces.append(str(s.get("rationale", "")))
+    text = " ".join(pieces)
+    return any(marker in text for marker in _AGENT_COLUMN_MARKERS)
+
+if TYPE_CHECKING:
+    from evalrx.agent_runtime.cli_types import CliAgentConfig
+    from evalrx.core.model import Model
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class Hypothesis:
+    """One M3-proposed, falsifiable candidate explanation. Proposed only —
+    ``test_design`` names how it *could* be checked, not a verdict."""
+
+    statement: str = ""
+    plain_statement: str = ""
+    basis: str = ""
+    test_design: str = ""
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "statement": self.statement,
+            "plain_statement": self.plain_statement,
+            "basis": self.basis,
+            "test_design": self.test_design,
+        }
+
+
+class HypothesisAgent:
+    """M3: propose hypotheses from an ``ExploratoryAnalysisReport`` (as a dict).
+
+    Args:
+        judge:      LLM-like object with ``generate(prompt) -> str``.
+        cli_config: Optional CLI coding-agent backend (same as
+                    ``ExploratoryAnalysisAgent``'s ``cli_config``); used
+                    instead of ``judge`` when its provider isn't ``"llm"``.
+        timeout_sec: Timeout for a single generation call.
+    """
+
+    def __init__(
+        self,
+        judge: "Model | None" = None,
+        cli_config: "CliAgentConfig | None" = None,
+        timeout_sec: int = 90,
+    ) -> None:
+        self._judge = judge
+        self._cli_config = cli_config
+        self._timeout_sec = timeout_sec
+
+    @property
+    def available(self) -> bool:
+        return self._judge is not None or (
+            self._cli_config is not None and self._cli_config.provider != "llm"
+        )
+
+    def propose(self, report: dict[str, Any], *, max_items: int = 8) -> list[Hypothesis]:
+        """Read *report* (an ``ExploratoryAnalysisReport.to_dict()``-shaped
+        dict) and return 0-3 proposed hypotheses. Never raises — a backend
+        failure just yields no hypotheses."""
+        if not self.available:
+            return []
+
+        takeaways = [t for t in report.get("takeaways") or [] if isinstance(t, dict)][:max_items]
+        takeaways_text = "\n".join(
+            f"- {t.get('title', '')}: {t.get('analysis', '')}" for t in takeaways
+        ) or "(none recorded)"
+        observations = [str(o) for o in report.get("observations") or []][:max_items]
+        observations_text = "\n".join(f"- {o}" for o in observations) or "(none recorded)"
+        signals = [s for s in report.get("candidate_signals") or [] if isinstance(s, dict)][:max_items]
+        signals_text = "\n".join(
+            f"- {s.get('display_name') or s.get('name')}: {s.get('rationale', '')}" for s in signals
+        ) or "(none recorded)"
+
+        if not takeaways and not observations and not signals:
+            return []
+
+        prompt = _PROPOSE_PROMPT.format(
+            question=report.get("question") or "Explore this dataset.",
+            takeaways_text=takeaways_text,
+            observations_text=observations_text,
+            signals_text=signals_text,
+        )
+        if _is_agent_report(report):
+            prompt += "\n\n" + _AGENT_TRAJECTORY_HINT
+        try:
+            raw = self._generate(prompt)
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("HypothesisAgent: generation failed: %s", exc)
+            return []
+        hypotheses = _parse_hypotheses(raw)
+        return self._repair_plain_language(raw, hypotheses)
+
+    def _repair_plain_language(self, raw: str, hypotheses: list[Hypothesis]) -> list[Hypothesis]:
+        """One bounded retry: if any PLAIN line is missing or still jargon-y,
+        ask the backend to rewrite just those lines. Never raises — a repair
+        failure just keeps the original (possibly jargon-y) hypotheses."""
+        if not hypotheses:
+            return hypotheses
+        violations = [
+            f"hypothesis {i}'s PLAIN line {reason}: {h.plain_statement!r}"
+            for i, h in enumerate(hypotheses, start=1)
+            for reason in [jargon_violation(h.plain_statement, h.statement)]
+            if reason
+        ]
+        if not violations:
+            return hypotheses
+        try:
+            repaired_raw = self._generate(
+                _PLAIN_REPAIR_PROMPT.format(raw=raw, violations="\n".join(f"- {v}" for v in violations))
+            )
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("HypothesisAgent: plain-language repair failed: %s", exc)
+            return hypotheses
+        repaired = _parse_hypotheses(repaired_raw)
+        return repaired or hypotheses
+
+    def _generate(self, prompt: str) -> str:
+        if self._cli_config is not None and self._cli_config.provider != "llm":
+            from evalrx.agent_runtime.codegen import CodegenRunner
+
+            with tempfile.TemporaryDirectory(prefix="evalrx_m3_") as tmp:
+                res = CodegenRunner(self._cli_config).run(
+                    prompt,
+                    workdir=Path(tmp),
+                    timeout_sec=self._timeout_sec,
+                )
+                return res.raw_output or ""
+        return str(self._judge.generate(prompt))  # type: ignore[union-attr]
+
+
+def _parse_hypotheses(raw: str) -> list[Hypothesis]:
+    """Parse ``HYPOTHESIS:``/``PLAIN:``/``BASIS:``/``TEST:`` blocks out of
+    *raw*. ``PLAIN:`` is optional (older/repaired responses may omit it),
+    in which case ``plain_statement`` stays "".
+
+    For the CLI-agent backend, *raw* is the full rendered tool-call
+    trajectory, not just a final answer (``CliAgentResult.raw_output``'s
+    docstring: "assistant text, every Bash/Edit/Write/Read tool call +
+    result") — an agent that narrates its plan before giving a final answer
+    can restate the same hypothesis twice in one trajectory. Dedupe by
+    statement (case-insensitive), keeping the order hypotheses first
+    appeared but the most recent basis/test for a repeated statement, since
+    later restatements tend to be the more refined ones.
+    """
+    if "HYPOTHESIS:" not in raw.upper():
+        return []
+    by_statement: dict[str, dict[str, str]] = {}
+    order: list[str] = []
+    cur: dict[str, str] | None = None
+    cur_key: str = ""
+
+    def _flush() -> None:
+        if cur and cur.get("statement"):
+            if cur_key not in by_statement:
+                order.append(cur_key)
+            by_statement[cur_key] = cur
+
+    for line in raw.splitlines():
+        line = line.strip()
+        upper = line.upper()
+        if upper.startswith("HYPOTHESIS:"):
+            _flush()
+            statement = line.split(":", 1)[1].strip()
+            cur = {"statement": statement, "plain_statement": "", "basis": "", "test_design": ""}
+            cur_key = statement.lower()
+        elif upper.startswith("PLAIN:") and cur is not None:
+            cur["plain_statement"] = line.split(":", 1)[1].strip()
+        elif upper.startswith("BASIS:") and cur is not None:
+            cur["basis"] = line.split(":", 1)[1].strip()
+        elif upper.startswith("TEST:") and cur is not None:
+            cur["test_design"] = line.split(":", 1)[1].strip()
+    _flush()
+    return [Hypothesis(**by_statement[key]) for key in order]