evalrx 0.1.2__py3-none-any.whl

evalrx/analyzers/reasoning/self_repair.py

@@ -0,0 +1,246 @@
+"""Self-repair — can the model detect its own error, and can it fix it without breaking what worked?
+
+Three capabilities get conflated as "self-correction", and they dissociate:
+
+  * **detection** — does the model know the answer is wrong when asked?
+  * **correction** — given that it says so, does the revision land on the gold?
+  * **damage** — does the same revision pass BREAK answers that were already right?
+
+The third is the one that decides deployment.  A self-critique loop with a 40%
+repair rate and a 25% damage rate is a net loss on any batch that is mostly
+passing, and reporting repair alone hides that completely — which is why this
+probe deliberately spends its generations on PASS cases too.
+
+Cost: 2 generations per case (critique + blind revision), 3 with
+``revise_with_critique=True``.
+
+References:
+- Large Language Models Cannot Self-Correct Reasoning Yet — Huang et al.,
+  ICLR 2024 — arXiv:2310.01798 (intrinsic self-correction degrades accuracy)
+- Self-Refine: Iterative Refinement with Self-Feedback — Madaan et al.,
+  NeurIPS 2023 — arXiv:2303.17651
+- Teaching Large Language Models to Self-Debug — Chen et al., ICLR 2024 —
+  arXiv:2304.05128
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import re
+from typing import TYPE_CHECKING, Any, Callable, Optional
+
+from evalrx.analyzers.reasoning._text import (
+    answer_equal,
+    extract_answer,
+    normalize_answer,
+)
+from evalrx.core.analyzer import Analyzer
+from evalrx.core.capability import Capability
+from evalrx.core.registry import register_analyzer
+from evalrx.core.result import Result
+
+if TYPE_CHECKING:
+    from evalrx.core.case import CaseBatch, FailureCase
+    from evalrx.core.model import Model
+
+_CRITIQUE = (
+    "Here is a candidate answer to the question above:\n\n{answer}\n\n"
+    "Is this answer correct? Reply with exactly one word: CORRECT or INCORRECT."
+)
+_REVISE_BLIND = (
+    "Here is a candidate answer to the question above:\n\n{answer}\n\n"
+    "Review it and produce your best final answer. Give the answer on its own "
+    "last line as 'Answer: <answer>'."
+)
+_REVISE_TOLD = (
+    "Here is a candidate answer to the question above:\n\n{answer}\n\n"
+    "This answer contains an error. Find it and produce a corrected final "
+    "answer on its own last line as 'Answer: <answer>'."
+)
+
+
+@register_analyzer("self_repair")
+class SelfRepairAnalyzer(Analyzer):
+    """Measure self-detection, repair rate, and — the part that decides deployment — damage rate.
+
+    Hyper-parameters:
+        revise_with_critique: also run the "you made an error" revision
+                              (+1 generation; measures repair under an oracle
+                              error signal the deployed loop will not have).
+        max_cases:            label-stratified cap; 0 (the default) = every case.
+                              PASS cases are REQUIRED here, they are what makes
+                              damage measurable.
+        generate_missing:     produce a baseline answer when ``observed`` is empty.
+        grader:               ``callable(prediction, case) -> bool | None``.
+        answer_fn:            ``callable(text) -> str`` answer extractor.
+    """
+
+    name = "self_repair"
+    requires = frozenset({Capability.GENERATE})
+    applies_to_modalities = frozenset({"text", "image"})
+
+    def __init__(
+        self,
+        revise_with_critique: bool = False,
+        max_cases: int = 0,
+        generate_missing: bool = True,
+        grader: Optional[Callable[[Any, "FailureCase"], Optional[bool]]] = None,
+        answer_fn: Optional[Callable[[Any], str]] = None,
+    ) -> None:
+        super().__init__(
+            revise_with_critique=revise_with_critique,
+            max_cases=max_cases,
+            generate_missing=generate_missing,
+        )
+        # ctor names, so sklearn-style get_params() reflection works
+        self.grader = grader or _default_grader
+        self.answer_fn = answer_fn or extract_answer
+
+    def _run(self, model: "Model", cases: "CaseBatch") -> Result:
+        per_case: list[dict[str, Any]] = []
+        for case in cases.stratified_head(self.max_cases):
+            baseline = case.observed
+            if not str(baseline or "").strip():
+                if not self.generate_missing:
+                    per_case.append(
+                        {"sample_id": case.id, "skipped": "no stored answer to revise"}
+                    )
+                    continue
+                baseline = str(model.generate(case.inputs))
+            entry = self._probe_case(model, case, str(baseline))
+            per_case.append(entry)
+
+        graded = [c for c in per_case if "baseline_correct" in c]
+        failed = [c for c in graded if c["baseline_correct"] == 0]
+        passed = [c for c in graded if c["baseline_correct"] == 1]
+        # A real model sometimes answers the critique in prose that parses to
+        # nothing; those cases carry self_says_incorrect=None and must be
+        # dropped from the detection rates rather than counted as "said correct".
+        detected = [c for c in graded if c.get("detection_correct") is not None]
+        verdicts = [c for c in passed if c.get("self_says_incorrect") is not None]
+
+        repair_rate = (
+            round(sum(c["repaired"] for c in failed) / len(failed), 4) if failed else None
+        )
+        damage_rate = (
+            round(sum(c["damaged"] for c in passed) / len(passed), 4) if passed else None
+        )
+        findings: dict[str, Any] = {
+            "n_cases": len(per_case),
+            "n_graded": len(graded),
+            "n_baseline_fail": len(failed),
+            "n_baseline_pass": len(passed),
+            "detection_accuracy": (
+                round(sum(c["detection_correct"] for c in detected) / len(detected), 4)
+                if detected
+                else None
+            ),
+            "n_critique_unparsed": sum(
+                1 for c in graded if c.get("self_says_incorrect") is None
+            ),
+            "false_alarm_rate": (
+                round(sum(c["self_says_incorrect"] for c in verdicts) / len(verdicts), 4)
+                if verdicts
+                else None
+            ),
+            "repair_rate": repair_rate,
+            "damage_rate": damage_rate,
+            "net_revision_gain": (
+                round(
+                    (sum(c["repaired"] for c in failed) - sum(c["damaged"] for c in passed))
+                    / len(graded),
+                    4,
+                )
+                if graded
+                else None
+            ),
+            "per_case": per_case,
+            "_caveat": (
+                "repair_rate WITHOUT damage_rate is not a result: a revision "
+                "pass is only a fix if net_revision_gain > 0 on the deployed "
+                "PASS/FAIL mix, and this batch is label-stratified, so its mix "
+                "is NOT the deployment mix — re-weight before promoting an L1 "
+                "self-critique fix. Cases needing PASS mass: damage_rate is "
+                "None when the batch has no passing cases, and a repair-only "
+                "reading is then unsafe. With revise_with_critique=True the "
+                "told_* columns use an ORACLE error signal (we told it there is "
+                "an error) and overstate what a deployed loop achieves. "
+                "INTERVENTIONAL — held-out confirmation must re-run every call."
+            ),
+        }
+        return Result(analyzer=self.name, model=repr(model), cases=cases, findings=findings)
+
+    # ------------------------------------------------------------------
+    def _probe_case(self, model: "Model", case: "FailureCase", baseline: str) -> dict[str, Any]:
+        entry: dict[str, Any] = {"sample_id": case.id}
+        base_grade = self.grader(baseline, case)
+
+        critique = str(model.generate(self._ask(case, _CRITIQUE, baseline)))
+        says_incorrect = _parse_verdict(critique)
+        revised = str(model.generate(self._ask(case, _REVISE_BLIND, baseline)))
+        revised_grade = self.grader(revised, case)
+
+        entry["self_says_incorrect"] = (
+            int(says_incorrect) if says_incorrect is not None else None
+        )
+        entry["critique_parsed"] = int(says_incorrect is not None)
+        entry["changed_answer"] = int(
+            normalize_answer(self.answer_fn(revised))
+            != normalize_answer(self.answer_fn(baseline))
+        )
+        if base_grade is None or revised_grade is None:
+            entry["skipped"] = "ungradable (no gold answer)"
+            return entry
+
+        entry.update(
+            {
+                "baseline_correct": int(base_grade),
+                "revised_correct": int(revised_grade),
+                "repaired": int(not base_grade and revised_grade),
+                "damaged": int(base_grade and not revised_grade),
+            }
+        )
+        if says_incorrect is not None:
+            entry["detection_correct"] = int(says_incorrect == (not base_grade))
+
+        if self.revise_with_critique:
+            told = str(model.generate(self._ask(case, _REVISE_TOLD, baseline)))
+            told_grade = self.grader(told, case)
+            if told_grade is not None:
+                entry["told_correct"] = int(told_grade)
+                entry["told_repaired"] = int(not base_grade and told_grade)
+                entry["told_damaged"] = int(base_grade and not told_grade)
+        return entry
+
+    @staticmethod
+    def _ask(case: "FailureCase", template: str, answer: str):
+        prompt = f"{case.inputs.prompt or ''}\n\n{template.format(answer=answer)}"
+        return dataclasses.replace(case.inputs, prompt=prompt)
+
+
+def _default_grader(prediction: Any, case: "FailureCase") -> Optional[bool]:
+    if case.expected is None:
+        return None
+    return answer_equal(extract_answer(prediction), case.expected)
+
+
+#: Negated forms matter: "not correct" is a verdict of INCORRECT, and a plain
+#: search for "correct" reads it backwards.  ``\bcorrect\b`` cannot match inside
+#: "incorrect" (no word boundary there), so the two patterns stay disjoint.
+_SAYS_INCORRECT = re.compile(
+    r"\b(?:incorrect|wrong|false|not\s+(?:correct|right)|isn'?t\s+(?:correct|right))\b"
+)
+_SAYS_CORRECT = re.compile(r"\b(?:correct|right|yes)\b")
+
+
+def _parse_verdict(text: Any) -> Optional[bool]:
+    """``True`` = the model called the answer incorrect; ``None`` = unparseable."""
+    lowered = str(text or "").lower()
+    # Take whichever verdict appears FIRST — models restate the question
+    # ("is this correct? ...") before delivering the verdict.
+    hits: list[tuple[int, bool]] = [
+        (m.start(), True) for m in _SAYS_INCORRECT.finditer(lowered)
+    ] + [(m.start(), False) for m in _SAYS_CORRECT.finditer(lowered)]
+    if not hits:
+        return None
+    return min(hits)[1]

evalrx/analyzers/reasoning/step_rollout_value.py

@@ -0,0 +1,216 @@
+"""Step-value rollouts — *where* did the reasoning chain break?
+
+Knowing that a chain failed is not actionable; knowing it was already doomed at
+step 2 is.  Math-Shepherd's insight is that a step's value can be estimated
+without any human step labels: continue from that prefix ``n`` times and count
+how often the completion reaches the gold answer.  The step where that value
+collapses is the break point — everything after it is downstream noise, and a
+fix targeted at the last visible mistake is aimed at the wrong place.
+
+This is the text counterpart of the agent-side ``first_error_judge``: same
+question (which step is the first error?), answered by rollout statistics
+instead of an LLM judge, so it carries no judge bias — at the cost of
+``n_steps × n_rollouts`` generations — on a large batch pass ``max_cases`` to
+bound it (the default, like every analyzer, measures every case).
+
+References:
+- Math-Shepherd: Verify and Reinforce LLMs Step-by-step without Human
+  Annotations — Wang et al., ACL 2024 — arXiv:2312.08935
+- Let's Verify Step by Step — Lightman et al., ICLR 2024 — arXiv:2305.20050
+- Improve Mathematical Reasoning with Process Supervision — Luo et al., 2024 —
+  arXiv:2406.06592 (binary-search variant of the same estimator)
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import re
+from typing import TYPE_CHECKING, Any, Callable, Optional
+
+from evalrx.analyzers.reasoning._text import answer_equal, extract_answer
+from evalrx.core.analyzer import Analyzer
+from evalrx.core.capability import Capability
+from evalrx.core.registry import register_analyzer
+from evalrx.core.result import Result
+
+if TYPE_CHECKING:
+    from evalrx.core.case import CaseBatch, FailureCase
+    from evalrx.core.model import Model
+
+_COT_SUFFIX = (
+    "Think step by step, one step per line. After the reasoning, give the final "
+    "answer on its own last line as 'Answer: <answer>'."
+)
+_CONTINUE = (
+    "Reasoning so far:\n{prefix}\n\nContinue from here and finish the solution. "
+    "Give the final answer on its own last line as 'Answer: <answer>'."
+)
+_ANSWER_LINE = re.compile(r"^\s*(?:final\s+)?answer\s*[:=]", re.IGNORECASE)
+
+
+def split_steps(text: Any, max_steps: int) -> list[str]:
+    """Split a chain into reasoning steps (lines first, sentences as fallback)."""
+    raw = str(text or "")
+    lines = [ln.strip() for ln in raw.splitlines()]
+    steps = [ln for ln in lines if ln and not _ANSWER_LINE.match(ln)]
+    if len(steps) < 2:  # single-paragraph chain — fall back to sentences
+        body = _ANSWER_LINE.split(raw)[0]
+        steps = [s.strip() for s in re.split(r"(?<=[.!?])\s+", body) if s.strip()]
+    if len(steps) <= max_steps:
+        return steps
+    # keep the shape of the chain: evenly spaced cut points, always incl. the last
+    idx = sorted({round(i * (len(steps) - 1) / (max_steps - 1)) for i in range(max_steps)})
+    return [steps[i] for i in idx]
+
+
+@register_analyzer("step_rollout_value")
+class StepRolloutValueAnalyzer(Analyzer):
+    """Estimate a per-step success value by rollouts and locate the break step.
+
+    Hyper-parameters:
+        n_rollouts:     completions sampled per step prefix.
+        max_steps:      cap on probed steps (the chain is subsampled evenly).
+        drop_threshold: value drop between consecutive steps that counts as the break.
+        max_cases:      label-stratified cap — cost is n_steps × n_rollouts each; 0 (the default) = every case.
+        gen_kwargs:     passed to ``model.generate`` (rollouts NEED temperature > 0;
+                        a deterministic model makes every rollout identical and the
+                        values collapse to 0/1 — the caveat says so).
+        grader/answer_fn: grading of each rollout against ``case.expected``.
+    """
+
+    name = "step_rollout_value"
+    requires = frozenset({Capability.GENERATE})
+    applies_to_modalities = frozenset({"text", "image"})
+
+    def __init__(
+        self,
+        n_rollouts: int = 3,
+        max_steps: int = 7,
+        drop_threshold: float = 0.34,
+        max_cases: int = 0,
+        gen_kwargs: Optional[dict] = None,
+        grader: Optional[Callable[[Any, "FailureCase"], Optional[bool]]] = None,
+        answer_fn: Optional[Callable[[Any], str]] = None,
+    ) -> None:
+        super().__init__(
+            n_rollouts=n_rollouts,
+            max_steps=max_steps,
+            drop_threshold=drop_threshold,
+            max_cases=max_cases,
+            gen_kwargs=dict(gen_kwargs or {}),
+        )
+        self.grader = grader or _default_grader
+        self.answer_fn = answer_fn or extract_answer
+
+    def _run(self, model: "Model", cases: "CaseBatch") -> Result:
+        per_case: list[dict[str, Any]] = []
+        step_values_by_case: dict[str, list[float]] = {}
+        for case in cases.stratified_head(self.max_cases):
+            if case.expected is None:
+                per_case.append({"sample_id": case.id, "skipped": "no gold answer"})
+                continue
+            chain = case.observed
+            if not str(chain or "").strip():
+                chain = str(
+                    model.generate(
+                        dataclasses.replace(
+                            case.inputs, prompt=f"{case.inputs.prompt or ''}\n\n{_COT_SUFFIX}"
+                        )
+                    )
+                )
+            entry = self._probe_case(model, case, str(chain))
+            # Contract: a per-case row carries one level of scalars — a numeric
+            # vector there looks like a signal and reaches no statistic. The
+            # trajectory moves to findings["step_values_by_case"]; every scalar
+            # the stats read (initial/final/min/max_value_drop/break_step_idx)
+            # stays on the row.
+            values = entry.pop("step_values", None)
+            if values is not None:
+                step_values_by_case[entry["sample_id"]] = values
+            per_case.append(entry)
+
+        scored = [c for c in per_case if "break_step_idx" in c]
+        breaks = [c["break_step_idx"] for c in scored if c["break_step_idx"] is not None]
+        findings: dict[str, Any] = {
+            "n_cases": len(per_case),
+            "n_scored": len(scored),
+            "n_rollouts": self.n_rollouts,
+            "gen_kwargs": dict(self.gen_kwargs),
+            "mean_initial_value": _mean([c["initial_value"] for c in scored]),
+            "mean_final_value": _mean([c["final_value"] for c in scored]),
+            "mean_break_depth": _mean([c["break_depth"] for c in scored
+                                       if c.get("break_depth") is not None]),
+            "n_with_break": len(breaks),
+            "per_case": per_case,
+            "step_values_by_case": step_values_by_case,
+            "_caveat": (
+                "Step values are Monte-Carlo estimates from n_rollouts "
+                "completions: with n_rollouts=3 a value is one of {0, .33, .67, "
+                "1} and a single-step 'drop' of 0.33 is within sampling noise — "
+                "raise n_rollouts before trusting a break_step_idx on an "
+                "individual case, and prefer the batch-level mean_break_depth. "
+                "The estimator REQUIRES sampling: at temperature 0 every rollout "
+                "from a prefix is identical, values are 0/1, and break_step_idx "
+                "degenerates into 'the first step whose continuation is wrong'. "
+                "initial_value is the model's own unconditional success rate — a "
+                "chain that starts near 0 has no break point to find, it never "
+                "worked. INTERVENTIONAL: held-out confirmation must re-roll."
+            ),
+        }
+        return Result(analyzer=self.name, model=repr(model), cases=cases, findings=findings)
+
+    # ------------------------------------------------------------------
+    def _probe_case(self, model: "Model", case: "FailureCase", chain: str) -> dict[str, Any]:
+        steps = split_steps(chain, self.max_steps)
+        entry: dict[str, Any] = {"sample_id": case.id, "n_steps": len(steps)}
+        if not steps:
+            entry["skipped"] = "no reasoning steps to roll out from"
+            return entry
+
+        values: list[float] = []
+        for i in range(len(steps)):
+            prefix = "\n".join(steps[: i + 1])
+            prompt = f"{case.inputs.prompt or ''}\n\n{_CONTINUE.format(prefix=prefix)}"
+            hits = 0
+            graded = 0
+            for _ in range(self.n_rollouts):
+                out = model.generate(
+                    dataclasses.replace(case.inputs, prompt=prompt), **self.gen_kwargs
+                )
+                verdict = self.grader(out, case)
+                if verdict is None:
+                    continue
+                graded += 1
+                hits += int(verdict)
+            values.append(round(hits / graded, 4) if graded else 0.0)
+
+        entry["step_values"] = values
+        entry["initial_value"] = values[0]
+        entry["final_value"] = values[-1]
+        entry["min_value"] = min(values)
+        drops = [values[i - 1] - values[i] for i in range(1, len(values))]
+        worst = max(drops) if drops else 0.0
+        entry["max_value_drop"] = round(worst, 4)
+        break_idx = next(
+            (i + 1 for i, d in enumerate(drops) if d >= self.drop_threshold), None
+        )
+        entry["break_step_idx"] = break_idx
+        entry["break_depth"] = (
+            round(break_idx / max(len(steps) - 1, 1), 4) if break_idx is not None else None
+        )
+        entry["break_step_text"] = steps[break_idx][:200] if break_idx is not None else None
+        # A chain whose value never recovers is unsalvageable by continuation;
+        # one that dips and recovers is a detour, not a break.
+        entry["recoverable"] = int(values[-1] > 0)
+        return entry
+
+
+def _mean(values: list) -> Optional[float]:
+    clean = [v for v in values if v is not None]
+    return round(sum(clean) / len(clean), 4) if clean else None
+
+
+def _default_grader(prediction: Any, case: "FailureCase") -> Optional[bool]:
+    if case.expected is None:
+        return None
+    return answer_equal(extract_answer(prediction), case.expected)