evalrx 0.1.2__py3-none-any.whl

evalrx/eval_agent/hypothesis.py

@@ -0,0 +1,172 @@
+"""Hypotheses — the unit the self-evolving agent proposes, tests, and mutates.
+
+A hypothesis is a falsifiable claim about *when/why* a model fails (e.g. "Qwen
+mis-binds entities when two names share a surname"). The loop turns it into
+cases + an experiment, runs it, and updates its status from the findings.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any
+
+
+class HypothesisStatus(str, Enum):
+    PROPOSED = "proposed"        # generated, not yet tested
+    TESTING = "testing"          # experiment in flight
+    SUPPORTED = "supported"      # evidence backs it
+    REFUTED = "refuted"          # evidence contradicts it
+    INCONCLUSIVE = "inconclusive"
+
+
+@dataclass
+class Hypothesis:
+    """A falsifiable claim about a model's failure behaviour.
+
+    Attributes:
+        statement:              Natural-language claim (LLM-generated/readable).
+        target_model:           Registered model name the claim is about.
+        predicted_failure_mode: Tag/description of the expected failure.
+        plain_statement:        Plain-language explanation understandable by a layperson.
+        test_design:            How to verify this claim — analyzer / per-case
+                                signal / strategy-contrast keywords proposed by
+                                M3 (e.g. ``"relative_attention.max_relative_weight"``,
+                                ``"prompt_contrast describe_first"``).  M4 uses
+                                it to route evidence deterministically and M1
+                                uses it for cycle-2 targeted probing.
+        expected_association:   Pre-registered direction for the named test:
+                                ``higher_on_failures`` or ``lower_on_failures``.
+                                This prevents a protective-valued signal (for
+                                example ``n_correct``) from being interpreted
+                                backwards after its effect is observed.
+        status:                 Lifecycle state (:class:`HypothesisStatus`).
+        parent_id:              Hypothesis this one was mutated from, if any.
+        id:                     Stable identifier.
+        evidence:               Result/case ids accumulated while testing.
+        metadata:               Free-form extras.
+    """
+
+    statement: str
+    target_model: str
+    predicted_failure_mode: str
+    plain_statement: str = ""
+    test_design: str = ""
+    expected_association: str = ""
+    status: HypothesisStatus = HypothesisStatus.PROPOSED
+    parent_id: str | None = None
+    id: str = ""
+    evidence: list[str] = field(default_factory=list)
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+class HypothesisGenerator:
+    """Proposes new hypotheses — from scratch or by mutating prior ones.
+
+    Stage 2: an LLM-backed generator that reads the store (past supported/refuted
+    hypotheses and their evidence) and proposes the next ones to test.
+    """
+
+    def propose(self, context: Any = None) -> list[Hypothesis]:
+        raise NotImplementedError("HypothesisGenerator is planned for Stage 2.")
+
+    def mutate(self, hypothesis: Hypothesis, feedback: Any = None) -> list[Hypothesis]:
+        """Derive refined/adjacent hypotheses from one that was tested."""
+        raise NotImplementedError("HypothesisGenerator.mutate is planned for Stage 2.")
+
+
+# ---------------------------------------------------------------------------
+# Serialization helpers (needed by JsonlStore and loop checkpointing)
+# ---------------------------------------------------------------------------
+
+
+def hypothesis_to_dict(h: Hypothesis) -> dict[str, Any]:
+    """Serialize a Hypothesis to a JSON-compatible dict."""
+    plain = h.plain_statement or (h.metadata.get("plain_statement", "") if h.metadata else "")
+    return {
+        "statement": h.statement,
+        "plain_statement": plain,
+        "target_model": h.target_model,
+        "predicted_failure_mode": h.predicted_failure_mode,
+        "test_design": h.test_design,
+        "expected_association": h.expected_association,
+        "status": h.status.value if h.status else HypothesisStatus.PROPOSED.value,
+        "parent_id": h.parent_id,
+        "id": h.id,
+        "evidence": list(h.evidence),
+        "metadata": dict(h.metadata),
+    }
+
+
+def hypothesis_from_dict(data: dict[str, Any]) -> Hypothesis:
+    """Deserialize a Hypothesis from a dict (e.g., loaded from JSONL)."""
+    raw_status = data.get("status", HypothesisStatus.PROPOSED.value)
+    try:
+        status = HypothesisStatus(raw_status)
+    except ValueError:
+        status = HypothesisStatus.PROPOSED
+    plain = data.get("plain_statement") or (data.get("metadata", {}).get("plain_statement", "") if isinstance(data.get("metadata"), dict) else "")
+    return Hypothesis(
+        statement=str(data.get("statement", "")),
+        target_model=str(data.get("target_model", "")),
+        predicted_failure_mode=str(data.get("predicted_failure_mode", "")),
+        plain_statement=plain,
+        test_design=str(data.get("test_design", "")),
+        expected_association=str(data.get("expected_association", "")),
+        status=status,
+        parent_id=data.get("parent_id"),
+        id=data.get("id", ""),
+        evidence=list(data.get("evidence", [])),
+        metadata=dict(data.get("metadata", {})),
+    )
+
+
+def _hyp_val(obj: Any, name: str, default: Any = None) -> Any:
+    """Read an attribute or mapping key, whichever *obj* is."""
+    if isinstance(obj, dict):
+        return obj.get(name, default)
+    return getattr(obj, name, default)
+
+
+def hypothesis_id(hypothesis: Any) -> str:
+    """Stable join key for one hypothesis, shared by every M3/M4/M5 log entry.
+
+    ``Hypothesis.id`` defaults to ``""`` and nothing in the M1-M5 loop fills it
+    in, so every stage that names a hypothesis has to derive one — deriving it
+    per call site is how the same object ends up with two different names from
+    two different stages, and the join between a claim and its verdict goes
+    silently empty. Falls back to a hash of the statement, mirroring
+    ``eval_agent.loop._hyp_key`` — deterministic, so two stages holding the
+    same hypothesis always agree. ``evalrx.contract.emit.hypothesis_id``
+    delegates here so the optional contract sidecar and the primary log never
+    disagree on the id.
+    """
+    hid = str(_hyp_val(hypothesis, "id", "") or "").strip()
+    if hid:
+        return hid
+    statement = str(_hyp_val(hypothesis, "statement", "") or "").strip()
+    if statement:
+        return "h-" + hashlib.sha1(statement.encode("utf-8")).hexdigest()[:12]
+    return "unknown"
+
+
+class ManualHypothesisGenerator(HypothesisGenerator):
+    """A non-LLM generator: drains a fixed queue, or calls an injected ``proposer``.
+
+    Lets the loop run + be unit-tested without an LLM; swap in an LLM-backed
+    generator later without touching the loop.
+    """
+
+    def __init__(self, hypotheses: list[Hypothesis] | None = None, proposer: Any = None) -> None:
+        self._queue: list[Hypothesis] = list(hypotheses or [])
+        self._proposer = proposer
+
+    def propose(self, context: Any = None) -> list[Hypothesis]:
+        if self._proposer is not None:
+            return list(self._proposer(context))
+        out, self._queue = self._queue, []
+        return out
+
+    def mutate(self, hypothesis: Hypothesis, feedback: Any = None) -> list[Hypothesis]:
+        return []

evalrx/eval_agent/label_quarantine.py

@@ -0,0 +1,209 @@
+"""Hold a run directory's label-bearing artifacts in memory while the fix
+stage runs, so nothing the repair coder or its sandbox can read from disk
+reveals gold answers or per-case pass/fail.
+
+Why this exists
+---------------
+The fix stage already withholds labels at the *prompt* level: the proposer sees
+EXPLORE cases only, the coded-pipeline payload carries ``id`` / ``prompt`` /
+``baseline_output`` and nothing else, and every FixAgent prompt says so. But the
+coder CLI runs with ``Bash Edit Write Read`` and the pipeline sandbox is a plain
+subprocess, both rooted *inside* the run directory — where, by the time the fix
+stage starts, M1..M4 have written the same labels many times over:
+
+* ``baseline.json`` and ``logs/report/discovery_cases.json`` — every case with
+  ``expected`` and ``label``, CONFIRM cases included;
+* ``logs/run.json`` — ``case_record`` events with ``expected``, ``label``
+  and ``metadata.gold``;
+* ``logs/artifacts/*.result.json``, ``logs/contract/c*.m1.json``,
+  ``explore/records.json`` — per-case analyzer signals, among them
+  ``gold_yes``, which *is* the gold answer on a yes/no task;
+* ``logs/workspace/post_m5/cases.json`` — the M5 experiment's cases with gold.
+
+``cat ../../../baseline.json`` from a fix workspace is two directory levels
+away. A candidate that joins its cases to any of these by id can hard-code the
+held-out answers; the frozen-model control catches the crude lookup-table form
+but not one that also manufactures consensus support through the bridge.
+
+What it does
+------------
+:func:`quarantine_run_dir` is a context manager. On entry it reads every
+regular file under the run directory into memory and unlinks it (directory
+names stay; they carry nothing). Files handed in as ``append_logs`` — the
+structured event log, which a :class:`logging.FileHandler` holds open in
+append mode — are truncated in place instead, so events the fix stage writes
+still land in the same file. On exit (also on an exception) everything is put
+back byte-for-byte; an append log becomes ``old + what the fix stage wrote``;
+a path the fix stage re-created with new content keeps the new content and
+the pre-fix bytes are saved beside it as ``<name>.pre_fix`` — nothing
+is ever lost. A ``fix_quarantine.json`` manifest is written on exit so the run
+records exactly which files were hidden.
+
+What it cannot do
+-----------------
+It only governs what *this run wrote*. A dataset manifest on a bind mount
+(``/app/work/data/<dataset>/manifest.json`` in the benchmark containers)
+carries ``gold`` too and cannot be hidden from a root process in the same
+container — that needs uid separation or host-side label delivery, which is
+out of scope here. The gap is deliberate and documented, not forgotten.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+from collections.abc import Iterable, Iterator
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+MANIFEST_NAME = "fix_quarantine.json"
+
+
+@dataclass
+class QuarantineReport:
+    """What was hidden and how it was restored (also written as JSON)."""
+
+    root: Path
+    hidden: list[str] = field(default_factory=list)
+    truncated: list[str] = field(default_factory=list)
+    kept: list[str] = field(default_factory=list)
+    restored: list[str] = field(default_factory=list)
+    merged: list[str] = field(default_factory=list)
+    conflicts: list[str] = field(default_factory=list)
+    n_bytes: int = 0
+
+    def to_json(self) -> dict:
+        return {
+            "root": str(self.root),
+            "n_hidden": len(self.hidden),
+            "n_bytes": self.n_bytes,
+            "hidden": sorted(self.hidden),
+            "truncated_in_place": sorted(self.truncated),
+            "kept_visible": sorted(self.kept),
+            "merged_append_logs": sorted(self.merged),
+            "conflicts_saved_as_pre_fix": sorted(self.conflicts),
+        }
+
+
+def _under(rel: str, prefixes: tuple[str, ...]) -> bool:
+    return any(rel == p or rel.startswith(p.rstrip("/") + "/") for p in prefixes)
+
+
+def _rel(path: Path, root: Path) -> str:
+    return path.relative_to(root).as_posix()
+
+
+@contextmanager
+def quarantine_run_dir(
+    run_dir: "str | os.PathLike[str]",
+    *,
+    append_logs: Iterable["str | os.PathLike[str]"] = (),
+    rewrite_logs: Iterable["str | os.PathLike[str]"] = (),
+    keep: Iterable[str] = (),
+    write_manifest: bool = True,
+) -> Iterator[QuarantineReport]:
+    """Hide every file under *run_dir* for the duration of the block.
+
+    Args:
+        run_dir:      The run directory (``baseline.json``, ``logs/``, ...).
+        append_logs:  Files that a live handler holds open in append mode.
+                      They are truncated to zero length instead of unlinked
+                      and restored as ``old + new``. RunLoggerV2 rewrites its
+                      documents atomically instead of appending, so its
+                      callers pass ``rewrite_logs`` (see
+                      ``RunLoggerV2.managed_json_paths``), not this.
+        keep:         Relative path prefixes (POSIX style) to leave visible,
+                      e.g. ``("logs/fixes",)`` when resuming a fix search.
+        write_manifest: Write ``fix_quarantine.json`` into *run_dir* on exit.
+
+    Yields the :class:`QuarantineReport`, filled in as the block runs.
+    """
+    root = Path(run_dir).resolve()
+    keep_prefixes = tuple(k.strip("/") for k in keep if k.strip("/"))
+    append_rel = set()
+    for p in append_logs:
+        p = Path(p).resolve()
+        try:
+            append_rel.add(_rel(p, root))
+        except ValueError:
+            logger.warning("label quarantine: append log %s is outside %s; ignored", p, root)
+    rewrite_rel = set()
+    for p in rewrite_logs:
+        p = Path(p).resolve()
+        try:
+            rewrite_rel.add(_rel(p, root))
+        except ValueError:
+            logger.warning("label quarantine: rewrite log %s is outside %s; ignored", p, root)
+
+    report = QuarantineReport(root=root)
+    stash: dict[str, bytes] = {}
+
+    if root.is_dir():
+        for dirpath, _dirnames, filenames in os.walk(root):
+            for name in filenames:
+                path = Path(dirpath) / name
+                if path.is_symlink():
+                    continue
+                rel = _rel(path, root)
+                if rel == MANIFEST_NAME or _under(rel, keep_prefixes):
+                    report.kept.append(rel)
+                    continue
+                data = path.read_bytes()
+                stash[rel] = data
+                report.n_bytes += len(data)
+                if rel in append_rel:
+                    # A logging.FileHandler appends through an O_APPEND fd:
+                    # after truncation its next write lands at offset 0.
+                    with path.open("r+b") as handle:
+                        handle.truncate(0)
+                    report.truncated.append(rel)
+                else:
+                    path.unlink()
+                report.hidden.append(rel)
+    logger.info(
+        "label quarantine: hid %d file(s), %d bytes, under %s for the fix stage",
+        len(report.hidden), report.n_bytes, root,
+    )
+
+    try:
+        yield report
+    finally:
+        for rel, old in stash.items():
+            path = root / rel
+            path.parent.mkdir(parents=True, exist_ok=True)
+            if rel in append_rel:
+                new = path.read_bytes() if path.exists() else b""
+                path.write_bytes(old + new)
+                report.merged.append(rel)
+            elif rel in rewrite_rel and path.exists() and path.stat().st_size > 0:
+                # Atomic JSON loggers rewrite a complete in-memory document;
+                # that new document already includes everything in `old`.
+                # Keeping it avoids a redundant, non-JSON `.pre_fix` sibling.
+                report.merged.append(rel)
+            elif path.exists() and path.stat().st_size > 0:
+                # Appended AFTER the extension so a `*.json` scan (the contract
+                # index) never mistakes the copy for a payload.
+                side = path.with_name(path.name + ".pre_fix")
+                side.write_bytes(old)
+                report.conflicts.append(rel)
+                logger.warning(
+                    "label quarantine: %s was rewritten during the fix stage; "
+                    "pre-fix content saved as %s", rel, side.name,
+                )
+            else:
+                path.write_bytes(old)
+            report.restored.append(rel)
+        if write_manifest and root.is_dir():
+            payload = report.to_json()
+            payload["restored_at"] = datetime.now(timezone.utc).isoformat()
+            (root / MANIFEST_NAME).write_text(
+                json.dumps(payload, indent=2) + "\n", encoding="utf-8"
+            )
+
+
+__all__ = ["MANIFEST_NAME", "QuarantineReport", "quarantine_run_dir"]