evalrx 0.1.2__py3-none-any.whl

evalrx/eval_agent/prompts/hypothesis_tester.py

@@ -0,0 +1,15 @@
+"""Prompt templates for M4 hypothesis testing."""
+
+_CONSISTENCY_PROMPT = """\
+Experiment protocol:
+{protocol_text}
+
+Hypothesis under review:
+Statement: {statement}
+Predicted failure mode: {failure_mode}
+
+Does this hypothesis address a failure mode that is relevant to the experiment protocol above?
+Answer YES or NO on the first line, then give a one-sentence reason.
+
+Answer YES if the hypothesis explains a failure that the protocol would care about.
+Answer NO if the hypothesis is about something the protocol does not mention or test."""

evalrx/eval_agent/prompts/nl_runner.py

@@ -0,0 +1,38 @@
+"""Prompt templates for natural-language diagnosis scaffolding."""
+
+_AGENT_PROMPT_TEMPLATE = """\
+Generate a complete `run.py` for an EvalRX diagnosis experiment.
+
+DESCRIPTION OF FAILURE TO DIAGNOSE
+===================================
+{description}
+
+TARGET MODEL
+============
+{model_key}
+
+REQUIREMENTS
+============
+1. Import from `evalrx` and `evalrx.eval_agent`.
+2. Create an `ExperimentProtocol` whose `description` field captures the
+   failure described above (verbatim or paraphrased).
+3. Build a `CaseBatch` with 6-10 `FailureCase` objects that probe the
+   described failure.  Include at least two "easy" control cases that the
+   model should PASS and several "hard" cases that expose the failure.
+   Use `Inputs(prompt=..., image=...)` — image is optional for text-only failures.
+4. Load the model with `compose("{model_key}", "hf_local", ...)`.
+5. Run `VLDiagnoseLoop` with `ProbeAgent`, `StatsAnalysisAgent`,
+   `DiagnosisAgent`, and `AgyModel` as judge (with a try/except fallback
+   when the agy binary is absent).
+6. Accept CLI flags: --model, --device, --dtype, --max-cycles,
+   --max-analyzers, --smoke-test, --run-dir.
+7. Write outputs via `RunContext(args.run_dir)` (from `evalrx.eval_agent`),
+   defaulting `--run-dir` to `Path(__file__).parent / "outputs"`. This
+   defaults to the V2 logger (`run.json` + `M1..M5/log.json`, no flat
+   `run_log.jsonl`, no `README.txt`/`manifest.json`) — don't assume those
+   V1-only files exist when printing a completion summary.
+8. Include a `--smoke-test` path with a `_SmokeModel` stand-in so the
+   script can be verified without a GPU.
+
+Write ONLY `run.py` — nothing else.  No markdown, no explanation.
+"""

evalrx/eval_agent/prompts/probe_agent.py

@@ -0,0 +1,25 @@
+"""Prompt templates for M1 probe-agent analyzer selection."""
+
+_SELECTION_PROMPT_TMPL = """\
+You are selecting diagnostic analyzers for a model evaluation experiment.
+Choose the analyzers that would surface the most useful evidence given the \
+researcher's description.
+
+EXPERIMENT DESCRIPTION:
+{description}
+{task_domain_section}
+{success_criteria_section}
+{failure_patterns_section}{cases_section}
+MODEL TYPE: {model_kind}
+
+AVAILABLE ANALYZERS ({n_available} compatible with this model):
+{analyzer_list}
+{failed_section}{prior_hypotheses_section}
+Select up to {max_n} analyzers. Return ONLY a JSON object, no other text:
+{{"analyzers": ["name1", "name2", ...], "rationale": "one sentence explaining the selection"}}
+
+If the analyzers you can actually rely on (excluding any listed as failed)
+cannot adequately surface the suspected mechanism, additionally include
+"need_custom": "<one line describing the probe to generate>" in the JSON —
+a bespoke probe will be code-generated and run in a sandbox.
+"""

evalrx/eval_agent/prompts/probe_candidate_generator.py

@@ -0,0 +1,14 @@
+"""Prompt for ProbeLLM-style Macro/Micro question paraphrasing (VLM probe search)."""
+
+from __future__ import annotations
+
+PARAPHRASE_PROMPT = """\
+You are helping stress-test a vision-language model. Below is a question that
+was originally asked about an image. Rewrite it as {style}, while preserving
+its exact meaning and its correct answer — do not change what is being asked
+about, only how it is phrased.
+
+Original question: {question}
+
+Reply with ONLY the rewritten question text — no quotes, no explanation, no
+"Question:" prefix."""

evalrx/eval_agent/prompts/probe_generator.py

@@ -0,0 +1,35 @@
+"""Prompt templates for generated code-writing stages."""
+
+from __future__ import annotations
+
+_INPUT_FILENAME = "m1_probe_input.json"
+_RESULT_MARKER = "PROBE_RESULT_JSON="
+_MAX_OUTPUT_CHARS = 4000
+
+_GENERATE_PROMPT = """\
+You are writing a self-contained Python script that probes a model's text outputs
+for a specific failure pattern.
+
+GOAL (what to probe for):
+{need}
+
+DATA: a JSON file named "{input_filename}" sits in the current working directory:
+{{
+  "cases": [
+    {{"id": "<case id>", "prompt": "<input prompt>", "expected": <expected or null>,
+     "label": "pass"|"fail"|"unknown"|null, "output": "<the model's text output>"}}
+  ]
+}}
+
+REQUIREMENTS:
+- Read "{input_filename}" from the current directory; do NOT hardcode the data.
+- Compute, per case, a numeric or boolean signal for the failure pattern above
+  (e.g. 1 if the output refuses / drifts language / breaks format, else 0).
+- Standard library + `import numpy` only. No network, no file writes, no model calls.
+- The LAST line of stdout MUST be exactly:
+  {marker}{{"findings": {{"<aggregate_metric>": <number>}}, "per_case": [{{"sample_id": "<case id>", "<signal_name>": <number_or_bool>}}, ...]}}
+- Every per_case entry MUST carry "sample_id" equal to the case "id".
+- Print NOTHING after that line. Keep the script under ~60 lines.
+
+Return ONLY the Python code{fences_hint}."""
+

evalrx/eval_agent/prompts/whitebox_probe_generator.py

@@ -0,0 +1,38 @@
+"""Prompt templates for generated code-writing stages."""
+
+from __future__ import annotations
+
+_MANIFEST = "m1_whitebox_manifest.json"
+_DUMP_DIR = "m1_whitebox"
+_RESULT_MARKER = "PROBE_RESULT_JSON="
+
+_GENERATE_PROMPT = """\
+You are writing a self-contained Python script that probes a vision-language
+model's INTERNAL attention for a specific failure mechanism.
+
+GOAL (what mechanism to probe for):
+{need}
+
+DATA in the current working directory:
+- "{manifest}": {{"cases": [{{"id": str, "label": "pass"|"fail"|null,
+   "seq_len": int, "n_layers": int, "tokens": [str, ...]}}, ...]}}
+- "{dump_dir}/<case_id>.npz" per case with arrays:
+   - attn_last:        float16 (n_layers, seq) — head-averaged attention from
+                       the LAST query position to every key position, per layer
+   - image_token_mask: bool (seq,) — True at image-patch token positions
+  The "tokens" list in the manifest gives the token string at each position —
+  use it to locate structural/special tokens (e.g. positions whose token
+  contains "im_start", "image_pad", "vision").
+
+REQUIREMENTS:
+- numpy only (`import numpy as np`); no network, no other file writes.
+- Compute ONE per-case numeric signal for the mechanism above (e.g. attention
+  mass on structural tokens, image-vs-text attention ratio at a layer,
+  attention entropy inside the image region).
+- The LAST line of stdout MUST be exactly:
+  {marker}{{"findings": {{"<aggregate_metric>": <number>}}, "per_case": [{{"sample_id": "<case id>", "<signal_name>": <number>}}, ...]}}
+- Every per_case entry MUST use "sample_id" equal to the case "id".
+- Print NOTHING after that line. Keep the script under ~60 lines.
+
+Return ONLY the Python code{fences_hint}."""
+

evalrx/eval_agent/report.py

@@ -0,0 +1,58 @@
+"""Diagnostic report — turn a statistical verdict (+ its pre-registration) into output.
+
+Carries the corrected decision, effect size + CI, e-value, the underpowered flag,
+and the pre-registration hash that proves the hypothesis preceded the test.
+"""
+
+from __future__ import annotations
+
+from typing import Optional
+
+from evalrx.stats import StatResult
+
+
+class DiagnosticReport:
+    """Format an A/B / hypothesis-test conclusion as a dict or markdown."""
+
+    @staticmethod
+    def from_comparison(
+        stat: StatResult,
+        *,
+        title: str = "A/B comparison",
+        hypothesis=None,
+        prereg_hash: Optional[str] = None,
+        split: Optional[str] = None,
+    ) -> dict:
+        return {
+            "title": title,
+            "decision": "REJECT H0" if stat.reject else "inconclusive",
+            "effect": stat.effect,
+            "ci": list(stat.ci),
+            "e_value": stat.e_value,
+            "underpowered": stat.underpowered,
+            "method": stat.method,
+            "alpha": stat.alpha,
+            "split": split,
+            "preregistered": prereg_hash is not None,
+            "prereg_hash": prereg_hash,
+            "hypothesis": getattr(hypothesis, "statement", None) or getattr(hypothesis, "predicate", None),
+            "details": stat.details,
+        }
+
+    @staticmethod
+    def to_markdown(report: dict) -> str:
+        lines = [
+            f"## {report['title']}",
+            f"- **Decision**: {report['decision']}"
+            + ("  ⚠️ underpowered" if report.get("underpowered") else ""),
+            f"- **Effect (B−A)**: {report['effect']:+.4f}  CI {report['ci'][0]:+.4f}..{report['ci'][1]:+.4f}",
+            f"- **e-value**: {report['e_value']:.3g}" if report.get("e_value") is not None else "- e-value: n/a",
+            f"- **Method**: {report['method']} (α={report['alpha']})",
+        ]
+        if report.get("hypothesis"):
+            lines.append(f"- **Hypothesis**: {report['hypothesis']}")
+        if report.get("preregistered"):
+            lines.append(f"- **Pre-registered**: yes (hash `{report['prereg_hash']}`, split `{report.get('split')}`)")
+        else:
+            lines.append("- **Pre-registered**: NO — exploratory, not confirmatory")
+        return "\n".join(lines)

evalrx/eval_agent/run_context.py

@@ -0,0 +1,354 @@
+"""RunContext — single owner of one diagnosis run's output directory.
+
+Historically each ``examples/*/run.py`` glued together four independent output
+producers by hand: the example wrote report files at the run root, the run
+logger was buried under a ``logs/`` subdir, :class:`StatsAnalysisAgent`
+figures went to a hand-built ``logs/figures/`` path, and the sandbox wrote
+experiment code into an ephemeral temp dir.  ``RunContext`` replaces that
+gluing with one library-owned object that owns the whole run directory and
+hands every producer its subdirectory.
+
+The logger is :class:`~evalrx.eval_agent.run_logger_v2.RunLoggerV2` (see
+``evalrx/eval_agent/RUN_LOGGER_V2.md``): one ``run.json`` plus one
+``M1/log.json``..``M5/log.json`` per stage, no flat event file, no persisted
+``prompts/``/``experiments/``/``tools/``/``workspace/``/``fixes/`` — generated
+text/code is captured inline into the relevant stage's JSON via an ephemeral
+runtime tree that :meth:`finalize` deletes, and no ``manifest.json``/
+``README.txt`` is written (the manifest lives inside ``run.json`` instead).
+
+Layout::
+
+    <root>/
+    ├── run.json           run-wide events: run_start, cases, diagnose_reports, manifest, …
+    ├── M1/log.json …      one JSON document per stage, plus each stage's own artifacts/
+    │   M5/log.json
+    ├── contract/          one validated JSON per stage (see evalrx.contract), if emitted
+    ├── artifacts/         M1 heavy numeric data (.npy / .json) written outside any stage
+    └── langfuse_trace.json
+
+``fixes/`` and ``experiments/`` attempts (:meth:`RunContext.new_trial`) — one
+numbered attempt per fix candidate or M5 experiment — live under an ephemeral
+:attr:`runtime_root` instead: generated code, the sandbox it ran in, and
+judge prompt/output are inlined into the owning stage's JSON, and the
+runtime tree is deleted at :meth:`finalize`, so "what did attempt #14 do" is
+one entry in that stage's log, not a filename-slug hunt across loose folders.
+
+Usage::
+
+    from evalrx.eval_agent import RunContext, VLDiagnoseLoop
+
+    with RunContext("examples/foo/outputs", verbose=True) as ctx:
+        stats_agent = StatsAnalysisAgent(judge=judge, figure_dir=str(ctx.figures_dir))
+        loop = VLDiagnoseLoop(..., run_logger=ctx.logger)
+        report = loop.run(cases)
+        ctx.write_diagnose_report(report, cases, discovery=discovery_rows)
+    # run.json/M*/log.json are flushed incrementally throughout the run;
+    # finalize() just closes them out and inlines the runtime tree.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import shutil
+import tempfile
+from datetime import datetime
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from evalrx.eval_agent.run_logger_v2 import RunLoggerV2
+
+
+class Trial:
+    """One self-contained attempt — a fix candidate or an M5 experiment.
+
+    Returned by :meth:`RunContext.new_trial`.  Everything about this one
+    attempt (generated code, the sandbox it ran in, judge prompt/output, and
+    its result/record) is written under :attr:`root`, so reviewing "what did
+    attempt #14 do" never requires hopping across run-global category dirs.
+    :attr:`root` lives under :attr:`RunContext.runtime_root` — an ephemeral
+    tree captured into the owning stage's JSON and deleted at
+    :meth:`RunContext.finalize`, not a permanent directory.
+
+    Directory creation is lazy: nothing is written to disk until the first
+    call to :meth:`write` / :attr:`workspace`, so an attempt that is discarded
+    before producing anything leaves no empty folder behind.
+    """
+
+    def __init__(self, root: Path) -> None:
+        self.root = root
+        self._workspace: "Path | None" = None
+
+    @property
+    def workspace(self) -> Path:
+        """``<trial>/workspace/`` — where the sandbox for this attempt runs."""
+        if self._workspace is None:
+            self._workspace = self.root / "workspace"
+            self._workspace.mkdir(parents=True, exist_ok=True)
+        return self._workspace
+
+    def write(self, name: str, content: "str | bytes") -> Path:
+        """Write *content* to ``<trial>/<name>``; return the path."""
+        self.root.mkdir(parents=True, exist_ok=True)
+        path = self.root / name
+        if isinstance(content, bytes):
+            path.write_bytes(content)
+        else:
+            path.write_text(content, encoding="utf-8")
+        return path
+
+    def write_record(self, markdown: str) -> Path:
+        """Write the human-readable summary to ``<trial>/record.md``."""
+        return self.write("record.md", markdown)
+
+    def write_result(self, data: "dict[str, Any]") -> Path:
+        """Write the machine-readable outcome to ``<trial>/result.json``."""
+        return self.write("result.json", json.dumps(data, indent=2, default=str))
+
+    def __repr__(self) -> str:
+        return f"Trial(root={str(self.root)!r})"
+
+
+class RunContext:
+    """Owns one run's output directory and every producer's subdirectory.
+
+    Args:
+        root:     Run root directory.  Created if missing.  Defaults to
+                  ``runs/<YYYYMMDD_HHMMSS>/`` relative to cwd.
+        run_id:   Optional identifier recorded in the manifest; defaults to the
+                  root directory name.
+        verbose:  Forwarded to the logger (human-readable stdout).
+        narrate:  Forwarded to the logger — live, aligned M1-M5 terminal
+                  narration instead of `verbose`'s raw one-liner (see
+                  :class:`evalrx.eval_agent.narration.LoopNarrator`).
+                  Supersedes *verbose* rather than stacking with it.
+        config:   Optional run-configuration dict recorded verbatim in the
+                  manifest (model, judge, protocol, …).
+
+    The logger allocates producer sandboxes in an external ephemeral runtime
+    tree; their text is inlined into JSON and their media is copied before
+    finalization removes the tree.
+    """
+
+    def __init__(
+        self,
+        root: "str | Path | None" = None,
+        *,
+        run_id: "str | None" = None,
+        verbose: bool = False,
+        narrate: bool = False,
+        config: "dict[str, Any] | None" = None,
+        observability_mode: str | None = None,
+    ) -> None:
+        if root is None:
+            root = Path("runs") / datetime.now().strftime("%Y%m%d_%H%M%S")
+        # Resolved to absolute: sandbox subprocesses run with cwd=<some
+        # workdir under root> *and* a script path built from that same
+        # workdir (see ExperimentSandbox._run_script / run_coded_pipeline) —
+        # if root stayed relative, the child process would resolve that
+        # script path a second time relative to its new cwd, doubling it.
+        self.root = Path(root).resolve()
+        self.root.mkdir(parents=True, exist_ok=True)
+        self.run_id = run_id or self.root.name
+        self.config = dict(config or {})
+        self._verbose = verbose
+        self._narrate = narrate
+        self._observability_mode = observability_mode
+        self._logger: "RunLoggerV2 | None" = None
+        self._workdir_seq = 0
+        self._trial_seq: "dict[str, int]" = {}
+        self._runtime_root: "Path | None" = None
+        self._finalized = False
+
+    @property
+    def runtime_root(self) -> Path:
+        """Ephemeral execution workspace used by producers.
+
+        Generated text/code is captured by the logger before this tree is
+        removed; it never becomes part of the portable run artifact.
+        """
+        if self._runtime_root is None:
+            self._runtime_root = Path(tempfile.mkdtemp(prefix=f"evalrx-{self.run_id}-"))
+        return self._runtime_root
+
+    # ------------------------------------------------------------------
+    # Directory properties — each lazily created on first access.
+    # ------------------------------------------------------------------
+
+    def _sub(self, name: str) -> Path:
+        d = self.root / name
+        d.mkdir(parents=True, exist_ok=True)
+        return d
+
+    @property
+    def figures_dir(self) -> Path:
+        return self._sub("M2/artifacts")
+
+    @property
+    def explore_dir(self) -> Path:
+        """The in-cycle explore step's report, tables and rendered figures
+        (``VLDiagnoseLoop(explorer=..., explore_dir=ctx.explore_dir)``; the
+        loop derives the same path from ``ctx.logger`` when not given).
+        Ephemeral — captured into ``M2/log.json`` and deleted at
+        :meth:`finalize`, not a permanent directory."""
+        d = self.runtime_root / "explore"
+        d.mkdir(parents=True, exist_ok=True)
+        return d
+
+    @property
+    def artifacts_dir(self) -> Path:
+        return self._sub("artifacts")
+
+    # ------------------------------------------------------------------
+    # Logging component
+    # ------------------------------------------------------------------
+
+    @property
+    def logger(self) -> "RunLoggerV2":
+        """The logger bound to this context (created on first use)."""
+        if self._logger is None:
+            from evalrx.eval_agent.run_logger_v2 import RunLoggerV2
+
+            self._logger = RunLoggerV2(
+                run_dir=self.root, verbose=self._verbose, narrate=self._narrate,
+                observability_mode=self._observability_mode,
+                context=self,
+            )
+        return self._logger
+
+    # ------------------------------------------------------------------
+    # Producer-facing path allocation
+    # ------------------------------------------------------------------
+
+    def new_workdir(self, label: str) -> Path:
+        """Return a fresh sandbox working directory under the runtime tree.
+
+        Replaces ``tempfile.mkdtemp()`` so the experiment code the agent
+        writes is captured with the rest of the run instead of vanishing
+        untracked. *label* is slugified; a monotonic counter guarantees
+        uniqueness.
+        """
+        self._workdir_seq += 1
+        slug = re.sub(r"[^a-zA-Z0-9]+", "_", label).strip("_") or "work"
+        d = self.runtime_root / "workspace" / f"{self._workdir_seq:02d}_{slug}"
+        d.mkdir(parents=True, exist_ok=True)
+        return d
+
+    def new_trial(self, category: str, label: str) -> "Trial":
+        """Allocate one self-contained attempt folder under the runtime tree.
+
+        A *trial* is one fix candidate or one M5 experiment: its generated
+        code, the sandbox it actually ran in, judge prompt/output, and its
+        result/record all live together under one numbered folder.
+        *category* is ``"fixes"`` or ``"experiments"``; numbering is
+        monotonic per category.
+
+        The trial's own folder (and its ``workspace/``) is created lazily on
+        first write — a candidate discarded before producing anything (e.g. a
+        deduped fix proposal) leaves no empty folder on disk; the numbering
+        still advances, so a gap in the sequence honestly means "proposed,
+        then discarded," not a missing record.
+        """
+        if category not in ("fixes", "experiments"):
+            raise ValueError(
+                f"new_trial: unknown category {category!r} "
+                "(expected 'fixes' or 'experiments')"
+            )
+        seq = self._trial_seq.get(category, 0) + 1
+        self._trial_seq[category] = seq
+        slug = re.sub(r"[^a-zA-Z0-9]+", "_", label).strip("_") or "trial"
+        return Trial(self.runtime_root / category / f"{seq:02d}_{slug}")
+
+    def figure_path(self, name: str) -> Path:
+        """Return ``figures_dir/<name>`` (created if needed)."""
+        if not name.lower().endswith((".png", ".jpg", ".jpeg", ".svg", ".pdf")):
+            name = f"{name}.png"
+        return self.figures_dir / name
+
+    # ------------------------------------------------------------------
+    # Report API — absorbs the per-example boilerplate.
+    # ------------------------------------------------------------------
+
+    def write_diagnose_report(
+        self,
+        report: Any,
+        cases: "list[Any]",
+        *,
+        discovery: "list[dict[str, Any]] | None" = None,
+    ) -> None:
+        """Inline the standard post-diagnosis report into ``run.json``.
+
+        *report* is an :class:`AutoDiagnoseReport` (all three loops return
+        this one unified class — ``VLDiagnoseReport`` is an alias for it).
+        *discovery* (optional) is a list of already-serialised case rows —
+        examples that compute task-specific columns (e.g. parsed yes/no)
+        build the rows themselves. Delegates to
+        :meth:`~evalrx.eval_agent.run_logger_v2.RunLoggerV2.log_diagnose_report`.
+        """
+        self.logger.log_diagnose_report(report, cases, discovery=discovery)
+
+    def publish_report(
+        self,
+        *,
+        model: "Any | None" = None,
+        example_dir: "str | Path | None" = None,
+    ) -> "Any":
+        """Publish the dynamic report after all diagnostic/fix stages finish."""
+        from evalrx.reporting.dynamic import publish_report
+
+        return publish_report(
+            self.root, example_dir=example_dir, model=model, run_logger=self._logger,
+        )
+
+    # ------------------------------------------------------------------
+    # Lifecycle
+    # ------------------------------------------------------------------
+
+    def finalize(self) -> None:
+        """Close the logger, inline the runtime tree, and delete it.  Idempotent."""
+        if self._finalized:
+            return
+        if self._logger is not None or self._runtime_root is not None:
+            logger = self.logger
+            if self._runtime_root is not None:
+                logger.log_runtime_snapshot(self._runtime_root)
+            logger.close()
+            # close() materializes the trace bundle and every M1-M5 log;
+            # index only afterwards so the manifest is complete.
+            logger.log_manifest(run_id=self.run_id, config=self.config)
+            # write_contract_index() only looks for an on-disk contract/ dir
+            # and the logger's trace_id. Without it, a run that emitted
+            # contract/ payloads (evalrx.contract.emit, independent of
+            # RunContext) never gets the index.json a reader opening the run
+            # without its producer needs.
+            self.write_contract_index()
+        if self._runtime_root is not None:
+            shutil.rmtree(self._runtime_root, ignore_errors=True)
+        self._finalized = True
+
+    def write_contract_index(self) -> "Path | None":
+        """Write ``contract/index.json`` when any stage emitted a payload.
+
+        Written here rather than by the emitter so it describes the finished
+        directory: a reader opening this run — or a zip of it — gets one file
+        naming every payload, its stage and cycle, and the wire model that
+        decodes it, without having to infer any of that from filenames.
+        """
+        if not (self.root / "contract").is_dir():
+            return None
+        try:
+            from evalrx.contract.emit import write_index
+        except ImportError:  # contract extra not installed
+            return None
+        trace_id = getattr(self._logger, "trace_id", "") if self._logger else ""
+        return write_index(self.root, trace_id=trace_id)
+
+    def __enter__(self) -> "RunContext":
+        return self
+
+    def __exit__(self, *_: Any) -> None:
+        self.finalize()
+
+    def __repr__(self) -> str:
+        return f"RunContext(root={str(self.root)!r})"