evalrx 0.1.2__py3-none-any.whl

evalrx/eval_agent/stages/probe_candidate_generator.py

@@ -0,0 +1,128 @@
+"""VLM probe candidate generator — ProbeLLM Macro/Micro generators (Sec 3.3),
+scoped to VLM QA and plugged into :class:`~evalrx.analysis.probe_search.ProbeSearch`.
+
+Scope (v1): both Macro and Micro produce a *paraphrase* of an existing seed's
+question over the SAME image, never new imagery and never an altered
+semantic target — so the seed's ``expected`` answer stays valid for the new
+candidate without needing a vision-capable judge or an image/answer
+generation tool (the paper's tool-augmented generation, which invokes web/code
+tools to obtain or verify a *new* gold answer, is out of scope here — see
+``ProbeSearch``'s docstring: a richer generator with those tools can be
+substituted without touching the search algorithm itself).
+
+- **Macro** (broad coverage): picks the seed question least similar (by token
+  overlap) to what the macro tree has already explored, then paraphrases it —
+  diversifies which part of the fixed image pool gets visited next.
+- **Micro** (local refinement): paraphrases the *current search node's own*
+  case — same image, same gold answer, different wording — probing surface
+  robustness (does the model's correctness flip on a reworded but
+  semantically identical question) rather than the paper's full
+  entity/attribute substitution.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from evalrx.core.case import CaseBatch, FailureCase, Inputs
+from evalrx.eval_agent.prompts.probe_candidate_generator import PARAPHRASE_PROMPT
+
+if TYPE_CHECKING:
+    from evalrx.analysis.probe_search import ProbeNode
+    from evalrx.core.model import Model
+
+logger = logging.getLogger(__name__)
+
+
+def _tokenize(text: str) -> set[str]:
+    return set(re.findall(r"[a-z0-9]+", text.lower()))
+
+
+def _jaccard_distance(a: str, b: str) -> float:
+    """1.0 = no shared tokens (maximally distinct), 0.0 = identical token sets."""
+    ta, tb = _tokenize(a), _tokenize(b)
+    union = ta | tb
+    if not union:
+        return 0.0
+    return 1.0 - len(ta & tb) / len(union)
+
+
+def _extract_question(raw: str) -> str:
+    text = raw.strip()
+    if text.startswith("```"):
+        text = re.sub(r"^```\w*\n?", "", text)
+        text = re.sub(r"\n?```\s*$", "", text)
+    text = re.sub(r"^(question|paraphrase)\s*:\s*", "", text, flags=re.IGNORECASE)
+    return text.strip().strip('"')
+
+
+@dataclass
+class VLMProbeCandidateGenerator:
+    """ProbeLLM-style Macro/Micro candidate generation over a fixed VLM seed pool.
+
+    Args:
+        seed_pool: Existing (image, question, expected) cases — e.g. loaded via
+                   ``VLMQADataset`` — the only source of images/gold answers
+                   (Scope v1: no new imagery, no new gold synthesis).
+        judge:     Text-only judge used to paraphrase questions (only the
+                   question text is sent — never the image, so no
+                   vision-capable judge is required). Required for either
+                   regime to produce candidates; ``available`` is False and
+                   both ``macro``/``micro`` return ``None`` without one
+                   (mirrors ``ProbeGenerator.available``).
+        max_repairs: Retries if the judge echoes the question unchanged.
+    """
+
+    seed_pool: CaseBatch
+    judge: "Model | None" = None
+    max_repairs: int = 1
+
+    @property
+    def available(self) -> bool:
+        return self.judge is not None and len(self.seed_pool) > 0
+
+    def macro(self, node: "ProbeNode", explored: "list[ProbeNode]") -> "FailureCase | None":
+        """Diversify: paraphrase the pool seed least similar to what the macro
+        tree has already visited (paper Eq.11's "under-represented" frontier,
+        approximated here by question-token overlap rather than embeddings)."""
+        if not self.available:
+            return None
+        explored_prompts = [n.case.inputs.prompt for n in explored]
+        seed = max(
+            self.seed_pool,
+            key=lambda c: min(
+                (_jaccard_distance(c.inputs.prompt, p) for p in explored_prompts),
+                default=1.0,
+            ),
+        )
+        return self._paraphrase(seed, style="a very differently worded question")
+
+    def micro(self, node: "ProbeNode") -> "FailureCase | None":
+        """Refine: paraphrase the search node's own case for local
+        surface-robustness probing (same image, same gold answer)."""
+        if not self.available:
+            return None
+        return self._paraphrase(
+            node.case, style="a lightly reworded variant (synonyms/reordering)"
+        )
+
+    def _paraphrase(self, base: FailureCase, *, style: str) -> "FailureCase | None":
+        prompt = PARAPHRASE_PROMPT.format(question=base.inputs.prompt, style=style)
+        for _attempt in range(self.max_repairs + 1):
+            try:
+                raw = self.judge.generate(prompt)  # type: ignore[union-attr]
+            except Exception as exc:  # noqa: BLE001 — generation is best-effort
+                logger.warning("VLMProbeCandidateGenerator: judge.generate failed: %s", exc)
+                return None
+            question = _extract_question(str(raw))
+            if question and question.strip().lower() != base.inputs.prompt.strip().lower():
+                return FailureCase(
+                    inputs=Inputs(prompt=question, image=base.inputs.image),
+                    expected=base.expected,
+                    tags={"probe_search_candidate"},
+                    metadata={"seed_case_id": base.id},
+                )
+        return None

evalrx/eval_agent/stages/probe_generator.py

@@ -0,0 +1,326 @@
+"""M1 tier (b) — ProbeGenerator: synthesise a new black-box probe on demand.
+
+When no registered analyzer in the catalog targets the observed failure, this
+generator creates a bespoke *probe* — but adapted to M1's reality: a probe needs
+the model, which (unlike M2's data-only stats tools) cannot be shipped into a
+subprocess.  So the split is:
+
+1. **Host collects** the model's outputs on the cases (the host owns the loaded
+   model) into ``m1_probe_input.json``.
+2. A **sandboxed generated script** reads that JSON and computes a per-case probe
+   metric over the *outputs* (refusal detection, language drift, format/printf
+   adherence, length, keyword presence, answer-extraction failure, …), printing a
+   strict ``PROBE_RESULT_JSON=`` line.
+3. The host wraps the parsed findings into a
+   :class:`~evalrx.core.result.Result` whose ``per_case`` entries flow into
+   M2 (stats tools) → M4 exactly like any catalog analyzer's output.
+
+This keeps the M2-tier(b) safety model: generated code never touches the repo
+source, never sees the weights, and runs in an
+:class:`~evalrx.agent_runtime.sandbox.ExperimentSandbox` subprocess.
+
+Scope (v1): probes that are **functions over a single forward-pass output**.
+Multi-sample (self-consistency), perturbation, and white-box probes need the
+model handle and are out of scope here — they belong to an in-process path.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from evalrx.agent_runtime.sandbox import ExperimentSandbox
+from evalrx.core.result import Result
+from evalrx.eval_agent.prompts.probe_generator import (
+    _GENERATE_PROMPT,
+    _INPUT_FILENAME,
+    _MAX_OUTPUT_CHARS,
+    _RESULT_MARKER,
+)
+
+if TYPE_CHECKING:
+    from evalrx.agent_runtime.cli_types import CliAgentConfig
+    from evalrx.core.case import CaseBatch
+    from evalrx.core.model import Model
+
+logger = logging.getLogger(__name__)
+
+
+
+@dataclass
+class GeneratedProbe:
+    """A validated, sandbox-executed probe that can be re-run on new cases.
+
+    Attributes:
+        name:   Short identifier (becomes ``generated:<name>``).
+        code:   The Python source that was written and executed.
+        need:   The natural-language failure pattern it probes for.
+        source: Which backend wrote it (``"cli:<provider>"`` or ``"llm"``).
+    """
+
+    name: str
+    code: str
+    need: str = ""
+    source: str = ""
+
+
+class ProbeGenerator:
+    """Generate, run, and cache bespoke black-box probes in a sandbox.
+
+    Args:
+        judge:       LLM used for the single-pass code-writing path.
+        cli_config:  CLI coding-agent config (``provider != "llm"``) used before
+                     the judge when present.
+        sandbox:     Execution sandbox (fresh temp-dir when ``None``).
+        timeout_sec: Hard wall-clock limit per sandbox run.
+        max_cases:   Cap on cases whose outputs are collected (cost guard); 0 (the default) = every case.
+    """
+
+    def __init__(
+        self,
+        judge: "Model | None" = None,
+        cli_config: "CliAgentConfig | None" = None,
+        sandbox: "ExperimentSandbox | None" = None,
+        timeout_sec: int = 60,
+        max_cases: int = 0,
+        run_logger: "Any | None" = None,
+    ) -> None:
+        self._judge = judge
+        self._cli_config = cli_config
+        self._timeout_sec = timeout_sec
+        self._max_cases = max_cases
+        self._sandbox = sandbox or ExperimentSandbox()
+        # Optional RunLogger — when set, every code-writing attempt (the prompt,
+        # the code produced, the backend used, and the pass/fail outcome) is
+        # recorded as a "tool_codegen" event so tool synthesis is fully traceable.
+        self.run_logger = run_logger
+        self._last_prompt: str = ""
+        self._last_raw: str = ""
+        self._last_raw_stream: str = ""
+        self._last_usage: dict | None = None
+
+    @property
+    def available(self) -> bool:
+        return self._judge is not None or (
+            self._cli_config is not None and self._cli_config.provider != "llm"
+        )
+
+    # ------------------------------------------------------------------
+    # Public interface
+    # ------------------------------------------------------------------
+
+    def generate(
+        self,
+        need: str,
+        model: "Model",
+        cases: "CaseBatch",
+        name: str = "custom",
+    ) -> "tuple[Result | None, GeneratedProbe | None]":
+        """Collect outputs, write+run a probe over them, return (Result, probe)."""
+        if not self.available:
+            logger.debug("ProbeGenerator: no code-writing backend configured")
+            return None, None
+
+        self._collect_outputs(model, cases)
+        self._last_prompt = ""
+        self._last_raw = ""
+        self._last_raw_stream = ""
+        self._last_usage = None
+        try:
+            code, source = self._write_code(need)
+        except Exception as exc:
+            logger.warning("ProbeGenerator: code writing failed: %s", exc)
+            self._emit_codegen(name, need, "", "", ok=False, error=f"code writing failed: {exc}")
+            return None, None
+        if not code.strip():
+            self._emit_codegen(name, need, source, "", ok=False, error="empty code produced")
+            return None, None
+
+        result = self._run_code(code, name, model, cases)
+        self._emit_codegen(
+            name, need, source, code, ok=result is not None,
+            error="" if result is not None else "sandbox produced no parseable result",
+        )
+        if result is None:
+            return None, None
+        probe = GeneratedProbe(name=name, code=code, need=need, source=source)
+        return result, probe
+
+    def _emit_codegen(
+        self, name: str, need: str, source: str, code: str, *, ok: bool, error: str = ""
+    ) -> None:
+        """Record one code-writing attempt to the RunLogger, if attached."""
+        if self.run_logger is None:
+            return
+        extra = ({"cli_usage": self._last_usage}
+                 if source.startswith("cli:") and self._last_usage else None)
+        try:
+            self.run_logger.log_tool_codegen(
+                module="m1_probe", name=name, need=need, source=source, ok=ok,
+                code=code, prompt=self._last_prompt, raw_output=self._last_raw,
+                raw_stream=self._last_raw_stream, error=error,
+                extra=extra,
+            )
+        except Exception as exc:  # logging must never break generation
+            logger.debug("ProbeGenerator: log_tool_codegen failed: %s", exc)
+
+    def run_cached(
+        self,
+        probe: GeneratedProbe,
+        model: "Model",
+        cases: "CaseBatch",
+    ) -> "Result | None":
+        """Re-run an already-generated probe on fresh cases (no LLM call)."""
+        self._collect_outputs(model, cases)
+        return self._run_code(probe.code, probe.name, model, cases)
+
+    # ------------------------------------------------------------------
+    # Internals
+    # ------------------------------------------------------------------
+
+    def _collect_outputs(self, model: "Model", cases: "CaseBatch") -> None:
+        """Run the model on each case and serialise outputs to the sandbox dir."""
+        records: list[dict[str, Any]] = []
+        selected = list(cases)
+        if self._max_cases > 0:  # 0 = every case
+            selected = selected[: self._max_cases]
+        if getattr(self.run_logger, "preserve_full_model_io", False):
+            from evalrx.eval_agent.model_instrumentation import InstrumentedModel
+
+            model = InstrumentedModel(
+                model, self.run_logger,
+                cycle=int(getattr(self.run_logger, "current_cycle", -1)),
+                analyzer="generated_probe_input_collection",
+                case_prompts={c.inputs.prompt: c.id for c in selected},
+                batch_case_ids=[c.id for c in selected],
+            )
+        for case in selected:
+            inp = getattr(case, "inputs", None)
+            try:
+                output = str(model.generate(inp)) if inp is not None else ""
+            except Exception as exc:  # a probe over partial outputs is still useful
+                logger.debug("ProbeGenerator: generate failed for %s: %s", case.id, exc)
+                output = ""
+            label = getattr(case, "label", None)
+            records.append({
+                "id": case.id,
+                "prompt": str(getattr(inp, "prompt", "")) if inp is not None else "",
+                "expected": getattr(case, "expected", None),
+                "label": getattr(label, "value", None),
+                "output": output[:_MAX_OUTPUT_CHARS],
+            })
+        path = Path(self._sandbox.workdir) / _INPUT_FILENAME
+        path.write_text(json.dumps({"cases": records}, default=str), encoding="utf-8")
+
+    def _write_code(self, need: str) -> tuple[str, str]:
+        if self._cli_config is not None and self._cli_config.provider != "llm":
+            code = self._write_code_cli(need)
+            if code:
+                return code, f"cli:{self._cli_config.provider}"
+        prompt = self._build_prompt(need, fenced=True)
+        self._last_prompt = prompt
+        raw = self._judge.generate(prompt)  # type: ignore[union-attr]
+        self._last_raw = str(raw)
+        return _extract_code(str(raw)), "llm"
+
+    def _write_code_cli(self, need: str) -> str:
+        from evalrx.agent_runtime.codegen import CodegenRunner
+
+        prompt = self._build_prompt(need, fenced=False)
+        self._last_prompt = prompt
+        result = CodegenRunner(self._cli_config).write_code(  # type: ignore[arg-type]
+            prompt,
+            workdir=Path(self._sandbox.workdir),
+            timeout_sec=self._timeout_sec,
+            preferred_filenames=("probe.py",),
+        )
+        self._last_raw = result.raw_output
+        self._last_raw_stream = ""
+        if result.raw_stream_path:
+            try:
+                self._last_raw_stream = (
+                    Path(self._sandbox.workdir) / result.raw_stream_path
+                ).read_text(encoding="utf-8")
+            except OSError:
+                pass
+        self._last_usage = result.usage
+        return result.code
+
+    def _build_prompt(self, need: str, *, fenced: bool) -> str:
+        return _GENERATE_PROMPT.format(
+            need=need.strip() or "Detect outputs that fail the task.",
+            input_filename=_INPUT_FILENAME,
+            marker=_RESULT_MARKER,
+            fences_hint=" inside a ```python code block" if fenced else
+                        ", written to a file named probe.py",
+        )
+
+    def _run_code(
+        self,
+        code: str,
+        name: str,
+        model: "Model",
+        cases: "CaseBatch",
+    ) -> "Result | None":
+        sandbox_result = self._sandbox.run(code, timeout_sec=self._timeout_sec)
+        if not sandbox_result.ok:
+            logger.warning(
+                "ProbeGenerator: sandbox run failed (rc=%s): %s",
+                sandbox_result.returncode, (sandbox_result.stderr or "").strip()[:200],
+            )
+            return None
+        return _parse_result(sandbox_result.stdout, name, model, cases)
+
+
+# ---------------------------------------------------------------------------
+# Module helpers
+# ---------------------------------------------------------------------------
+
+def _extract_code(raw: str) -> str:
+    cleaned = re.sub(r"<think>.*?</think>", "", raw, flags=re.DOTALL)
+    fence = re.search(r"```(?:python)?\s*\n(.*?)```", cleaned, flags=re.DOTALL)
+    if fence:
+        return fence.group(1).strip()
+    return cleaned.strip()
+
+
+def _parse_result(
+    stdout: str,
+    name: str,
+    model: "Model",
+    cases: "CaseBatch",
+) -> "Result | None":
+    """Parse the last ``PROBE_RESULT_JSON=`` line into a Result, or None."""
+    marker_line = None
+    for line in stdout.splitlines():
+        s = line.strip()
+        if s.startswith(_RESULT_MARKER):
+            marker_line = s[len(_RESULT_MARKER):]
+    if marker_line is None:
+        logger.warning("ProbeGenerator: no PROBE_RESULT_JSON line in probe output")
+        return None
+    try:
+        data = json.loads(marker_line)
+    except json.JSONDecodeError as exc:
+        logger.warning("ProbeGenerator: unparseable PROBE_RESULT_JSON: %s", exc)
+        return None
+
+    findings: dict[str, Any] = {}
+    raw_findings = data.get("findings")
+    if isinstance(raw_findings, dict):
+        findings.update(raw_findings)
+    per_case = data.get("per_case")
+    if isinstance(per_case, list):
+        findings["per_case"] = [e for e in per_case if isinstance(e, dict)]
+
+    return Result(
+        analyzer=f"generated:{name}",
+        model=repr(model),
+        cases=cases,
+        findings=findings,
+        metadata={"generated": True, "probe": name},
+    )

evalrx/eval_agent/stages/probe_search_agent.py

@@ -0,0 +1,106 @@
+"""ProbeSearchAgent — wires ProbeLLM's hierarchical MCTS
+(:mod:`evalrx.analysis.probe_search`) to a real target model + judge.
+
+``analysis.probe_search`` stays standalone (a generic tree search over
+injected callables); this eval_agent-layer module supplies those callables
+from real components:
+
+- verifier V              -> :class:`~evalrx.eval_agent.stages.case_discovery.CaseDiscoveryAgent`
+- Macro/Micro generators   -> :class:`~evalrx.eval_agent.stages.probe_candidate_generator.VLMProbeCandidateGenerator`
+
+The discovered failure cases (``ProbeSearchResult.failure_cases``) are plain
+``FailureCase`` objects and feed directly into
+:func:`evalrx.analysis.failure_modes.cluster_failures` for failure-mode
+synthesis, or into M1-M4 like any other labeled batch.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any
+
+from evalrx.analysis.probe_search import ProbeSearch, ProbeSearchResult
+from evalrx.eval_agent.stages.case_discovery import CaseDiscoveryAgent
+from evalrx.eval_agent.stages.probe_candidate_generator import VLMProbeCandidateGenerator
+
+if TYPE_CHECKING:
+    from evalrx.core.case import CaseBatch, FailureCase
+    from evalrx.core.model import Model
+    from evalrx.eval_agent.stages.protocol import ExperimentProtocol
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ProbeSearchAgent:
+    """Run a hierarchical Macro/Micro MCTS probe search against a target model.
+
+    Args:
+        judge:    Text-only judge used both to score PASS/FAIL (via
+                  ``CaseDiscoveryAgent``) and to paraphrase Macro/Micro
+                  candidates (via ``VLMProbeCandidateGenerator``). Required —
+                  without it, generation is unavailable and the search finds
+                  nothing (``ProbeSearchResult.n_simulations == 0``).
+        protocol: Optional experiment protocol passed to the discovery judge
+                  for scoring context.
+        budget:   Total simulations (T_max in the paper's Eq.4).
+        beta:     UCB exploration constant (Eq.7).
+        w_max:    Max children per search-tree node before progressive
+                  widening forces a deeper descent instead of a new sibling.
+    """
+
+    judge: "Model"
+    protocol: "ExperimentProtocol | None" = None
+    budget: int = 20
+    beta: float = 1.0
+    w_max: int = 3
+    run_logger: Any | None = None
+
+    def __post_init__(self) -> None:
+        if self.judge is None:
+            raise ValueError(
+                "ProbeSearchAgent requires a judge (e.g. ClaudeModel() or AgyModel()) "
+                "— without one, candidate generation is unavailable and the search "
+                "would silently discover nothing."
+            )
+
+    def run(self, model: "Model", seed_pool: "CaseBatch") -> ProbeSearchResult:
+        judge = self.judge
+        if getattr(self.run_logger, "preserve_full_model_io", False):
+            from evalrx.eval_agent.model_instrumentation import InstrumentedModel
+
+            cycle = int(getattr(self.run_logger, "current_cycle", -1))
+            ids = [case.id for case in seed_pool]
+            prompts = {case.inputs.prompt: case.id for case in seed_pool}
+            judge = InstrumentedModel(
+                judge, self.run_logger, cycle=cycle, analyzer="probe_search_judge",
+                batch_case_ids=ids,
+            )
+            model = InstrumentedModel(
+                model, self.run_logger, cycle=cycle, analyzer="probe_search_target",
+                case_prompts=prompts, batch_case_ids=ids,
+            )
+        discovery = CaseDiscoveryAgent(judge=judge)
+        generator = VLMProbeCandidateGenerator(seed_pool=seed_pool, judge=judge)
+
+        def verify(case: "FailureCase") -> "FailureCase":
+            report = discovery.discover(model, [case], protocol=self.protocol)
+            cases = list(report.cases)
+            return cases[0] if cases else case
+
+        search = ProbeSearch(
+            generate_macro=generator.macro,
+            generate_micro=generator.micro,
+            verify=verify,
+            seeds=seed_pool,
+            budget=self.budget,
+            beta=self.beta,
+            w_max=self.w_max,
+        )
+        result = search.run()
+        logger.info(
+            "ProbeSearchAgent: %d simulation(s) (macro=%d micro=%d), error_rate=%.2f",
+            result.n_simulations, result.n_macro, result.n_micro, result.error_rate,
+        )
+        return result

evalrx/eval_agent/stages/protocol.py

@@ -0,0 +1,112 @@
+"""ExperimentProtocol — user-supplied description of what an experiment tests.
+
+The protocol is the human prior that anchors the self-evolving loop:
+
+- **M1** passes the protocol to :class:`~evalrx.eval_agent.probe_agent.ProbeAgent`,
+  which uses an LLM judge to select analyzers from the description.
+- **M2** uses the protocol to frame its statistical narrative.
+- **M4** uses it to verify that a hypothesis is consistent with what the user
+  actually set out to investigate.
+
+The description should be written in plain researcher language describing the
+*task* and *observed behaviour*.  No failure-mode tags or internal jargon are
+needed — the judge LLM interprets the text and selects relevant analyzers.
+
+Usage::
+
+    protocol = ExperimentProtocol(
+        description=(
+            "We test QwenVL on spatial reasoning. Given an image with two "
+            "objects, the model frequently gives wrong left/right and "
+            "above/below positions, and sometimes names objects not visible "
+            "in the image at all."
+        ),
+        task_domain="spatial reasoning",
+        success_criteria="Positions and object names must match what is visible.",
+        target_modalities=frozenset({"text", "image"}),
+    )
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class ExperimentProtocol:
+    """Natural-language description of an evaluation experiment.
+
+    This is the *human prior* passed into the loop so it can make
+    informed, targeted decisions rather than running every analyzer
+    blindly.
+
+    Attributes:
+        description:        What the experiment tests — free text (required).
+        task_domain:        Short label, e.g. ``"spatial reasoning"``,
+                            ``"GUI navigation"``.
+        success_criteria:   What counts as a pass (used by M4 verifier).
+        failure_patterns:   Optional free-text observations about what the
+                            researcher has already noticed — passed verbatim
+                            to the LLM judge as additional context.
+        target_modalities:  ``{"text", "image"}`` for VLMs;
+                            ``{"text"}`` for text-only LLMs.
+        output_contract:    Optional machine-readable response contract.  It
+                            prevents a valid short answer from being confused
+                            with a truncated reasoning trace.
+        metadata:           Free-form extras (dataset names, hyperparams …).
+    """
+
+    description: str
+    task_domain: str = ""
+    success_criteria: str = ""
+    failure_patterns: str = ""
+    target_modalities: frozenset[str] = field(
+        default_factory=lambda: frozenset({"text"})
+    )
+    output_contract: dict[str, Any] = field(default_factory=dict)
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "description": self.description,
+            "task_domain": self.task_domain,
+            "success_criteria": self.success_criteria,
+            "failure_patterns": self.failure_patterns,
+            "target_modalities": sorted(self.target_modalities),
+            "output_contract": self.output_contract,
+            "metadata": self.metadata,
+        }
+
+    def prompt_text(self) -> str:
+        """Serialize the complete evaluation contract for judge prompts.
+
+        ``description`` alone is not enough for short-answer benchmarks: the
+        success criteria and output contract determine whether a terse answer
+        is valid.  Keeping one formatter prevents M2/M3 prompt handoffs from
+        silently dropping those fields.
+        """
+        return json.dumps(self.to_dict(), ensure_ascii=False, indent=2, default=str)
+
+
+@dataclass
+class ProbingSchema:
+    """What M1 decided to probe and why.
+
+    Returned by :meth:`~evalrx.eval_agent.probe_agent.ProbeAgent.probe_with_schema`
+    alongside the raw ``{analyzer: Result}`` dict so callers can understand
+    *why* those analyzers were chosen.
+
+    Attributes:
+        selected_analyzers: Analyzers to run, in priority order.
+        rationale:          NL explanation of the selection.
+        custom_params:      Per-analyzer parameter overrides (e.g.
+                            ``{"attention": {"layer": -1}}``).
+        protocol:           The protocol that shaped this schema, if any.
+    """
+
+    selected_analyzers: list[str]
+    rationale: str = ""
+    custom_params: dict[str, dict[str, Any]] = field(default_factory=dict)
+    protocol: ExperimentProtocol | None = None