evalrx 0.1.2__py3-none-any.whl

evalrx/contract/m5.py

@@ -0,0 +1,292 @@
+"""M5 — intervention and repair. Two different jobs under one number.
+
+``SurgeryAgent`` INTERVENES: it changes one variable, re-runs, and reads the
+result as causal evidence about *why*. Its output is a verdict, not a deliverable
+-- the modified inputs are discarded.
+
+``FixAgent`` REPAIRS: it proposes changes intended to survive, and validates them
+paired against the unmodified baseline on the same cases.
+
+Conflating them produces the pipeline's worst failure mode: presenting a
+diagnostic manipulation as a shippable fix.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Literal
+
+from pydantic import Field, model_validator
+
+from evalrx.contract.common import (
+    CaseBatchRef,
+    HypothesisStatus,
+    StageEnvelope,
+    WireModel,
+)
+from evalrx.contract.methodology import MethodologyWire
+
+# ===========================================================================
+# M5a — SurgeryAgent: verify the mechanism
+# ===========================================================================
+
+class SurgeryInput(WireModel):
+    """One hypothesis at a time. Surgery is per-claim by construction."""
+
+    hypothesis_id: str
+    results_ref: str = Field(description="Path to the M1 ProbeOutput, for per-case signal extraction.")
+    data: CaseBatchRef
+    model_name: str
+
+
+class InterventionOutput(StageEnvelope):
+    """``SurgeryAgent.operate`` result — causal evidence, not a repair.
+
+    ``fixed`` here means "the intervention completely separated failing from
+    passing cases", i.e. the mechanism is confirmed. It does NOT mean a usable
+    repair exists; that claim belongs to :class:`FixOutput` alone.
+    """
+
+    hypothesis_id: str = Field(min_length=1, description="Points into the M3 DiagnosisOutput.")
+    hypothesis_status: HypothesisStatus = Field(
+        description="Verdict on the hypothesis. Named apart from the envelope's `status`, "
+                    "which is this STAGE's execution state — they answer different questions."
+    )
+    strategy: Literal["verify_fn", "analyzer_params", "experiment_writer", "passive_correlation"] = (
+        "passive_correlation"
+    )
+    fixed: bool = Field(
+        default=False, description="Mechanism fully separates the groups. NOT a shippable repair."
+    )
+    confidence_score: float = Field(default=0.0, ge=0.0, le=1.0)
+    evidence_dimensions: dict[str, float] = Field(
+        default_factory=dict, description="Breakdown: evidence gap / sample adequacy / control cleanliness."
+    )
+    evidence: dict[str, Any] = Field(default_factory=dict)
+    methodology: MethodologyWire | None = Field(
+        default=None,
+        description="The manipulation performed: which variable was changed, what was held fixed, "
+                    "what was compared. This is what makes the causal claim auditable.",
+    )
+    trial_root: str | None = Field(
+        default=None,
+        description="Attempt folder when the experiment-writer strategy ran: generated code, "
+                    "stdout/stderr, workspace and record all sit there.",
+    )
+    new_data: CaseBatchRef | None = Field(
+        default=None,
+        description="When SUPPORTED: the cases NOT in the signal group — the refined subset for the "
+                    "next M1 cycle. null means no refinement, which differs from an empty batch.",
+    )
+
+    @property
+    def is_intervention(self) -> bool:
+        """Whether this run can ground a causal claim.
+
+        ``passive_correlation`` only observes, so it cannot — which makes this a
+        function of ``strategy``, not an independent fact worth storing.
+        """
+        return self.strategy != "passive_correlation"
+
+
+# ===========================================================================
+# M5b — FixAgent: propose and validate repairs
+# ===========================================================================
+
+class FixInput(WireModel):
+    """Repairs are attempted only on hypotheses M4 verified. ``max_tier`` bounds
+    how invasive the search may get and is the caller's decision, never the
+    agent's."""
+
+    hypothesis_ids: list[str]
+    data: CaseBatchRef
+    model_name: str
+    max_tier: Literal["L0", "L1", "L2", "L3a", "L3b", "L4"] = Field(
+        description="L0 runtime config / L1 prompt / L2 scaffold / L3a internals-read / "
+                    "L3b internals-write / L4 params.\n\n"
+                    "L0 is in this list because `FixTier` has it — decoding settings are a "
+                    "real, and the least invasive, place to intervene. Omitting it made a "
+                    "tier the pipeline can produce unrepresentable."
+    )
+    max_repair_rounds: int = Field(
+        default=1, ge=1, description="Feedback-driven propose->validate rounds. 1 = no loop."
+    )
+    prior_attempts_ref: str | None = None
+
+
+class FixAttemptWire(WireModel):
+    """One candidate validated against the unmodified baseline on the SAME cases.
+
+    Every count here is a PAIRED flip, never a rate difference. "repaired 18 /
+    broke 1" is a statement about individual cases changing sign; a pass-rate
+    delta is not the same claim and must never be displayed as if it were.
+    """
+
+    tier: Literal["L0", "L1", "L2", "L3a", "L3b", "L4"]
+
+    ref: str = Field(
+        default="",
+        description="The reader-facing identifier, e.g. `R3`. Assigned by the producer over the "
+                    "whole run and STABLE across `selection` and `attempted`, so a reader who "
+                    "sees R3 in the sweep can find R3's card and know it is the same repair.\n\n"
+                    "This exists because `name` cannot do the job. Consumers that numbered rows "
+                    "by list position gave the same repair two different numbers in the two "
+                    "lists, which is worse than no number at all. Empty means the producer "
+                    "assigned none; a consumer may then number by position, but only within one "
+                    "list and never as a cross-list identity.",
+    )
+    name: str = Field(
+        description="Machine slug, e.g. `audio_evidence_then_answer`. It is the JOIN KEY — "
+                    "`best`, `selected_on_explore` and `ebh_survivors` all name a candidate this "
+                    "way — and it is what the attempt folder on disk is called.\n\n"
+                    "It is NOT the label to show a reader. These slugs are chosen by the agent "
+                    "while naming its log files, so they read as identifiers rather than as "
+                    "language. Show `ref` to point at a candidate and `headline` to say what it "
+                    "does.",
+    )
+    headline: str = Field(
+        default="",
+        description="One plain sentence saying what this repair changes, written for a reader "
+                    "with no background in the field: no tier names, no metric names, no jargon, "
+                    "no slug echoed back. \"Ask the model to describe what it hears before "
+                    "answering\" — not \"audio-evidence-first prompt scaffold\".\n\n"
+                    "Empty when the producer had nothing readable to offer, which a consumer "
+                    "should render as an honest blank rather than by humanising the slug: "
+                    "`audio_evidence_then_answer` -> \"Audio Evidence Then Answer\" invents "
+                    "fluency the run never had.",
+    )
+    kind: str | None = None
+    source: str | None = None
+    methodology: MethodologyWire | None = Field(
+        default=None,
+        description="What this candidate DOES, as a graph plus a rendered draw.io diagram. The "
+                    "payload is the implementation; this is the explanation. Required for any "
+                    "candidate a reader is asked to accept — a repair nobody can follow cannot "
+                    "be reviewed, only trusted.",
+    )
+    trial_root: str | None = Field(
+        default=None,
+        description="Self-contained attempt folder, RELATIVE TO THE RUN ROOT. Preferred join "
+                    "key across rounds — name alone is not unique.\n\n"
+                    "It holds `outputs.jsonl`, one line per validated case: "
+                    "`{case_id, status, output}` where status is fixed / broken / unchanged. "
+                    "That is what a reader needs to study a repair case by case — the "
+                    "candidate's actual answer beside the baseline's — and it is kept out of "
+                    "this payload because it is one row per case per candidate. Recorded "
+                    "relative because an absolute producer path resolves nowhere else, which "
+                    "is the same reason MediaRef paths are relative.",
+    )
+
+    n_pairs: int = 0
+    n_baseline_correct: int = 0
+    n_fixed: int = 0
+    n_broken: int = 0
+    fixed_cases: list[str] = Field(default_factory=list)
+    broken_cases: list[str] = Field(default_factory=list)
+
+    n_applicable: int = Field(default=0, description="Cases the candidate actually touched.")
+    coverage: float | None = Field(default=None, description="applicable FAILs / total FAILs.")
+    n_unstable: int = Field(
+        default=0, description="Dropped as baseline-unstable, so sampling noise is not read as regression."
+    )
+    n_model_independent: int = Field(
+        default=0,
+        description="Cases the scaffold solved by its own computation under frozen_model_control — "
+                    "excluded from the paired test rather than counted as fixed.",
+    )
+
+    effect: float | None = None
+    e_value: float | None = None
+    reject: bool = Field(default=False, description="This candidate's own gate. Not sufficient to win.")
+    verdict: Literal[
+        "fixed", "partial", "unsafe", "regressed", "no_effect", "not_executed", "model_independent"
+    ] = Field(
+        description="The candidate's outcome. Subsumes a separate `fixed` boolean "
+                    "(verdict == 'fixed') and a separate exec_error ('not_executed')."
+    )
+    summary: str = ""
+
+
+class FixOutput(StageEnvelope):
+    """``FixAgent.propose_and_validate`` result.
+
+    What each candidate literally applied — the prompt text, the generated code —
+    is not here. It is the ``result.json`` under ``trial_root``, which is also
+    where the per-case outputs and the execution record sit; inlining a code blob
+    would put a second, drifting copy in every report a reader loads.
+    """
+
+    max_tier: Literal["L0", "L1", "L2", "L3a", "L3b", "L4"]
+    routed: list[dict[str, str]] = Field(
+        default_factory=list, description="Which hypothesis went to which tier, and why."
+    )
+
+    attempted: list[FixAttemptWire] = Field(
+        default_factory=list,
+        description="The CONFIRMATION evidence: candidates validated on the held-out split. "
+                    "With a confirm split in play this is exactly one frozen candidate — "
+                    "selecting and testing on the same cases is the error the two-stage "
+                    "protocol exists to prevent.",
+    )
+    selection: list[FixAttemptWire] = Field(
+        default_factory=list,
+        description="The candidates tried while CHOOSING one, on the diagnosis split. "
+                    "Descriptive only — never confirmation evidence, and a reader must not "
+                    "add these effect sizes to the ones above.\n\n"
+                    "Carried because omitting it misrepresents the search: a run that tried "
+                    "seven candidates across L1 and L2 and confirmed one L1 reported a "
+                    "single L1 row, and every reader concluded L2 was never attempted. "
+                    "What was ruled out is part of what the run found — an `unsafe` "
+                    "candidate that repaired 7 cases and broke 8 is a result.",
+    )
+    selected_on_explore: str | None = Field(
+        default=None,
+        description="Which candidate the selection phase froze for confirmation. Names a "
+                    "row in `selection`; `best` names one in `attempted`.",
+    )
+
+    best: str | None = Field(
+        default=None, description="Winning candidate NAME. The row itself is in `attempted`."
+    )
+    fixed: bool = False
+    ebh_survivors: list[str] = Field(
+        default_factory=list,
+        description="Candidate names surviving e-BH across the whole tested family. A winner must be "
+                    "BOTH individually `fixed` AND in this set — best-of-N needs its own correction.",
+    )
+    repair_rounds: int = Field(default=0, ge=0)
+    recommendation: dict[str, Any] | None = Field(
+        default=None,
+        description="{recommend_tier, reason} when nothing validated. Escalation is NEVER automatic.",
+    )
+    refine_signal: dict[str, Any] | None = Field(
+        default=None,
+        description="Set when a candidate helped one subset and hurt another — evidence the hypothesis "
+                    "should be re-scoped, not that no fix exists. Feedback edge back into M3.",
+    )
+
+    @model_validator(mode="after")
+    def _fixed_requires_confirmed_survivor(self) -> "FixOutput":
+        if not self.fixed:
+            return self
+        if self.best is None:
+            raise ValueError("fixed=True requires a `best` candidate")
+        row = next((a for a in self.attempted if a.name == self.best), None)
+        if row is None:
+            raise ValueError(f"best={self.best!r} is not among the validated attempts")
+        if row.methodology is None:
+            raise ValueError(
+                "fixed=True without a methodology on the winning candidate: a shipped repair "
+                "must carry an explanation a reviewer can read"
+            )
+        if self.ebh_survivors and self.best not in self.ebh_survivors:
+            raise ValueError(
+                f"best={self.best!r} is not an e-BH survivor; best-of-N selection without "
+                "family correction is not a validated fix"
+            )
+        return self
+
+
+__all__ = [
+    "SurgeryInput", "InterventionOutput",
+    "FixInput", "FixAttemptWire", "FixOutput",
+]

evalrx/contract/methodology.py

@@ -0,0 +1,76 @@
+"""Methodology — what an M5 repair or intervention actually DOES.
+
+A candidate's method currently survives only as a code blob or a prompt string.
+Neither answers the question a reader has ("what does this do to my model?"), so
+the contract carries a diagram instead.
+
+The contract is the draw.io XML and nothing else: a frontend embeds
+``drawio_xml`` directly, with no toolchain and no intermediate format to
+interpret. How a producer arrives at that XML is its own business —
+``examples/contract/drawio_builder.py`` offers a step-graph with automatic
+layout for producers that want one.
+
+The XML is still validated on its own terms. A diagram whose edge points at a
+cell that does not exist renders as a perfectly ordinary picture, so nothing
+downstream — and no reader — would ever notice; that check has to happen here or
+not at all.
+"""
+
+from __future__ import annotations
+
+import xml.etree.ElementTree as ET
+
+from pydantic import Field, model_validator
+
+from evalrx.contract.common import WireModel
+
+
+class MethodologyWire(WireModel):
+    """A repair or intervention, as an embeddable diagram plus plain language."""
+
+    title: str = Field(min_length=1, max_length=120)
+    summary: str = Field(
+        default="", max_length=1000,
+        description="Two or three sentences a non-specialist can follow. Not the code.",
+    )
+    tier: str = Field(default="", description="L1..L4 for a fix; empty for an intervention.")
+    drawio_xml: str = Field(
+        min_length=1,
+        description="Uncompressed mxfile document, embeddable as-is. Compressed diagrams are "
+                    "rejected: they cannot be diffed, reviewed, or repaired by hand.",
+    )
+    n_model_calls: int | None = Field(
+        default=None,
+        description="Model calls this method makes per case. A large claimed gain alongside few "
+                    "model calls is the shape of a model-independent result, where the scaffold "
+                    "rather than the model did the work.",
+    )
+
+    @model_validator(mode="after")
+    def _xml_is_renderable(self) -> "MethodologyWire":
+        try:
+            root = ET.fromstring(self.drawio_xml)
+        except ET.ParseError as exc:
+            raise ValueError(f"drawio_xml is not well-formed XML: {exc}") from exc
+        if root.tag != "mxfile":
+            raise ValueError(f"drawio_xml root must be <mxfile>, got <{root.tag}>")
+        cells = list(root.iter("mxCell"))
+        if not cells:
+            raise ValueError("drawio_xml contains no mxCell elements")
+        ids = [c.get("id") for c in cells if c.get("id")]
+        dupes = {i for i in ids if ids.count(i) > 1}
+        if dupes:
+            raise ValueError(f"duplicate mxCell ids render unpredictably: {sorted(dupes)}")
+        known = set(ids)
+        for c in cells:
+            for end in ("source", "target"):
+                ref = c.get(end)
+                if ref and ref not in known:
+                    raise ValueError(
+                        f"edge {c.get('id')!r} has {end}={ref!r}, which is not a cell in this "
+                        "diagram — it draws as an arrow into empty space"
+                    )
+        return self
+
+
+__all__ = ["MethodologyWire"]

evalrx/contract/pre_m1.py

@@ -0,0 +1,58 @@
+"""Pre-M1 — probe search: synthesize new test cases instead of analyzing old ones.
+
+Optional and structurally different from every other stage: its output is DATA,
+not a measurement or a verdict, so it is the one stage whose result can be fed
+back in as another stage's input.
+"""
+
+from __future__ import annotations
+
+from pydantic import Field, model_validator
+
+from evalrx.contract.common import CaseBatchRef, StageEnvelope, WireModel
+from evalrx.contract.m1 import ModelRef, ProtocolWire
+
+
+class ProbeSearchInput(WireModel):
+    """Seeds plus a budget. The search anchors both regime trees on the seeds."""
+
+    model: ModelRef
+    seed_pool: CaseBatchRef
+    protocol: ProtocolWire | None = None
+    budget: int = Field(default=20, ge=1, description="Total simulations (T_max).")
+    beta: float = Field(default=1.0, description="UCB exploration constant.")
+    w_max: int = Field(default=3, ge=1, description="Max children before progressive widening.")
+
+
+class ProbeSearchOutput(StageEnvelope):
+    """Discovered cases plus search accounting.
+
+    ``all_cases`` — not ``failure_cases`` — is what M1 should normally receive:
+    a batch of nothing but FAILs has no control group, and every downstream
+    comparison degenerates.
+
+    The in-memory result also holds ``macro_root`` / ``micro_root``, two trees
+    whose nodes point at both parent and children. That graph is cyclic and
+    cannot be serialized naively, which is why the search trajectory is carried
+    here as an explicit flat edge list rather than a nested object.
+    """
+
+    n_simulations: int = Field(ge=0)
+    n_macro: int = Field(ge=0)
+    n_micro: int = Field(ge=0)
+
+    all_cases: CaseBatchRef = Field(description="Every evaluated case, PASS and FAIL. Feed this to M1.")
+    failure_cases: CaseBatchRef = Field(description="The FAIL subset. Feed this to cluster_failures.")
+
+    @property
+    def error_rate(self) -> float:
+        return self.failure_cases.n_cases / self.n_simulations if self.n_simulations else 0.0
+
+    @model_validator(mode="after")
+    def _counts_are_consistent(self) -> "ProbeSearchOutput":
+        if self.failure_cases.n_cases > self.all_cases.n_cases:
+            raise ValueError("failure_cases cannot exceed all_cases")
+        return self
+
+
+__all__ = ["ProbeSearchInput", "ProbeSearchOutput"]

evalrx/contract/typescript.py

@@ -0,0 +1,140 @@
+"""JSON Schema -> TypeScript for the stage contract.
+
+Written in Python on purpose. ``export.py`` documented ``npx
+json-schema-to-typescript`` as the way to get types, which puts a node toolchain
+and a network fetch between a Python field and the frontend that has to decode
+it — and a generation step that can fail for reasons unrelated to the contract
+is a generation step CI will end up skipping. One language, one command, one
+committed artifact that a diff check can police.
+
+The output is a single ``.d.ts``: every stage's payload and every type it
+references, so a frontend imports one file and gets the whole pipeline.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+_SCALARS = {
+    "string": "string", "integer": "number", "number": "number",
+    "boolean": "boolean", "null": "null",
+}
+
+
+def _ts_type(schema: dict[str, Any]) -> str:
+    """One JSON-Schema node as a TypeScript type expression."""
+    if not schema:
+        return "unknown"
+    if "$ref" in schema:
+        return schema["$ref"].rsplit("/", 1)[-1]
+    if "const" in schema:
+        return _literal(schema["const"])
+    if "enum" in schema:
+        return " | ".join(_literal(v) for v in schema["enum"]) or "never"
+    for key in ("anyOf", "oneOf"):
+        if key in schema:
+            parts = [_ts_type(s) for s in schema[key]]
+            # dict.fromkeys, not set: a stable union order keeps the generated
+            # file diffable, and a diff check is the only thing that makes a
+            # generated artifact trustworthy.
+            return " | ".join(dict.fromkeys(parts)) or "unknown"
+    if "allOf" in schema and len(schema["allOf"]) == 1:
+        return _ts_type(schema["allOf"][0])
+
+    kind = schema.get("type")
+    if isinstance(kind, list):
+        return " | ".join(dict.fromkeys(_SCALARS.get(k, "unknown") for k in kind))
+    if kind == "array":
+        items = schema.get("items") or schema.get("prefixItems")
+        if isinstance(items, list):   # tuple, e.g. ci: (float, float)
+            return "[" + ", ".join(_ts_type(i) for i in items) + "]"
+        return f"({_ts_type(items or {})})[]"
+    if kind == "object":
+        extra = schema.get("additionalProperties")
+        value = _ts_type(extra) if isinstance(extra, dict) else "unknown"
+        return f"{{ [key: string]: {value} }}"
+    return _SCALARS.get(kind, "unknown")
+
+
+def _literal(value: Any) -> str:
+    if isinstance(value, str):
+        return '"' + value.replace('"', '\\"') + '"'
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if value is None:
+        return "null"
+    return str(value)
+
+
+def _doc(schema: dict[str, Any], indent: str) -> str:
+    """Carry the Python docstring/description across. A generated type with no
+    prose sends the reader back to the Python file, which is the drift this
+    whole contract exists to avoid."""
+    text = (schema.get("description") or "").strip()
+    if not text:
+        return ""
+    lines = [ln.rstrip() for ln in text.splitlines()]
+    body = "\n".join(f"{indent} * {ln}".rstrip() for ln in lines)
+    return f"{indent}/**\n{body}\n{indent} */\n"
+
+
+def _interface(name: str, schema: dict[str, Any]) -> str:
+    if "enum" in schema and "properties" not in schema:
+        return f"{_doc(schema, '')}export type {name} = {_ts_type(schema)};\n"
+    props = schema.get("properties") or {}
+    required = set(schema.get("required") or ())
+    out = [_doc(schema, ""), f"export interface {name} {{\n"]
+    for field, sub in props.items():
+        opt = "" if field in required else "?"
+        out.append(_doc(sub, "  "))
+        out.append(f"  {field}{opt}: {_ts_type(sub)};\n")
+    if schema.get("additionalProperties") is True:
+        out.append(
+            "  /** Producer-defined keys. The envelope is closed; payload areas are not. */\n"
+            "  [key: string]: unknown;\n"
+        )
+    out.append("}\n")
+    return "".join(out)
+
+
+def render(
+    schemas: dict[str, dict[str, Any]], *, schema_version: int,
+    aliases: "dict[str, tuple[Any, ...]] | None" = None,
+) -> str:
+    """Render every stage schema plus its ``$defs`` into one declaration file.
+
+    ``aliases`` names literal unions that pydantic inlines at every use site
+    (``Modality`` is one). A frontend that wants to write ``slot: Modality``
+    cannot, unless the name survives generation — and passing them explicitly
+    beats inferring which inline unions "look like" a named type.
+    """
+    defs: dict[str, dict[str, Any]] = {}
+    roots: dict[str, dict[str, Any]] = {}
+    for schema in schemas.values():
+        for name, sub in (schema.get("$defs") or {}).items():
+            defs.setdefault(name, sub)
+        root = {k: v for k, v in schema.items() if k != "$defs"}
+        roots.setdefault(root.get("title") or "Unnamed", root)
+
+    parts = [
+        "// GENERATED by `python -m evalrx.contract.export --out docs/contract`.\n"
+        "// Do not edit: Python is the source of truth (evalrx/contract/).\n"
+        f"// schema_version {schema_version}\n\n"
+        "/** Bumped only when a field's MEANING changes. Unknown fields must be\n"
+        " *  ignored by readers, never rejected. */\n"
+        f"export const SCHEMA_VERSION = {schema_version};\n\n",
+    ]
+    for name, values in sorted((aliases or {}).items()):
+        parts.append(
+            f"export type {name} = " + " | ".join(_literal(v) for v in values) + ";\n\n"
+        )
+    for name in sorted(defs):
+        if name in roots:
+            continue
+        parts.append(_interface(name, defs[name]) + "\n")
+    for name in sorted(roots):
+        parts.append(_interface(name, roots[name]) + "\n")
+    return "".join(parts)
+
+
+__all__ = ["render"]

evalrx/core/__init__.py

@@ -0,0 +1,85 @@
+"""EvalRX core — the sklearn-like substrate.
+
+Public contracts every other module builds on:
+
+  Capability   the vocabulary connecting models and analyzers
+  Model        analyzable model:  generate() + forward(capture)->Trace
+  Analyzer     sklearn-style estimator:  Analyzer(**params).run(model, data)->Result
+  FailureCase  the central data unit;  CaseBatch is a collection of them
+  Result       uniform, agent-readable output (findings + artifacts)
+  registry     discovery: list/match models and analyzers by capability
+  Pipeline     compose analyzers
+  Experiment   declarative spec the agent layer targets
+"""
+
+from evalrx.core.analyzer import Analyzer
+from evalrx.core.capability import Capability, CapabilityError
+from evalrx.core.case import (
+    CaseBatch,
+    FailureCase,
+    Inputs,
+    Label,
+    Provenance,
+    Source,
+    Step,
+    StepRole,
+    Trajectory,
+    as_casebatch,
+)
+from evalrx.core.experiment import Experiment, ExperimentRunner
+from evalrx.core.model import CaptureSpec, Model, TokenLogprob, Trace
+from evalrx.core.pipeline import Pipeline
+from evalrx.core.registry import (
+    Registry,
+    register_analyzer,
+    register_model,
+    registry,
+)
+from evalrx.core.result import Result
+from evalrx.core.spec import (
+    AttnSemantics,
+    AudioSpec,
+    ModelSpec,
+    ModulePaths,
+    VisionSpec,
+)
+from evalrx.core.tokentype import TokenTypeMap, build_token_type_map
+from evalrx.core.tool import ChatTurn, Tool, ToolCall
+
+__all__ = [
+    "Capability",
+    "CapabilityError",
+    "Model",
+    "Trace",
+    "TokenLogprob",
+    "CaptureSpec",
+    "Analyzer",
+    "FailureCase",
+    "CaseBatch",
+    "Inputs",
+    "Label",
+    "Provenance",
+    "Source",
+    "Step",
+    "StepRole",
+    "Trajectory",
+    "as_casebatch",
+    "ModelSpec",
+    "VisionSpec",
+    "AudioSpec",
+    "ModulePaths",
+    "AttnSemantics",
+    "Tool",
+    "ToolCall",
+    "ChatTurn",
+    "TokenTypeMap",
+    "build_token_type_map",
+    "Result",
+    "registry",
+    "Registry",
+    "register_model",
+    "register_analyzer",
+    "Pipeline",
+    "Experiment",
+    "ExperimentRunner",
+]