open-interpretability 0.0.20__py3-none-any.whl

openinterp/planner_effects.py

@@ -0,0 +1,89 @@
+"""Explore draws weighted by a pole's effect on the loss (issue #3486).
+
+On GPT-2, VERIFIED poles bring back no more of a layer's loss than random poles do
+(``docs/analyses/registry-coverage.md``). So the registry may favour poles that are easy
+to explain over poles that matter. So a share of explore draws (``[tool.openinterp]
+effect_share``) picks a pole in proportion to its estimated effect. The effects come from
+the model's table in ``data/pole-effects/`` (``open-interpretability pole-effects``). These
+draws record ``selection: attribution``, so they never count towards the registry
+statistics, which use only random draws. A model with no table always draws uniformly.
+"""
+
+from __future__ import annotations
+
+import json
+import random
+from pathlib import Path
+
+EFFECTS_DIR = Path("data") / "pole-effects"
+DEFAULT_EFFECT_SHARE = 0.3
+
+
+def table_path(root: Path, model: str) -> Path:
+    """Where ``model``'s table of pole effects is kept, under the repository ``root``."""
+    return root / EFFECTS_DIR / f"{model.replace('/', '--')}.json"
+
+
+def effect_share(pyproject: Path) -> float:
+    """``[tool.openinterp] effect_share``: the share of explore draws weighted by effect."""
+    from openinterp.planner import _config
+
+    return float(_config(pyproject).get("effect_share", DEFAULT_EFFECT_SHARE))
+
+
+def unit_slug(model: str, row: dict) -> str:
+    """The slug of the pole that a table row names."""
+    from openinterp.core.identifiers import NeuronIdentifier
+
+    return NeuronIdentifier(
+        model_name=model, layer=row["layer"], neuron_idx=row["neuron"], pole=row["pole"]
+    ).slug
+
+
+def weighted_pole(rng: random.Random, model: str, rows: list[dict], taken: set[str]):
+    """A row drawn in proportion to its effect, among untaken poles that raise the loss.
+
+    ``taken`` holds studied slugs and the claim keys of open PRs.
+    """
+    from openinterp.planner import claim_key
+
+    slugs = {r_id: unit_slug(model, r) for r_id, r in enumerate(rows)}
+    free = [
+        r
+        for r_id, r in enumerate(rows)
+        if r["effect"] > 0 and not {slugs[r_id], claim_key(slugs[r_id])} & taken
+    ]
+    if not free:
+        return None
+    return rng.choices(free, weights=[r["effect"] for r in free], k=1)[0]
+
+
+def effect_draw(rng: random.Random, model: str, pyproject: Path, taken: set[str], window):
+    """An explore Candidate weighted by effect, or None for a uniform draw.
+
+    It takes one draw from ``rng`` for the share, and one more if it weights.
+    """
+    from openinterp.planner import SKILLS, Candidate, _audit_options
+
+    path = table_path(pyproject.parent, model)
+    if rng.random() >= effect_share(pyproject) or not path.exists():
+        return None
+    row = weighted_pole(rng, model, json.loads(path.read_text())["poles"], taken)
+    if row is None:
+        return None
+    audit = (
+        f"open-interpretability audit --model {model} --layer {row['layer']} "
+        f"--neuron {row['neuron']} --pole {row['pole']} --selection attribution"
+    ) + (f" --window-tokens {window}" if window else "")
+    return Candidate(
+        kind="explore",
+        model=model,
+        weight=0.0,
+        rationale=f"new pole drawn by its estimated loss effect ({row['effect']:.2e}); "
+        "selection attribution, so it doesn't count towards registry statistics",
+        commands=[audit],
+        slug=unit_slug(model, row),
+        skills=SKILLS["explore"],
+        layer=row["layer"],
+        audit=_audit_options([audit]),
+    )

openinterp/planner_heads.py

@@ -0,0 +1,92 @@
+"""The unit of an explore draw: a random neuron pole or a random attention head (#3248).
+
+Before this, explore draws always sampled an MLP neuron pole. So heads were audited only
+by hand, and the registry had no head records for follow-ups to start from. Now, after the
+model draw, an explore draw becomes a random head audit with probability ``head_share``.
+
+The share is configured (``[tool.openinterp] head_share``), not weighted by unit count.
+Qwen2.5-0.5B has 336 heads and about 233,000 poles. By count, heads would be under 0.2%
+of draws, so they would almost never come up.
+
+The kind is drawn from a generator seeded by the audit seed, not from the planner's own.
+So the planner's sequence of draws doesn't shift. A seed that drew a neuron still draws
+that neuron, unless it falls in the head share.
+"""
+
+from __future__ import annotations
+
+import random
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from openinterp.planner import Candidate
+
+DEFAULT_HEAD_SHARE = 0.1  # of explore draws; [tool.openinterp] head_share overrides it
+_KIND_SALT = 0x4EAD  # mixes the audit seed into the kind draw's own generator
+
+
+def head_share(pyproject: Path = Path("pyproject.toml")) -> float:
+    """Return ``[tool.openinterp] head_share``, or the default."""
+    from openinterp.planner import _config
+
+    return float(_config(pyproject).get("head_share", DEFAULT_HEAD_SHARE))
+
+
+def explore_audit(model: str, seed: int, window: int | None, unit: str | None = None) -> str:
+    """The ``open-interpretability audit`` command of a random explore draw."""
+    audit = f"open-interpretability audit --model {model} --random --seed {seed}"
+    if unit:
+        audit += f" --unit {unit}"
+    if window:
+        audit += f" --window-tokens {window}"
+    return audit
+
+
+def is_head_draw(audit_seed: int, share: float) -> bool:
+    """Whether the explore draw with ``audit_seed`` studies a head, at ``share``."""
+    return random.Random(audit_seed ^ _KIND_SALT).random() < share
+
+
+def head_draw(model: str, audit_seed: int, pyproject: Path, window: int | None) -> Candidate | None:
+    """A random head audit of ``model``, or None when the draw stays a neuron pole."""
+    from openinterp.planner import Candidate, _audit_options
+
+    share = head_share(pyproject)
+    if not is_head_draw(audit_seed, share):
+        return None
+    audit = explore_audit(model, audit_seed, window, unit="head")
+    return Candidate(
+        kind="explore",
+        model=model,
+        weight=0.0,
+        rationale=f"uniformly random attention head (unbiased among heads); heads get "
+        f"{share:.0%} of explore draws ([tool.openinterp] head_share)",
+        commands=[audit],
+        skills=["attention-head-audit"],  # the explore skills are for neuron poles
+        audit=_audit_options([audit]),
+    )
+
+
+def neuron_draw(
+    model: str, audit_seed: int, window: int | None, chance: float | None, why: str
+) -> Candidate:
+    """A random neuron-pole audit of ``model``: the default explore draw.
+
+    ``chance`` is the model's probability in the model draw, or None with one model.
+    """
+    from openinterp.planner import SKILLS, Candidate, _audit_options
+
+    rationale = "uniformly random new neuron (unbiased; counts towards registry statistics)"
+    if chance is not None:
+        rationale += f"; {model} drawn with probability {chance:.0%} by {why}"
+    audit = explore_audit(model, audit_seed, window)
+    return Candidate(
+        kind="explore",
+        model=model,
+        weight=0.0,
+        rationale=rationale,
+        commands=[audit],
+        skills=SKILLS["explore"],
+        audit=_audit_options([audit]),
+    )

openinterp/planner_models.py

@@ -0,0 +1,174 @@
+"""Which model an explore draw studies: the smallest one below its coverage target.
+
+One fully interpretable model is worth more than a little coverage of many. So the
+models are ordered by size (MLP neurons, ``layers × width``). The smallest one whose
+verified share is below the target gets most of the explore draws: the *focus*. A
+neuron is verified when the latest record of one of its poles is VERIFIED. A pole that
+the weights prove dead also counts (``openinterp.harness.dead``). When the focus passes
+the target, the focus moves to the next model. The other models share
+the rest of the draws, ``∝ 1/√(1 + neurons covered)``, so sister draws and cross-model
+comparisons still happen. Neurons within a model are still drawn at random, so each
+model's statistics stay unbiased.
+
+A neuron is covered when it has a record or an open PR claims it. A model of unknown
+size can't be the focus. When no model can be, every model is weighted by coverage.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+import re
+from collections.abc import Iterable, Sequence
+from dataclasses import dataclass
+
+from openinterp.core.identifiers import NeuronIdentifier, model_slug
+from openinterp.core.schemas import AuditRecord, ComponentIdentifier
+
+# The focus moves on when this share of its neurons is verified.
+DEFAULT_COVERAGE_TARGET = 0.8
+# The share of explore draws left for the other models. ``[tool.openinterp]`` can set both.
+DEFAULT_OTHERS_SHARE = 0.2
+
+
+@dataclass(frozen=True)
+class ModelCoverage:
+    """How much of one model's MLP the registry and open claims cover."""
+
+    model: str
+    covered: int  # neurons with a record, a stopped scan or an open claim
+    total: int | None  # every MLP neuron (layers × width); None when unknown
+    verified: int = 0  # neurons with a VERIFIED pole
+
+    @property
+    def verified_share(self) -> float | None:
+        """Verified share of the model's neurons, or None when its size is unknown."""
+        return self.verified / self.total if self.total else None
+
+    def __str__(self) -> str:
+        """``319 covered, 62/36,864 verified (0.17%)``; no total when the size is unknown."""
+        if not self.total:
+            return f"{self.covered:,} covered, {self.verified:,} verified"
+        return (
+            f"{self.covered:,} covered, {self.verified:,}/{self.total:,} verified "
+            f"({self.verified_share:.2%})"
+        )
+
+
+def model_config(model: str, download: bool = True) -> dict | None:
+    """The model's ``config.json``, or None if it can't be read.
+
+    With ``download`` False, read only the Hugging Face cache: no network. The file is in
+    the cache once the model has been used. Reading it never loads weights.
+    """
+    from huggingface_hub import hf_hub_download, try_to_load_from_cache
+
+    try:
+        path = hf_hub_download(model, "config.json") if download else None
+        path = path or try_to_load_from_cache(model, "config.json")
+        if not isinstance(path, str):  # not cached, or cached as missing
+            return None
+        with open(path) as fh:
+            return json.load(fh)
+    except Exception:  # offline with nothing cached, or an unknown model
+        return None
+
+
+def model_dims(model: str, download: bool = True) -> tuple[int, int] | None:
+    """(layers, MLP width) from the model's ``config.json``, or None if it can't be read.
+
+    ``download`` is as for :func:`model_config`.
+    """
+    config = model_config(model, download)
+    if config is None:
+        return None
+    return config_dims(config)
+
+
+def model_heads(model: str, download: bool = True) -> tuple[int, int] | None:
+    """(layers, attention heads per layer) from ``config.json``, or None if it can't be read."""
+    config = model_config(model, download)
+    if config is None:
+        return None
+    layers = config.get("num_hidden_layers") or config.get("n_layer")
+    heads = config.get("num_attention_heads") or config.get("n_head")
+    return (layers, heads) if layers and heads else None
+
+
+def config_dims(config: dict) -> tuple[int, int] | None:
+    """(layers, MLP width) from a ``config.json``, or None if it does not give them."""
+    layers = config.get("num_hidden_layers") or config.get("n_layer")
+    width = config.get("intermediate_size") or config.get("n_inner")
+    if width is None and config.get("n_embd"):  # GPT-2: n_inner null means 4 × n_embd
+        width = 4 * config["n_embd"]
+    return (layers, width) if layers and width else None
+
+
+def model_coverage(
+    pool: Sequence[str],
+    records: Iterable[AuditRecord],
+    claims: Sequence[str],
+    dead: Iterable[ComponentIdentifier] = (),
+    stopped: Iterable[ComponentIdentifier] = (),
+) -> dict[str, ModelCoverage]:
+    """Coverage of each model in ``pool``.
+
+    ``records`` are the latest record of each pole. A neuron is verified when one of
+    them is VERIFIED. A pole in ``dead`` (proven dead by the weights) also counts. A pole
+    in ``stopped`` (a scan stopped on it: dead or default-on) counts as covered. The
+    size comes from the records (``model_num_layers × model_mlp_width``), else from the
+    model's cached ``config.json``.
+    """
+    neurons: dict[str, set[tuple[int, int]]] = {m: set() for m in pool}
+    verified: dict[str, set[tuple[int, int]]] = {m: set() for m in pool}
+    dims: dict[str, tuple[int, int]] = {}
+    for record in records:
+        sub, ident = record.submission, record.submission.identifier
+        if ident.model_name not in neurons or not isinstance(ident, NeuronIdentifier):
+            continue
+        neurons[ident.model_name].add((ident.layer, ident.neuron_idx))
+        if record.verdict.status == "VERIFIED":
+            verified[ident.model_name].add((ident.layer, ident.neuron_idx))
+        if sub.model_num_layers and sub.model_mlp_width:
+            dims[ident.model_name] = (sub.model_num_layers, sub.model_mlp_width)
+    for ident in dead:
+        if ident.model_name in neurons and isinstance(ident, NeuronIdentifier):
+            neurons[ident.model_name].add((ident.layer, ident.neuron_idx))
+            verified[ident.model_name].add((ident.layer, ident.neuron_idx))
+    for ident in stopped:
+        if ident.model_name in neurons and isinstance(ident, NeuronIdentifier):
+            neurons[ident.model_name].add((ident.layer, ident.neuron_idx))
+    coverage = {}
+    for model in pool:
+        claim = re.compile(rf"{re.escape(model_slug(model))}-L(\d+)-N(\d+)-")  # see claim_key
+        for c in claims:
+            neurons[model].update((int(m[1]), int(m[2])) for m in claim.finditer(c))
+        layers_width = dims.get(model) or model_dims(model, download=False)
+        total = layers_width[0] * layers_width[1] if layers_width else None
+        coverage[model] = ModelCoverage(model, len(neurons[model]), total, len(verified[model]))
+    return coverage
+
+
+def focus_model(coverage: Iterable[ModelCoverage], target: float) -> str | None:
+    """The smallest model of known size whose verified share is below ``target``, if any."""
+    below = [c for c in coverage if c.total and c.verified / c.total < target]
+    return min(below, key=lambda c: (c.total, c.model)).model if below else None
+
+
+def explore_weights(
+    coverage: dict[str, ModelCoverage], target: float, others_share: float
+) -> dict[str, float]:
+    """Explore probability per model: ``1 − others_share`` for the focus model.
+
+    The other models share ``others_share``, ``∝ 1/√(1 + neurons covered)``.
+    """
+
+    def by_coverage(models: Sequence[str]) -> dict[str, float]:
+        weights = {m: 1.0 / math.sqrt(1 + coverage[m].covered) for m in models}
+        return {m: w / sum(weights.values()) for m, w in weights.items()}
+
+    focus = focus_model(coverage.values(), target)
+    if focus is None or len(coverage) == 1:
+        return by_coverage(list(coverage))
+    others = by_coverage([m for m in coverage if m != focus])
+    return {focus: 1.0 - others_share, **{m: others_share * w for m, w in others.items()}}

openinterp/planner_sisters.py

@@ -0,0 +1,122 @@
+"""``sister`` draws: VERIFIED poles' explanations tested in other models.
+
+``open-interpretability next`` makes these draws.
+:func:`openinterp.planner.plan_next` adds these to the follow-up candidates. The weights
+are in docs/PROTOCOL.md (Sister study).
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from pathlib import Path
+
+from openinterp.planner import (
+    BASE_WEIGHTS,
+    SKILLS,
+    Candidate,
+    _decay,
+    claim_key,
+    configured_windows,
+)
+
+# A match of a VERIFIED pole that isn't its sister can be drawn too. Its weight is this
+# share of a sister's weight, times its correlation. Any link can be tested. But the
+# draws lean heavily towards the sisters, where the explanation is likeliest to carry over.
+LINK_WEIGHT = 0.05
+# So can any pole of another configured model, found by the search or not. For each
+# VERIFIED pole and model, one draw picks a random pole there. Its weight is this share
+# of a sister's weight.
+# It is also the control for the matched draws.
+UNMATCHED_WEIGHT = 0.01
+
+
+def link_affinity(row, match) -> float:
+    """Return how strongly to draw a study of ``match``.
+
+    1 for the pole's sister, else :data:`LINK_WEIGHT` × its correlation. Never zero, as
+    for every weight here.
+    """
+    return 1.0 if row.sister is match else LINK_WEIGHT * max(match.pearson, 0.01)
+
+
+def sister_candidates(
+    states, claims, now, pool: Sequence[str], root: Path, seed: int = 0
+) -> list[Candidate]:
+    """Return the ``sister`` draws for the models of ``pool``.
+
+    A sister draw audits a pole of another model with a VERIFIED pole's explanation,
+    unchanged. The draws are every unstudied match that ``open-interpretability sisters`` found
+    (read from ``root``), weighted by :func:`link_affinity`. Each model also gets a
+    random pole (:data:`UNMATCHED_WEIGHT`, seeded from ``seed``). Only models in the pool
+    are drawn, so CI can reproduce the record.
+    """
+    from openinterp.sisters import DEFAULT_PATH, load_rows
+
+    rows, windows = load_rows(root / DEFAULT_PATH), configured_windows(root / "pyproject.toml")
+    out = []
+    for slug, state in states.items():
+        ident = state.record.submission.identifier
+        if state.record.verdict.status != "VERIFIED" or ident.kind != "mlp_neuron_pole":
+            continue
+        for row in rows.get(slug, []):
+            for m in row.matches if row.other in pool else []:
+                if m.slug(row.other) not in states:
+                    out.append(_sister(state, row, m, windows.get(row.other), claims, now))
+        for other in pool:
+            if other != ident.model_name:
+                out.append(_unmatched(state, other, windows.get(other), seed, claims, now))
+    return out
+
+
+def _unmatched(state, other: str, window: int | None, seed: int, claims, now) -> Candidate:
+    """The draw that audits a random pole of ``other`` with ``state``'s explanation."""
+    import hashlib
+
+    digest = hashlib.sha256(f"{seed} {state.slug} {other}".encode()).digest()
+    audit = f"--model {other} --random --seed {int.from_bytes(digest[:4])}"
+    if window:
+        audit += f" --window-tokens {window}"
+    audit += f" --sister-of {state.record_path.as_posix()}"
+    factor, notes = _decay(state, claims, now)
+    why = (
+        f"a random pole of {other}, matched by no search, with {state.slug}'s explanation: "
+        "the control for its matches"
+    )
+    verify = "open-interpretability verify --audit-file <new submission> --corpus <scan corpus>"
+    return Candidate(
+        kind="sister",
+        model=other,
+        weight=BASE_WEIGHTS["sister"] * UNMATCHED_WEIGHT * factor,
+        rationale="; ".join([why, *notes]),
+        commands=[f"open-interpretability audit {audit}", verify],
+        skills=SKILLS["sister"],
+        record=state.record_path.as_posix(),
+        audit=audit,
+    )
+
+
+def _sister(state, row, m, window: int | None, claims, now) -> Candidate:
+    """The draw that audits ``m`` with ``state``'s explanation."""
+    target = m.slug(row.other)
+    factor, notes = _decay(state, claims, now)
+    if any(claim_key(target) in c for c in claims):
+        factor, notes = factor * 0.05, [*notes, "an open PR already claims the match"]
+    audit = f"--model {row.other} --layer {m.layer} --neuron {m.neuron_idx} --pole {m.pole}"
+    if window:
+        audit += f" --window-tokens {window}"
+    audit += f" --sister-of {state.record_path.as_posix()}"
+    link = "sister" if row.sister is m else "correlated match (not a sister)"
+    why = f"{link} of {state.slug} (r {m.pearson:+.2f}): does its explanation carry over?"
+    verify = "open-interpretability verify --audit-file <new submission> --corpus <scan corpus>"
+    return Candidate(
+        kind="sister",
+        model=row.other,
+        weight=BASE_WEIGHTS["sister"] * link_affinity(row, m) * factor,
+        rationale="; ".join([why, *notes]),
+        commands=[f"open-interpretability audit {audit}", verify],
+        slug=target,
+        skills=SKILLS["sister"],
+        layer=m.layer,
+        record=state.record_path.as_posix(),
+        audit=audit,
+    )

openinterp/planner_states.py

@@ -0,0 +1,57 @@
+"""What the planner knows of each studied unit: its latest counted record and its history.
+
+:func:`load_states` reads the archive with the errata of each record applied
+(:mod:`openinterp.core.errata`, issue #3271). So the planner sees the standings of the site.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from pathlib import Path
+
+from openinterp.core.agent_names import canonical_agent
+from openinterp.core.schemas import AuditRecord
+
+
+@dataclass
+class NeuronState:
+    """What the registry holds on one unit: its latest record and how much it has been studied."""
+
+    slug: str
+    record: AuditRecord
+    record_path: Path
+    experiments: int = 0
+    reviewers: set[str] = field(default_factory=set)
+    independent_reviews: int = 0  # experiments reviewed by a model other than the proposer's
+    last: datetime | None = None
+
+
+def load_states(wiki_dir: Path) -> dict[str, NeuronState]:
+    """One state per neuron pole, derived from all of its archived experiments.
+
+    Each record has the edits of its errata applied (:mod:`openinterp.core.errata`).
+    Follow-ups are planned from the most recent experiment that counts toward the status.
+    An experiment that an erratum sets aside doesn't count. The exception is a pole whose
+    experiments are all set aside, as in the standing rule. The count, reviewers and recency
+    cover every experiment on the neuron.
+    """
+    from openinterp.builder.wiki import load_archive_paths  # Jinja: keep the CLI start light
+    from openinterp.core.errata import corrected, load_errata
+
+    errata = load_errata(wiki_dir)
+    states: dict[str, NeuronState] = {}
+    counted: set[str] = set()  # poles with an experiment that isn't set aside
+    for path, raw in load_archive_paths(Path(wiki_dir)):  # oldest first
+        record, aside = corrected(raw, errata)
+        slug = record.submission.identifier.slug
+        state = states.setdefault(slug, NeuronState(slug, record, path))
+        if not aside or slug not in counted:
+            state.record, state.record_path = record, path
+        if not aside:
+            counted.add(slug)
+        state.experiments += 1
+        state.reviewers.add(canonical_agent(record.verdict.reviewer_agent_model))
+        state.independent_reviews += not record.self_reviewed
+        state.last = record.verdict.timestamp
+    return states

openinterp/planner_superpositions.py

@@ -0,0 +1,114 @@
+"""``superposition`` draws: audits of the superpositions defined in ``data/superpositions/``.
+
+A superposition is a sparse direction over one layer's MLP neurons (issue #3215).
+``open-interpretability superpositions`` writes the definitions, one per line (issue #3686).
+:func:`openinterp.planner.plan_next` adds a draw for each definition that has no
+experiment and no claim (issue #3687).
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Sequence
+from pathlib import Path
+
+from openinterp.core.paths import DEFAULT_EXCHANGE_DIR, DEFAULT_WIKI_DIR
+from openinterp.planner import BASE_WEIGHTS, SKILLS, Candidate, claim_key, configured_windows
+
+DEFINITIONS_DIR = Path("data/superpositions")
+SHOW_COMMAND = "open-interpretability superpositions show"
+# A superposition draw has the weight of one sister draw (the full sister weight, not
+# LINK_WEIGHT or UNMATCHED_WEIGHT). Both test a ready unit that no experiment has studied yet.
+SUPERPOSITION_WEIGHT = BASE_WEIGHTS["sister"]
+
+
+def load_definitions(path: Path) -> list[dict]:
+    """Return the definitions in the JSONL file ``path``, one per non-empty line."""
+    lines = path.read_text(encoding="utf-8").splitlines()
+    return [json.loads(line) for line in lines if line.strip()]
+
+
+def definition_identifier(definition: dict):
+    """Return the :class:`SuperpositionIdentifier` of one definition."""
+    from openinterp.core.identifiers import SuperpositionIdentifier
+    from openinterp.core.profiles import SuperpositionProfile, superposition_id
+
+    fields = {k: v for k, v in definition.items() if k not in ("model", "layer")}
+    profile = SuperpositionProfile.model_validate(fields)
+    return SuperpositionIdentifier(
+        model_name=definition["model"],
+        layer=definition["layer"],
+        superposition_id=superposition_id(profile.members),
+    )
+
+
+def default_definitions(model: str) -> Path:
+    """Return the file that holds ``model``'s definitions, relative to the repo root."""
+    from openinterp.core.identifiers import model_slug
+
+    return DEFINITIONS_DIR / f"{model_slug(model)}.jsonl"
+
+
+def definition_file(slug: str) -> str:
+    """Return the path where a draw of the superposition ``slug`` writes its definition."""
+    return f"{DEFAULT_EXCHANGE_DIR}/{claim_key(slug)}/superposition.json"
+
+
+def superposition_candidates(states, claims: Sequence[str], root: Path) -> list[Candidate]:
+    """Return a ``superposition`` draw for each unstudied definition under ``root``.
+
+    The definitions are in every ``root/data/superpositions/*.jsonl``. A definition is
+    left out when ``states`` has an experiment on it, or when an open PR in ``claims``
+    claims it. It is also left out when a scan of it stopped (``wiki/stops/``): it is
+    dead or default-on, and no audit can follow. A missing folder gives no draws.
+    """
+    stopped = _stopped(root / DEFAULT_WIKI_DIR)
+    folder = root / DEFINITIONS_DIR
+    files = sorted(folder.glob("*.jsonl")) if folder.is_dir() else []
+    windows = configured_windows(root / "pyproject.toml")
+    out = []
+    for path in files:
+        for definition in load_definitions(path):
+            ident = definition_identifier(definition)
+            if ident.slug in states or ident.slug in stopped:
+                continue
+            if any(claim_key(ident.slug) in c for c in claims):
+                continue
+            source = path.relative_to(root)
+            out.append(_candidate(ident, definition, source, windows.get(ident.model_name)))
+    return out
+
+
+def _stopped(wiki_dir: Path) -> set[str]:
+    """The slugs of the chosen units whose scans stopped."""
+    from openinterp.core.draws import load_stops
+
+    return {s.identifier.slug for s in load_stops(wiki_dir)} if wiki_dir.is_dir() else set()
+
+
+def _candidate(ident, definition: dict, source: Path, window: int | None) -> Candidate:
+    """The draw that audits the superposition ``ident``, defined in the file ``source``."""
+    target = definition_file(ident.slug)
+    show = f"{SHOW_COMMAND} {ident.superposition_id} --model {ident.model_name} --out {target}"
+    if source != default_definitions(ident.model_name):
+        show += f" --definitions {source.as_posix()}"
+    audit = f"--superposition {target}"
+    if window:
+        audit += f" --window-tokens {window}"
+    members = len(definition["members"])
+    why = (
+        f"an unstudied superposition of {members} neurons on layer {ident.layer} "
+        f"({definition['method']}, from {definition['source']})"
+    )
+    verify = "open-interpretability verify --audit-file <new submission> --corpus <scan corpus>"
+    return Candidate(
+        kind="superposition",
+        model=ident.model_name,
+        weight=SUPERPOSITION_WEIGHT,
+        rationale=why,
+        commands=[show, f"open-interpretability audit {audit}", verify],
+        slug=ident.slug,
+        skills=SKILLS["superposition"],
+        layer=ident.layer,
+        audit=audit,
+    )