open-interpretability 0.0.20__py3-none-any.whl

openinterp/corpus_layouts.py

@@ -0,0 +1,108 @@
+"""How a dataset's rows become documents, for the layouts of ``open-interpretability corpus``."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Sequence
+from itertools import pairwise
+
+# WikiText's top-level heading row (" = Title = \n") starts an article; sections are
+# " = = Section = = \n". Its text is tokenised: "1 @,@ 000", "well @-@ known", "lobster ,".
+_WIKITEXT_TITLE = re.compile(r" = [^=].* = \n?")
+_WIKITEXT_HEADING = re.compile(r"(?:= )+(.*?)(?: =)+")
+_WIKITEXT_DETOKENIZE = [
+    (re.compile(r" @(.)@ "), r"\1"),  # number separators and hyphens
+    (re.compile(r" ([.,;:!?%)\]}])"), r"\1"),
+    (re.compile(r"([(\[{$]) "), r"\1"),
+    (re.compile(r" (n't|'s|'re|'ve|'m|'ll|'d)\b"), r"\1"),
+    (re.compile(r" ' s\b"), "'s"),  # possessive after punctuation: "Inc. ' s"
+    (re.compile(r"s ' "), "s' "),  # plural possessive
+    (re.compile(r'" ([^"]*?) "'), r'"\1"'),
+]
+
+
+def wikitext_articles(rows: Sequence[str | None]) -> list[str]:
+    """Return the WikiText articles that start and end within ``rows``, detokenised.
+
+    Headings become plain lines, and blank lines separate paragraphs. An article cut by
+    the start or end of ``rows`` (a row group's edge) is dropped, not truncated.
+    """
+    starts = [i for i, row in enumerate(rows) if _WIKITEXT_TITLE.fullmatch(row or "")]
+    return [_wikitext_article(rows[a:b]) for a, b in pairwise(starts)]
+
+
+def _wikitext_article(rows: Sequence[str | None]) -> str:
+    paragraphs = []
+    for row in rows:
+        text = (row or "").strip()
+        if heading := _WIKITEXT_HEADING.fullmatch(text):
+            text = heading.group(1)
+        for pattern, repl in _WIKITEXT_DETOKENIZE:
+            text = pattern.sub(repl, text)
+        if text:
+            paragraphs.append(text)
+    return "\n\n".join(paragraphs)
+
+
+# The parts of a book that are left out: its first 10% and its last 5%. They hold its front
+# matter (producer credits, title page, contents, preface) and back matter (notes, index).
+BOOK_FRONT = 0.10
+BOOK_BACK = 0.05
+PASSAGE_CHARS = (
+    3000  # a passage's length, at most: the default of `open-interpretability corpus --max-chars`
+)
+_PARAGRAPH_BREAK = re.compile(r"\n[ \t]*\n")
+_LINE_BREAK = re.compile(r"[ \t]*\n[ \t]*")
+
+
+def book_passages(rows: Sequence[str | None]) -> list[str]:
+    """Return passages from the body of each book in ``rows``, in order.
+
+    A book is one row, often hundreds of thousands of characters. Cut to its start, it
+    would give only front matter. So each book loses its first :data:`BOOK_FRONT` and last
+    :data:`BOOK_BACK`. The paragraphs at those cuts can be partial, so they are dropped
+    too. The paragraphs left join into passages of at most :data:`PASSAGE_CHARS`, which
+    always start and end at a paragraph break. A longer paragraph is a passage alone. The
+    books' lines are wrapped at about 70 characters, so a line break within a paragraph
+    becomes a space. Verse loses its line breaks too.
+    """
+    passages = []
+    for book in rows:
+        text = book or ""
+        body = text[int(len(text) * BOOK_FRONT) : len(text) - int(len(text) * BOOK_BACK)]
+        paragraphs = [_LINE_BREAK.sub(" ", p).strip() for p in _PARAGRAPH_BREAK.split(body)]
+        passage: list[str] = []
+        for paragraph in filter(None, paragraphs[1:-1]):
+            if passage and len("\n\n".join([*passage, paragraph])) > PASSAGE_CHARS:
+                passages.append("\n\n".join(passage))
+                passage = []
+            passage.append(paragraph)
+        if passage:
+            passages.append("\n\n".join(passage))
+    return passages
+
+
+# Row layouts other than one document per row: rows of one row group -> its documents.
+LAYOUTS = {"wikitext": wikitext_articles, "book_passages": book_passages}
+
+
+def chat_formatter(model: str, revision: str | None = None):
+    """Return a function that writes a conversation with ``model``'s chat template.
+
+    Also returns the model commit that the template comes from. A conversation is a list
+    of ``{"role": ..., "content": ...}`` messages. The text keeps the template's special
+    tokens as text (e.g. ``<|im_start|>``), which the tokenizer reads back as the same
+    tokens. Raises ``ValueError`` if the model's tokenizer has no chat template.
+    """
+    from huggingface_hub import HfApi
+    from transformers import AutoTokenizer
+
+    revision = HfApi().model_info(model, revision=revision).sha
+    tokenizer = AutoTokenizer.from_pretrained(model, revision=revision)
+    if not tokenizer.chat_template:
+        raise ValueError(f"{model}'s tokenizer has no chat template")
+
+    def render(messages) -> str:
+        return tokenizer.apply_chat_template(list(messages or []), tokenize=False)
+
+    return render, revision

openinterp/corpus_presets.py

@@ -0,0 +1,168 @@
+"""The datasets that the corpus command can sample by name, and how to read them.
+
+The command is ``open-interpretability corpus --dataset``.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class Preset:
+    """A dataset that ``open-interpretability corpus --dataset`` can sample, and how to read it."""
+
+    dataset: str
+    files: str  # glob within the dataset repo; comma-separated globs are sampled equally
+    text_column: str = "text"  # comma-separated columns join into one document
+    revision: str | None = None  # default branch or tag; pinned to a commit on use
+    layout: str | None = None  # how rows become documents (see LAYOUTS); None: one per row
+    docs_per_row_group: int = 20
+    # A regular expression whose group, matched in each file glob, is its documents' `lang`.
+    lang_pattern: str | None = None
+    # The dataset column that gives each document's programming language (`code_language`).
+    code_language_column: str | None = None
+    # A model ID: `text_column` holds conversations, and this model's chat template writes
+    # each as text (unless `open-interpretability corpus --chat-template` names another model).
+    chat_template: str | None = None
+
+
+WIKIPEDIA_LANGUAGES = ("en", "de", "fr", "ru", "el", "ar", "hi", "zh", "ja", "ko")
+
+# The languages of the `fineweb-2` preset. A key is FineWeb-2's folder name: an ISO 639-3
+# code and a script. Its value is the code that a document's `lang` records. Wikipedia's
+# folders and lid.176 (openinterp.langid) use the same codes. So one language has one
+# label in a blended corpus. Add entries, never change one: a corpus rebuilt from its
+# sidecar reads this table. FineWeb-2 has no English (that is FineWeb). The set spans 12
+# scripts. Some pairs share a script (Spanish and Portuguese, Russian and Ukrainian,
+# Arabic and Persian). The reviewer uses such pairs for hard negatives.
+FINEWEB2_LANGUAGES = {
+    "spa_Latn": "es", "por_Latn": "pt", "ita_Latn": "it", "nld_Latn": "nl",
+    "pol_Latn": "pl", "tur_Latn": "tr", "vie_Latn": "vi", "ind_Latn": "id",
+    "swh_Latn": "sw", "rus_Cyrl": "ru", "ukr_Cyrl": "uk", "arb_Arab": "ar",
+    "fas_Arab": "fa", "cmn_Hani": "zh", "jpn_Jpan": "ja", "kor_Hang": "ko",
+    "hin_Deva": "hi", "ben_Beng": "bn", "tha_Thai": "th", "heb_Hebr": "he",
+    "ell_Grek": "el", "tam_Taml": "ta",
+}  # fmt: skip
+
+# The values of the github-code preset's `language` column (the dataset card's list).
+# `open-interpretability corpus --detect-code` (openinterp.codeid) gives only these names too,
+# so one programming language has one label in exact and detected corpora.
+GITHUB_CODE_LANGUAGES = (
+    "Assembly", "Batchfile", "C", "C#", "C++", "CMake", "CSS", "Dockerfile", "FORTRAN",
+    "GO", "HTML", "Haskell", "Java", "JavaScript", "Julia", "Lua", "Makefile", "Markdown",
+    "PHP", "Perl", "PowerShell", "Python", "Ruby", "Rust", "SQL", "Scala", "Shell", "TeX",
+    "TypeScript", "Visual Basic",
+)  # fmt: skip
+
+# Licences of datasets whose cards state none: copies of a dataset, with its licence.
+UNSTATED_LICENSES = {"emozilla/pg19": "apache-2.0"}  # deepmind/pg19's card
+
+PRESETS = {
+    # Clean, educational web text (FineWeb filtered by an educational-quality classifier),
+    # closer to recent models' pretraining mix than the Pile. ODC-By: attribute excerpts.
+    "fineweb-edu": Preset("HuggingFaceFW/fineweb-edu", "sample/10BT/*.parquet"),
+    # The first 10k Pile documents; the registry's original corpus.
+    "pile-10k": Preset("NeelNanda/pile-10k", "data/*.parquet"),
+    # The Pile without Books3, BookCorpus2, OpenSubtitles, YTSubtitles and OWT2: ~90x
+    # pile-10k's documents, for rare concepts. The main branch is zstd JSONL, so read the
+    # Hub's Parquet conversion (its first 891,748 training documents). Licence "other": each
+    # Pile component keeps its own terms.
+    "pile-uncopyrighted": Preset(
+        "monology/pile-uncopyrighted",
+        "default/partial-train/*.parquet",
+        revision="refs/convert/parquet",
+    ),
+    # C4 English validation split: the standard web baseline of the SAE literature. The
+    # main branch is gzipped JSON, so read the Hub's Parquet conversion. ODC-By.
+    "c4": Preset("allenai/c4", "en/partial-validation/*.parquet", revision="refs/convert/parquet"),
+    # Synthetic short stories written with a 3-4 year-old's vocabulary: a low-confound control
+    # corpus for small models. Parquet is on the main branch. CDLA-Sharing-1.0: share excerpts
+    # under the same licence.
+    "tinystories": Preset("roneneldan/TinyStories", "data/train-*.parquet"),
+    # WikiText-103: 28,475 Good and Featured Wikipedia articles, long and clean, for windows
+    # of 512+ tokens and entity/fact neurons. One row per paragraph or heading, so rows are
+    # joined back into articles. CC BY-SA 3.0 and GFDL: attribute and share excerpts alike.
+    "wikitext": Preset(
+        "Salesforce/wikitext", "wikitext-103-raw-v1/train-*.parquet", layout="wikitext"
+    ),
+    # GitHub source files in 32 languages, deduplicated and filtered, for syntax, bracket
+    # and indentation neurons. This is codeparrot/github-code without autogenerated, very
+    # long-lined or mostly non-alphanumeric files. Row groups are ~26 MB of 10,000 files from ~9,000
+    # repos in shuffled order, so take 100 from each. The card says Apache-2.0, but each
+    # file keeps its repo's licence (its `license` column: mostly MIT and GPL).
+    "github-code": Preset(
+        "codeparrot/github-code-clean",
+        "data/train-*.parquet",
+        "code",
+        docs_per_row_group=100,
+        code_language_column="language",
+    ),
+    # Wikipedia (November 2023 dump) in ten languages and eight scripts, with an equal share
+    # of documents each, for language-identity and script neurons. Latin script: English,
+    # German and French. Other scripts: Russian (Cyrillic), Greek, Arabic, Hindi
+    # (Devanagari), Chinese (Han), Japanese (kana and kanji) and Korean (Hangul). Row groups
+    # are 1,000 articles of 10-22 MB, so take 50 from each: 500 documents read one row group
+    # per language. CC BY-SA 3.0 and GFDL: attribute and share excerpts alike.
+    "wikipedia-multilingual": Preset(
+        "wikimedia/wikipedia",
+        ",".join(f"20231101.{lang}/train-*.parquet" for lang in WIKIPEDIA_LANGUAGES),
+        docs_per_row_group=50,
+        lang_pattern=r"^\d+\.([\w-]+)/",
+    ),
+    # Web text in 22 languages besides English (FINEWEB2_LANGUAGES), with an equal share of
+    # documents each. It is FineWeb's pipeline run on Common Crawl for other languages. It
+    # gives the blend non-English text in more registers than encyclopedic Wikipedia. Only
+    # each language's first file is read. It holds 1.2M-4M documents from many crawls, in
+    # row groups of 1,000 documents (~3.6 MB) from one crawl each. Reading the footers of
+    # every file (~2,000) took 10 minutes. ODC-By 1.0: attribute excerpts. The card also
+    # requires users to obey the Common Crawl terms of use.
+    "fineweb-2": Preset(
+        "HuggingFaceFW/fineweb-2",
+        ",".join(f"data/{folder}/train/000_00000.parquet" for folder in FINEWEB2_LANGUAGES),
+        lang_pattern=r"^data/([a-z]{3}_[A-Z][a-z]{3})/",
+    ),
+    # Scientific papers from Semantic Scholar (peS2o v2), for academic prose, citations and
+    # section structure. Half the documents are abstracts with their titles (`s2ag`). Half
+    # are full texts (`s2orc`): title, abstract, then the body with its section headings
+    # and in-text citations. The main branch is gzipped JSON, so read the Hub's Parquet
+    # conversion. Its train split holds only abstracts, so full texts come from its
+    # validation split: 3 files of 41,032 papers, all full texts. Row groups are
+    # 1,000 documents (~1 MB of abstracts, ~32 MB of full texts). ODC-By 1.0: attribute
+    # excerpts.
+    "pes2o": Preset(
+        "allenai/peS2o",
+        "v2/partial-train/*.parquet,v2/partial-validation/000[1-3].parquet",
+        revision="refs/convert/parquet",
+    ),
+    # Mathematical web text from Common Crawl, with LaTeX, for number, operator and LaTeX
+    # neurons. Row groups are 1,000 documents of ~3.6 MB. ODC-By 1.0: attribute excerpts.
+    # The card also requires users to obey the Common Crawl terms of use.
+    "open-web-math": Preset("open-web-math/open-web-math", "data/train-*.parquet"),
+    # Grade-school word problems and their worked solutions (train split), for arithmetic
+    # neurons. A document is a problem, then its solution. A solution keeps the dataset's
+    # calculator annotations ("<<48/2=24>>24") and final answer ("#### 72"). MIT.
+    "gsm8k": Preset("openai/gsm8k", "main/train-*.parquet", "question,answer"),
+    # Multi-turn conversations between a user and ChatGPT (the SFT train split of UltraChat,
+    # filtered and truecased), for instruct models. Qwen2.5's chat template writes each, so
+    # every model scans the same text (`--chat-template MODEL` writes it with another
+    # model's). Row groups are 1,000 conversations of ~6.5 MB. MIT.
+    # Project Gutenberg books published before 1919 (PG-19), for narrative, dialogue and
+    # older English. deepmind/pg19 has only a loading script, so read emozilla/pg19, its
+    # Parquet copy. Only its validation and test splits: 150 books in ~35 MB. A training
+    # row group is 1,000 books (~380 MB). Each book is one row, of ~270,000 characters
+    # (median), so the book_passages layout gives passages from its body
+    # (openinterp.corpus_layouts). Public-domain texts; PG-19 is Apache-2.0
+    # (UNSTATED_LICENSES).
+    "pg19": Preset(
+        "emozilla/pg19",
+        "data/validation-*.parquet,data/test-*.parquet",
+        layout="book_passages",
+    ),
+    "ultrachat": Preset(
+        "HuggingFaceH4/ultrachat_200k",
+        "data/train_sft-*.parquet",
+        "messages",
+        chat_template="Qwen/Qwen2.5-0.5B",
+    ),
+}

openinterp/corpus_rebuild.py

@@ -0,0 +1,93 @@
+"""Rebuild the corpus that a record scanned (`open-interpretability corpus --same-as`).
+
+A follow-up review draws held-out windows from the scan's corpus (`open-interpretability score`).
+Corpora are git-ignored, and a scan can read a corpus that only one experiment used. So
+the record's source is what brings the same bytes back.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from openinterp.blends import build_blend
+from openinterp.core.schemas import COMMITTED_SOURCES, CorpusSource
+from openinterp.corpus import build_from_hub, load_sources, source_path
+
+COMMITTED_SIDECARS = tuple(Path(name) for name in COMMITTED_SOURCES)
+
+
+def scanned_source(submission) -> CorpusSource:
+    """Return the source of the corpus that ``submission`` scanned.
+
+    A record without its own source can match a committed sidecar by SHA-256. Raises
+    ``ValueError`` if neither says how to rebuild the corpus.
+    """
+    provenance = submission.provenance
+    if provenance is None:
+        raise ValueError("the record has no provenance, so it names no corpus")
+    source = provenance.corpus_source or load_sources(COMMITTED_SIDECARS).get(
+        provenance.corpus_sha256
+    )
+    if source is None:
+        raise ValueError(
+            f"the record doesn't say how to rebuild its corpus ({provenance.corpus}, "
+            f"SHA-256 {provenance.corpus_sha256})"
+        )
+    return source
+
+
+def rebuild(source: CorpusSource, out: Path) -> CorpusSource:
+    """Write the corpus that ``source`` describes to ``out``, at its pinned dataset commits.
+
+    Raises ``ValueError``, and leaves no corpus, if the bytes differ. A scan's held-out
+    windows must come from its own text, so a near match is not acceptable.
+    """
+    built = _build(source, out)
+    if built.sha256 != source.sha256:
+        out.unlink()
+        source_path(out).unlink()
+        raise ValueError(
+            f"the rebuilt corpus has SHA-256 {built.sha256}, but the record scanned "
+            f"{source.sha256}. Its dataset or preset may have changed."
+        )
+    return built
+
+
+def _build(source: CorpusSource, out: Path) -> CorpusSource:
+    if source.blend:
+        return build_blend(
+            source.blend,
+            out,
+            documents=source.documents,
+            seed=source.seed,
+            max_chars=source.max_chars,
+            ascii=source.ascii,
+            revisions=_revisions(source.components or []),
+        )
+    return build_from_hub(
+        source.dataset,
+        out,
+        files=source.files,
+        text_column=source.text_column,
+        revision=source.revision,
+        documents=source.documents,
+        seed=source.seed,
+        max_chars=source.max_chars,
+        docs_per_row_group=source.docs_per_row_group,
+        layout=source.layout,
+        first=source.first,
+        ascii=source.ascii,
+        lang_pattern=source.lang_pattern,
+        detect_lang=source.lang_detector is not None,
+        code_language_column=source.code_language_column,
+        detect_code=source.code_detector is not None,
+        chat_template=source.chat_template,
+        chat_template_revision=source.chat_template_revision,
+    )
+
+
+def _revisions(components: list[CorpusSource]) -> dict[str, str | None]:
+    """A blend's pinned commits: each dataset's, and each chat template model's."""
+    revisions = {part.dataset: part.revision for part in components}
+    templates = {p.chat_template: p.chat_template_revision for p in components if p.chat_template}
+    return revisions | templates

openinterp/coverage.py

@@ -0,0 +1,155 @@
+"""Registry coverage: how much of each MLP layer's effect on the loss the VERIFIED poles keep.
+
+For one layer, mean-ablating every neuron raises the model's next-token loss. The rise
+is what that layer contributes. The coverage of the layer is the share of that rise that
+comes back when only the VERIFIED poles keep their real activations:
+
+    recovered = (loss_ablated - loss_kept) / (loss_ablated - loss_clean)
+
+A neuron's activation has two poles: its positive part and its negative part. Each part is
+replaced by its own mean over a separate sample of the corpus, unless that pole is
+VERIFIED. So a neuron whose positive pole is VERIFIED keeps its positive part only.
+
+The same number of poles, drawn at random from the layer's other poles, gives a baseline.
+It is the mean over several draws.
+VERIFIED poles that recover no more than random ones explain no more of the loss than
+any poles would.
+
+This is a running measure of how much of the model the registry explains. It is
+reported, not gated. ``open-interpretability coverage`` writes one row per layer.
+"""
+
+from __future__ import annotations
+
+import random
+from collections.abc import Sequence
+from pathlib import Path
+
+import torch
+
+from openinterp.core.paths import DEFAULT_WIKI_DIR
+from openinterp.harness.hooks import NeuronHookRunner
+
+CE_CHUNK = 512  # token positions per cross-entropy chunk: Qwen2.5's vocabulary is 151,936
+
+
+def verified_poles(model_name: str, wiki_dir: Path = DEFAULT_WIKI_DIR) -> dict[int, set]:
+    """The neuron poles that stand VERIFIED, as ``{layer: {(neuron_idx, pole)}}``.
+
+    Standing uses the site's rules, errata included (``builder/standing.py``).
+    """
+    from openinterp.builder.compiler import load_records
+    from openinterp.builder.tree import build_tree
+    from openinterp.core.errata import load_errata
+    from openinterp.core.paths import experiments_dir
+
+    records = load_records(experiments_dir(Path(wiki_dir)))
+    out: dict[int, set] = {}
+    for model in build_tree(records, load_errata(wiki_dir)):
+        if model.name != model_name:
+            continue
+        for layer in model.layers:
+            for neuron in layer.neurons:
+                for side, standing in neuron.standings.items():
+                    if standing.verified:
+                        out.setdefault(layer.layer, set()).add((neuron.idx, side))
+    return out
+
+
+def windows(runner: NeuronHookRunner, texts: Sequence[str], count: int, tokens: int, seed: int):
+    """Two disjoint sets of ``count`` windows of ``tokens`` tokens: one to fit, one to test.
+
+    Each window is the start of one document long enough to fill it.
+    """
+    ids = [runner.tokenizer(t)["input_ids"] for t in texts]
+    starts = [row[:tokens] for row in ids if len(row) >= tokens]
+    random.Random(seed).shuffle(starts)
+    if len(starts) < 2 * count:
+        raise ValueError(f"only {len(starts)} documents have {tokens} tokens; need {2 * count}")
+    fit, test = starts[:count], starts[count : 2 * count]
+    return torch.tensor(fit), torch.tensor(test)
+
+
+def pole_means(runner: NeuronHookRunner, ids: torch.Tensor, layer: int, batch: int):
+    """The mean positive part and mean negative part of each neuron's activation."""
+    pos = neg = None
+    for chunk in ids.split(batch):
+        act = runner.layer_activations(chunk, [layer])[layer].float().flatten(0, -2)
+        p, n = act.clamp(min=0).sum(0), act.clamp(max=0).sum(0)
+        pos, neg = (p, n) if pos is None else (pos + p, neg + n)
+    total = ids.numel()
+    return pos / total, neg / total
+
+
+def keep_poles(means, keep: set, width: int):
+    """An intervention that mean-ablates every pole outside ``keep``."""
+    m_pos, m_neg = means
+    keep_pos = torch.zeros(width, dtype=torch.bool)
+    keep_neg = torch.zeros(width, dtype=torch.bool)
+    for idx, pole in keep:
+        (keep_pos if pole == "positive" else keep_neg)[idx] = True
+
+    def intervene(act: torch.Tensor) -> torch.Tensor:
+        dev = act.device
+        pos = torch.where(keep_pos.to(dev), act.clamp(min=0), m_pos.to(dev, act.dtype))
+        neg = torch.where(keep_neg.to(dev), act.clamp(max=0), m_neg.to(dev, act.dtype))
+        return pos + neg
+
+    return intervene
+
+
+@torch.no_grad()
+def mean_loss(runner: NeuronHookRunner, ids: torch.Tensor, batch: int, interventions=None):
+    """Mean next-token cross-entropy over ``ids``, with optional MLP interventions."""
+    total, count = 0.0, 0
+    for chunk in ids.split(batch):
+        chunk = chunk.to(runner.device)
+        logits = runner.run_with_interventions({"input_ids": chunk}, interventions)
+        flat = logits[:, :-1].flatten(0, 1)
+        target = chunk[:, 1:].flatten()
+        for lo in range(0, target.numel(), CE_CHUNK):
+            part = flat[lo : lo + CE_CHUNK].float()
+            total += torch.nn.functional.cross_entropy(
+                part, target[lo : lo + CE_CHUNK], reduction="sum"
+            ).item()
+        count += target.numel()
+    return total / count
+
+
+def random_poles(verified: set, width: int, seed: int) -> set:
+    """As many poles as ``verified``, drawn from the layer's other poles (or all of them)."""
+    others = [
+        (i, p) for i in range(width) for p in ("positive", "negative") if (i, p) not in verified
+    ]
+    return set(random.Random(seed).sample(others, min(len(verified), len(others))))
+
+
+def recovered(clean: float, ablated: float, kept: float) -> float | None:
+    """The share of the ablation's loss rise that keeping the poles brings back."""
+    rise = ablated - clean
+    return None if rise <= 0 else (ablated - kept) / rise
+
+
+def layer_row(runner, fit, test, layer: int, verified: set, clean: float, batch: int, seed: int,
+              draws: int = 5):  # fmt: skip
+    """Measure one layer: ablated, kept-VERIFIED and kept-random losses, and shares.
+
+    The random baseline is the mean over ``draws`` random sets of poles.
+    """
+    width = runner.mlp_width(layer)
+    means = pole_means(runner, fit, layer, batch)
+    ablated = mean_loss(runner, test, batch, {layer: keep_poles(means, set(), width)})
+    row = {"layer": layer, "width": width, "verified_poles": len(verified),
+           "loss_clean": clean, "loss_ablated": ablated}  # fmt: skip
+    if verified:
+        kept = mean_loss(runner, test, batch, {layer: keep_poles(means, verified, width)})
+        shares = []
+        for d in range(draws):
+            rand = random_poles(verified, width, seed + 1000 * layer + d)
+            loss = mean_loss(runner, test, batch, {layer: keep_poles(means, rand, width)})
+            shares.append(recovered(clean, ablated, loss))
+        valid = [x for x in shares if x is not None]
+        row |= {"loss_kept_verified": kept, "recovered": recovered(clean, ablated, kept),
+                "recovered_random": sum(valid) / len(valid) if valid else None,
+                "recovered_random_draws": shares}  # fmt: skip
+    return row