open-interpretability 0.0.20__py3-none-any.whl

openinterp/harness/wiring_activations.py

@@ -0,0 +1,285 @@
+"""A unit's wiring from activations, at the peaks of prompts where it fires.
+
+Upstream, it uses attribution patching (Syed et al., 2023). Downstream, it removes the
+unit's write from each later unit's input. :class:`~openinterp.core.wiring.Wiring`
+defines each number. The unit is described as in :mod:`openinterp.harness.wiring`.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from contextlib import suppress
+
+import torch
+
+from openinterp.core.exceptions import ArchitectureNotSupportedError
+from openinterp.core.wiring import Wiring
+from openinterp.harness.hooks import NeuronHookRunner, _StopForward
+from openinterp.harness.units import Unit
+from openinterp.harness.wiring import (
+    QKV,
+    Target,
+    _attn_read_point,
+    _downstream_layers,
+    _has_attention,
+    _head_outputs,
+    _matrix,
+    _mlp_activations,
+    _mlp_read_point,
+    _project,
+    _qkv_modules,
+    _top,
+    _upstream_layers,
+    target_of,
+)
+
+
+def _clean_pass(runner, t: Target, encoded, attention: bool) -> dict:
+    """One forward pass keeping what both directions need, all float32 ``[b, seq, ...]``.
+
+    The pass stops after the deepest layer it reads, so it computes no logits.
+    """
+    up_n, up_h = _upstream_layers(t)
+    down_n, down_h = _downstream_layers(runner, t)
+    cap = {"acts": {}, "z": {}, "mlp_in": {}, "attn_in": {}}
+
+    def keep(store, i, out=False):
+        if out:  # a decoder layer may return a tuple whose first item is the residual
+            return lambda _m, _a, o: store.__setitem__(
+                i, (o[0] if isinstance(o, tuple) else o).detach().float()
+            )
+        return lambda _m, args: store.__setitem__(i, args[0].detach().float())
+
+    hooks = [(runner.down_proj(i), keep(cap["acts"], i)) for i in {*up_n, t.layer}]
+    hooks += [(_mlp_read_point(runner, i), keep(cap["mlp_in"], i)) for i in {*down_n, t.layer}]
+    if attention:
+        hooks += [(runner.attn_out(i), keep(cap["z"], i)) for i in {*up_h, *down_h, t.layer}]
+        hooks += [
+            (_attn_read_point(runner, i), keep(cap["attn_in"], i)) for i in {*down_h, t.layer}
+        ]
+    handles = [m.register_forward_pre_hook(fn) for m, fn in hooks]
+    written: dict[int, torch.Tensor] = {}  # the read of a unit reading its layer's output
+    if t.site in ("mlp_out", "resid_post"):
+        module = runner.mlp_write(t.layer) if t.site == "mlp_out" else runner.layers[t.layer]
+        handles.append(module.register_forward_hook(keep(written, t.layer, out=True)))
+    deepest = max({*up_n, *up_h, *down_n, *down_h, t.layer})
+
+    def stop(*_) -> None:
+        raise _StopForward
+
+    handles.append(runner.layers[deepest].register_forward_hook(stop))
+    try:
+        with torch.no_grad(), suppress(_StopForward):
+            runner.forward(encoded)
+    finally:
+        for h in handles:
+            h.remove()
+    reads = {"mlp_in": cap["mlp_in"], "attn_in": cap["attn_in"]}
+    cap["read"] = reads.get(t.site, written)[t.layer]
+    return cap
+
+
+def _head_strength(runner, t: Target, encoded, x: torch.Tensor) -> torch.Tensor:
+    """A head's strength ``[b, seq]`` with its layer's attention input replaced by ``x``."""
+    z: list[torch.Tensor] = []
+
+    def capture(_m, args):
+        z.append(args[0])
+        raise _StopForward
+
+    handles = [
+        _attn_read_point(runner, t.layer).register_forward_pre_hook(
+            lambda _m, args: (x.to(args[0].dtype), *args[1:])
+        ),
+        runner.attn_out(t.layer).register_forward_pre_hook(capture),
+    ]
+    try:
+        with suppress(_StopForward):
+            runner.forward(encoded)
+    finally:
+        for h in handles:
+            h.remove()
+    out = z[-1].reshape(*z[-1].shape[:-1], runner.n_heads, -1)[..., t.head, :].float()
+    w_o = t.write.to(out.device)
+    return ((out - t.mean_output.to(out.device)) @ w_o.T).norm(dim=-1)
+
+
+def _downstream_head_effects(runner, t: Target, encoded, cap, delta, layer, peaks) -> torch.Tensor:
+    """Return ``[b, n_heads, 3]``: how far each head's write at the peak moves.
+
+    It moves when ``delta`` is removed from what the head's ``q``, ``k`` or ``v`` reads,
+    one channel at a time.
+    """
+    modules = _qkv_modules(runner.layers[layer])
+    w_o = _head_outputs(runner, layer)  # [n, d, hd]
+    rows = torch.arange(len(peaks))
+    clean = cap["z"][layer][rows, peaks].reshape(len(peaks), runner.n_heads, -1)
+    effects = []
+    for c in QKV:
+        module, part = modules[c]
+        z = _z_without(runner, layer, module, part, delta, encoded)
+        changed = z[rows, peaks].reshape(len(peaks), runner.n_heads, -1)
+        moved = torch.einsum("ndh,bnh->bnd", w_o.to(changed.device), clean - changed)
+        effects.append(moved.norm(dim=-1))
+    return torch.stack(effects, -1)
+
+
+def _z_without(runner, layer: int, module, part, delta, encoded) -> torch.Tensor:
+    """Return the layer's head outputs (float32) when one channel reads without ``delta``.
+
+    The channel is one of ``q``, ``k`` or ``v``: ``module``, or its ``part`` of a fused
+    projection.
+    """
+    norm = _attn_read_point(runner, layer)
+    state: dict = {}
+
+    def before(_m, args):
+        if not state.get("busy"):
+            state["busy"] = True
+            try:
+                state["normed"] = norm(args[0] - delta.to(args[0].dtype))
+            finally:
+                state["busy"] = False
+
+    def swap(_m, _args, out):
+        alt = _project(module, state["normed"])
+        if part is None:
+            return alt
+        out = out.clone()
+        out[..., part] = alt[..., part]
+        return out
+
+    z: list[torch.Tensor] = []
+
+    def capture(_m, args):
+        z.append(args[0].float())
+        raise _StopForward
+
+    handles = [
+        norm.register_forward_pre_hook(before),
+        module.register_forward_hook(swap),
+        runner.attn_out(layer).register_forward_pre_hook(capture),
+    ]
+    try:
+        with torch.no_grad(), suppress(_StopForward):
+            runner.forward(encoded)
+    finally:
+        for h in handles:
+            h.remove()
+    return z[-1]
+
+
+def activation_wiring(
+    runner: NeuronHookRunner,
+    unit: Unit,
+    prompts: Sequence[str],
+    threshold: float,
+    *,
+    k: int = 10,
+    batch_size: int = 8,
+) -> Wiring | None:
+    """Return the unit's top connections by attribution, at its peak on each firing prompt.
+
+    A prompt fires when the unit reaches ``threshold``. ``None`` if it reaches it on none.
+    See :class:`Wiring`.
+    """
+    t = target_of(runner, unit)
+    attention = _has_attention(runner)
+    if t.site == "attn_in" and not attention:
+        raise ArchitectureNotSupportedError("a head's wiring needs supported attention")
+    up_n, up_h = _upstream_layers(t)
+    down_n, down_h = _downstream_layers(runner, t)
+    if not attention:
+        up_h = down_h = range(0)
+    w_out = {i: _matrix(runner.down_proj(i)) for i in up_n}  # [d, width]
+    w_o = {i: _head_outputs(runner, i) for i in up_h} if attention else {}
+    dtype = next(runner.model.parameters()).dtype
+    acc = _Sums()
+    for start in range(0, len(prompts), batch_size):
+        encoded = runner.encode(list(prompts[start : start + batch_size]))
+        cap = _clean_pass(runner, t, encoded, attention)
+        found = _peak_gradient(runner, t, unit, encoded, cap, dtype, threshold)
+        if found is None:
+            continue
+        grad, peaks, fired = found
+        acc.used += int(fired.sum())
+        with torch.no_grad():
+            _add_upstream(acc, runner, cap, grad, fired, up_n, up_h, w_out, w_o)
+            _add_downstream(acc, runner, t, encoded, cap, fired, peaks, down_n, down_h, dtype)
+    if not acc.used:
+        return None
+    return Wiring(
+        method="activations",
+        upstream_neurons=_top("mlp_neuron", acc.mean("up_n", up_n), [""], k),
+        upstream_heads=_top("attn_head", acc.mean("up_h", up_h), [""], k) if attention else None,
+        downstream_neurons=_top("mlp_neuron", acc.mean("down_n", down_n), [""], k),
+        downstream_heads=_top("attn_head", acc.mean("down_h", down_h), list(QKV), k)
+        if attention
+        else None,
+        prompts=acc.used,
+    )
+
+
+class _Sums:
+    """Per-(direction, layer) sums over the prompts where the unit fired, and their count."""
+
+    def __init__(self) -> None:
+        self.sums: dict[tuple[str, int], torch.Tensor] = {}
+        self.used = 0
+
+    def add(self, key, value) -> None:
+        self.sums[key] = self.sums[key] + value if key in self.sums else value
+
+    def mean(self, kind: str, layers) -> list[tuple[int, torch.Tensor]]:
+        blocks = []
+        for i in layers:
+            v = self.sums[(kind, i)] / self.used
+            blocks.append((i, v if v.dim() == 2 else v[:, None]))
+        return blocks
+
+
+def _peak_gradient(runner, t: Target, unit: Unit, encoded, cap, dtype, threshold: float):
+    """Return the read gradients, the peaks, and the mask of prompts that fired.
+
+    Each gradient is of a firing prompt's peak strength with respect to the unit's read.
+    ``None`` if no prompt fired.
+    """
+    mask = encoded.get("attention_mask", torch.ones_like(encoded["input_ids"])).bool()
+    x = cap["read"].to(dtype).requires_grad_(True)
+    with torch.enable_grad():
+        s = t.strength(x) if t.strength is not None else _head_strength(runner, t, encoded, x)
+        measured = mask & (mask.cumsum(-1) > int(unit.skip_first))  # protocol 0.8
+        s = s.float().masked_fill(~measured.to(s.device), -1.0)
+        peak_values, peaks = s.max(-1)
+        fired = peak_values.detach() >= threshold
+        if not fired.any():
+            return None
+        (grad,) = torch.autograd.grad(peak_values[fired].sum(), x)
+    return grad.float()[fired], peaks[fired], fired
+
+
+def _add_upstream(acc: _Sums, runner, cap, grad, fired, up_n, up_h, w_out, w_o) -> None:
+    for i in up_n:
+        per_neuron = (grad @ w_out[i].to(grad.device)) * cap["acts"][i][fired]
+        acc.add(("up_n", i), per_neuron.sum((0, 1)))
+    for i in up_h:
+        z = cap["z"][i][fired].reshape(*grad.shape[:2], runner.n_heads, -1)
+        per_head = torch.einsum("bsd,ndh,bsnh->n", grad, w_o[i].to(grad.device), z)
+        acc.add(("up_h", i), per_head)
+
+
+def _add_downstream(acc: _Sums, runner, t, encoded, cap, fired, peaks, down_n, down_h, dtype):
+    delta = t.written(cap)
+    rows = torch.arange(len(peaks))
+    at_peak = delta[fired][rows, peaks]
+    for i in down_n:
+        resid = cap["mlp_in"][i][fired][rows, peaks]
+        both = torch.cat([resid, resid - at_peak]).to(dtype)
+        after = _mlp_activations(runner, i, both).float()
+        acc.add(("down_n", i), (after[: len(peaks)] - after[len(peaks) :]).sum(0))
+    if down_h:  # empty without supported attention
+        sub = {key: v[fired] for key, v in encoded.items()}
+        kept = {**cap, "z": {i: v[fired] for i, v in cap["z"].items()}}
+        for i in down_h:
+            effects = _downstream_head_effects(runner, t, sub, kept, delta[fired], i, peaks)
+            acc.add(("down_h", i), effects.sum(0))

openinterp/langid.py

@@ -0,0 +1,73 @@
+"""A document's language, for corpora whose dataset doesn't say it.
+
+Used by ``open-interpretability corpus --detect-lang``. The detector is fastText's ``lid.176``
+(compressed, 176 languages). We compared it with lingua, langdetect and langid.py on
+labelled Wikipedia (``docs/analyses/language-detection.md``). It was the most accurate
+at every length and the fastest, and it alone is unsure of source code.
+
+A document gets a label only when the detector's confidence reaches ``MIN_CONFIDENCE``.
+So code, markup and mixed text mostly stay unlabelled, and the scan leaves them out of
+the language confound. Detection is per document, never per window, because a window of
+a few dozen tokens is too short to detect reliably.
+
+The model (CC BY-SA 3.0) is downloaded once, checked against ``MODEL_SHA256`` and cached
+in ``$XDG_CACHE_HOME/openinterp`` (else ``~/.cache/openinterp``).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import urllib.request
+from collections.abc import Sequence
+from functools import cache
+from pathlib import Path
+
+MODEL_URL = "https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.ftz"
+MODEL_SHA256 = "8f3472cfe8738a7b6099e8e999c3cbfae0dcd15696aac7d7738a8039db603e83"
+# Below this the document is left unlabelled. On labelled Wikipedia it keeps 98.5% of
+# documents at 99.9% accuracy, and labels 9% of source files (0.5 would label 45%).
+MIN_CONFIDENCE = 0.8
+DETECTOR = f"fasttext lid.176.ftz (sha256 {MODEL_SHA256[:12]}), confidence >= {MIN_CONFIDENCE}"
+
+
+def model_path() -> Path:
+    """The cached model, downloaded and checked on first use."""
+    base = Path(os.environ.get("XDG_CACHE_HOME") or Path.home() / ".cache")
+    path = base / "openinterp" / Path(MODEL_URL).name
+    if not path.exists():
+        path.parent.mkdir(parents=True, exist_ok=True)
+        with urllib.request.urlopen(MODEL_URL) as response:  # noqa: S310  (fixed https URL)
+            data = response.read()
+        if hashlib.sha256(data).hexdigest() != MODEL_SHA256:
+            raise ValueError(f"{MODEL_URL} does not match its pinned SHA-256")
+        partial = path.with_suffix(".part")
+        partial.write_bytes(data)
+        partial.replace(path)
+    return path
+
+
+@cache
+def _model():
+    import fasttext
+
+    return fasttext.load_model(str(model_path()))
+
+
+def predict(text: str) -> tuple[str, float]:
+    """Return the most likely language of ``text`` and the detector's confidence in it.
+
+    The language is the model's code, for example ``"en"`` or ``"ceb"``.
+    """
+    # fastText reads one line; ``f.predict`` returns [(probability, label)] without numpy.
+    ((p, label),) = _model().f.predict(" ".join(text.split()), 1, 0.0, "strict")
+    return label.removeprefix("__label__"), float(p)
+
+
+def detect(texts: Sequence[str]) -> list[str | None]:
+    """Each document's language, or ``None`` below :data:`MIN_CONFIDENCE`."""
+    out: list[str | None] = []
+    for text in texts:
+        label, p = predict(text)
+        out.append(label if p >= MIN_CONFIDENCE else None)
+    return out

openinterp/names.py

@@ -0,0 +1,129 @@
+"""Find people's names in a document, and tag its words' parts of speech.
+
+The names are the scan's ``names`` label set, and the tags its ``part of speech`` set
+(issue #3223). One pass of the same spaCy model gives both.
+
+The label is a model's guess, for experiments only. Nothing removes text because of it.
+On a 500-document sample it matched names in 47% of documents. Most are public and
+fictional people, so a match does not say that the text is personal data.
+
+The detector is spaCy's ``en_core_web_trf``, the best of spaCy's four English models. On
+CoNLL-2003's person names they had F1 0.67, 0.71, 0.78 and 0.89. We also compared
+them on a sample of web text, code and other languages. There the smaller models also
+called code and non-English phrases names. They matched 77%, 71% and 62% of documents.
+This model is the slowest, at about 180 ms a document on CPU. So each document's result
+stays in a cache on disk. It is an English model, so it reads other languages less well.
+
+The model (MIT) is about 460 MB. It is downloaded once, checked against ``MODEL_SHA256``
+and cached in ``$XDG_CACHE_HOME/openinterp`` (else ``~/.cache/openinterp``).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import shutil
+import urllib.request
+import zipfile
+from functools import cache
+from pathlib import Path
+
+MODEL = "en_core_web_trf-3.8.0"
+MODEL_URL = (
+    f"https://github.com/explosion/spacy-models/releases/download/{MODEL}/{MODEL}-py3-none-any.whl"
+)
+MODEL_SHA256 = "272a31e9d8530d1e075351d30a462d7e80e31da23574f1b274e200f3fff35bf5"
+DETECTOR = f"spaCy {MODEL} (sha256 {MODEL_SHA256[:12]})"
+PERSON = "person"  # the label of a match
+# The parser and lemmatizer are slower, and neither label set needs them. The tagger and
+# the attribute ruler give each word its universal part-of-speech tag (UPOS).
+_PIPES = ("transformer", "tagger", "attribute_ruler", "ner")
+
+
+def _cache_dir() -> Path:
+    return Path(os.environ.get("XDG_CACHE_HOME") or Path.home() / ".cache") / "openinterp"
+
+
+def _download(wheel: Path) -> None:
+    """Write the model's wheel to ``wheel``, or raise if it does not match its pinned SHA-256."""
+    digest = hashlib.sha256()
+    partial = wheel.with_suffix(".part")
+    with urllib.request.urlopen(MODEL_URL) as response, partial.open("wb") as out:
+        while chunk := response.read(1 << 20):
+            digest.update(chunk)
+            out.write(chunk)
+    if digest.hexdigest() != MODEL_SHA256:
+        partial.unlink()
+        raise ValueError(f"{MODEL_URL} does not match its pinned SHA-256")
+    partial.replace(wheel)
+
+
+def model_path() -> Path:
+    """The unpacked model, downloaded and checked on first use."""
+    root = _cache_dir() / MODEL
+    path = root / "en_core_web_trf" / MODEL
+    if not path.exists():
+        root.parent.mkdir(parents=True, exist_ok=True)
+        wheel = root.with_suffix(".whl")
+        _download(wheel)
+        unpacking = root.with_suffix(".unpacking")
+        shutil.rmtree(unpacking, ignore_errors=True)
+        with zipfile.ZipFile(wheel) as archive:
+            archive.extractall(unpacking)
+        unpacking.replace(root)
+        wheel.unlink()
+    return path
+
+
+@cache
+def _model():
+    import spacy
+
+    nlp = spacy.load(model_path())
+    return spacy.load(model_path(), exclude=[p for p in nlp.pipe_names if p not in _PIPES])
+
+
+def _cached(kind: str, text: str) -> Path:
+    key = hashlib.sha256(text.encode()).hexdigest()
+    return _cache_dir() / kind / MODEL_SHA256[:12] / f"{key}.json"
+
+
+def _write(path: Path, value) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    partial = path.with_suffix(f".{os.getpid()}.part")  # scans can run side by side
+    partial.write_text(json.dumps(value))
+    partial.replace(path)
+
+
+def _annotate(text: str) -> None:
+    """Run the model on ``text`` once, and cache both its names and its words' tags."""
+    doc = _model()(text)
+    names = sorted((e.start_char, e.end_char) for e in doc.ents if e.label_ == "PERSON")
+    tags = [(t.idx, t.idx + len(t.text), t.pos_) for t in doc if t.pos_ and not t.is_space]
+    _write(_cached("names", text), names)
+    _write(_cached("pos", text), tags)
+
+
+def person_matches(text: str) -> tuple[tuple[int, int, str], ...]:
+    """Return ``(start, end, PERSON)`` of each name in ``text``, sorted.
+
+    The result stays on disk by the text's SHA-256, so a later scan of a corpus does not
+    run the model again.
+    """
+    path = _cached("names", text)
+    if not path.exists():
+        _annotate(text)
+    return tuple((start, end, PERSON) for start, end in json.loads(path.read_text()))
+
+
+def pos_matches(text: str) -> tuple[tuple[int, int, str], ...]:
+    """Return ``(start, end, UPOS)`` of each word of ``text``, in order (issue #3223).
+
+    Whitespace has no tag. A document whose names are already cached gets its tags from
+    one more pass of the model, and both stay on disk after that.
+    """
+    path = _cached("pos", text)
+    if not path.exists():
+        _annotate(text)
+    return tuple((start, end, tag) for start, end, tag in json.loads(path.read_text()))

openinterp/neuronpedia_baseline.py

@@ -0,0 +1,88 @@
+"""Neuronpedia's explanations as a blind proposer baseline.
+
+The command is `open-interpretability neuronpedia-baseline`.
+
+Does Neuronpedia's explanation of a dictionary latent explain it as well as the agent's
+hypothesis? The rule was fixed before scoring. Each experiment on a latent that
+Neuronpedia hosts takes the latent's first machine-written explanation, word for word
+(:func:`openinterp.crossref.machine_explanation`). Human-written ones are
+left out, so the baseline is one automatic explainer.
+
+The explanation is then scored on the record's own held-out windows, so the two scores
+are paired. The Patchscope baseline is scored the same way (:mod:`openinterp.baseline`).
+Nothing is sent to Neuronpedia. The explanations are read once through its public API
+and kept in the cache of :mod:`openinterp.crossref`. Each row credits Neuronpedia and the model that
+wrote the explanation, and links to the latent's page. The author's account is not kept.
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from collections.abc import Sequence
+
+from openinterp.baseline import METRICS, BaselineRow, paired_report
+
+CREDIT = "Neuronpedia"
+
+
+def reading(explainer: str | None) -> str:
+    """The ``reading`` of a row: who wrote the baseline, or that there was none."""
+    return f"{CREDIT}, {explainer}" if explainer else f"no {CREDIT} explanation"
+
+
+def report(rows: Sequence[BaselineRow], links: dict[str, str | None]) -> str:
+    """Return the comparison as a Markdown page.
+
+    ``links`` maps each row's slug to its latent's Neuronpedia page. The page gives the
+    rule, the paired table, and every explanation with its credit and link.
+    """
+    have = [r for r in rows if r.baseline_explanation is not None]
+    judged = sum(r.baseline_judge_model is not None for r in rows)
+    units = {r.slug for r in rows}
+    covered = {r.slug for r in have}
+    explainers = Counter(r.reading.removeprefix(f"{CREDIT}, ") for r in have)
+    credits = ", ".join(f"{k} on {n} experiments" for k, n in explainers.items()) or "none"
+    intro = (
+        f"Explanations from [Neuronpedia](https://www.neuronpedia.org), credited to the "
+        f"model that wrote them ({credits}). "
+        f"The rule, fixed before scoring: each latent's first machine-written explanation, "
+        f"word for word. Human-written ones are left out. Neuronpedia has one for "
+        f"{len(covered)} of the {len(units)} audited latents it hosts. Each is scored on the "
+        f"same held-out windows as the agent's explanation. Detection and fuzzing pair only "
+        f"experiments whose agent explanation was judged by the same model as the "
+        f"baseline's ({judged} of {len(rows)}). Embedding uses the record's own encoder. "
+        f"The sign test counts experiments, and a unit with several experiments counts "
+        f"more than once."
+    )
+    after = [
+        "Per experiment, agent → Neuronpedia on each metric (— where unpaired). The "
+        "agent's explanation is on the unit's page.",
+        "",
+        "| Unit | Status | Neuronpedia's explanation | Detection | Fuzzing | Embedding |",
+        "| --- | --- | --- | --- | --- | --- |",
+    ]
+    for r in sorted(rows, key=lambda r: (r.slug, r.experiment_id)):
+        link = links.get(r.slug)
+        unit = f"[{r.slug}]({link})" if link else r.slug
+        theirs = _cell(r.baseline_explanation) if r.baseline_explanation else "—"
+        pairs = " | ".join(_pair(r, name) for name in METRICS)
+        after.append(f"| {unit} | {r.status} | {theirs} | {pairs} |")
+    return paired_report(
+        rows,
+        title="Neuronpedia explanations as a blind proposer baseline",
+        generated_by="`open-interpretability neuronpedia-baseline`",
+        method="`openinterp/neuronpedia_baseline.py` and docs/PROTOCOL.md §4",
+        intro=intro,
+        after=after,
+    )
+
+
+def _pair(row: BaselineRow, name: str) -> str:
+    """``agent → baseline`` on metric ``name``, or ``—`` if either is missing."""
+    a, b = row.agent[name], row.baseline[name]
+    return "—" if a is None or b is None else f"{a:.2f} → {b:.2f}"
+
+
+def _cell(text: str) -> str:
+    """``text`` made safe for one Markdown table cell."""
+    return " ".join(text.split()).replace("|", "\\|").replace("<", "&lt;")