quantum-cortex 1.0.0__py3-none-any.whl

cortex_c2b/lm_bridge.py

@@ -0,0 +1,338 @@
+"""cortex_c2b.lm_bridge -- the journal in the decode loop (ADR-008; REPRISE step 3).
+
+Until 2026-09-12 the journal was measured as an ORGAN: a hash-seeded cue encoder
+and a reference skill probe stood in for the model. This module wires the organ
+into the language model itself, under the nine invariants validated by the
+founder before any code was written (ADR-008):
+
+  1. N1 intact at step zero: the reader's output projection is zero-initialised,
+     so a model with the journal on produces N1's logits at initialisation,
+     window or no window; a parent checkpoint loads its weights unchanged.
+  2. One read path, the organ's: cues go to `JournalPath.retrieve`; nothing is
+     duplicated on the model side, nothing scans.
+  3. The journal enters as CONTENT, through a zero-init cross attention over
+     the retrieved bytes embedded with the model's own byte embeddings -- not
+     as prompt tokens. Journal OFF = the reader absent = N1 exactly.
+  4. The router decides the read, on the floor: the reader's contribution is
+     multiplied by the path 4 gate; with no score the gate is zero.
+  5. The cue encoder is learned (a projection of the mean pooled residual
+     stream to a unit vector), trained by contrast: the cue of a query must be
+     nearest to the cue of its own statement among the batch. One encoder for
+     writing and reading; the DG/CA3/CA1 write gate is unchanged.
+  6. Organ use is taught, not assumed: `cortex_c2b.organ_use` builds the
+     curriculum; the target of every example is computed from what the journal
+     actually returned (cite what was shown, else abstain).
+  7. Two processes, persistence declared: session A writes, session B reopens
+     the sealed journal and the checkpoint from the disk alone.
+  8. A failure is attributable: retrieval hit (the encoder), attention mass on
+     the retrieved bytes (the reader), attribute exactness given a valid
+     citation (the answer) are reported next to the verdict.
+  9. Everything is a flag, a ledger record, and a CPU smoke test.
+
+The contract of an episodic answer (ADR-008, point d, decided 2026-09-12):
+  `<EPI>` marks an episodic query; the answer is `<CITE>` + a one byte LOCAL
+  LABEL of the retrieved episode (the harness maps it back to the pointer and
+  verifies it) + `<ANS>` + the attribute bytes + newline, or `<UNKNOWN>` +
+  newline. Four of the eight reserved oracle tokens (NOW-1) are taken; the
+  vocabulary size does not change, so the N1 checkpoint loads as is.
+
+Pure torch over the numpy organ. Written from scratch (our filon).
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from cortex_c2b import Journal, CUE_DIM
+from cortex_c2b.write_path import WritePath, WriteReport
+from cortex_c2b.read_path import JournalPath
+
+# ---- the contract tokens: 4 of the 8 reserved oracle tokens (ids 256..263) ----
+TOK_EPI, TOK_CITE, TOK_ANS, TOK_UNKNOWN = 256, 257, 258, 259
+CONTRACT_TOKENS = {"EPI": TOK_EPI, "CITE": TOK_CITE, "ANS": TOK_ANS, "UNKNOWN": TOK_UNKNOWN}
+LABELS = b"ABCDEFGH"                     # local labels of the retrieved episodes, at most 8 per read
+NEWLINE = 10
+
+
+# ---------------------------------------------------------------------------- #
+# The cue encoder -- learned, one for writing and reading (invariant 5)        #
+# ---------------------------------------------------------------------------- #
+class CueEncoder(nn.Module):
+    """Mean pooled residual stream -> linear -> unit vector of `cue_dim`."""
+
+    def __init__(self, n_embd: int, cue_dim: int = CUE_DIM):
+        super().__init__()
+        self.proj = nn.Linear(n_embd, cue_dim, bias=False)
+
+    def forward(self, h: torch.Tensor, mask: torch.Tensor | None = None) -> torch.Tensor:
+        if mask is None:
+            pooled = h.mean(dim=1)
+        else:
+            m = mask.to(h.dtype).unsqueeze(-1)
+            pooled = (h * m).sum(dim=1) / m.sum(dim=1).clamp(min=1.0)
+        return F.normalize(self.proj(pooled), dim=-1, eps=1e-6)
+
+
+def contrastive_loss(query_cues: torch.Tensor, statement_cues: torch.Tensor,
+                     temperature: float = 0.1) -> torch.Tensor:
+    """InfoNCE in both directions: row i of the queries must pick row i of the
+    statements among the batch (and back). The signal that teaches the encoder
+    to put a question and its own episode at the same address."""
+    logits = query_cues @ statement_cues.t() / temperature
+    target = torch.arange(logits.size(0), device=logits.device)
+    return 0.5 * (F.cross_entropy(logits, target) + F.cross_entropy(logits.t(), target))
+
+
+# ---------------------------------------------------------------------------- #
+# The reader -- zero-init cross attention over the read window (invariant 3)   #
+# ---------------------------------------------------------------------------- #
+class JournalReader(nn.Module):
+    """Queries from the residual stream, keys and values from the embedded bytes
+    of the read window (plus a learned null slot: "nothing to read"). The output
+    projection starts at zero, so the model begins as N1 and learns to read.
+    `forward` also returns the attention mass on the real bytes (excluding the
+    null slot): the reader's diagnostic (invariant 8)."""
+
+    def __init__(self, n_embd: int, n_head: int, window_len: int):
+        super().__init__()
+        self.h = n_head
+        self.ln_x = nn.LayerNorm(n_embd)
+        self.ln_w = nn.LayerNorm(n_embd)
+        self.q = nn.Linear(n_embd, n_embd, bias=False)
+        self.kv = nn.Linear(n_embd, 2 * n_embd, bias=False)
+        self.out = nn.Linear(n_embd, n_embd, bias=False)
+        self.null = nn.Parameter(torch.zeros(1, 1, n_embd))
+        self.wpe = nn.Embedding(window_len, n_embd)          # order within the window
+        nn.init.zeros_(self.out.weight)                       # the no-op law
+        self.scale = (n_embd // n_head) ** -0.5
+
+    def forward(self, x: torch.Tensor, window_emb: torch.Tensor, window_mask: torch.Tensor):
+        b, t, c = x.shape
+        r = window_emb.size(1)
+        pos = torch.arange(r, device=x.device)
+        w = torch.cat([self.null.expand(b, 1, c), window_emb + self.wpe(pos)[None]], dim=1)   # (b, r+1, c)
+        m = torch.cat([torch.ones(b, 1, dtype=torch.bool, device=x.device), window_mask.bool()], dim=1)
+        hq = self.q(self.ln_x(x)).view(b, t, self.h, c // self.h).transpose(1, 2)               # (b,h,t,d)
+        k, v = self.kv(self.ln_w(w)).split(c, dim=2)
+        k = k.view(b, r + 1, self.h, c // self.h).transpose(1, 2)
+        v = v.view(b, r + 1, self.h, c // self.h).transpose(1, 2)
+        att = (hq @ k.transpose(-2, -1)) * self.scale
+        att = att.masked_fill(~m[:, None, None, :], float("-inf"))
+        p = torch.softmax(att, dim=-1)
+        read = (p @ v).transpose(1, 2).contiguous().view(b, t, c)
+        mass = p[..., 1:].sum(dim=-1).mean()                   # attention on real bytes, null slot excluded
+        self.last_attended = read                              # RES-21: what was read, before the zero init projection
+        self.last_attn = p.detach()                            # (b, h, t, r + 1): the pointer policy reads it at decode
+        return self.out(read), mass
+
+
+class MatchHead(nn.Module):
+    """RES-21 (ADR-008, amendment of 2026-09-14): a small head that answers ONE question at
+    the decision position, "is the queried entity's episode in the window?", from what
+    the model had before reading (the residual stream at the journal block) and what it
+    read (the reader's attended values, before the zero init projection). Trained
+    directly on that label, which the harness knows exactly for every curriculum example
+    (the own pointer among the window's labels, never the ground truth alone), and
+    coupled to the decision token's logits: + coupling * m on <CITE>, - coupling * m on
+    <UNKNOWN>. The last layer starts at zero, so the model begins as its parent (m = 0,
+    no bias) and learns the comparison from there."""
+
+    def __init__(self, n_embd: int):
+        super().__init__()
+        self.ln_pre = nn.LayerNorm(n_embd)
+        self.ln_read = nn.LayerNorm(n_embd)
+        self.up = nn.Linear(2 * n_embd, n_embd)
+        self.out = nn.Linear(n_embd, 1)
+        nn.init.zeros_(self.out.weight); nn.init.zeros_(self.out.bias)   # the no-op law
+
+    def forward(self, pre: torch.Tensor, attended: torch.Tensor) -> torch.Tensor:
+        z = torch.cat([self.ln_pre(pre), self.ln_read(attended)], dim=-1)
+        return self.out(torch.nn.functional.gelu(self.up(z))).squeeze(-1)  # (b,) a logit: present > 0
+
+
+# ---------------------------------------------------------------------------- #
+# The router's gate on the read (invariant 4)                                  #
+# ---------------------------------------------------------------------------- #
+PATH_JOURNAL = 4                          # the journal is path 4 of the RES-18 router (ADR-007)
+
+
+def journal_gate(route: str, decision=None) -> float:
+    """The multiplier on the reader's contribution. `pinned_on` / `pinned_off`
+    are the two arms of the protocol (declared, recorded); `router` takes the
+    path 4 weight of a RouteDecision -- a hard decision for another path, or no
+    decision at all, gives zero: the floor guarantee at the model scale."""
+    if route == "pinned_on":
+        return 1.0
+    if route == "pinned_off":
+        return 0.0
+    if route != "router":
+        raise ValueError(f"unknown journal_route {route!r}")
+    if decision is None:
+        return 0.0
+    if decision.weights is not None:
+        return float(decision.weights[PATH_JOURNAL]) if len(decision.weights) > PATH_JOURNAL else 0.0
+    return float(decision.confidence) if decision.path == PATH_JOURNAL else 0.0
+
+
+# ---------------------------------------------------------------------------- #
+# The read window -- labelled, budgeted, in a seeded random order              #
+# ---------------------------------------------------------------------------- #
+def build_read_window(items: list[tuple[str, bytes]], budget: int, rng: np.random.Generator,
+                      marks: list[float] | None = None) -> tuple[bytes, dict[int, str]]:
+    """`items` = (pointer, payload) as retrieved. Returns the window bytes
+    (`A:<payload>\\n` per item, order shuffled with `rng` so a position is never a
+    shortcut, each payload cut to its share of `budget`) and the label byte ->
+    pointer map the harness uses to verify a citation.
+    `marks` (RES-23, the familiarity mark, v9): one number per item, the organ's
+    own similarity for that line; the prefix becomes `A(0.97):` (two decimals,
+    clamped at zero) and each payload gives six more bytes to it inside the same
+    budget. None keeps the window byte for byte as every earlier run wrote it."""
+    items = list(items)[: len(LABELS)]
+    if not items:
+        return b"", {}
+    if marks is not None and len(marks) != len(items):
+        raise ValueError("one mark per item")
+    order = rng.permutation(len(items))
+    per = max(1, budget // len(items) - (9 if marks is not None else 3))   # 'X:' or 'X(0.97):', and the newline
+    out, labels = [], {}
+    for slot, i in enumerate(order):
+        pointer, payload = items[i]
+        label = LABELS[slot]
+        labels[label] = pointer
+        prefix = bytes([label]) + (f"({max(0.0, min(1.0, float(marks[i]))):.2f})".encode("ascii") if marks is not None else b"") + b":"
+        out.append(prefix + payload[:per] + b"\n")
+    return b"".join(out), labels
+
+
+def encode_query(text: str) -> list[int]:
+    return [TOK_EPI] + list(text.encode("utf-8"))
+
+
+def encode_target(label: int | None, attr: str | None) -> list[int]:
+    """The contract: cite a label and answer, or abstain."""
+    if label is None:
+        return [TOK_UNKNOWN, NEWLINE]
+    return [TOK_CITE, label, TOK_ANS] + list(attr.encode("utf-8")) + [NEWLINE]
+
+
+def parse_contract(tokens: list[int]) -> tuple[str, int | None, str | None]:
+    """('unknown'|'cite'|'malformed', label, attribute). A malformed answer is a
+    claim without the contract's shape -- counted against the model, never for it."""
+    toks = list(tokens)
+    if toks and toks[0] == TOK_UNKNOWN:
+        return "unknown", None, None
+    if len(toks) >= 3 and toks[0] == TOK_CITE and toks[2] == TOK_ANS and toks[1] in LABELS:
+        body = toks[3:]
+        if NEWLINE in body:
+            body = body[: body.index(NEWLINE)]
+        try:
+            return "cite", toks[1], bytes(b for b in body if b < 256).decode("utf-8")
+        except UnicodeDecodeError:
+            return "malformed", toks[1], None
+    return "malformed", None, None
+
+
+# ---------------------------------------------------------------------------- #
+# The bridge -- the torch model above, the numpy organ below (invariant 2)     #
+# ---------------------------------------------------------------------------- #
+@dataclass
+class ReadResult:
+    window: bytes
+    labels: dict[int, str]            # label byte -> pointer
+    pointers: list[str]               # what the organ returned, in retrieval order
+    scores: list[float]
+
+
+class JournalBridge:
+    """Holds the organ (journal, write path, router path) and talks to the model
+    through two calls it must expose: `journal_hidden(idx, mask)` (the residual
+    stream at the journal block, before the reader) and `cue_encoder`.
+
+    The model never sees a pointer or a payload except through the read window;
+    the organ never sees a tensor except the cue."""
+
+    def __init__(self, model, journal: Journal, write_path: WritePath | None = None,
+                 journal_path: JournalPath | None = None, k: int = 4, budget_bytes: int = 256,
+                 seed: int = 0, shuffle_seed: int | None = None, device=None, mark: bool = False):
+        """`seed` is the scope's separation seed (DG) and the index seed -- it must
+        be the same at every open of a journal (Decision 8: a journal written under
+        another separation is refused). `shuffle_seed` only drives the order of
+        the read window and may differ between sessions."""
+        self.model = model
+        self.j = journal
+        if model.cue_encoder is None or model.cue_encoder.proj.out_features != journal.cue_dim:
+            raise ValueError("the model's cue encoder must produce the journal's cue dimension "
+                             f"({journal.cue_dim}); set journal_cue_dim accordingly")
+        self.wp = write_path or WritePath(journal, seed=seed)
+        self.jp = journal_path or JournalPath(journal, seed=seed)
+        self.k, self.budget = k, budget_bytes
+        self.mark = mark                                        # RES-23: the organ's similarity in every window line
+        self.rng = np.random.default_rng(seed if shuffle_seed is None else shuffle_seed)
+        self.device = device or next(model.parameters()).device
+        self.pointer_of: dict[str, str] = {}                   # entity -> pointer, for the curriculum's targets
+
+    # ---- tensors ----------------------------------------------------------------
+    def tokens(self, sequences: list[list[int]]) -> tuple[torch.Tensor, torch.Tensor]:
+        n = max(len(s) for s in sequences)
+        idx = torch.zeros(len(sequences), n, dtype=torch.long)
+        mask = torch.zeros(len(sequences), n, dtype=torch.bool)
+        for i, s in enumerate(sequences):
+            idx[i, : len(s)] = torch.tensor(s, dtype=torch.long); mask[i, : len(s)] = True
+        return idx.to(self.device), mask.to(self.device)
+
+    def cue_tensor(self, sequences: list[list[int]]) -> torch.Tensor:
+        """Cues with gradient (for the contrastive loss)."""
+        idx, mask = self.tokens(sequences)
+        h = self.model.journal_hidden(idx)
+        return self.model.cue_encoder(h, mask)
+
+    def cues(self, sequences: list[list[int]]) -> np.ndarray:
+        """Cues for the organ (no gradient)."""
+        with torch.no_grad():
+            return self.cue_tensor(sequences).float().cpu().numpy()
+
+    # ---- the organ ---------------------------------------------------------------
+    def write(self, text: str, schema: str, now: float, entity: str | None = None) -> WriteReport:
+        cue = self.cues([list(text.encode("utf-8"))])[0]
+        rep = self.wp.write(cue, text.encode("utf-8"), schema, now=now)
+        if rep.admitted:
+            self.jp.on_write(rep.entry)
+            if entity is not None:
+                self.pointer_of[entity] = rep.entry.pointer
+        return rep
+
+    def read(self, cue: np.ndarray) -> ReadResult:
+        hits = self.jp.retrieve(cue, k=self.k)
+        items = [(e.pointer, payload) for e, payload, _ in hits]
+        scores = [float(s) for _, _, s in hits]
+        window, labels = build_read_window(items, self.budget, self.rng, marks=scores if self.mark else None)
+        return ReadResult(window=window, labels=labels, pointers=[p for p, _ in items], scores=scores)
+
+    def marks_for(self, cue: np.ndarray, pointers: list[str]) -> list[float]:
+        """The organ's similarity between `cue` and each episode, whatever route put
+        the line in a window (a plain read, a paired negative, a forced bootstrap,
+        a shape filler): the same computation as the index's scoring."""
+        return [self.jp.cue_similarity(cue, p) for p in pointers]
+
+    def window_for(self, cue: np.ndarray, items: list[tuple[str, bytes]], rng: np.random.Generator
+                   ) -> tuple[bytes, dict[int, str]]:
+        """A window over arbitrary items, marked with the organ's similarities when
+        the bridge marks, unmarked otherwise; the curriculum's windows go through here."""
+        marks = self.marks_for(cue, [p for p, _ in items]) if self.mark else None
+        return build_read_window(items, self.budget, rng, marks=marks)
+
+    def window_tensors(self, windows: list[bytes], window_len: int) -> tuple[torch.Tensor, torch.Tensor]:
+        """Byte windows -> (tokens, mask), each cut to `window_len`; an empty
+        window is one masked position (the reader keeps its null slot)."""
+        r = max(1, min(window_len, max((len(w) for w in windows), default=1)))
+        tok = torch.zeros(len(windows), r, dtype=torch.long)
+        mask = torch.zeros(len(windows), r, dtype=torch.bool)
+        for i, w in enumerate(windows):
+            w = w[:r]
+            if w:
+                tok[i, : len(w)] = torch.tensor(list(w), dtype=torch.long); mask[i, : len(w)] = True
+        return tok.to(self.device), mask.to(self.device)

cortex_c2b/organ_use.py

@@ -0,0 +1,254 @@
+"""cortex_c2b.organ_use -- teaching the model to use its own organ (NOW-8; ADR-008 invariant 6).
+
+The rule that combines abstention and citation, decided by the founder on
+2026-09-12: **the target of every example is computed from what the journal
+actually returned, never from the ground truth alone.** If the episode that
+holds the answer is among the retrieved ones, the target cites its local label
+and gives the attribute; otherwise the target is `<UNKNOWN>`, even though the
+fact exists in the journal. The model is never rewarded for citing what it was
+not shown, so no invented citation enters the signal; while the cue encoder is
+poor the model learns silence first (one token), and as the contrastive loss
+improves retrieval, citations replace abstentions by themselves.
+
+Three mitigations, all declared and hashed in the run's configuration:
+  - an asymmetric weight on the decision token of an abstention target
+    (answering instead of abstaining costs more than the reverse);
+  - negative examples (entities never planted) from the first step;
+  - a bootstrap for the cold start: on a missed retrieval, with a declared
+    probability that decays to zero, the right episode is FORCED into the
+    window and the example is marked `forced` -- counted in the diagnostics,
+    never as recall.
+
+Decontamination (invariant 5): the training facts come from the same generator
+family as the frozen protocol, with disjoint seeds, and every entity name is
+checked against the protocol's facts and its negative control; a collision is
+removed and reported. Written from scratch (our filon).
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+import numpy as np
+
+from cortex_c2b.hm_protocol import Fact, generate_facts, N_FACTS, N_NEGCTRL
+from cortex_c2b.lm_bridge import (JournalBridge, encode_query, encode_target, build_read_window,
+                                  TOK_UNKNOWN)
+
+EVAL_SEEDS = (0, 10_000)                  # the frozen protocol's facts and its negative control
+
+
+# ---------------------------------------------------------------------------- #
+# Facts: disjoint seeds, checked entities                                       #
+# ---------------------------------------------------------------------------- #
+def decontaminate(train: list[Fact], eval_facts: list[Fact]) -> tuple[list[Fact], list[str]]:
+    """Remove every training fact whose entity appears in the evaluation facts."""
+    banned = {f.entity for f in eval_facts}
+    kept = [f for f in train if f.entity not in banned]
+    removed = [f.entity for f in train if f.entity in banned]
+    return kept, removed
+
+
+def training_facts(n: int, seed: int = 100_000, n_negatives: int = 50) -> tuple[list[Fact], list[Fact], dict]:
+    """Planted facts and never-planted negatives for training, both disjoint
+    from the frozen protocol's sets (seeds 0 and 10_000). Returns
+    (facts, negatives, report)."""
+    if seed in EVAL_SEEDS or seed + 1 in EVAL_SEEDS:
+        raise ValueError("training seeds must differ from the frozen protocol's seeds (0 and 10_000)")
+    eval_facts = [f for s in EVAL_SEEDS for f in generate_facts(N_FACTS if s == 0 else N_NEGCTRL, s)[0]]
+    facts, gen_hash = generate_facts(n, seed)
+    negs, _ = generate_facts(n_negatives, seed + 1)
+    facts, removed_f = decontaminate(facts, eval_facts)
+    negs, removed_n = decontaminate(negs, eval_facts + facts)
+    report = {"generator_hash": gen_hash, "seed": seed, "n_facts": len(facts), "n_negatives": len(negs),
+              "removed_collisions": removed_f + removed_n}
+    return facts, negs, report
+
+
+# ---------------------------------------------------------------------------- #
+# The pool: facts planted through the normal gated path under the CURRENT encoder
+# ---------------------------------------------------------------------------- #
+def plant_pool(bridge: JournalBridge, facts: list[Fact], now: float = 0.0) -> dict:
+    """Write every fact as an episode (statement bytes, schema, cue from the
+    model). Returns admission statistics; `bridge.pointer_of` maps entities to
+    their pointers, which is how a target knows whether the right episode was
+    among the retrieved ones."""
+    admitted = 0
+    for i, f in enumerate(facts):
+        rep = bridge.write(f.statement, f.schema, now=now + i, entity=f.entity)
+        admitted += int(rep.admitted)
+    return {"planted": len(facts), "admitted": admitted}
+
+
+# ---------------------------------------------------------------------------- #
+# Examples                                                                       #
+# ---------------------------------------------------------------------------- #
+@dataclass
+class Example:
+    kind: str                          # answer | abstain | negative | forced
+    query: list[int]                   # <EPI> + question bytes
+    target: list[int]                  # the contract
+    window: bytes
+    labels: dict[int, str] = field(default_factory=dict)
+    retrieved_ok: bool = False         # the right episode was among the retrieved (the encoder's success)
+    entity: str = ""
+    attr: str = ""
+    statement: list[int] = field(default_factory=list)   # the episode's bytes, for the contrastive pair
+
+    @property
+    def tokens(self) -> list[int]:
+        return self.query + self.target
+
+    def weights(self, asym: float) -> list[float]:
+        """Loss on the target only; the decision token of an abstention target
+        carries the asymmetric weight."""
+        w = [0.0] * len(self.query) + [1.0] * len(self.target)
+        if self.target and self.target[0] == TOK_UNKNOWN:
+            w[len(self.query)] = float(asym)
+        return w
+
+
+def _label_of(labels: dict[int, str], pointer: str) -> int | None:
+    for label, p in labels.items():
+        if p == pointer:
+            return label
+    return None
+
+
+def make_example(bridge: JournalBridge, fact: Fact, negative: bool, forced_frac: float,
+                 rng: np.random.Generator, paraphrase: bool = False) -> Example:
+    """One example, its target computed from the actual read (the rule above)."""
+    question = fact.paraphrase if paraphrase else fact.query
+    query = encode_query(question)
+    cue = bridge.cues([query])[0]
+    read = bridge.read(cue)
+    statement = list(fact.statement.encode("utf-8"))
+    if negative:
+        return Example("negative", query, encode_target(None, None), read.window, read.labels,
+                       False, fact.entity, fact.attr, statement)
+    pointer = bridge.pointer_of.get(fact.entity)
+    label = _label_of(read.labels, pointer) if pointer else None
+    if label is not None:
+        return Example("answer", query, encode_target(label, fact.attr), read.window, read.labels,
+                       True, fact.entity, fact.attr, statement)
+    if pointer and forced_frac > 0 and rng.random() < forced_frac:
+        # the bootstrap: the right episode is forced into the window (replacing the last slot)
+        items = [(p, bridge.j.payloads.get(p)) for p in read.pointers][: max(0, bridge.k - 1)]
+        items.append((pointer, bridge.j.payloads.get(pointer)))
+        window, labels = bridge.window_for(cue, items, rng)
+        return Example("forced", query, encode_target(_label_of(labels, pointer), fact.attr), window, labels,
+                       False, fact.entity, fact.attr, statement)
+    return Example("abstain", query, encode_target(None, None), read.window, read.labels,
+                   False, fact.entity, fact.attr, statement)
+
+
+def make_paired_negative(bridge: JournalBridge, fact: Fact, rng: np.random.Generator,
+                         paraphrase: bool = False, keep_shape: bool = False) -> Example:
+    """The minimal pair (v5 lever, run 6f10e8cbfebf): the SAME planted fact, its own episode
+    WITHHELD from the window; target abstain. Four runs showed the decision to cite or abstain
+    following the share of negatives seen in training (a class prior) far more than the
+    window's content; with pairs the prior earns nothing, and the only way down the loss is
+    to check whether the queried entity is in the read.
+
+    `keep_shape=False` is the pair AS RUN in v5 (`db028e6a4262`, INVALID): the own pointer
+    removed from the k retrieved, the window built from what is left, so it has k - 1 lines
+    whenever the episode had been retrieved (over 90 % of the time late in the run). That is
+    a shape cue, cheaper than a comparison, and absent from the frozen protocol's windows,
+    which always have k lines. The pair was not minimal; kept as written so the v5 run stays
+    reproducible under its hash. `keep_shape=True` is the v6 lever: k + 1 retrieved, the own
+    dropped, k lines kept, so the pair's two halves have the same shape and differ only by the
+    episode's presence."""
+    question = fact.paraphrase if paraphrase else fact.query
+    query = encode_query(question)
+    cue = bridge.cues([query])[0]
+    own = bridge.pointer_of.get(fact.entity)
+    items = withheld_items(bridge, cue, own, rng, keep_shape=keep_shape)
+    window, labels = bridge.window_for(cue, items, rng)
+    return Example("negative", query, encode_target(None, None), window, labels, False,
+                   fact.entity, fact.attr, list(fact.statement.encode("utf-8")))
+
+
+def withheld_items(bridge: JournalBridge, cue: np.ndarray, own: str | None, rng: np.random.Generator,
+                   keep_shape: bool) -> list[tuple[str, bytes]]:
+    """The window items of a paired negative. `keep_shape=False`: the read's items minus the own
+    pointer, one line fewer than the read whenever the own was retrieved (v5 as run). `keep_shape=True`:
+    the same number of lines as the read would have shown: the k + 1 th candidate fills the slot when the
+    index returns one, else another planted episode drawn at random (the index is a locality sensitive
+    hash and may return fewer than k candidates, so the fill is by construction, not by luck)."""
+    if not keep_shape:
+        read = bridge.read(cue)
+        return [(p, bridge.j.payloads.get(p)) for p in read.pointers if p != own]
+    hits = bridge.jp.retrieve(cue, k=bridge.k + 1)
+    top = [(e.pointer, payload) for e, payload, _ in hits]
+    shown = min(len(top), bridge.k)                                     # what a plain read shows
+    items = [(p, pl) for p, pl in top if p != own][: bridge.k]
+    if len(items) < shown:                                              # the own was shown and no k + 1 th candidate exists
+        seen = {p for p, _ in top}
+        others = sorted(p for p in bridge.pointer_of.values() if p != own and p not in seen)
+        if others:
+            p = others[int(rng.integers(len(others)))]
+            items.append((p, bridge.j.payloads.get(p)))
+    return items
+
+
+def make_batch(bridge: JournalBridge, facts: list[Fact], negatives: list[Fact], n: int,
+               rng: np.random.Generator, negatives_frac: float = 0.25, forced_frac: float = 0.5,
+               paraphrase_frac: float = 0.5, paired: bool = False, keep_shape: bool = False) -> list[Example]:
+    """`paired`: negatives are minimal pairs of the planted facts (the episode withheld)
+    instead of separate never planted entities; `keep_shape`: the pair keeps k lines (v6)."""
+    out = []
+    for _ in range(n):
+        if rng.random() < negatives_frac and (paired or negatives):
+            if paired:
+                f = facts[int(rng.integers(len(facts)))]
+                out.append(make_paired_negative(bridge, f, rng, paraphrase=rng.random() < paraphrase_frac,
+                                                keep_shape=keep_shape))
+            else:
+                f = negatives[int(rng.integers(len(negatives)))]
+                out.append(make_example(bridge, f, True, 0.0, rng, paraphrase=rng.random() < paraphrase_frac))
+        else:
+            f = facts[int(rng.integers(len(facts)))]
+            out.append(make_example(bridge, f, False, forced_frac, rng, paraphrase=rng.random() < paraphrase_frac))
+    return out
+
+
+def own_in_window(e: Example) -> bool:
+    """RES-21's label: the queried entity's own episode is among the window's labels. Computed
+    from what the journal returned (the pointer is the content hash of the statement's bytes),
+    never from the ground truth alone: a positive whose episode was not retrieved is a 0."""
+    from cortex_c2b import content_hash
+    return content_hash(bytes(e.statement)) in set(e.labels.values())
+
+
+def match_tensors(examples: list[Example], device):
+    """(targets (b,) float 0/1, decision positions (b,) long): the position at which the
+    decision token is predicted is the last token of the query."""
+    import torch
+    t = torch.tensor([1.0 if own_in_window(e) else 0.0 for e in examples], device=device)
+    d = torch.tensor([len(e.query) - 1 for e in examples], dtype=torch.long, device=device)
+    return t, d
+
+
+def batch_stats(examples: list[Example]) -> dict:
+    n = max(1, len(examples))
+    kinds = {k: sum(e.kind == k for e in examples) for k in ("answer", "abstain", "negative", "forced")}
+    planted = [e for e in examples if e.kind != "negative"]
+    return {**{f"n_{k}": v for k, v in kinds.items()},
+            "retrieval_hit": (sum(e.retrieved_ok for e in planted) / len(planted)) if planted else 0.0,
+            "abstain_target_rate": sum(e.target[0] == TOK_UNKNOWN for e in examples) / n}
+
+
+def collate(bridge: JournalBridge, examples: list[Example], asym: float, window_len: int):
+    """Tensors for one training step: (idx, targets, weights, window, window_mask,
+    query_sequences, statement_sequences). Targets are the next token; the
+    weight of a position is the weight of the token it predicts."""
+    import torch
+    seqs = [e.tokens for e in examples]
+    idx, _ = bridge.tokens([s[:-1] for s in seqs])
+    tgt, _ = bridge.tokens([s[1:] for s in seqs])
+    n = idx.size(1)
+    w = torch.zeros(len(examples), n)
+    for i, e in enumerate(examples):
+        ww = e.weights(asym)[1:]                                   # aligned with the predicted token
+        w[i, : len(ww)] = torch.tensor(ww)
+    window, wmask = bridge.window_tensors([e.window for e in examples], window_len)
+    return idx, tgt, w.to(idx.device), window, wmask, [e.query for e in examples], [e.statement for e in examples]