zikaron 0.1.0__py3-none-any.whl

zikaron/core/retrieval/query.py

@@ -0,0 +1,327 @@
+"""Query construction: a prompt is not an FTS5 expression, and it is not always under 512 tokens.
+
+`retrieval.md` §"Query construction" is normative. Two failures live here, one per arm, and both
+were underspecified until that section was written:
+
+- **Lexical.** A raw prompt is not a safe `MATCH` expression — a quote, a parenthesis or a bare
+  `OR` changes the query's meaning or raises `fts5: syntax error`, and a syntax error on the push
+  path would send every user message to the degraded path. The rule splits on boundaries only and
+  hands each fragment to FTS5 **as a quoted string literal**, so FTS5's own tokenizer does the case
+  folding and diacritic stripping. Reimplementing `unicode61` was the alternative and it is a
+  silent-divergence bug in the one component that must agree with the index.
+- **Dense.** A long prompt, or consolidation's group-gist concatenation, can exceed the model's
+  512-token input, and BGE truncates silently. So the read side gets the same preflight the write
+  side has — through the same tokenizer — and resolves it the other way: it **truncates and records
+  it**, because a write can be handed back to an agent that still holds its text while refusing a
+  user's prompt would mean refusing to retrieve at all.
+
+`retrieval.md` §"Two kinds of query" then makes both rules serve two callers. An **external** query
+is text from outside the store; an **internal** query is a memory querying other memories, whose
+dense side is its own stored first-chunk embedding and whose lexical side is its own `gist +
+content` through the identical constructor. One rule called on stored prose instead of on a prompt,
+never a second rule.
+"""
+
+from dataclasses import dataclass
+from enum import StrEnum
+from typing import Final
+
+import aiosqlite
+
+from zikaron.core.errors import BadConfigSource, ErrorCode, ZikaronError
+from zikaron.core.indexing.encoder import Encoder, assembled_tokens, token_head
+from zikaron.core.indexing.vectors import serialize
+
+#: What separates the two prose fields of an internal query's lexical side. Any non-alphanumeric
+#: character would do — the constructor keeps only alphanumeric runs — so the choice is legibility.
+_INTERNAL_JOIN: Final = "\n"
+
+
+class QueryOrigin(StrEnum):
+    """Where a query's dense vector came from, which decides whether a cosine may be read off it.
+
+    An `INTERNAL` query reuses a stored first-chunk embedding, which the write path L2-normalized,
+    so `cos = 1 - d^2/2` over `vec0`'s distance is exact and the three cosine cutoffs mean what
+    they say. An `EXTERNAL` query's vector is whatever the embedder returned for the prompt and is
+    deliberately **not** renormalized: ordering by distance to unit documents is monotone in cosine
+    for any fixed query vector, so the ranking is unaffected, and no external query is ever
+    thresholded — all three cutoffs are `s(X -> Y)` between two stored memories. Carrying the origin
+    is what keeps that from being a fact somebody has to remember: a cosine is offered only where it
+    is exact.
+    """
+
+    EXTERNAL = "external"
+    INTERNAL = "internal"
+
+
+@dataclass(frozen=True, slots=True)
+class LexicalQuery:
+    """One query's lexical side: the terms selected, and the expression bound into `MATCH`.
+
+    `expression` is `None` when no term survived, which is `retrieval.md`'s `lexical_skipped`: a
+    prompt of pure punctuation runs dense-only rather than raising or matching everything.
+    """
+
+    terms: tuple[str, ...]
+    expression: str | None
+
+    @property
+    def skipped(self) -> bool:
+        """Whether the lexical arm has nothing to run — zero surviving terms."""
+        return self.expression is None
+
+
+@dataclass(frozen=True, slots=True)
+class PreparedQuery:
+    """Everything the retrieval algorithm needs of a query, whichever kind produced it.
+
+    The algorithm is identical for both kinds (`retrieval.md` §"Two kinds of query"), so this is
+    the seam: construction differs, retrieval does not.
+    """
+
+    origin: QueryOrigin
+    vector: bytes
+    lexical: LexicalQuery
+
+
+@dataclass(frozen=True, slots=True)
+class ExternalQuery:
+    """A prepared external query plus the two facts only its dense preflight can report.
+
+    `query_tokens` is the pre-truncation token count of the query text **alone**, excluding the BGE
+    prefix — the prefix is charged against the budget on the other side of the subtraction, so
+    counting it here would double it and make the recorded number disagree with the same field on a
+    store configured with an empty prefix. Both fields go on the `search`/`surface_call` event.
+    """
+
+    prepared: PreparedQuery
+    query_tokens: int
+    query_truncated: bool
+
+
+def _quote(term: str) -> str:
+    """One term as an FTS5 string literal.
+
+    Quoting is what neutralizes operators — a quoted `OR` is the word "or", a quoted `-x` is "x" —
+    and, more importantly, what hands the term's *contents* to the table's own tokenizer, so case
+    folding and diacritic stripping happen on both sides by the same code. The internal-quote
+    doubling `retrieval.md` step 3 requires is unconditional rather than guarded: a term is a run of
+    alphanumerics and so cannot contain a quote today, and an unconditional `replace` costs nothing
+    while a conditional one would be a branch nothing can reach.
+    """
+    escaped = term.replace('"', '""')
+    return f'"{escaped}"'
+
+
+def _alphanumeric_runs(text: str) -> list[str]:
+    """Every maximal run of Unicode alphanumerics in `text`, in order.
+
+    `str.isalnum()` per character rather than a tokenizer of our own: `retrieval.md` bounds the
+    residual risk of this being *our* boundary rule rather than SQLite's to one term missing one
+    match — checked against `fts5vocab` on nine technical strings with zero disagreements — which is
+    a strictly smaller exposure than reproducing `unicode61`'s folding tables.
+    """
+    runs: list[str] = []
+    current: list[str] = []
+    for character in text:
+        if character.isalnum():
+            current.append(character)
+        elif current:
+            runs.append("".join(current))
+            current = []
+    if current:
+        runs.append("".join(current))
+    return runs
+
+
+def lexical_terms(text: str, *, max_terms: int) -> tuple[str, ...]:
+    """The terms `text` contributes to a lexical query, in the order they will be joined.
+
+    Deduplicated on the exact string, then sorted **longest-first with ties broken by first
+    occurrence**, then cut to `max_terms`. Longest-first is the right truncation for this corpus
+    rather than an arbitrary one: in prose about code the long tokens are the identifiers, error
+    strings and paths, while the terms it drops are short common words BM25 would have scored near
+    zero anyway.
+
+    Deduplication compares the text as written, and deliberately does not fold case: folding is
+    FTS5's, on both sides, and a second folding rule here is exactly the silent divergence the
+    quoted-literal design avoids. Two differently-cased occurrences of one word therefore spend two
+    of the `max_terms` slots, which costs a slot and cannot cost correctness.
+    """
+    seen: dict[str, int] = {}
+    for index, run in enumerate(_alphanumeric_runs(text)):
+        seen.setdefault(run, index)
+    ranked = sorted(seen.items(), key=lambda pair: (-len(pair[0]), pair[1]))
+    return tuple(term for term, _ in ranked[:max_terms])
+
+
+def lexical_query(text: str, *, max_terms: int) -> LexicalQuery:
+    """`text` as an FTS5 `MATCH` expression, or a skipped arm if it yields no term.
+
+    The expression is `OR` over quoted terms, in selection order, and is always **bound as a
+    parameter** rather than formatted into SQL text. BM25 already ranks a document matching several
+    terms above one matching a single term, so `OR` loses nothing a phrase would gain — while a
+    phrase would require adjacency and so would silently drop every non-adjacent match on dotted
+    identifiers, paths and error strings, which is precisely the query shape this store serves.
+    """
+    terms = lexical_terms(text, max_terms=max_terms)
+    expression = " OR ".join(_quote(term) for term in terms) if terms else None
+    return LexicalQuery(terms=terms, expression=expression)
+
+
+def internal_lexical_text(gist: str, content: str) -> str:
+    """The prose an internal query's lexical side is built from: the memory's own `gist + content`.
+
+    A function rather than a call-site concatenation because the 64-term cap does real work here —
+    a memory has far more terms than a prompt does — so which text the cap is applied to is part of
+    the contract rather than an incidental detail of one caller.
+    """
+    return f"{gist}{_INTERNAL_JOIN}{content}"
+
+
+def _prefix_too_long(prefix_tokens: int, cap: int) -> ZikaronError:
+    return ZikaronError(
+        ErrorCode.BAD_CONFIG,
+        source=BadConfigSource.FILE,
+        file="<effective config>",
+        key="embedding.embed_prefix_query",
+        value=f"{prefix_tokens} tokens",
+        expected=f"a prefix leaving at least one token of the model's {cap}-token input for the "
+        "query itself — a longer prefix leaves no query to embed, so the store cannot be read",
+    )
+
+
+def _prefix_leaves_no_query_token(prefix_tokens: int, cap: int) -> ZikaronError:
+    """`bad_config` for a prefix beside which not even this query's first token fits.
+
+    Distinct from `_prefix_too_long` only in how it was discovered: that one is arithmetic on the
+    two counts, this one is what the assembled sequence turned out to cost once the boundary between
+    them was tokenized. The claim is deliberately about *this* query rather than about every query,
+    because the boundary's cost depends on the text either side of it and a different first token
+    could fuse more cheaply. Same key, same remedy, both reachable only by configuration.
+    """
+    return ZikaronError(
+        ErrorCode.BAD_CONFIG,
+        source=BadConfigSource.FILE,
+        file="<effective config>",
+        key="embedding.embed_prefix_query",
+        value=f"{prefix_tokens} tokens",
+        expected="a prefix beside which this query's first token still fits once the two are "
+        f"tokenized together — assembled with even one of its tokens the sequence exceeds the "
+        f"model's {cap}-token input",
+    )
+
+
+def _fit_to_cap(text: str, *, prefix: str, budget: int, encoder: Encoder) -> str:
+    """The nominal-budget head of `text`, shrunk until its **assembled** sequence fits the cap.
+
+    Called only once the whole text is known not to fit. The budget subtraction chooses the first
+    candidate; this is what settles it, by counting `prefix + head` as one string at each step
+    rather than adding the two pieces' counts. Each step drops one query token, so the loop
+    terminates, and its floor is one token: a prefix beside which not even this query's first token
+    fits is a configuration error rather than a query silently reduced to the prefix alone.
+
+    Not "the longest head that fits", and the distinction is worth stating because retokenization at
+    the boundary can *reduce* a count as well as raise it — so a head one token past the nominal
+    budget could in principle fit. The search deliberately does not look there: `retrieval.md` sets
+    the budget as what the query may keep, and going above it to reclaim a token the boundary
+    happened to absorb would make the kept length depend on the prefix in a way no field records.
+
+    Raises:
+        ZikaronError: `BAD_CONFIG` naming `embedding.embed_prefix_query` if not even this query's
+            first token fits beside the prefix.
+    """
+    for tokens in range(min(budget, encoder.count_tokens(text)), 0, -1):
+        kept = token_head(text, tokens=tokens, encoder=encoder)
+        if assembled_tokens(prefix, kept, encoder=encoder) <= encoder.max_sequence_tokens:
+            return kept
+    raise _prefix_leaves_no_query_token(encoder.count_tokens(prefix), encoder.max_sequence_tokens)
+
+
+def external_query(text: str, *, encoder: Encoder, prefix: str, max_terms: int) -> ExternalQuery:
+    """Build both arms of a query from text outside the store, embedding it under the token budget.
+
+    The dense side is embedded through the deployed artifact with D20's BGE query-instruction
+    prefix; the lexical side is the term constructor above, over the **whole** text whether or not
+    the dense side had to be truncated.
+
+    Blocking work — one embedding call — so a caller on an event loop is responsible for keeping
+    this off it, exactly as the write path's `embed_chunks` is.
+
+    Args:
+        text: the query as the caller received it: a user prompt, an agent's `search` string, or
+            consolidation's group-gist concatenation.
+        encoder: the deployed artifact. Its tokenizer counts, and its model embeds.
+        prefix: `embed_prefix_query`. Query-side only, per BGE's asymmetric convention; the empty
+            string disables it.
+        max_terms: `fts_query_max_terms`.
+
+    Raises:
+        ZikaronError: `BAD_CONFIG` if the configured prefix leaves no room for the query — either by
+            the budget arithmetic or once the two are tokenized as one string — or if the artifact's
+            token count and token spans disagree.
+    """
+    cap = encoder.max_sequence_tokens
+    budget = cap - encoder.n_special_tokens - encoder.count_tokens(prefix)
+    if budget < 1:
+        raise _prefix_too_long(encoder.count_tokens(prefix), cap)
+
+    query_tokens = encoder.count_tokens(text)
+    # The assembled sequence is counted first, so the common path pays one tokenizer call and the
+    # search below runs only when the input genuinely overflows. Checking it this way also keeps
+    # `query_truncated` exact: a head slices from the first token's start to the last token's end,
+    # so calling it on a query that already fits would drop leading or trailing punctuation and
+    # report a truncation that did not happen.
+    fits_whole = assembled_tokens(prefix, text, encoder=encoder) <= cap
+    kept = text if fits_whole else _fit_to_cap(text, prefix=prefix, budget=budget, encoder=encoder)
+    (vector,) = encoder.embed([f"{prefix}{kept}"])
+    return ExternalQuery(
+        prepared=PreparedQuery(
+            origin=QueryOrigin.EXTERNAL,
+            vector=serialize(vector),
+            lexical=lexical_query(text, max_terms=max_terms),
+        ),
+        query_tokens=query_tokens,
+        # What was dropped, not what the budget subtraction predicted would be dropped: the two
+        # differ exactly when the prefix boundary retokenizes, which is the case this preflight now
+        # settles by counting the assembled string instead of the sum of its parts.
+        query_truncated=kept != text,
+    )
+
+
+async def internal_query(
+    db: aiosqlite.Connection, *, memory_uuid: str, max_terms: int
+) -> PreparedQuery:
+    """Build both arms of one memory's query against the others, reading no model at all.
+
+    The dense side is the memory's **stored first-chunk embedding** (`part_index = 0`): it already
+    exists, it is already normalized, it cannot overflow the model's input because the write-side
+    preflight proved it fits, and reusing it is what makes planning a pure function of the store
+    rather than of a second inference. The lexical side is the memory's own `gist + content` through
+    the identical constructor an external query uses, where the 64-term cap does real work.
+
+    Assumes the caller's own open transaction: an internal query's whole point is to run inside the
+    write or planning transaction that produced the row it is querying from.
+
+    Raises:
+        ZikaronError: `NOT_FOUND` if `memory_uuid` names no row, or names one with no `part_index=0`
+            chunk. Invariant 12 makes the second impossible for a row with content, so it is
+            reported rather than papered over with a second embed call.
+    """
+    rows = await db.execute_fetchall(
+        "SELECT m.gist, m.content, v.embedding "
+        "FROM memory m "
+        "JOIN memory_chunk c ON c.memory_uuid = m.uuid AND c.part_index = 0 "
+        "JOIN memory_vec v ON v.rowid = c.chunk_id "
+        "WHERE m.uuid = ?",
+        (memory_uuid,),
+    )
+    found = list(rows)
+    if not found:
+        raise ZikaronError(ErrorCode.NOT_FOUND, uuid=memory_uuid)
+    gist, content, embedding = found[0]
+    return PreparedQuery(
+        origin=QueryOrigin.INTERNAL,
+        vector=bytes(embedding),
+        lexical=lexical_query(internal_lexical_text(str(gist), str(content)), max_terms=max_terms),
+    )

zikaron/core/retrieval/ranking.py

@@ -0,0 +1,260 @@
+"""Fusion, demotion, the total order, and the supersession repair — all pure, no store.
+
+`retrieval.md` §"The fused score, written out", §"Total order" and §"Supersession: eligible,
+demoted, and labelled" are normative. Four steps, in this order, and the order is observable:
+
+1. **RRF.** `Σ 1 / (rrf_k + rank)` over the arms that returned the row, ranks 1-based. An arm that
+   did not return it contributes nothing — not a term at a notional worst rank.
+2. **Penalties.** A demoted row's fused score is multiplied by `supersession_penalty` or
+   `retired_penalty`. On the *immediate* edge, not on graph depth: nothing measured says a
+   twice-superseded record is more dangerous than a once-superseded one. The two states are mutually
+   exclusive, so the penalties never compound.
+3. **The total order**, five steps deep, because fused ties are pervasive rather than rare —
+   measured: every hybrid query has at least one exact tie, and 12.5% have one *inside the top
+   five*, where order fell to whichever arm happened to be appended first.
+4. **The supersession repair**, a stable topological pass that puts every retrieved replacement
+   ahead of every record it replaced.
+
+The caller cuts to `limit` **after** all four, which is what lets the repair promote a replacement
+into the returned set rather than merely reshuffling what already made the cut. That is the case the
+measurement is about: a superseded record outranks its own replacement 28-50% of the time on the
+polarity stubs, and in those cases the correct answer is provably present and losing.
+
+Nothing here touches the store, so determinism is testable by calling it twice.
+"""
+
+from collections.abc import Iterable, Mapping, Sequence
+from dataclasses import dataclass
+from typing import Final
+
+from zikaron.core.errors import ErrorCode, RowState, ZikaronError
+from zikaron.core.events import Demotion
+from zikaron.core.records.memory import Tier
+from zikaron.core.records.supersession import BadSupersessionReason
+from zikaron.core.retrieval.arms import ArmOutcome, ArmRow
+
+#: Which tier wins a fused tie: consolidated prose has been reviewed by the consolidator, and
+#: `~/Memory` reached the same tiebreak independently ("cohesive memory over raw journal line").
+_TIER_ORDER: Final[Mapping[Tier, int]] = {Tier.LONG_TERM: 0, Tier.JOURNAL: 1}
+
+#: The recency component for a long-term row. Step 4 is D27's *journal-local* recency tiebreak, so a
+#: long-term row skips it and falls through to `uuid` — nothing about a long-term record's age is a
+#: claim about its truth. A constant is well defined here precisely because step 3 has already
+#: partitioned the remaining ties by tier: two rows still tied at step 4 share a tier, so this value
+#: is only ever compared against another long-term row's copy of it.
+_NO_RECENCY: Final = -1
+
+
+@dataclass(frozen=True, slots=True)
+class PoolRow:
+    """One candidate's row facts — exactly the columns the read path needs, and no more.
+
+    `content` and `version` are deliberately absent. `search` returns neither (a version would imply
+    a licence to write that the agent has not earned), the injected block shows only the gist, and
+    selecting long text for every row of a fused pool would spend I/O on the latency-critical path
+    for prose nobody reads.
+    """
+
+    uuid: str
+    tier: Tier
+    gist: str
+    active: bool
+    superseded_by: str | None
+    created_at: str
+    updated_at: str
+
+    @property
+    def state(self) -> RowState:
+        """This row's display state, per `schema.md` §"Retrieval eligibility"'s three states."""
+        if self.active:
+            return RowState.LIVE
+        if self.superseded_by is not None:
+            return RowState.SUPERSEDED
+        return RowState.RETIRED
+
+    @property
+    def demotion(self) -> Demotion | None:
+        """Why this row is demoted, or `None` for a live row that is not."""
+        if self.active:
+            return None
+        return Demotion.SUPERSEDED if self.superseded_by is not None else Demotion.RETIRED
+
+
+@dataclass(frozen=True, slots=True)
+class Penalties:
+    """The two demotion multipliers, which are separate config keys with equal defaults.
+
+    Equal by honesty rather than by symmetry: nothing measured distinguishes a superseded row from
+    an outright-retired one, so they start equal and can diverge once something does.
+    """
+
+    superseded: float
+    retired: float
+
+    def factor(self, demotion: Demotion | None) -> float:
+        """The multiplier for one row's demotion state — 1.0 for a row that is not demoted."""
+        if demotion is None:
+            return 1.0
+        return self.superseded if demotion is Demotion.SUPERSEDED else self.retired
+
+
+@dataclass(frozen=True, slots=True)
+class RankedMemory:
+    """One candidate, scored and placed: the row, its fused score, and where each arm found it.
+
+    `best_rank` is the minimum over the arms that returned it, so a row only one arm found is ranked
+    on that arm's number rather than penalized for the other's silence. `best_distance` is the dense
+    arm's rolled-up minimum, kept because the three cosine cutoffs are read off it — and only
+    meaningful for a query whose vector is unit length, which `query.QueryOrigin` is what tracks.
+    """
+
+    row: PoolRow
+    fused_score: float
+    dense_rank: int | None
+    lexical_rank: int | None
+    best_distance: float | None
+
+    @property
+    def best_rank(self) -> int:
+        """The best rank any arm gave this row. Both arms cannot be absent: it would not be here."""
+        ranks = [rank for rank in (self.dense_rank, self.lexical_rank) if rank is not None]
+        return min(ranks)
+
+    @property
+    def demoted(self) -> bool:
+        """Whether this row was demoted, which is exactly whether it is inactive."""
+        return self.row.demotion is not None
+
+
+def rrf_score(ranks: Iterable[int], *, rrf_k: int) -> float:
+    """Reciprocal Rank Fusion over the ranks one row achieved, in the arms that returned it.
+
+    `Σ 1 / (rrf_k + rank)`, ranks **1-based** — the formula every quality figure in `retrieval.md`
+    was measured through. Rebasing to 0 would leave `rrf_k = 60` naming a different denominator, so
+    the config default and the measurements would silently stop describing the same pipeline.
+    """
+    return sum(1.0 / (rrf_k + rank) for rank in ranks)
+
+
+def _by_uuid(outcome: ArmOutcome) -> Mapping[str, ArmRow]:
+    return {row.uuid: row for row in outcome.rows}
+
+
+def _journal_recency(rows: Iterable[PoolRow]) -> Mapping[str, int]:
+    """Each journal row's position when the pool's timestamps are ordered newest first.
+
+    An index rather than the timestamp itself, so step 4 needs no string inversion and no timestamp
+    parsing — `created_at` is opaque text this layer never has to interpret, and a mapping from
+    distinct value to descending position is monotone, so comparing indices ascending *is* comparing
+    timestamps descending. Ties on the timestamp share an index and fall through to `uuid`, which is
+    what step 5 is for.
+    """
+    journal = [row for row in rows if row.tier is Tier.JOURNAL]
+    descending = sorted({row.created_at for row in journal}, reverse=True)
+    position = {value: index for index, value in enumerate(descending)}
+    return {row.uuid: position[row.created_at] for row in journal}
+
+
+def _order_key(
+    ranked: RankedMemory, recency: Mapping[str, int]
+) -> tuple[float, int, int, int, str]:
+    """`retrieval.md`'s five-step total order, as one comparable tuple.
+
+    Ascending on every component, so the fused score is negated. The components are, in order: the
+    fused score descending; the best rank any arm gave it ascending, since a document one arm ranked
+    first is a better bet than one both arms ranked tenth; tier, long-term first; `created_at`
+    descending **inside a journal block only**; and `uuid`, which is arbitrary and is the point —
+    it guarantees a total order, so the same store and query always produce the same list.
+    """
+    return (
+        -ranked.fused_score,
+        ranked.best_rank,
+        _TIER_ORDER[ranked.row.tier],
+        recency.get(ranked.row.uuid, _NO_RECENCY),
+        ranked.row.uuid,
+    )
+
+
+def _repair_supersession(ordered: Sequence[RankedMemory]) -> tuple[RankedMemory, ...]:
+    """Reorder so every retrieved replacement precedes every record it replaced.
+
+    Not "immediately above" — that rule is unsatisfiable, because `merge` absorbing `A` and `B` into
+    `C` writes `A→C` *and* `B→C`, so no linear order can put `C` immediately above both. The
+    realizable rule is precedence, enforced as a stable topological pass: scan in order and emit the
+    first not-yet-emitted row whose `superseded_by` target is either absent from the pool or already
+    emitted. It handles `A→B→C` transitively, giving `C, B, A`.
+
+    Adjacency is neither preserved nor needed: the injected block gives the agent the replacement's
+    uuid explicitly, which is a stronger cue than proximity and is the part that survives a group of
+    rows sharing one replacement.
+
+    Raises:
+        ZikaronError: `BAD_SUPERSESSION` with `reason='cycle'` if no row is emittable, which
+            invariant 6 makes impossible — every row waits on at most one other and the graph is
+            acyclic, so the wait-for relation is a forest. Refused rather than worked around: a
+            fallback to the pre-repair order would answer from a corrupted graph with a *plausible*
+            list, and putting the replacement first is the whole point of the pass.
+    """
+    in_pool = {ranked.row.uuid for ranked in ordered}
+    waiting = list(ordered)
+    emitted: list[RankedMemory] = []
+    emitted_uuids: set[str] = set()
+    while waiting:
+        for index, ranked in enumerate(waiting):
+            target = ranked.row.superseded_by
+            if target is None or target not in in_pool or target in emitted_uuids:
+                emitted.append(waiting.pop(index))
+                emitted_uuids.add(ranked.row.uuid)
+                break
+        else:
+            stalled = waiting[0].row
+            raise ZikaronError(
+                ErrorCode.BAD_SUPERSESSION,
+                uuid=stalled.uuid,
+                target=stalled.superseded_by,
+                reason=BadSupersessionReason.CYCLE,
+            )
+    return tuple(emitted)
+
+
+def rank(
+    *,
+    dense: ArmOutcome,
+    lexical: ArmOutcome,
+    rows: Mapping[str, PoolRow],
+    rrf_k: int,
+    penalties: Penalties,
+) -> tuple[RankedMemory, ...]:
+    """Fuse both arms into one ordered pool: RRF, penalties, the total order, then the repair.
+
+    The returned pool is **not** cut to any output budget; the caller cuts it, because the repair
+    must run first if it is to promote a replacement into the returned set.
+
+    Args:
+        rows: the row facts for every uuid either arm returned. A uuid missing from this mapping is
+            a row that vanished between the arm query and the pool load, which one read transaction
+            makes impossible; it is dropped rather than guessed at.
+    """
+    dense_rows = _by_uuid(dense)
+    lexical_rows = _by_uuid(lexical)
+    scored: list[RankedMemory] = []
+    for uuid in [*dense_rows, *(uuid for uuid in lexical_rows if uuid not in dense_rows)]:
+        row = rows.get(uuid)
+        if row is None:
+            continue
+        found = [arm[uuid] for arm in (dense_rows, lexical_rows) if uuid in arm]
+        fused = rrf_score((arm_row.rank for arm_row in found), rrf_k=rrf_k)
+        dense_row = dense_rows.get(uuid)
+        lexical_row = lexical_rows.get(uuid)
+        scored.append(
+            RankedMemory(
+                row=row,
+                fused_score=fused * penalties.factor(row.demotion),
+                dense_rank=None if dense_row is None else dense_row.rank,
+                lexical_rank=None if lexical_row is None else lexical_row.rank,
+                best_distance=None if dense_row is None else dense_row.distance,
+            )
+        )
+    recency = _journal_recency(ranked.row for ranked in scored)
+    ordered = sorted(scored, key=lambda ranked: _order_key(ranked, recency))
+    return _repair_supersession(ordered)