moca-reader 0.6.0__py3-none-any.whl

moca_reader/audience.py

@@ -0,0 +1,49 @@
+"""Audience (package spec §5.3, Reader contract §8, moca-spec ADR-0016).
+
+- A node's label is its ``moca.audience``, a non-empty string.
+- A translation with no ``moca.audience`` of its own takes the label of the node's default representation.
+- A label that is present but not a non-empty string is malformed: it never matches any audience set, even when the
+  host allows unlabelled content.
+- When the host supplies an audience set, a record with no label is outside it unless the host sets
+  ``allow_unlabelled``.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import Any
+
+
+def audience_of(node: Any, rep: Any) -> tuple[str | None, bool]:
+    """(label, invalid) for a representation."""
+    own = rep.frontmatter.get("moca")
+    if isinstance(own, dict) and "audience" in own:
+        source: Any = own
+    else:
+        reps = getattr(node, "representations", None) or []
+        source = reps[0].frontmatter.get("moca") if reps else None
+    if not isinstance(source, dict) or "audience" not in source:
+        return None, False
+    value = source["audience"]
+    return (value, False) if isinstance(value, str) and value != "" else (None, True)
+
+
+def audience_allowed(
+    label: str | None, invalid: bool, audiences: Sequence[str] | None, allow_unlabelled: bool = False
+) -> bool:
+    """Whether a label passes the host's audience set."""
+    if audiences is None:
+        return True
+    if invalid:
+        return False
+    if label is None:
+        return allow_unlabelled
+    return label in audiences
+
+
+def record_allowed(record: Any, audiences: Sequence[str] | None, allow_unlabelled: bool = False) -> bool:
+    """The audience rule for a citation record, for the gate and store backends."""
+    label = record.get("audience")
+    return audience_allowed(
+        label if isinstance(label, str) else None, record.get("audienceInvalid") is True, audiences, allow_unlabelled
+    )

moca_reader/backends/__init__.py

@@ -0,0 +1 @@
+"""Search backends: lexical (BM25), dense over sidecar vectors, an in-memory store, and hybrid fusion."""

moca_reader/backends/bm25.py

@@ -0,0 +1,118 @@
+"""BM25 over plain text, shared by the lexical and in-memory store backends.
+
+Tokens. Every Reader's built-in lexical backend tokenises the same way, so they return the same results:
+1. NFKC-normalise the text, then lower-case it as package spec §6.3 does (one code point at a time).
+2. Letters, digits and combining marks are word characters; anything else separates words. Keeping marks inside
+   words keeps Devanagari, Arabic and other scripts with vowel signs whole.
+3. Scripts written without spaces (Chinese, Japanese kana, Thai, Lao, Khmer, Myanmar) are split into units, each
+   character with the combining marks after it, and indexed as each unit and each pair of neighbouring units.
+No stemming: hosts that need it supply their own lexical backend.
+"""
+
+from __future__ import annotations
+
+import math
+import unicodedata
+from collections.abc import Sequence
+from typing import Any
+
+from .._js import round_scaled, utf16_key
+from ..unicode import lower_code_point, normalize
+
+UNSPACED = (
+    (0x0E00, 0x0EFF),  # Thai, Lao
+    (0x1000, 0x109F),  # Myanmar
+    (0x1780, 0x17FF),  # Khmer
+    (0x3040, 0x30FF),  # Hiragana, Katakana
+    (0x31F0, 0x31FF),  # Katakana phonetic extensions
+    (0x3400, 0x4DBF),  # CJK extension A
+    (0x4E00, 0x9FFF),  # CJK unified ideographs
+    (0xF900, 0xFAFF),  # CJK compatibility ideographs
+    (0x20000, 0x3FFFF),  # CJK extensions B and later
+)
+
+
+def _unspaced(cp: int) -> bool:
+    return any(lo <= cp <= hi for lo, hi in UNSPACED)
+
+
+def _word_char(ch: str) -> bool:
+    return len(ch) == 1 and unicodedata.category(ch)[0] in "LNM"
+
+
+def _mark(ch: str) -> bool:
+    return len(ch) == 1 and unicodedata.category(ch)[0] == "M"
+
+
+def tokens(text: str) -> list[str]:
+    out: list[str] = []
+    run: list[str] = []
+    run_unspaced = False
+
+    def flush() -> None:
+        if not run:
+            return
+        if not run_unspaced:
+            out.append("".join(run))
+        else:
+            for i, unit in enumerate(run):
+                out.append(unit)
+                if i + 1 < len(run):
+                    out.append(unit + run[i + 1])
+        run.clear()
+
+    for ch in normalize("NFKC", text):
+        c = lower_code_point(ch)
+        if not _word_char(c):
+            flush()
+            continue
+        u = _unspaced(ord(c))
+        if u and run_unspaced and run and _mark(c):
+            run[-1] += c
+            continue
+        if run and u != run_unspaced:
+            flush()
+        run_unspaced = u
+        run.append(c)
+    flush()
+    return out
+
+
+def count_terms(text: str) -> tuple[dict[str, int], int]:
+    """A text's term frequencies and length in tokens: all BM25 needs of it."""
+    t = tokens(text)
+    terms: dict[str, int] = {}
+    for term in t:
+        terms[term] = terms.get(term, 0) + 1
+    return terms, len(t)
+
+
+def bm25(docs: Sequence[dict[str, Any]], query: str, k1: float = 1.2, b: float = 0.75) -> list[dict[str, Any]]:
+    """``docs`` are ``{item, text, key}``; key breaks ties. Best first, zero scores dropped."""
+    return bm25_counted(
+        [{"item": d["item"], "key": d["key"], "counts": count_terms(d["text"])} for d in docs], query, k1, b
+    )
+
+
+def bm25_counted(docs: Sequence[dict[str, Any]], query: str, k1: float = 1.2, b: float = 0.75) -> list[dict[str, Any]]:
+    """bm25() over documents already counted (``{item, key, counts}``), so a backend can count each document once.
+    The same arithmetic in the same order, so scores are identical."""
+    q = list(dict.fromkeys(tokens(query)))
+    if not q or not docs:
+        return []
+    avg = sum(d["counts"][1] for d in docs) / len(docs) or 1
+    df = {term: sum(1 for d in docs if term in d["counts"][0]) for term in q}
+    scored = []
+    for d in docs:
+        terms, length = d["counts"]
+        score = 0.0
+        for term in q:
+            f = terms.get(term, 0)
+            if f == 0:
+                continue
+            idf = math.log(1 + (len(docs) - df[term] + 0.5) / (df[term] + 0.5))
+            score += idf * ((f * (k1 + 1)) / (f + k1 * (1 - b + (b * length) / avg)))
+        scored.append({"item": d["item"], "key": d["key"], "score": round_scaled(score, 1e4)})
+    kept = [s for s in scored if s["score"] > 0]
+    kept.sort(key=lambda s: (-s["score"], utf16_key(s["key"])))
+    return [{"item": s["item"], "score": s["score"]} for s in kept]

moca_reader/backends/dense.py

@@ -0,0 +1,146 @@
+"""Dense backend over sidecar vectors (Reader contract §9.3), scored by the index's ``model.distance`` (cosine by
+default). The host supplies the embedder. Every loaded package it does not search is reported: a sidecar whose model
+does not match the embedder (S006), or no vectors at all (S007)."""
+
+from __future__ import annotations
+
+import json
+import math
+from typing import TYPE_CHECKING, Any
+
+from .._js import get, js_string, round_scaled, truthy
+from ..audience import audience_allowed, audience_of
+from ..citation import pick_representation
+from ..diagnostics import Diagnostics
+
+if TYPE_CHECKING:
+    from ..library import Library
+    from ..search import Embedder
+
+
+class DenseBackend:
+    features = ("dense", "filterPushdown")
+
+    def __init__(self, library: Library, embedder: Embedder) -> None:
+        if embedder is None or not callable(getattr(embedder, "embed", None)):
+            raise TypeError("DenseBackend needs a host embedder")
+        self.library = library
+        self.embedder = embedder
+        self._checks = Diagnostics()
+        self.diagnostics = self._checks.items
+        self.checked: dict[str, bool] = {}
+        self._check()
+
+    def search(self, query: str, limit: int, filters: dict[str, Any]) -> list[dict[str, Any]]:
+        self._check()
+        digests = filters.get("digests")
+        usable = [
+            p
+            for p in self.library.packages
+            if self.checked.get(p.result.digest or "") and (digests is None or p.result.digest in digests)
+        ]
+        if not usable:
+            return []
+        q = self.embedder.embed([query], "query")[0]
+        excluded = {f"{x['digest']} {x['node']}" for x in filters.get("exclude") or []}
+        audiences, concepts, locale = filters.get("audiences"), filters.get("concepts"), filters.get("locale")
+        scored: list[dict[str, Any]] = []
+        for p in usable:
+            result = p.result
+            distance = get(get(p.index, "model"), "distance")
+            by_path = {n.path: n for n in result.nodes}
+            for c in p.chunks or []:
+                node = by_path.get(c["path"])
+                rep = (
+                    next((r for r in node.representations if (r.locale or None) == (c.get("locale") or None)), None)
+                    if node
+                    else None
+                )
+                if node is None or rep is None or not isinstance(c.get("vector"), list):
+                    continue
+                node_id = f"{js_string(result.manifest.get('id'))}#{node.path}"
+                if f"{result.digest} {node_id}" in excluded:
+                    continue
+                if locale and pick_representation(node, locale) is not rep:
+                    continue
+                label, invalid = audience_of(node, rep)
+                if not audience_allowed(label, invalid, audiences, filters.get("allowUnlabelled") is True):
+                    continue
+                if concepts and not any(x in concepts for x in rep.concepts or []):
+                    continue
+                hit: dict[str, Any] = {"digest": result.digest, "node": node_id, "path": node.path}
+                if rep.locale:
+                    hit["locale"] = rep.locale
+                hit["span"] = {"start": c["start"], "end": c["end"]}
+                hit["score"] = round_scaled(similarity(distance, q, c["vector"]), 1e4)
+                scored.append(hit)
+        scored.sort(key=lambda h: -h["score"])
+        return scored[:limit]
+
+    def _check(self) -> None:
+        """Compares the embedder with each loaded sidecar's model, once per package digest."""
+        for p in self.library.packages:
+            digest = p.result.digest or ""
+            if digest in self.checked:
+                continue
+            vectors = [c for c in p.chunks or [] if isinstance(c.get("vector"), list)]
+            name = f"{js_string(p.result.manifest.get('id'))}@{js_string(p.result.manifest.get('version'))}"
+            if not truthy(p.index) or not vectors:
+                self._checks.add("S007_NO_VECTORS", f"{name}: no sidecar vectors; the package is not searched")
+                self.checked[digest] = False
+                continue
+            problem = model_mismatch(self.embedder, get(p.index, "model"), len(vectors[0]["vector"]))
+            if problem:
+                self._checks.add("S006_MODEL_MISMATCH", f"{name}: {problem}")
+            self.checked[digest] = not problem
+
+
+def model_mismatch(embedder: Any, model: Any, vector_length: int) -> str | None:
+    """Why an embedder cannot search vectors made by ``model``, or None when it can."""
+    name = get(model, "name")
+    if not truthy(name):
+        return "the index names no model"
+    if name != embedder.name:
+        return f'index model "{name}" is not the host embedder "{embedder.name}"'
+    version, own_version = model.get("version"), getattr(embedder, "version", None)
+    if truthy(version) and truthy(own_version) and version != own_version:
+        return f"index model version {version} is not {own_version}"
+    dims = model["dimensions"] if model.get("dimensions") is not None else vector_length
+    if dims != embedder.dimensions:
+        return f"index has {dims} dimensions; the host embedder has {embedder.dimensions}"
+    # Prefixes are compared, never applied (Reader contract §9.3).
+    for key, attr in (("queryPrefix", "query_prefix"), ("documentPrefix", "document_prefix")):
+        theirs, ours = model.get(key), getattr(embedder, attr, None)
+        # A prefix declared on one side only is a mismatch too.
+        if theirs != ours:
+            return f"index {key} {json.dumps(theirs, ensure_ascii=False)} is not the host embedder's {json.dumps(ours, ensure_ascii=False)}"
+    return None
+
+
+def similarity(distance: Any, a: Any, b: Any) -> float:
+    """How well vector ``b`` matches query ``a``: cosine (the default), the dot product, or the negative euclidean
+    distance. A higher score is always a better match."""
+    if not isinstance(a, list) or not isinstance(b, list) or len(a) != len(b):
+        return 0.0
+    if distance == "dot":
+        total = 0.0
+        for x, y in zip(a, b, strict=True):
+            total += x * y
+        return total
+    if distance == "euclidean":
+        squares = 0.0
+        for x, y in zip(a, b, strict=True):
+            squares += (x - y) ** 2
+        return 0 - math.sqrt(squares)
+    return cosine(a, b)
+
+
+def cosine(a: list[float], b: list[float]) -> float:
+    if len(a) != len(b):
+        return 0.0
+    dot = na = nb = 0.0
+    for x, y in zip(a, b, strict=True):
+        dot += x * y
+        na += x * x
+        nb += y * y
+    return 0.0 if na == 0 or nb == 0 else dot / math.sqrt(na * nb)

moca_reader/backends/hybrid.py

@@ -0,0 +1,73 @@
+"""Hybrid backend: a lexical and a dense backend merged by Reciprocal Rank Fusion (Reader contract §9.6).
+
+    score(hit) = sum of weight_b / (k + rank_b(hit))
+
+rank_b is the hit's 1-based position in backend b's results. k (default 60) and the weights (default 1) are the
+host's; a package never sets them. A package the dense backend cannot search (S006, S007) still comes back through
+the lexical side.
+"""
+
+from __future__ import annotations
+
+import math
+from typing import Any
+
+from .._js import round_scaled, utf16_key
+
+
+def _key(h: dict[str, Any]) -> str:
+    span = h.get("span") or {}
+    return (
+        f"{h.get('digest')}\0{h.get('node')}\0{h.get('locale') or ''}\0{span.get('start', '')}\0{span.get('end', '')}"
+    )
+
+
+def _tie(h: dict[str, Any]) -> tuple[bytes, bytes, float]:
+    """Ties: node id, then locale, then span start; the same order in every Reader."""
+    return utf16_key(str(h.get("node"))), utf16_key(h.get("locale") or ""), (h.get("span") or {}).get("start", -1)
+
+
+def _weight(name: str, w: Any) -> float:
+    if isinstance(w, bool) or not isinstance(w, (int, float)) or not math.isfinite(w) or w < 0:
+        raise ValueError(f"weights.{name} must be a number of at least 0")
+    return float(w)
+
+
+class HybridBackend:
+    def __init__(self, lexical: Any, dense: Any, k: float = 60, weights: dict[str, float] | None = None) -> None:
+        if not callable(getattr(lexical, "search", None)) or not callable(getattr(dense, "search", None)):
+            raise TypeError("HybridBackend needs a lexical and a dense backend")
+        if isinstance(k, bool) or not isinstance(k, (int, float)) or not math.isfinite(k) or k < 0:
+            raise ValueError("k must be a number of at least 0")
+        self.lexical, self.dense, self.k = lexical, dense, k
+        w = weights or {}
+        self.weights = {
+            "lexical": _weight("lexical", w.get("lexical", 1)),
+            "dense": _weight("dense", w.get("dense", 1)),
+        }
+        if self.weights["lexical"] == 0 and self.weights["dense"] == 0:
+            raise ValueError("at least one weight must be above 0")
+        pushdown = all("filterPushdown" in (getattr(b, "features", None) or []) for b in (lexical, dense))
+        self.features = ("lexical", "dense", *(("filterPushdown",) if pushdown else ()))
+
+    @property
+    def diagnostics(self) -> list[Any]:
+        """Both backends' findings, such as S006 and S007 from the dense side."""
+        return [*(getattr(self.lexical, "diagnostics", None) or []), *(getattr(self.dense, "diagnostics", None) or [])]
+
+    def search(self, query: str, limit: int, filters: dict[str, Any]) -> list[dict[str, Any]]:
+        lexical = self.lexical.search(query, limit, filters)
+        dense = self.dense.search(query, limit, filters)
+        fused: dict[str, dict[str, Any]] = {}
+
+        def add(hits: Any, weight: float) -> None:
+            if weight == 0 or not isinstance(hits, list):
+                return
+            for i, hit in enumerate(hits):
+                entry = fused.setdefault(_key(hit), {"hit": hit, "score": 0.0})
+                entry["score"] += weight / (self.k + i + 1)
+
+        add(lexical, self.weights["lexical"])
+        add(dense, self.weights["dense"])
+        ranked = sorted((e for e in fused.values() if e["score"] > 0), key=lambda e: (-e["score"], _tie(e["hit"])))
+        return [{**e["hit"], "score": round_scaled(e["score"], 1e6)} for e in ranked[:limit]]

moca_reader/backends/lexical.py

@@ -0,0 +1,112 @@
+"""Lexical (BM25) backend over the packages in a library: each package's sidecar chunks when it was added with them,
+otherwise its nodes. With a locale, the representation package spec §5 selects for it; without one, every
+representation, so content in any language can be found, and the best-scoring one for each node is returned."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from .._js import js_string
+from ..audience import audience_allowed, audience_of
+from ..citation import citation_for, pick_representation
+from ..sidecar import chunk_text
+from .bm25 import bm25_counted, count_terms
+
+if TYPE_CHECKING:
+    from ..library import Library
+
+
+class LexicalBackend:
+    features = ("lexical", "filterPushdown")
+
+    def __init__(self, library: Library) -> None:
+        self.library = library
+        self.diagnostics: list[Any] = []
+        # Each document's term counts, per package digest, kept after its first search and dropped when the package
+        # leaves the library; scores are the same as counting afresh.
+        self._counted: dict[str, dict[str, tuple[dict[str, int], int]]] = {}
+
+    def search(self, query: str, limit: int, filters: dict[str, Any]) -> list[dict[str, Any]]:
+        excluded = {f"{x['digest']} {x['node']}" for x in filters.get("exclude") or []}
+        digests = set(filters["digests"]) if filters.get("digests") is not None else None
+        audiences, concepts, locale = filters.get("audiences"), filters.get("concepts"), filters.get("locale")
+        docs: list[dict[str, Any]] = []
+        loaded = {p.result.digest or "" for p in self.library.packages}
+        for gone in [d for d in self._counted if d not in loaded]:
+            del self._counted[gone]
+        for p in self.library.packages:
+            result = p.result
+            if digests is not None and result.digest not in digests:
+                continue
+            counted = self._counted.setdefault(result.digest or "", {})
+            by_path = {n.path: n for n in result.nodes}
+            pid = js_string(result.manifest.get("id"))
+
+            def keep(node: Any, rep: Any, pid: str = pid, digest: Any = result.digest) -> str | None:
+                node_id = f"{pid}#{node.path}"
+                if f"{digest} {node_id}" in excluded:
+                    return None
+                label, invalid = audience_of(node, rep)
+                if not audience_allowed(label, invalid, audiences, filters.get("allowUnlabelled") is True):
+                    return None
+                if concepts and not any(c in concepts for c in rep.concepts or []):
+                    return None
+                return node_id
+
+            if p.chunks is not None:
+                for c in p.chunks:
+                    node = by_path.get(c["path"])
+                    rep = (
+                        next((r for r in node.representations if (r.locale or None) == (c.get("locale") or None)), None)
+                        if node
+                        else None
+                    )
+                    if node is None or rep is None:
+                        continue
+                    if locale and pick_representation(node, locale) is not rep:
+                        continue
+                    node_id = keep(node, rep)
+                    if not node_id:
+                        continue
+                    key = f"{node_id}\0{js_string(c['start']).rjust(12, '0')}"
+                    # The cache key adds the locale: chunks of two representations can start at the same offset.
+                    cache_key = f"{key}\0{c.get('locale') or ''}"
+                    if cache_key not in counted:
+                        title = citation_for(result, node, rep)["node"].get("title") or ""
+                        counted[cache_key] = count_terms(f"{title} {chunk_text(c, rep)}")
+                    item: dict[str, Any] = {"digest": result.digest, "node": node_id, "path": node.path}
+                    if rep.locale:
+                        item["locale"] = rep.locale
+                    item["span"] = {"start": c["start"], "end": c["end"]}
+                    docs.append({"item": item, "counts": counted[cache_key], "key": key})
+            else:
+                for node in result.nodes:
+                    reps = [pick_representation(node, locale)] if locale else node.representations
+                    for rep in reps:
+                        node_id = keep(node, rep)
+                        if not node_id:
+                            continue
+                        key = f"{node_id}\0{rep.locale or ''}"
+                        if key not in counted:
+                            fm = rep.frontmatter
+                            tags = fm["tags"] if isinstance(fm.get("tags"), list) else []
+                            parts = [fm.get("title"), fm.get("description"), *tags, rep.body]
+                            counted[key] = count_terms(" ".join(x for x in parts if isinstance(x, str)))
+                        item = {"digest": result.digest, "node": node_id, "path": node.path}
+                        if rep.locale:
+                            item["locale"] = rep.locale
+                        docs.append({"item": item, "counts": counted[key], "key": key})
+        seen: set[str] = set()
+        out: list[dict[str, Any]] = []
+        for hit in bm25_counted(docs, query):
+            item = hit["item"]
+            if "span" not in item:
+                # Node ids repeat across versions of a package, so key on the digest too.
+                key = f"{item['digest']} {item['node']}"
+                if key in seen:
+                    continue
+                seen.add(key)
+            out.append({**item, "score": hit["score"]})
+            if len(out) >= limit:
+                break
+        return out

moca_reader/backends/memory_store.py

@@ -0,0 +1,122 @@
+"""A reference store backend, kept in memory (Reader contract §9.4).
+
+It stands in for a host's vector store in tests and shows what a store binding must record at ingest: the whole
+citation record, including the package digest. Search then resolves hits from those records, so the host never has
+to load every package to search its store.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from .._js import js_string, round_scaled
+from ..audience import record_allowed
+from ..citation import citation_for
+from ..sidecar import chunk_text
+from .bm25 import bm25
+from .dense import similarity
+
+if TYPE_CHECKING:
+    from ..library import Library
+    from ..search import Embedder
+
+
+class MemoryStoreBackend:
+    """Without an embedder the store is searched lexically. ``distance`` is how its vectors are compared."""
+
+    def __init__(
+        self, embedder: Embedder | None = None, filter_pushdown: bool = True, distance: str = "cosine"
+    ) -> None:
+        self.embedder = embedder
+        self.distance = distance
+        self.features = (
+            "dense" if embedder is not None else "lexical",
+            *(("filterPushdown",) if filter_pushdown else ()),
+        )
+        self.diagnostics: list[Any] = []
+        self.records: list[dict[str, Any]] = []
+
+    def ingest(self, library: Library) -> MemoryStoreBackend:
+        """Ingests every node representation (or sidecar chunk) of every package, through citation records."""
+        index = library.relation_index()
+        now = library.clock()
+        pending: list[dict[str, Any]] = []
+        for p in library.packages:
+            result = p.result
+            by_path = {n.path: n for n in result.nodes}
+            units: list[tuple[Any, Any, Any, Any, Any]]
+            if p.chunks is not None:
+                units = [
+                    (by_path.get(c["path"]), c.get("locale") or None, {"start": c["start"], "end": c["end"]}, c, None)
+                    for c in p.chunks
+                ]
+            else:
+                units = [
+                    (node, rep.locale, None, None, rep.body) for node in result.nodes for rep in node.representations
+                ]
+            for node, locale, span, chunk, body in units:
+                rep = next((r for r in node.representations if (r.locale or None) == locale), None) if node else None
+                if node is None or rep is None:
+                    continue
+                text = chunk_text(chunk, rep) if chunk is not None else body or ""
+                record = citation_for(result, node, rep, now, index, text)
+                if span is not None:
+                    record["span"] = dict(span)
+                hit: dict[str, Any] = {"digest": result.digest, "node": record["node"]["id"], "path": node.path}
+                if rep.locale:
+                    hit["locale"] = rep.locale
+                if span is not None:
+                    hit["span"] = span
+                hit["record"] = record
+                entry: dict[str, Any] = {
+                    "hit": hit,
+                    "text": f"{record['node'].get('title') or ''} {text}",
+                    "concepts": record.get("concepts") or [],
+                }
+                if record.get("audience"):
+                    entry["audience"] = record["audience"]
+                if record.get("audienceInvalid") is True:
+                    entry["audienceInvalid"] = True
+                pending.append(entry)
+        if self.embedder is not None and pending:
+            vectors = self.embedder.embed([e["text"] for e in pending], "document")
+            for entry, vector in zip(pending, vectors, strict=True):
+                entry["vector"] = vector
+        self.records.extend(pending)
+        return self
+
+    def remove(self, digest: str) -> MemoryStoreBackend:
+        """Removes everything ingested from a package digest."""
+        self.records = [r for r in self.records if r["hit"]["digest"] != digest]
+        return self
+
+    def search(self, query: str, limit: int, filters: dict[str, Any]) -> list[dict[str, Any]]:
+        candidates = self.records
+        if "filterPushdown" in self.features:
+            digests = set(filters.get("digests") or [])
+            excluded = {f"{x['digest']} {x['node']}" for x in filters.get("exclude") or []}
+            audiences, concepts = filters.get("audiences"), filters.get("concepts")
+            candidates = [
+                r
+                for r in candidates
+                if r["hit"]["digest"] in digests
+                and f"{r['hit']['digest']} {r['hit']['node']}" not in excluded
+                and record_allowed(r, audiences, filters.get("allowUnlabelled") is True)
+                and (not concepts or any(c in concepts for c in r["concepts"]))
+            ]
+        if self.embedder is None:
+            docs = [
+                {
+                    "item": r["hit"],
+                    "text": r["text"],
+                    "key": f"{r['hit']['node']}\0{r['hit'].get('locale') or ''}\0{js_string((r['hit'].get('span') or {}).get('start', 0))}",
+                }
+                for r in candidates
+            ]
+            return [{**h["item"], "score": h["score"]} for h in bm25(docs, query)[:limit]]
+        q = self.embedder.embed([query], "query")[0]
+        scored = [
+            {**r["hit"], "score": round_scaled(similarity(self.distance, q, r.get("vector")), 1e4)} for r in candidates
+        ]
+        scored.sort(key=lambda h: -h["score"])
+        return scored[:limit]