midas-memory 0.0.1__py3-none-any.whl

midas/__init__.py

@@ -0,0 +1,64 @@
+"""Midas — agentic memory.
+
+Open-core wedge: semantic recall + budgeted context assembly behind a small,
+store/embedder-agnostic API. This is the thing the eval harness benchmarks.
+"""
+from .embeddings import (
+    DiskCachedEmbedder,
+    Embedder,
+    HashingEmbedder,
+    LocalEmbedder,
+    LocalReranker,
+    OpenAIEmbedder,
+    configure_local_model_cache,
+    cosine,
+    tokenize,
+)
+from .importance import ContentImportance, StructuralImportance
+from .memory import CaptureResult, ContextBlock, Memory, Reranker, approx_tokens, format_record
+from .policy import AGENT_MEMORY_INSTRUCTIONS, DEFAULT_POLICY, MemoryPolicy, policy_summary
+from .store import InMemoryStore
+from .types import MEMORY_KINDS, MemoryKind, MemoryRecord, RecallHit
+
+try:
+    from .sqlite_store import SQLiteStore
+except ImportError:
+    SQLiteStore = None  # sqlite-vec not installed
+
+try:
+    from .ann import IVFIndex, IVFStore
+except ImportError:
+    IVFIndex = IVFStore = None  # numpy not installed (ANN backend is optional)
+
+__all__ = [
+    "Memory",
+    "ContentImportance",
+    "StructuralImportance",
+    "MemoryPolicy",
+    "DEFAULT_POLICY",
+    "AGENT_MEMORY_INSTRUCTIONS",
+    "policy_summary",
+    "CaptureResult",
+    "ContextBlock",
+    "Reranker",
+    "approx_tokens",
+    "format_record",
+    "MemoryRecord",
+    "RecallHit",
+    "MemoryKind",
+    "MEMORY_KINDS",
+    "Embedder",
+    "DiskCachedEmbedder",
+    "HashingEmbedder",
+    "LocalEmbedder",
+    "LocalReranker",
+    "OpenAIEmbedder",
+    "configure_local_model_cache",
+    "cosine",
+    "tokenize",
+    "InMemoryStore",
+    "SQLiteStore",
+    "IVFIndex",
+    "IVFStore",
+]
+__version__ = "0.0.1"

midas/ann.py

@@ -0,0 +1,156 @@
+"""Approximate nearest-neighbour search for scaling Midas past the exact in-memory scan.
+
+`InMemoryStore`'s cached scan is exact but O(N) per query (~230 ms at 1M x 768-d). `IVFIndex` is an
+**inverted-file** index built with **numpy only** (no native dependency, unlike faiss/hnswlib): it
+clusters the corpus into `nlist` cells via k-means, and a query then compares against only the
+`nprobe` nearest cells instead of the whole corpus. That makes search **sub-linear** -- at 1M with
+`nlist=1000, nprobe=8` a query scans ~8K vectors instead of 1M (~100x fewer) -- trading a little
+recall for a large speedup. `nprobe` tunes the recall/latency trade-off at query time (no rebuild).
+
+`IVFStore` wraps the index behind the same store surface as `InMemoryStore`, so `Memory` can use it
+unchanged. The index is built lazily and rebuilt on mutation, so it suits **read-heavy / batch-loaded**
+corpora (build once, query many); for write-heavy workloads keep `InMemoryStore`.
+"""
+from __future__ import annotations
+
+from typing import Callable, Sequence
+
+import numpy as np
+
+from .types import MemoryRecord
+
+
+def _kmeans(x: np.ndarray, k: int, *, n_iter: int, rng: np.random.Generator) -> np.ndarray:
+    """Spherical k-means (cosine): centroids are L2-normalized, assignment is by max dot product.
+
+    Inputs are assumed L2-normalized (Midas embeddings are), so dot product == cosine similarity.
+    """
+    # Init from k distinct points.
+    centroids = x[rng.choice(x.shape[0], k, replace=False)].copy()
+    for _ in range(n_iter):
+        assign = np.argmax(x @ centroids.T, axis=1)
+        for c in range(k):
+            members = x[assign == c]
+            if len(members):
+                v = members.sum(axis=0)
+                norm = float(np.linalg.norm(v))
+                if norm > 0.0:
+                    centroids[c] = v / norm
+            else:  # empty cell -> reseed from a random point so it can pick up members next round
+                centroids[c] = x[rng.integers(x.shape[0])]
+    return centroids
+
+
+class IVFIndex:
+    """Inverted-file ANN index over L2-normalized vectors (cosine similarity)."""
+
+    def __init__(self, nlist: int | None = None, *, n_iter: int = 10, train_sample: int = 50_000,
+                 seed: int = 0) -> None:
+        self.nlist = nlist
+        self.n_iter = n_iter
+        self.train_sample = train_sample
+        self.seed = seed
+        self._centroids: np.ndarray | None = None
+        self._lists: list[np.ndarray] = []      # cell -> row indices into _vectors
+        self._vectors: np.ndarray | None = None  # (N, d) float32, L2-normalized
+
+    def fit(self, vectors: Sequence[Sequence[float]] | np.ndarray) -> "IVFIndex":
+        x = np.asarray(vectors, dtype=np.float32)
+        if x.ndim != 2 or x.shape[0] == 0:
+            raise ValueError("fit() needs a non-empty (N, d) matrix")
+        n = x.shape[0]
+        nlist = self.nlist or max(1, int(round(np.sqrt(n))))
+        nlist = min(nlist, n)
+        self.nlist = nlist
+        rng = np.random.default_rng(self.seed)
+        # Train centroids on a sample (k-means cost is O(sample x nlist x d x iters)); assign all.
+        train = x if n <= self.train_sample else x[rng.choice(n, self.train_sample, replace=False)]
+        centroids = _kmeans(train, nlist, n_iter=self.n_iter, rng=rng)
+        assign = np.argmax(x @ centroids.T, axis=1)
+        self._centroids = centroids
+        self._lists = [np.where(assign == c)[0] for c in range(nlist)]
+        self._vectors = x
+        return self
+
+    def search(self, query: Sequence[float], *, k: int = 10, nprobe: int = 8,
+               allowed: np.ndarray | None = None) -> tuple[np.ndarray, np.ndarray]:
+        """Return (row indices, scores) for the top-k, highest cosine first.
+
+        `allowed` is an optional boolean mask over rows (predicate pushdown); candidates failing it
+        are dropped before the top-k cut.
+        """
+        if self._vectors is None or self._centroids is None:
+            return np.array([], dtype=int), np.array([], dtype=np.float32)
+        q = np.asarray(query, dtype=np.float32)
+        nprobe = max(1, min(nprobe, self.nlist))
+        csims = self._centroids @ q
+        cells = np.argpartition(-csims, nprobe - 1)[:nprobe]
+        candidates = np.concatenate([self._lists[c] for c in cells]) if len(cells) else np.empty(0, int)
+        if candidates.size and allowed is not None:
+            candidates = candidates[allowed[candidates]]
+        if candidates.size == 0:
+            return np.array([], dtype=int), np.array([], dtype=np.float32)
+        sims = self._vectors[candidates] @ q
+        kk = min(k, candidates.size)
+        part = np.argpartition(-sims, kk - 1)[:kk]
+        order = part[np.argsort(-sims[part], kind="stable")]
+        return candidates[order], sims[order].astype(np.float32)
+
+
+class IVFStore:
+    """Store with the `InMemoryStore` surface, backed by an `IVFIndex` for sub-linear search.
+
+    Build is lazy and triggered on the first search after a mutation. `nprobe` (recall/latency knob)
+    is set at construction. Best for read-heavy corpora; for write-heavy use `InMemoryStore`.
+    """
+
+    def __init__(self, *, nlist: int | None = None, nprobe: int = 8, seed: int = 0) -> None:
+        self._records: dict[str, MemoryRecord] = {}
+        self._nlist = nlist
+        self.nprobe = nprobe
+        self.seed = seed
+        self._index: IVFIndex | None = None
+        self._rows: list[MemoryRecord] = []  # row -> record (aligned with the fitted matrix)
+        self._dirty = True
+
+    def put(self, record: MemoryRecord) -> None:
+        self._records[record.id] = record
+        self._dirty = True
+
+    def get(self, record_id: str) -> MemoryRecord | None:
+        return self._records.get(record_id)
+
+    def delete(self, record_id: str) -> bool:
+        existed = self._records.pop(record_id, None) is not None
+        self._dirty = self._dirty or existed
+        return existed
+
+    def all(self) -> list[MemoryRecord]:
+        return list(self._records.values())
+
+    def clear(self) -> None:
+        self._records.clear()
+        self._index = None
+        self._rows = []
+        self._dirty = True
+
+    def _ensure_index(self) -> None:
+        if not self._dirty:
+            return
+        recs = [r for r in self._records.values() if r.embedding is not None]
+        self._rows = recs
+        self._index = IVFIndex(nlist=self._nlist, seed=self.seed).fit(
+            [r.embedding for r in recs]) if recs else None
+        self._dirty = False
+
+    def search(self, embedding: list[float], *, limit: int,
+               predicate: Callable[[MemoryRecord], bool] | None = None
+               ) -> list[tuple[float, MemoryRecord]]:
+        self._ensure_index()
+        if self._index is None:
+            return []
+        allowed = None
+        if predicate is not None:
+            allowed = np.fromiter((predicate(r) for r in self._rows), dtype=bool, count=len(self._rows))
+        rows, scores = self._index.search(embedding, k=limit, nprobe=self.nprobe, allowed=allowed)
+        return [(float(s), self._rows[i]) for i, s in zip(rows, scores)]

midas/bm25.py

@@ -0,0 +1,51 @@
+"""BM25 lexical ranking — pure-Python, zero-dep.
+
+Fused with semantic recall (see `Memory.recall(hybrid=True)`) to catch lexically-relevant memories
+the bi-encoder misses — a no-LLM lever to push retrieval past the embedding-only ceiling. Standard
+Okapi BM25 with non-negative IDF; rebuilt per recall over the candidate set (cheap at eval scale).
+"""
+from __future__ import annotations
+
+import math
+from collections import Counter
+
+from .embeddings import tokenize
+from .types import MemoryRecord
+
+_K1 = 1.5
+_B = 0.75
+
+
+class BM25:
+    def __init__(self, records: list[MemoryRecord]) -> None:
+        self.records = records
+        self._docs = [tokenize(r.content) for r in records]
+        self._tfs = [Counter(d) for d in self._docs]
+        self._dls = [len(d) for d in self._docs]
+        self._avgdl = (sum(self._dls) / len(self._dls)) if self._dls else 0.0
+        df: Counter = Counter()
+        for doc in self._docs:
+            for term in set(doc):
+                df[term] += 1
+        n = len(records)
+        # Non-negative IDF (the BM25+ form) so common terms never push scores negative.
+        self._idf = {t: math.log(1 + (n - c + 0.5) / (c + 0.5)) for t, c in df.items()}
+
+    def scores(self, query: str) -> dict[str, float]:
+        """Map record id -> BM25 score (only records with a positive score are included)."""
+        q_terms = tokenize(query)
+        avgdl = self._avgdl or 1.0
+        out: dict[str, float] = {}
+        for i, record in enumerate(self.records):
+            tf = self._tfs[i]
+            dl = self._dls[i]
+            s = 0.0
+            for term in q_terms:
+                f = tf.get(term, 0)
+                if not f:
+                    continue
+                idf = self._idf.get(term, 0.0)
+                s += idf * (f * (_K1 + 1)) / (f + _K1 * (1 - _B + _B * dl / avgdl))
+            if s > 0.0:
+                out[record.id] = s
+        return out

midas/embeddings.py

@@ -0,0 +1,334 @@
+"""Pluggable embeddings.
+
+`HashingEmbedder` is a dependency-free, deterministic, offline stand-in so the
+whole harness runs anywhere with zero setup. It is essentially a vectorized
+bag-of-words — good enough to exercise the pipeline, NOT a real semantic model.
+Swap in `OpenAIEmbedder` (or a local BGE/e5) for genuine semantic retrieval;
+that swap is the entire reason embeddings live behind a Protocol.
+"""
+from __future__ import annotations
+
+import hashlib
+import math
+import os
+import re
+import sqlite3
+import struct
+import threading
+from pathlib import Path
+from typing import Protocol, runtime_checkable
+
+_WORD = re.compile(r"\w+", re.UNICODE)
+
+
+def tokenize(text: str) -> list[str]:
+    """Lowercase word tokens of length > 2 (Unicode-aware)."""
+    return [w for w in _WORD.findall(text.lower()) if len(w) > 2]
+
+
+@runtime_checkable
+class Embedder(Protocol):
+    dim: int
+
+    def embed(self, text: str) -> list[float]: ...
+
+    def embed_many(self, texts: list[str]) -> list[list[float]]: ...
+
+
+class HashingEmbedder:
+    """Offline, deterministic bag-of-words hashed into a fixed-dim unit vector."""
+
+    def __init__(self, dim: int = 256) -> None:
+        self.dim = dim
+
+    def embed(self, text: str) -> list[float]:
+        vec = [0.0] * self.dim
+        for tok in tokenize(text):
+            digest = hashlib.md5(tok.encode("utf-8")).digest()
+            h = int.from_bytes(digest[:8], "big")
+            idx = h % self.dim
+            sign = 1.0 if (h >> 8) & 1 else -1.0
+            vec[idx] += sign
+        return l2_normalize(vec)
+
+    def embed_many(self, texts: list[str]) -> list[list[float]]:
+        return [self.embed(text) for text in texts]
+
+
+class OpenAIEmbedder:
+    """Real semantic embeddings via OpenAI. Requires `OPENAI_API_KEY` and the
+    `openai` package (`uv pip install openai`)."""
+
+    def __init__(self, model: str = "text-embedding-3-small") -> None:
+        from openai import OpenAI  # lazy import — optional dependency
+
+        self._client = OpenAI()
+        self.model = model
+        self.dim = 1536
+
+    def embed(self, text: str) -> list[float]:
+        resp = self._client.embeddings.create(model=self.model, input=text)
+        return l2_normalize(list(resp.data[0].embedding))
+
+    def embed_many(self, texts: list[str]) -> list[list[float]]:
+        if not texts:
+            return []
+        resp = self._client.embeddings.create(model=self.model, input=texts)
+        return [l2_normalize(list(item.embedding)) for item in resp.data]
+
+
+def _default_embedding_cache_path() -> Path:
+    return _default_cache_root() / "midas-embeddings.sqlite3"
+
+
+def _cache_namespace(embedder: object) -> str:
+    parts = [
+        f"{type(embedder).__module__}.{type(embedder).__qualname__}",
+        f"dim={getattr(embedder, 'dim', 'unknown')}",
+    ]
+    for attr in ("model_name", "max_text_chars"):
+        value = getattr(embedder, attr, None)
+        if value is not None:
+            parts.append(f"{attr}={value}")
+    return "|".join(parts)
+
+
+def _encode_vector(vec: list[float]) -> bytes:
+    return struct.pack(f"<{len(vec)}f", *(float(v) for v in vec))
+
+
+def _decode_vector(blob: bytes, *, dim: int) -> list[float]:
+    if len(blob) != dim * 4:
+        raise ValueError("cached embedding has an unexpected dimension")
+    return [float(v) for v in struct.unpack(f"<{dim}f", blob)]
+
+
+class DiskCachedEmbedder:
+    """Persistent SQLite cache wrapper for expensive embedders.
+
+    The key includes the embedder namespace and exact input text hash, so changing the
+    local model or truncation limit gets a separate cache namespace by default.
+    """
+
+    def __init__(
+        self,
+        inner: Embedder,
+        *,
+        path: str | Path | None = None,
+        namespace: str | None = None,
+    ) -> None:
+        self.inner = inner
+        self.dim = inner.dim
+        self.namespace = namespace or _cache_namespace(inner)
+        self.path = Path(path) if path is not None else _default_embedding_cache_path()
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self._lock = threading.Lock()
+        self._conn = sqlite3.connect(str(self.path), check_same_thread=False)
+        with self._lock:
+            self._conn.execute("PRAGMA journal_mode=WAL")
+            self._conn.execute(
+                """
+                CREATE TABLE IF NOT EXISTS embeddings (
+                    namespace TEXT NOT NULL,
+                    text_sha256 TEXT NOT NULL,
+                    dim INTEGER NOT NULL,
+                    embedding BLOB NOT NULL,
+                    created_at INTEGER NOT NULL DEFAULT (unixepoch()),
+                    PRIMARY KEY (namespace, text_sha256)
+                )
+                """
+            )
+            self._conn.commit()
+        self.hits = 0
+        self.misses = 0
+
+    def embed(self, text: str) -> list[float]:
+        return self.embed_many([text])[0]
+
+    def embed_many(self, texts: list[str]) -> list[list[float]]:
+        if not texts:
+            return []
+
+        keys = [hashlib.sha256(text.encode("utf-8")).hexdigest() for text in texts]
+        cached = self._read_many(set(keys))
+        missing: dict[str, str] = {}
+        for key, text in zip(keys, texts):
+            if key not in cached and key not in missing:
+                missing[key] = text
+
+        self.hits += len(texts) - sum(1 for key in keys if key not in cached)
+        self.misses += len(missing)
+
+        if missing:
+            missing_keys = list(missing)
+            missing_texts = [missing[key] for key in missing_keys]
+            embed_many = getattr(self.inner, "embed_many", None)
+            vectors = (
+                embed_many(missing_texts)
+                if callable(embed_many)
+                else [self.inner.embed(text) for text in missing_texts]
+            )
+            if len(vectors) != len(missing_keys):
+                raise ValueError("cached embedder inner returned the wrong number of embeddings")
+            self._write_many(zip(missing_keys, vectors))
+            cached.update(zip(missing_keys, vectors))
+
+        return [cached[key] for key in keys]
+
+    def _read_many(self, keys: set[str]) -> dict[str, list[float]]:
+        if not keys:
+            return {}
+        rows: dict[str, list[float]] = {}
+        ordered = list(keys)
+        with self._lock:
+            for i in range(0, len(ordered), 500):
+                chunk = ordered[i : i + 500]
+                placeholders = ",".join("?" for _ in chunk)
+                params = [self.namespace, *chunk]
+                cur = self._conn.execute(
+                    f"""
+                    SELECT text_sha256, dim, embedding
+                    FROM embeddings
+                    WHERE namespace = ? AND text_sha256 IN ({placeholders})
+                    """,
+                    params,
+                )
+                for key, dim, blob in cur.fetchall():
+                    if dim == self.dim:
+                        rows[key] = _decode_vector(blob, dim=self.dim)
+        return rows
+
+    def _write_many(self, items) -> None:
+        rows = [
+            (self.namespace, key, self.dim, _encode_vector(vec))
+            for key, vec in items
+        ]
+        if not rows:
+            return
+        with self._lock:
+            self._conn.executemany(
+                """
+                INSERT OR REPLACE INTO embeddings(namespace, text_sha256, dim, embedding)
+                VALUES (?, ?, ?, ?)
+                """,
+                rows,
+            )
+            self._conn.commit()
+
+
+def _default_cache_root() -> Path:
+    if root := os.getenv("MIDAS_CACHE_ROOT"):
+        return Path(root)
+    if os.name == "nt" and Path("D:/").exists():
+        return Path("D:/hf-cache")
+    return Path.home() / ".cache" / "midas"
+
+
+def _default_tmp_dir() -> Path:
+    if tmp := os.getenv("MIDAS_TMP_DIR"):
+        return Path(tmp)
+    if os.name == "nt" and Path("D:/").exists():
+        return Path("D:/tmp")
+    return _default_cache_root() / "tmp"
+
+
+def configure_local_model_cache(cache_dir: str | Path | None = None) -> str:
+    """Return a fastembed cache dir and set safe local cache env defaults."""
+    cache_root = _default_cache_root()
+    root = Path(cache_dir) if cache_dir is not None else cache_root / "fastembed"
+    tmp = _default_tmp_dir()
+    root.mkdir(parents=True, exist_ok=True)
+    tmp.mkdir(parents=True, exist_ok=True)
+
+    os.environ.setdefault("HF_HOME", str(cache_root))
+    os.environ.setdefault("HF_HUB_CACHE", str(cache_root / "hub"))
+    os.environ.setdefault("TMP", str(tmp))
+    os.environ.setdefault("TEMP", str(tmp))
+    os.environ.setdefault("HF_HUB_DISABLE_SYMLINKS_WARNING", "1")
+    return str(root)
+
+
+class LocalEmbedder:
+    """Real local semantic embeddings via fastembed (ONNX — no API key, no torch).
+    First use downloads a small model (~130 MB). Install: `uv pip install fastembed`."""
+
+    _MAX_TEXT_CHARS = 2000
+
+    def __init__(
+        self,
+        model_name: str = "BAAI/bge-base-en-v1.5",
+        *,
+        cache_dir: str | Path | None = None,
+        batch_size: int = 32,
+        max_text_chars: int | None = None,
+    ) -> None:
+        self.cache_dir = configure_local_model_cache(cache_dir)
+        from fastembed import TextEmbedding  # lazy import — optional dependency
+
+        self._model = TextEmbedding(model_name=model_name, cache_dir=self.cache_dir)
+        self.model_name = model_name
+        self.dim = 768
+        self.batch_size = batch_size
+        self.max_text_chars = max_text_chars or self._MAX_TEXT_CHARS
+
+    def embed(self, text: str) -> list[float]:
+        vec = next(iter(self._model.embed([text[: self.max_text_chars]], batch_size=1)))
+        return l2_normalize([float(x) for x in vec])
+
+    def embed_many(self, texts: list[str]) -> list[list[float]]:
+        if not texts:
+            return []
+        capped = [text[: self.max_text_chars] for text in texts]
+        return [
+            l2_normalize([float(x) for x in vec])
+            for vec in self._model.embed(capped, batch_size=self.batch_size)
+        ]
+
+
+class LocalReranker:
+    """Local cross-encoder reranker (fastembed/ONNX, no API key, no torch). Reorders
+    candidate documents by relevance to the query — sharper precision than bi-encoder
+    cosine. First use downloads a small model. Used by Midas to refine the recall pool."""
+
+    def __init__(
+        self,
+        model_name: str = "Xenova/ms-marco-MiniLM-L-6-v2",
+        *,
+        cache_dir: str | Path | None = None,
+    ) -> None:
+        self.cache_dir = configure_local_model_cache(cache_dir)
+        from fastembed.rerank.cross_encoder import TextCrossEncoder  # lazy, optional
+
+        self._model = TextCrossEncoder(model_name=model_name, cache_dir=self.cache_dir)
+        self.model_name = model_name
+
+    # Cross-encoders cap the query+doc PAIR at ~512 tokens; oversized input crashes the ONNX
+    # runtime (seen on long real turns). Cap defensively — affects only the rerank score, not
+    # the stored/assembled content.
+    _MAX_QUERY_CHARS = 400
+    _MAX_DOC_CHARS = 1200
+
+    def rerank(self, query: str, documents: list[str]) -> list[float]:
+        """Return a relevance score per document (higher = more relevant), in order.
+        Length-capped to stay within the cross-encoder context; degrades to neutral scores on
+        any runtime error rather than crashing the whole query."""
+        if not documents:
+            return []
+        q = query[: self._MAX_QUERY_CHARS]
+        docs = [d[: self._MAX_DOC_CHARS] for d in documents]
+        try:
+            return [float(s) for s in self._model.rerank(q, docs)]
+        except Exception:
+            return [0.0] * len(docs)
+
+
+def l2_normalize(vec: list[float]) -> list[float]:
+    norm = math.sqrt(sum(v * v for v in vec))
+    if norm == 0.0:
+        return vec
+    return [v / norm for v in vec]
+
+
+def cosine(a: list[float], b: list[float]) -> float:
+    """Cosine similarity. Inputs assumed L2-normalized; result clamped to [-1, 1]."""
+    return max(-1.0, min(1.0, sum(x * y for x, y in zip(a, b))))

midas/entity.py

@@ -0,0 +1,51 @@
+"""No-LLM entity-grounded abstention — a candidate lever for the *Calibrated* frontier.
+
+Diagnosed root cause (see docs §5/§7): the reader confabulates an answer drawn from a retrieved
+distractor that is about a DIFFERENT entity than the question asks about (e.g. Q asks the *hamster's*
+name, the retrieved turn is "I have a *cat* named Luna" → the reader answers "Luna"). Relevance, cosine
+and NLI-entailment all fail here because that distractor genuinely *entails* the wrong answer.
+
+This checks something orthogonal: is the answer's **source turn about the entity the question asks
+about?** We pull the question's focus nouns and test whether the source mentions them. Pure regex, no
+LLM. This module is **measured offline** before any end-to-end claim (the real test needs a capable
+reader; the local 1B model exhibits hallucination, not confab-from-distractor — see the benchmark notes).
+"""
+from __future__ import annotations
+
+import re
+
+# Generic scaffolding that is NOT the entity a question is about: question words, copulas, and the
+# attribute/relation words that recur across confab pairs ("favorite", "name") and so must not count
+# as a shared entity.
+_STOP = {
+    "what", "which", "who", "whom", "whose", "where", "when", "why", "how", "is", "are", "was", "were",
+    "the", "a", "an", "of", "to", "for", "in", "on", "at", "by", "with", "and", "or", "but", "do",
+    "does", "did", "have", "has", "had", "his", "her", "their", "my", "your", "our", "its", "this",
+    "that", "user", "users", "you", "i", "we", "they", "he", "she", "it", "name", "named", "call",
+    "called", "favorite", "favourite", "kind", "type", "about", "tell", "me", "please", "they're",
+}
+_WORD = re.compile(r"[A-Za-z][A-Za-z'-]+")
+
+
+def entities(text: str) -> set[str]:
+    """Salient content words (non-stopword, >=3 chars), lowercased — a turn's rough entity fingerprint.
+    Possessives are normalised to the base noun ("user's" -> "user") so scaffolding is filtered."""
+    out: set[str] = set()
+    for m in _WORD.finditer(text or ""):
+        w = m.group().lower()
+        if w.endswith("'s"):
+            w = w[:-2]  # possessive -> base noun
+        if len(w) >= 3 and w not in _STOP:
+            out.add(w)
+    return out
+
+
+def question_focus(question: str) -> set[str]:
+    """The entity/entities a question is about — its salient nouns minus the generic scaffolding."""
+    return entities(question)
+
+
+def entity_grounded(question: str, source: str, *, min_overlap: int = 1) -> bool:
+    """Is the source turn about the entity the question asks about? True if they share >= `min_overlap`
+    salient words. A confab drawn from a wrong-entity distractor shares none of the question's focus."""
+    return len(question_focus(question) & entities(source)) >= min_overlap