hypermind 0.11.0__py3-none-any.whl

hypermind/simlab/knowledge.py

@@ -0,0 +1,716 @@
+"""Knowledge — federation memory aggregated across all sims.
+
+Every sim under ``~/.hypermind/sims/`` produces durable, signed artefacts:
+claims, citations, disputes, syntheses, vouches, oracle resolutions,
+reputation snapshots. This module walks the registry, parses every
+trace.json, and builds an in-memory index that the UI can drill into.
+
+Public API:
+
+    build_knowledge_index() -> KnowledgeIndex
+    KnowledgeIndex.overview()      → high-level KPIs
+    KnowledgeIndex.list_claims(…)  → paginated, filterable
+    KnowledgeIndex.list_agents()   → federation roster (cumulative stats)
+    KnowledgeIndex.list_topics()   → category clusters
+    KnowledgeIndex.claim(kid)      → full provenance (citations + disputes
+                                     + source sim + outcome if resolved)
+
+The index is rebuilt lazily — call ``invalidate()`` after a sim
+finishes to force a re-scan on the next read.
+"""
+
+from __future__ import annotations
+
+import json
+import threading
+from collections import defaultdict
+from dataclasses import dataclass, field
+from typing import Any
+
+from .registry import sims_dir
+
+# ---------------------------------------------------------------------------
+# Record shapes (compact — full trace stays on disk)
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class ClaimRecord:
+    kid: str
+    agent: str
+    persona: str
+    query: str
+    confidence: float
+    ts_wall_ms: int
+    sim_id: str
+    citations_received: int = 0
+    disputes_received: int = 0
+    resolved_outcome: bool | None = None  # None = not yet resolved
+    ground_truth: float | None = None
+    category: str = "general"  # filled from question's category if available
+    reasoning: str = ""  # the agent's freetext reasoning (when available)
+    sim_topic: str = ""  # the sim's topic (when sim was topic-driven)
+    panel_mean: float | None = None  # the panel's aggregate mean for this Q
+    panel_spread: float | None = None  # spread σ for this Q
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "kid": self.kid,
+            "agent": self.agent,
+            "persona": self.persona,
+            "query": self.query,
+            "confidence": self.confidence,
+            "ts_wall_ms": self.ts_wall_ms,
+            "sim_id": self.sim_id,
+            "citations_received": self.citations_received,
+            "disputes_received": self.disputes_received,
+            "resolved_outcome": self.resolved_outcome,
+            "ground_truth": self.ground_truth,
+            "category": self.category,
+            "reasoning": self.reasoning,
+            "sim_topic": self.sim_topic,
+            "panel_mean": self.panel_mean,
+            "panel_spread": self.panel_spread,
+        }
+
+
+@dataclass
+class AgentAggregate:
+    name: str  # persona-replica id, e.g. "optimist-economist-01"
+    persona: str
+    runs_participated: int = 0
+    claims_published: int = 0
+    citations_received: int = 0
+    disputes_received: int = 0
+    citations_made: int = 0
+    disputes_made: int = 0
+    vouches_received: int = 0
+    vouches_given: int = 0
+    oracle_correct: int = 0
+    oracle_total: int = 0
+    last_reputation: float | None = None
+    last_seen_ts_ms: int = 0
+    sims: set[str] = field(default_factory=set)
+
+    @property
+    def accuracy(self) -> float | None:
+        return self.oracle_correct / self.oracle_total if self.oracle_total else None
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "name": self.name,
+            "persona": self.persona,
+            "runs_participated": self.runs_participated,
+            "claims_published": self.claims_published,
+            "citations_received": self.citations_received,
+            "disputes_received": self.disputes_received,
+            "citations_made": self.citations_made,
+            "disputes_made": self.disputes_made,
+            "vouches_received": self.vouches_received,
+            "vouches_given": self.vouches_given,
+            "oracle_correct": self.oracle_correct,
+            "oracle_total": self.oracle_total,
+            "accuracy": self.accuracy,
+            "last_reputation": self.last_reputation,
+            "last_seen_ts_ms": self.last_seen_ts_ms,
+            "sims": sorted(self.sims),
+        }
+
+
+@dataclass
+class TopicAggregate:
+    topic: str  # category slug (e.g. "regulation", "growth", "primary")
+    claim_count: int = 0
+    citation_count: int = 0
+    dispute_count: int = 0
+    mean_confidence_sum: float = 0.0  # accumulators; finalize on render
+    confidence_n: int = 0
+    sims: set[str] = field(default_factory=set)
+    sample_queries: list[str] = field(default_factory=list)
+
+    @property
+    def mean_confidence(self) -> float | None:
+        return self.mean_confidence_sum / self.confidence_n if self.confidence_n else None
+
+    @property
+    def contested_ratio(self) -> float:
+        return self.dispute_count / self.claim_count if self.claim_count else 0.0
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "topic": self.topic,
+            "claim_count": self.claim_count,
+            "citation_count": self.citation_count,
+            "dispute_count": self.dispute_count,
+            "mean_confidence": self.mean_confidence,
+            "contested_ratio": self.contested_ratio,
+            "n_sims": len(self.sims),
+            "sample_queries": self.sample_queries[:5],
+        }
+
+
+@dataclass
+class KnowledgeIndex:
+    claims: dict[str, ClaimRecord] = field(default_factory=dict)
+    agents: dict[str, AgentAggregate] = field(default_factory=dict)
+    topics: dict[str, TopicAggregate] = field(default_factory=dict)
+    # kid → list of citations/disputes against it (built once)
+    citations_to: dict[str, list[dict[str, Any]]] = field(default_factory=lambda: defaultdict(list))
+    disputes_to: dict[str, list[dict[str, Any]]] = field(default_factory=lambda: defaultdict(list))
+    n_sims: int = 0
+    last_built_ts_ms: int = 0
+
+    # ---- summaries ----
+    def overview(self) -> dict[str, Any]:
+        total_citations = sum(c.citations_received for c in self.claims.values())
+        total_disputes = sum(c.disputes_received for c in self.claims.values())
+        resolved = [c for c in self.claims.values() if c.resolved_outcome is not None]
+        accuracy = (
+            sum(1 for c in resolved if (c.confidence > 0.5) == c.resolved_outcome) / len(resolved)
+            if resolved
+            else None
+        )
+        # Top contributors
+        top_agents = sorted(self.agents.values(), key=lambda a: a.claims_published, reverse=True)[
+            :8
+        ]
+        top_topics = sorted(self.topics.values(), key=lambda t: t.claim_count, reverse=True)[:8]
+        return {
+            "n_sims": self.n_sims,
+            "n_agents": len(self.agents),
+            "n_claims": len(self.claims),
+            "n_resolved": len(resolved),
+            "total_citations": total_citations,
+            "total_disputes": total_disputes,
+            "n_topics": len(self.topics),
+            "panel_accuracy": accuracy,
+            "last_built_ts_ms": self.last_built_ts_ms,
+            "top_agents": [a.to_dict() for a in top_agents],
+            "top_topics": [t.to_dict() for t in top_topics],
+        }
+
+    def list_claims(
+        self,
+        *,
+        q: str = "",
+        persona: str = "",
+        topic: str = "",
+        sim_id: str = "",
+        limit: int = 100,
+        offset: int = 0,
+    ) -> dict[str, Any]:
+        items = list(self.claims.values())
+        if q:
+            ql = q.lower()
+            items = [c for c in items if ql in c.query.lower() or ql in c.kid.lower()]
+        if persona:
+            items = [c for c in items if c.persona == persona]
+        if topic:
+            items = [c for c in items if c.category == topic]
+        if sim_id:
+            items = [c for c in items if c.sim_id == sim_id]
+        items.sort(key=lambda c: c.ts_wall_ms, reverse=True)
+        total = len(items)
+        page = items[offset : offset + limit]
+        return {
+            "total": total,
+            "offset": offset,
+            "limit": limit,
+            "items": [c.to_dict() for c in page],
+        }
+
+    def list_agents(self) -> list[dict[str, Any]]:
+        return [
+            a.to_dict()
+            for a in sorted(
+                self.agents.values(),
+                key=lambda x: x.claims_published,
+                reverse=True,
+            )
+        ]
+
+    def list_topics(self) -> list[dict[str, Any]]:
+        return [
+            t.to_dict()
+            for t in sorted(
+                self.topics.values(),
+                key=lambda x: x.claim_count,
+                reverse=True,
+            )
+        ]
+
+    def ask(self, query: str, *, top_k: int = 20, min_overlap: int = 1) -> dict[str, Any]:
+        """Search across all claims with text similar to `query` and
+        compute an aggregated verdict. Plain word-overlap scoring — no
+        embeddings, no LLM round-trip. Cheap, deterministic, on-disk only.
+
+        Returns a payload the UI's "Ask the federation" tab renders:
+        {
+            query: str,
+            n_matched: int,
+            mean_confidence: float | None,
+            verdict_label: str,
+            agreement: str,
+            top_claims: [ClaimRecord.to_dict() ...],   # most-relevant
+            sim_ids: [str ...],                         # contributing sims
+            personas: {persona: count, ...},            # who weighed in
+        }
+        """
+        if not query.strip():
+            return {
+                "query": query,
+                "n_matched": 0,
+                "items": [],
+                "sim_ids": [],
+                "mean_confidence": None,
+                "verdict_label": "",
+                "agreement": "",
+                "personas": {},
+            }
+
+        # Simple bag-of-words overlap. Lowercase, drop punctuation,
+        # drop tiny stopwords. Score = number of overlapping content words.
+        STOP = {
+            "the",
+            "a",
+            "an",
+            "of",
+            "to",
+            "in",
+            "on",
+            "is",
+            "are",
+            "be",
+            "will",
+            "and",
+            "or",
+            "for",
+            "with",
+            "vs",
+            "as",
+            "by",
+            "this",
+            "that",
+            "do",
+            "does",
+            "did",
+            "would",
+            "could",
+            "should",
+            "have",
+            "has",
+            "had",
+            "from",
+            "at",
+            "if",
+            "than",
+            "what",
+            "when",
+            "who",
+            "how",
+        }
+
+        def tokenize(text: str) -> set[str]:
+            t = ""
+            for ch in text.lower():
+                t += ch if ch.isalnum() else " "
+            return {w for w in t.split() if len(w) > 2 and w not in STOP}
+
+        q_words = tokenize(query)
+        if not q_words:
+            return {
+                "query": query,
+                "n_matched": 0,
+                "items": [],
+                "sim_ids": [],
+                "mean_confidence": None,
+                "verdict_label": "",
+                "agreement": "",
+                "personas": {},
+            }
+
+        scored: list[tuple[int, ClaimRecord]] = []
+        for c in self.claims.values():
+            cw = tokenize(c.query)
+            overlap = len(q_words & cw)
+            if overlap >= min_overlap:
+                scored.append((overlap, c))
+
+        scored.sort(key=lambda x: (-x[0], -x[1].ts_wall_ms))
+        matched = [c for _, c in scored[:top_k]]
+        if not matched:
+            return {
+                "query": query,
+                "n_matched": 0,
+                "items": [],
+                "sim_ids": [],
+                "mean_confidence": None,
+                "verdict_label": "NO MATCH",
+                "agreement": "no prior claims found",
+                "personas": {},
+            }
+
+        # Aggregate: confidence-weighted mean (don't double-count
+        # identical claim copies from replicas — group by query).
+        confidences = [c.confidence for c in matched]
+        mean = sum(confidences) / len(confidences)
+        # Spread = max - min as a quick "agreement" proxy
+        spread = (max(confidences) - min(confidences)) if confidences else 0.0
+
+        if mean >= 0.7:
+            label, agreement = "YES", "strong agreement"
+        elif mean >= 0.55:
+            label, agreement = "LEAN YES", "leans yes"
+        elif mean > 0.45:
+            label, agreement = "SPLIT", "no clear consensus"
+        elif mean > 0.3:
+            label, agreement = "LEAN NO", "leans no"
+        else:
+            label, agreement = "NO", "strong disagreement"
+
+        if spread < 0.15:
+            agreement = "strong consensus"
+        elif spread > 0.35:
+            agreement = "sharp disagreement"
+
+        personas: dict[str, int] = {}
+        sims: dict[str, int] = {}
+        for c in matched:
+            personas[c.persona] = personas.get(c.persona, 0) + 1
+            sims[c.sim_id] = sims.get(c.sim_id, 0) + 1
+
+        return {
+            "query": query,
+            "n_matched": len(scored),
+            "items": [c.to_dict() for c in matched],
+            "sim_ids": list(sims.keys()),
+            "mean_confidence": mean,
+            "spread": spread,
+            "verdict_label": label,
+            "agreement": agreement,
+            "personas": personas,
+        }
+
+    def claim(self, kid: str) -> dict[str, Any] | None:
+        c = self.claims.get(kid)
+        if c is None:
+            # Allow partial-prefix lookup since UI shows truncated kids
+            for k, rec in self.claims.items():
+                if k.startswith(kid):
+                    c = rec
+                    break
+        if c is None:
+            return None
+        return {
+            "claim": c.to_dict(),
+            "citations": self.citations_to.get(c.kid, []),
+            "disputes": self.disputes_to.get(c.kid, []),
+        }
+
+
+# ---------------------------------------------------------------------------
+# Builder
+# ---------------------------------------------------------------------------
+
+
+def _category_for_query(query: str, questions: list[dict]) -> str:
+    """Look up the question's category by query text (since claims only
+    carry the query, not the category)."""
+    for q in questions:
+        if q.get("query") == query:
+            return str(q.get("category", "general"))
+    return "general"
+
+
+def _ingest_trace(idx: KnowledgeIndex, sim_id: str, trace: dict[str, Any]) -> None:
+    """Pull a single trace.json into the index. Side-effect: mutates idx."""
+    questions = trace.get("questions") or []
+    panel_records = trace.get("panel_records") or []
+
+    # Pre-build query→category lookup to avoid O(N×Q) linear scans
+    _query_to_cat: dict[str, str] = {
+        (q.get("query") or ""): str(q.get("category", "general"))
+        for q in questions
+    }
+
+    # Track which agents participated in this sim (for runs_participated)
+    sim_agents: set[str] = set()
+
+    # ---- Claims (from panel_records[].claims) ----
+    sim_topic = trace.get("topic") or ""
+    for pr in panel_records:
+        cat = _query_to_cat.get(pr.get("query") or "", "general")
+        # Build reasoning lookup: per_agent rows have name+reasoning;
+        # match to claims by agent name + query.
+        reasoning_by_agent: dict[str, str] = {}
+        for pa in pr.get("per_agent") or []:
+            name = pa.get("agent") or pa.get("name") or ""
+            r = pa.get("reasoning")
+            if name and r:
+                reasoning_by_agent[name] = r
+        panel_mean = pr.get("mean_confidence")
+        panel_spread = pr.get("spread")
+        for c in pr.get("claims") or []:
+            kid = c.get("kid")
+            if not kid:
+                continue
+            agent_name = c.get("agent", "")
+            rec = ClaimRecord(
+                kid=kid,
+                agent=agent_name,
+                persona=c.get("persona", ""),
+                query=c.get("query", ""),
+                confidence=float(c.get("confidence", 0.0)),
+                ts_wall_ms=int(c.get("ts_wall_ms", 0)),
+                sim_id=sim_id,
+                category=cat,
+                reasoning=reasoning_by_agent.get(agent_name, ""),
+                sim_topic=sim_topic,
+                panel_mean=panel_mean,
+                panel_spread=panel_spread,
+            )
+            idx.claims[kid] = rec
+            sim_agents.add(rec.agent)
+            # Touch agent aggregate
+            ag = idx.agents.setdefault(
+                rec.agent,
+                AgentAggregate(name=rec.agent, persona=rec.persona),
+            )
+            ag.persona = rec.persona  # keep persona fresh
+            ag.claims_published += 1
+            ag.last_seen_ts_ms = max(ag.last_seen_ts_ms, rec.ts_wall_ms)
+            ag.sims.add(sim_id)
+            # Touch topic
+            tp = idx.topics.setdefault(cat, TopicAggregate(topic=cat))
+            tp.claim_count += 1
+            tp.mean_confidence_sum += rec.confidence
+            tp.confidence_n += 1
+            tp.sims.add(sim_id)
+            if pr.get("query") and pr["query"] not in tp.sample_queries:
+                tp.sample_queries.append(pr["query"])
+
+    # ---- Citations (panel_records[].citations) ----
+    for pr in panel_records:
+        cat = _query_to_cat.get(pr.get("query") or "", "general")
+        for c in pr.get("citations") or []:
+            target_kid = c.get("to_claim_kid")
+            from_agent = c.get("from_agent", "")
+            to_agent = c.get("to_agent", "")
+            sim_agents.add(from_agent)
+            sim_agents.add(to_agent)
+            entry = {
+                "sim_id": sim_id,
+                "from_agent": from_agent,
+                "from_persona": c.get("from_persona", ""),
+                "to_agent": to_agent,
+                "to_persona": c.get("to_persona", ""),
+                "to_claim_kid": target_kid,
+                "their_conf": c.get("their_conf"),
+                "my_conf": c.get("my_conf"),
+                "delta": c.get("delta"),
+            }
+            if target_kid:
+                idx.citations_to[target_kid].append(entry)
+                if target_kid in idx.claims:
+                    idx.claims[target_kid].citations_received += 1
+            if to_agent:
+                idx.agents.setdefault(
+                    to_agent,
+                    AgentAggregate(name=to_agent, persona=c.get("to_persona", "")),
+                ).citations_received += 1
+            if from_agent:
+                idx.agents.setdefault(
+                    from_agent,
+                    AgentAggregate(name=from_agent, persona=c.get("from_persona", "")),
+                ).citations_made += 1
+            tp = idx.topics.setdefault(cat, TopicAggregate(topic=cat))
+            tp.citation_count += 1
+
+    # ---- Disputes (panel_records[].disputes) ----
+    for pr in panel_records:
+        cat = _query_to_cat.get(pr.get("query") or "", "general")
+        for d in pr.get("disputes") or []:
+            target_kid = d.get("target_claim_kid") or d.get("to_claim_kid")
+            from_agent = d.get("from_agent") or d.get("disputer", "")
+            target_agent = d.get("target_agent") or d.get("to_agent", "")
+            sim_agents.add(from_agent)
+            entry = {
+                "sim_id": sim_id,
+                "from_agent": from_agent,
+                "from_persona": d.get("from_persona", ""),
+                "target_agent": target_agent,
+                "target_persona": d.get("target_persona") or d.get("to_persona", ""),
+                "target_claim_kid": target_kid,
+                "delta": d.get("delta"),
+            }
+            if target_kid:
+                idx.disputes_to[target_kid].append(entry)
+                if target_kid in idx.claims:
+                    idx.claims[target_kid].disputes_received += 1
+            if target_agent:
+                idx.agents.setdefault(
+                    target_agent,
+                    AgentAggregate(name=target_agent, persona=d.get("target_persona") or ""),
+                ).disputes_received += 1
+            if from_agent:
+                idx.agents.setdefault(
+                    from_agent,
+                    AgentAggregate(name=from_agent, persona=d.get("from_persona") or ""),
+                ).disputes_made += 1
+            tp = idx.topics.setdefault(cat, TopicAggregate(topic=cat))
+            tp.dispute_count += 1
+
+    # ---- Vouches ----
+    for v in trace.get("vouches") or []:
+        voucher = v.get("voucher", "")
+        vouchee = v.get("vouchee", "")
+        if vouchee:
+            idx.agents.setdefault(
+                vouchee,
+                AgentAggregate(name=vouchee, persona=v.get("vouchee_persona", "")),
+            ).vouches_received += 1
+        if voucher:
+            idx.agents.setdefault(
+                voucher,
+                AgentAggregate(name=voucher, persona=v.get("voucher_persona", "")),
+            ).vouches_given += 1
+
+    # ---- Oracle resolutions: outcome + accuracy ----
+    for r in trace.get("oracle_resolutions") or []:
+        kid = r.get("resolved_kid")
+        outcome = r.get("outcome")
+        gt = r.get("ground_truth")
+        publisher = r.get("publisher", "")
+        predicted = r.get("predicted_confidence", 0.5)
+        if kid and kid in idx.claims:
+            idx.claims[kid].resolved_outcome = bool(outcome) if outcome is not None else None
+            if gt is not None:
+                idx.claims[kid].ground_truth = float(gt)
+        if publisher:
+            ag = idx.agents.setdefault(
+                publisher,
+                AgentAggregate(name=publisher, persona=r.get("publisher_persona", "")),
+            )
+            ag.oracle_total += 1
+            if outcome is not None and (predicted > 0.5) == bool(outcome):
+                ag.oracle_correct += 1
+
+    # ---- Reputation: pick the latest after_oracle score per agent ----
+    rs = trace.get("reputation_snapshots") or {}
+    for snap in rs.get("after_oracle") or []:
+        name = snap.get("agent")
+        if not name:
+            continue
+        ag = idx.agents.setdefault(
+            name,
+            AgentAggregate(name=name, persona=snap.get("persona", "")),
+        )
+        ag.last_reputation = float(snap.get("score", 0.0))
+
+    # Mark every agent that participated in this sim
+    for name in sim_agents:
+        if name:
+            ag = idx.agents.setdefault(name, AgentAggregate(name=name, persona=""))
+            ag.runs_participated += 1
+            ag.sims.add(sim_id)
+
+
+# ---------------------------------------------------------------------------
+# Public builder + cache
+# ---------------------------------------------------------------------------
+
+_CACHE: KnowledgeIndex | None = None
+_CACHE_LOCK = threading.Lock()
+_CACHE_SEEN_SIMS: set[str] = set()
+_INVALIDATION_COUNTER: int = 0
+_CACHE_COUNTER_AT_BUILD: int = -1
+
+# Per-sim trace cache: completed sims never change, so once parsed we
+# keep the raw trace dict to avoid re-reading trace.json on every rebuild.
+# Capped at 500 entries; oldest entry is evicted when full.
+_knowledge_cache: dict[str, Any] = {}  # sim_id -> parsed trace dict
+_knowledge_cache_lock = threading.Lock()
+_KNOWLEDGE_CACHE_MAX = 500
+
+
+def _get_cached_claims(sim_id: str) -> Any:
+    with _knowledge_cache_lock:
+        return _knowledge_cache.get(sim_id)
+
+
+def _set_cached_claims(sim_id: str, trace: Any) -> None:
+    with _knowledge_cache_lock:
+        if len(_knowledge_cache) >= _KNOWLEDGE_CACHE_MAX:
+            # Evict the oldest entry (first key in insertion order)
+            oldest = next(iter(_knowledge_cache))
+            del _knowledge_cache[oldest]
+        _knowledge_cache[sim_id] = trace
+
+
+def build_knowledge_index() -> KnowledgeIndex:
+    """Walk the registry and build a fresh index. Cached; cheap if no
+    new sims arrived since the last build.
+    """
+    global _CACHE, _CACHE_SEEN_SIMS, _CACHE_COUNTER_AT_BUILD
+
+    # O(1) fast path — nothing has changed since last build
+    with _CACHE_LOCK:
+        if _CACHE is not None and _CACHE_COUNTER_AT_BUILD == _INVALIDATION_COUNTER:
+            return _CACHE
+
+    sdir = sims_dir()
+
+    # Snapshot which sim-dirs are currently on disk + their states.
+    # We rebuild fully if the set of done sims changed.
+    current_done: set[str] = set()
+    for d in sdir.iterdir():
+        if not d.is_dir():
+            continue
+        status_path = d / "status.json"
+        if not status_path.exists():
+            continue
+        try:
+            status = json.loads(status_path.read_text())
+        except (OSError, json.JSONDecodeError):
+            continue
+        if status.get("state") == "done" and (d / "trace.json").exists():
+            current_done.add(d.name)
+
+    # Second fast path — done-sim set unchanged
+    with _CACHE_LOCK:
+        if _CACHE is not None and current_done == _CACHE_SEEN_SIMS:
+            return _CACHE
+
+    # Build index OUTSIDE the lock so concurrent readers aren't blocked
+    idx = KnowledgeIndex()
+    for sim_id in sorted(current_done):
+        # Check the per-sim trace cache first — completed sims never change.
+        trace = _get_cached_claims(sim_id)
+        if trace is None:
+            trace_path = sdir / sim_id / "trace.json"
+            try:
+                trace = json.loads(trace_path.read_text())
+            except (OSError, json.JSONDecodeError):
+                continue
+            _set_cached_claims(sim_id, trace)
+        _ingest_trace(idx, sim_id, trace)
+    idx.n_sims = len(current_done)
+    import time
+    idx.last_built_ts_ms = int(time.time() * 1000)
+
+    # Swap under lock with double-check to avoid clobbering a concurrent build
+    with _CACHE_LOCK:
+        if _CACHE is None or current_done != _CACHE_SEEN_SIMS:
+            _CACHE = idx
+            _CACHE_SEEN_SIMS = current_done
+            _CACHE_COUNTER_AT_BUILD = _INVALIDATION_COUNTER
+    return idx
+
+
+def invalidate() -> None:
+    """Force the next ``build_knowledge_index()`` call to rebuild."""
+    global _CACHE, _CACHE_SEEN_SIMS, _INVALIDATION_COUNTER
+    with _CACHE_LOCK:
+        _CACHE = None
+        _CACHE_SEEN_SIMS = set()
+        _INVALIDATION_COUNTER += 1