confluence-knowledge 0.4.0__tar.gz → 0.5.0__tar.gz

{confluence_knowledge-0.4.0 → confluence_knowledge-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: confluence-knowledge
-Version: 0.4.0
+Version: 0.5.0
 Summary: Local, GET-only Confluence subtree indexer + grounded-Q&A MCP service
 License: MIT
 Project-URL: Repository, https://github.com/nkunal1992/agents
@@ -19,7 +19,9 @@ Requires-Dist: keyring>=25.2
 Requires-Dist: mcp<2,>=1.6
 Requires-Dist: pip-system-certs>=5.0
 Provides-Extra: models
-Requires-Dist: sentence-transformers<4.0,>=3.0; extra == "models"
+Requires-Dist: onnxruntime>=1.17; extra == "models"
+Requires-Dist: tokenizers>=0.15; extra == "models"
+Requires-Dist: fastembed>=0.9; extra == "models"
 Requires-Dist: huggingface-hub>=0.26; extra == "models"
 Provides-Extra: files
 Requires-Dist: python-docx>=1.1; extra == "files"
@@ -38,9 +40,9 @@ See `../../docs/confluence-knowledge.md` for the full guide.
 
 - GET-only `ConfluenceClient` — non-GET raises `SecurityException` pre-flight.
 - Recursive subtree crawl (Cloud v2 + Data Center), resumable.
-- Local embeddings (`all-MiniLM-L6-v2`) + local reranker
-  (`ms-marco-MiniLM-L6-v2`) — Apache-2.0, pinned in `models.lock.json`,
-  never a cloud embedding call.
+- Local ONNX embeddings (`bge-m3`, quantized) + sparse leg (`Qdrant/bm25`)
+  + reranker (`bge-reranker-v2-m3`, quantized) — MIT/Apache-2.0, pinned in
+  `models.lock.json`, no torch, never a cloud embedding call.
 - SQLite + FTS5 + vector-blob index under `.conf_knowledge_index/`.
 - Credentials via OS keyring / env var / chmod-600 `.conf_auth.json`.
 

{confluence_knowledge-0.4.0 → confluence_knowledge-0.5.0}/README.md

@@ -5,9 +5,9 @@ See `../../docs/confluence-knowledge.md` for the full guide.
 
 - GET-only `ConfluenceClient` — non-GET raises `SecurityException` pre-flight.
 - Recursive subtree crawl (Cloud v2 + Data Center), resumable.
-- Local embeddings (`all-MiniLM-L6-v2`) + local reranker
-  (`ms-marco-MiniLM-L6-v2`) — Apache-2.0, pinned in `models.lock.json`,
-  never a cloud embedding call.
+- Local ONNX embeddings (`bge-m3`, quantized) + sparse leg (`Qdrant/bm25`)
+  + reranker (`bge-reranker-v2-m3`, quantized) — MIT/Apache-2.0, pinned in
+  `models.lock.json`, no torch, never a cloud embedding call.
 - SQLite + FTS5 + vector-blob index under `.conf_knowledge_index/`.
 - Credentials via OS keyring / env var / chmod-600 `.conf_auth.json`.
 

{confluence_knowledge-0.4.0 → confluence_knowledge-0.5.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "confluence-knowledge"
-version = "0.4.0"
+version = "0.5.0"
 description = "Local, GET-only Confluence subtree indexer + grounded-Q&A MCP service"
 readme = "README.md"
 requires-python = ">=3.11"
@@ -31,7 +31,10 @@ Issues = "https://github.com/nkunal1992/agents/issues"
 
 [project.optional-dependencies]
 models = [
-    "sentence-transformers>=3.0,<4.0",
+    # ONNX stack — no torch: ~40 MB runtime vs 2.4 GB, native ARM64 Windows
+    "onnxruntime>=1.17",
+    "tokenizers>=0.15",
+    "fastembed>=0.9",          # sparse leg (Qdrant/bm25 ONNX)
     "huggingface-hub>=0.26",
 ]
 files = [

{confluence_knowledge-0.4.0 → confluence_knowledge-0.5.0}/src/confluence_knowledge/cli.py

@@ -94,9 +94,21 @@ def _wizard() -> int:
 
 def _doctor() -> int:
     from .server import status
+    from .update import check
     try:
         s = status()
         print(json.dumps(s, indent=2))
+        upd = check("confluence-knowledge")
+        if upd["installed"]:
+            line = f"version: {upd['installed']}"
+            if upd["latest"]:
+                line += f" (latest: {upd['latest']})"
+            elif upd["latest"] is None:
+                line += " (latest: unchecked — offline or internal mirror)"
+            print(line)
+        if upd["update_available"]:
+            print("update available: run `confluence-knowledge update` "
+                  "(or pip install -U confluence-knowledge[models])")
         if s.get("configured"):
             from .server import _client_for  # internal but fine in CLI
             profile = SiteProfile.load(default_home())
@@ -122,6 +134,7 @@ def main() -> None:
     sub.add_parser("setup", help="guided credential + config setup")
     sub.add_parser("doctor", help="verify config, credentials, models")
     sub.add_parser("prepare-models", help="download the pinned local models")
+    sub.add_parser("update", help="upgrade this package via pip -U")
     reg = sub.add_parser(
         "register",
         help="write this server into the IDE's MCP config (no manual paste)")
@@ -145,6 +158,23 @@ def main() -> None:
         for line in prepare_models(default_home()):
             print(line)
         return
+    if args.cmd == "update":
+        import subprocess
+        cmd = [sys.executable, "-m", "pip", "install", "-U",
+               "confluence-knowledge[models]"]
+        if os.name == "nt":
+            # this exe is the running process on Windows — pip can't replace
+            # a file that's in use; hand the user the equivalent python -m
+            # pip command instead (same mechanism `pip install -U pip` uses)
+            print("on Windows, upgrade via the venv python directly:\n  "
+                  + " ".join(f'"{c}"' if " " in c else c for c in cmd))
+            return
+        r = subprocess.run(cmd, check=False)
+        if r.returncode == 0:
+            print("\nupdated — run `confluence-knowledge prepare-models` "
+                  "(new releases may pin new model artifacts) then "
+                  "`confluence-knowledge doctor`")
+        sys.exit(r.returncode)
     if args.cmd == "register":
         from .registration import register
         try:

{confluence_knowledge-0.4.0 → confluence_knowledge-0.5.0}/src/confluence_knowledge/crawl.py

@@ -111,13 +111,17 @@ def _fetch_page_dc(client: ConfluenceClient, page_id: str) -> dict:
     }
 
 
+from .retrieval import EMBED_MODEL_ID as _MODEL_ID
+
+
 class Crawler:
     def __init__(self, client: ConfluenceClient, index: Index, env: str,
-                 embedder=None, site_url: str | None = None):
+                 embedder=None, sparse=None, site_url: str | None = None):
         self.client = client
         self.index = index
         self.env = env
         self.embedder = embedder  # retrieval.LocalEmbedder or None
+        self.sparse = sparse      # retrieval.LocalSparseEmbedder or None
         # fallback base for page links when a response carries no _links.base
         if site_url and env == "cloud" and not site_url.rstrip("/").endswith("/wiki"):
             site_url = site_url.rstrip("/") + "/wiki"
@@ -208,15 +212,17 @@ class Crawler:
             chunks, warns = extract_chunks(page["storage"])
             for c in chunks:
                 c.chunk_id = f"{pid}:{content_hash[:8]}:{c.ordinal}"
-            vecs = None
+            vecs, sparse_vecs = None, None
             if self.embedder is not None and chunks:
-                vecs = self.embedder.embed(
-                    [f"{c.heading_path}\n{c.text}" for c in chunks])
+                texts = [f"{c.heading_path}\n{c.text}" for c in chunks]
+                vecs = self.embedder.embed(texts)
+                if self.sparse is not None:
+                    sparse_vecs = self.sparse.embed(texts)
             self.index.upsert_page({
                 "page_id": pid, "parent_id": None, "title": page["title"],
                 "url": page["url"], "level": levels.get(pid, 0),
                 "version": page["version"], "content_hash": content_hash,
-                "warnings": warns}, chunks, vecs)
+                "warnings": warns}, chunks, vecs, sparse_vecs)
             seen.add(pid)
             for cid, ctype in self._children(pid, is_folder=False):
                 if cid not in seen:
@@ -230,6 +236,7 @@ class Crawler:
             state = "running"
         else:
             self.index.publish(seen)
+            self.index.set_model_id(_MODEL_ID)
             self.index.save_job(job_id, "done", [], errors)
             state = "done"
         return {"job_id": job_id, "state": state, "remaining": len(queue),

{confluence_knowledge-0.4.0 → confluence_knowledge-0.5.0}/src/confluence_knowledge/index.py

@@ -27,6 +27,7 @@ CREATE TABLE IF NOT EXISTS chunks (
   chunk_id TEXT PRIMARY KEY, page_id TEXT, heading TEXT, kind TEXT,
   ordinal INTEGER, text TEXT, active INTEGER DEFAULT 0);
 CREATE TABLE IF NOT EXISTS vectors (chunk_id TEXT PRIMARY KEY, vec BLOB);
+CREATE TABLE IF NOT EXISTS sparse_vectors (chunk_id TEXT PRIMARY KEY, payload BLOB);
 CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
   chunk_id UNINDEXED, text);
 CREATE TABLE IF NOT EXISTS jobs (
@@ -110,7 +111,8 @@ class Index:
                 "errors": job["errors"]}
 
     # ---- content ----
-    def upsert_page(self, page: dict, chunks: list, vecs: np.ndarray | None) -> None:
+    def upsert_page(self, page: dict, chunks: list, vecs: np.ndarray | None,
+                    sparse: list[dict[int, float]] | None = None) -> None:
         with self.lock, self.db:
             self.db.execute(
                 "INSERT OR REPLACE INTO pages VALUES (?,?,?,?,?,?,?,?,?,0)",
@@ -123,6 +125,7 @@ class Index:
             for cid in old:
                 self.db.execute("DELETE FROM chunks_fts WHERE chunk_id=?", (cid,))
                 self.db.execute("DELETE FROM vectors WHERE chunk_id=?", (cid,))
+                self.db.execute("DELETE FROM sparse_vectors WHERE chunk_id=?", (cid,))
             self.db.execute("DELETE FROM chunks WHERE page_id=?", (page["page_id"],))
             for i, c in enumerate(chunks):
                 self.db.execute(
@@ -134,6 +137,37 @@ class Index:
                     self.db.execute(
                         "INSERT INTO vectors VALUES (?,?)",
                         (c.chunk_id, vecs[i].astype(np.float32).tobytes()))
+                if sparse is not None and i < len(sparse) and sparse[i]:
+                    idx_arr = np.fromiter(sparse[i].keys(), dtype=np.int32)
+                    val_arr = np.fromiter(sparse[i].values(), dtype=np.float32)
+                    self.db.execute(
+                        "INSERT INTO sparse_vectors VALUES (?,?)",
+                        (c.chunk_id,
+                         idx_arr.tobytes() + val_arr.tobytes()))
+
+    def sparse_for(self, chunk_ids: list[str]) -> dict[str, dict[int, float]]:
+        if not chunk_ids:
+            return {}
+        q = ",".join("?" * len(chunk_ids))
+        out = {}
+        with self.lock:
+            for cid, blob in self.db.execute(
+                    f"SELECT chunk_id,payload FROM sparse_vectors "
+                    f"WHERE chunk_id IN ({q})", chunk_ids):
+                # layout: int32[n] indices + float32[n] values → n = len/8
+                k = len(blob) // 8
+                idx = np.frombuffer(blob[:4 * k], dtype=np.int32)
+                vals = np.frombuffer(blob[4 * k:], dtype=np.float32)
+                out[cid] = dict(zip(idx.tolist(), vals.tolist()))
+        return out
+
+    def model_id(self) -> str | None:
+        return self.meta_get("embed_model")
+
+    def set_model_id(self, model_id: str) -> None:
+        with self.lock, self.db:
+            self.db.execute("INSERT OR REPLACE INTO meta VALUES ('embed_model', ?)",
+                            (model_id,))
 
     def publish(self, seen_page_ids: set[str]) -> None:
         """Atomically make exactly the crawled page set the active snapshot."""

confluence_knowledge-0.5.0/src/confluence_knowledge/models.lock.json

@@ -0,0 +1,30 @@
+{
+  "comment": "Pinned ONNX model artifacts. Retrieval is local-only — torch and cloud embedding APIs are never used. 'files' limits the snapshot to what the runtime needs.",
+  "models": {
+    "embed": {
+      "repo": "Xenova/bge-m3",
+      "revision": "4de13258303883538bd53b696b452bf8099f0858",
+      "license": "MIT",
+      "files": ["onnx/sentence_transformers_quantized.onnx", "tokenizer.json", "tokenizer_config.json", "config.json", "sentencepiece.bpe.model", "special_tokens_map.json", "README.md"]
+    },
+    "sparse": {
+      "repo": "Qdrant/bm25",
+      "revision": "22b8d2af71a76161e18dd432d2cee0eefa66e412",
+      "license": "Apache-2.0",
+      "note": "downloaded through fastembed's own cache (models/fastembed)"
+    },
+    "rerank": {
+      "repo": "onnx-community/bge-reranker-v2-m3-ONNX",
+      "revision": "6f5ff65298512715a1e669753bc754d2bc8f367b",
+      "license": "MIT",
+      "files": ["onnx/model_quantized.onnx", "tokenizer.json", "tokenizer_config.json", "config.json", "special_tokens_map.json", "quantize_config.json", "README.md"]
+    },
+    "rerank_light": {
+      "repo": "onnx-community/bge-reranker-base-ONNX",
+      "revision": "f131ae72114cb18333648a2ee6b0942849e8ab95",
+      "license": "MIT",
+      "files": ["onnx/model_quantized.onnx", "tokenizer.json", "tokenizer_config.json", "config.json", "special_tokens_map.json", "quantize_config.json", "README.md"],
+      "note": "downloaded only when *_RERANKER env selects it"
+    }
+  }
+}

confluence_knowledge-0.5.0/src/confluence_knowledge/retrieval.py

@@ -0,0 +1,347 @@
+"""Local hybrid retrieval: dense + sparse + FTS → RRF → ONNX cross-encoder.
+
+Everything runs on this machine via ONNX Runtime — no torch, no cloud.
+Models load local-only from pinned artifacts; a missing model is a setup
+error, never a silent fallback.
+
+Backend layout (all under <home>/models/):
+  embed  — Xenova/bge-m3  onnx/sentence_transformers_quantized.onnx
+           (dense 1024-dim, normalized, multilingual, 8k ctx)
+  sparse — Qdrant/bm25 via fastembed SparseTextEmbedding
+           (learned term-weight leg; BGE-M3's lexical head has no ONNX
+           export, so a purpose-built sparse model fills that slot)
+  rerank — onnx-community/bge-reranker-v2-m3-ONNX  onnx/model_quantized.onnx
+           (sequence-classification logits → relevance score)
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+from pathlib import Path
+
+import numpy as np
+
+from .index import Index
+from .settings import SetupError
+
+EMBED_REPO = "Xenova/bge-m3"                            # Apache-2.0/MIT
+EMBED_ONNX = "onnx/sentence_transformers_quantized.onnx"
+SPARSE_MODEL = "Qdrant/bm25"                            # MIT — via fastembed
+RERANK_REPO = "onnx-community/bge-reranker-v2-m3-ONNX"  # MIT
+RERANK_ONNX = "onnx/model_quantized.onnx"
+RERANK_BASE_REPO = "onnx-community/bge-reranker-base-ONNX"  # light override
+
+EMBED_MODEL_ID = "bge-m3-onnx-quantized"
+
+TOP_K_RETRIEVE = 30
+TOP_K_EVIDENCE = 8
+RERANK_MAX = 25          # bounds cross-encoder latency on CPU
+EMBED_MAX_TOKENS = 512   # quantized model input cap (chunks stay under)
+RERANK_MAX_TOKENS = 512
+
+
+def _models_dir(home: Path) -> Path:
+    return home / "models"
+
+
+def _model_home(home: Path, repo: str) -> Path:
+    return _models_dir(home) / repo.split("/")[-1]
+
+
+def _require(dest: Path, repo: str, cli: str, needed: list[str]) -> Path:
+    if not all((dest / f).exists() for f in needed):
+        raise SetupError(
+            f"local model '{repo}' not prepared — run `{cli} prepare-models`. "
+            "Cloud embeddings are never used.")
+    return dest
+
+
+def _ort_session(onnx_path: Path, cli: str):
+    try:
+        import onnxruntime as ort
+    except ImportError as e:
+        raise SetupError("onnxruntime isn't installed — reinstall the "
+                         f"service ({cli} bootstrap reinstalls deps)") from e
+    return ort.InferenceSession(
+        str(onnx_path), providers=["CPUExecutionProvider"],
+        sess_options=_ort_opts(ort))
+
+
+def _ort_opts(ort):
+    so = ort.SessionOptions()
+    so.intra_op_num_threads = max(1, (os.cpu_count() or 4) // 2)
+    so.inter_op_num_threads = 1
+    so.log_severity_level = 3  # errors only
+    return so
+
+
+def _tokenizer(dest: Path):
+    from tokenizers import Tokenizer
+    return Tokenizer.from_file(str(dest / "tokenizer.json"))
+
+
+class LocalEmbedder:
+    """BGE-M3 dense embeddings via ONNX (no torch). Same interface."""
+
+    def __init__(self, home: Path, cli: str = "confluence-knowledge"):
+        dest = _require(_model_home(home, EMBED_REPO), EMBED_REPO, cli,
+                        [EMBED_ONNX, "tokenizer.json"])
+        self._sess = _ort_session(dest / EMBED_ONNX, cli)
+        self._tok = _tokenizer(dest)
+        self._tok.enable_padding()
+        self._tok.enable_truncation(EMBED_MAX_TOKENS)
+
+    def embed(self, texts: list[str]) -> np.ndarray:
+        enc = self._tok.encode_batch(texts, add_special_tokens=True)
+        ids = np.asarray([e.ids for e in enc], dtype=np.int64)
+        mask = np.asarray([e.attention_mask for e in enc], dtype=np.int64)
+        out = self._sess.run(
+            ["sentence_embedding"],
+            {"input_ids": ids, "attention_mask": mask})[0]
+        v = np.asarray(out, dtype=np.float32)
+        n = np.linalg.norm(v, axis=1, keepdims=True)
+        return v / (n + 1e-9)  # defensive normalize — quantized graphs drift
+
+
+class LocalSparseEmbedder:
+    """Learned term-weight sparse vectors via fastembed (ONNX, no torch).
+
+    Fills the role BGE-M3's lexical head plays on torch — no ONNX export of
+    that head exists, so a purpose-built sparse model covers the same leg.
+    """
+
+    def __init__(self, home: Path, cli: str = "confluence-knowledge"):
+        try:
+            from fastembed import SparseTextEmbedding
+        except ImportError as e:
+            raise SetupError("fastembed isn't installed — reinstall the "
+                             f"service ({cli} bootstrap reinstalls deps)") from e
+        os.environ.setdefault("HF_HUB_OFFLINE", "1")
+        os.environ.setdefault("TRANSFORMERS_OFFLINE", "1")
+        dest = _models_dir(home) / "fastembed"
+        try:
+            self._m = SparseTextEmbedding(SPARSE_MODEL, cache_dir=str(dest),
+                                          local_files_only=True)
+        except Exception as e:
+            raise SetupError(
+                f"sparse model '{SPARSE_MODEL}' not prepared — run "
+                f"`{cli} prepare-models`.") from e
+
+    def embed(self, texts: list[str]) -> list[dict[int, float]]:
+        """One {token_idx: weight} dict per input — the sparse payload."""
+        out = []
+        for emb in self._m.embed(texts):
+            out.append(dict(zip(emb.indices.tolist(),
+                                emb.values.tolist())))
+        return out
+
+
+class LocalReranker:
+    """bge-reranker-v2-m3 via ONNX sequence-classification logits.
+
+    Override with env `*_EMBED_RERANKER` = "onnx-community/bge-reranker-base-ONNX"
+    (lighter, English) for slow machines — doctor surfaces the hint.
+    """
+
+    def __init__(self, home: Path, cli: str = "confluence-knowledge"):
+        # per-app override → "<APP>_RERANKER" (e.g. SP_KNOWLEDGE_RERANKER)
+        # plus a global KNOWLEDGE_RERANKER fallback
+        env_key = cli.upper().replace("-", "_") + "_RERANKER"
+        repo = os.environ.get(env_key, os.environ.get("KNOWLEDGE_RERANKER",
+                                                     RERANK_REPO))
+        dest = _require(_model_home(home, repo), repo, cli,
+                        [RERANK_ONNX, "tokenizer.json"])
+        self._sess = _ort_session(dest / RERANK_ONNX, cli)
+        self._tok = _tokenizer(dest)
+        self._tok.no_padding()
+        self._tok.enable_truncation(RERANK_MAX_TOKENS)
+        self.repo = repo
+
+    def score(self, query: str, docs: list[str]) -> np.ndarray:
+        pairs = [[query, d] for d in docs]
+        enc = self._tok.encode_batch(pairs, add_special_tokens=True)
+        maxlen = max(len(e.ids) for e in enc)
+        ids = np.zeros((len(enc), maxlen), dtype=np.int64)
+        mask = np.zeros((len(enc), maxlen), dtype=np.int64)
+        for i, e in enumerate(enc):
+            ids[i, :len(e.ids)] = e.ids
+            mask[i, :len(e.attention_mask)] = e.attention_mask
+        logits = self._sess.run(
+            ["logits"], {"input_ids": ids, "attention_mask": mask})[0]
+        s = logits.reshape(-1).astype(np.float32)
+        return 1.0 / (1.0 + np.exp(-s))  # sigmoid → 0..1 relevance
+
+
+def prepare_models(home: Path) -> list[str]:
+    """Download pinned ONNX artifacts once (public; no document text sent)."""
+    from huggingface_hub import snapshot_download
+
+    out = []
+    lock = _load_lock()
+    models = _models_dir(home)
+    models.mkdir(parents=True, exist_ok=True)
+
+    # dense + rerankers: direct HF snapshot (repo + revision pinned in the
+    # lock); rerank_light is fetched too so *_RERANKER overrides always work
+    for key in ("embed", "rerank", "rerank_light"):
+        spec = lock["models"][key]
+        dest = _models_dir(home) / spec["repo"].split("/")[-1]
+        if dest.exists():
+            complete = all((dest / f).exists() for f in spec["files"])
+            if complete:
+                out.append(f"already prepared: {spec['repo']}")
+                continue
+            shutil.rmtree(dest)  # partial/interrupted → refetch
+        snapshot_download(repo_id=spec["repo"], revision=spec["revision"],
+                          local_dir=dest, allow_patterns=spec["files"])
+        if not _license_ok(dest):
+            _license_ok_upstream(dest, spec, snapshot_download)
+        if not _license_ok(dest):
+            raise SetupError(
+                f"{spec['repo']} license artifact missing — refusing to ship")
+        out.append(f"prepared {spec['repo']}@{spec['revision'][:8]} "
+                   f"({spec['license']})")
+
+    # sparse: fastembed owns its cache layout — let it download into our dir
+    os.environ.pop("HF_HUB_OFFLINE", None)
+    spec = lock["models"]["sparse"]
+    try:
+        from fastembed import SparseTextEmbedding
+        SparseTextEmbedding(spec["repo"], cache_dir=str(models / "fastembed"))
+        out.append(f"prepared {spec['repo']} ({spec['license']})")
+    except Exception as e:
+        raise SetupError(f"couldn't fetch sparse model {spec['repo']}: {e}") from e
+    return out
+
+
+def _license_ok_upstream(dest: Path, spec: dict, snapshot_download) -> None:
+    """ONNX-community repackages sometimes drop the license line — if the card
+    points at a `base_model`, fetch that model's card and judge IT. The
+    upstream card is saved next to ours for audit."""
+    import re
+    for md in dest.glob("*.md"):
+        m = re.search(r"base_model:\s*(?:-\s*)?([\w.-]+/[\w.-]+)",
+                      md.read_text(errors="ignore")[:6000])
+        if not m:
+            continue
+        try:
+            path = snapshot_download(repo_id=m.group(1),
+                                     allow_patterns=["README.md", "LICENSE*"],
+                                     local_dir=dest / "_upstream")
+        except Exception:  # noqa: BLE001 — upstream card optional; license judge stays on local files
+            return
+        up = Path(path)
+        for f in list(up.glob("README.md")) + list(up.glob("LICENSE*")):
+            f.rename(dest / f"_upstream_{f.name}")
+        shutil.rmtree(up, ignore_errors=True)
+        return
+
+
+def _license_ok(dest: Path) -> bool:
+    for md in list(dest.glob("*.md")) + list(dest.glob("LICENSE*")):
+        head = md.read_text(errors="ignore")[:6000].lower()
+        if ("apache" in head or "mit license" in head or "license: mit" in head
+                or "cc-by" in head):
+            return True
+    return any(dest.glob("LICENSE*"))
+
+
+def _load_lock() -> dict:
+    import json
+
+    p = Path(__file__).parent / "models.lock.json"  # shipped inside the package
+    if p.exists():
+        return json.loads(p.read_text())
+    raise SetupError("models.lock.json missing from package — reinstall the service")
+
+
+def _cosine(q: np.ndarray, mat: dict[str, np.ndarray]) -> list[tuple[str, float]]:
+    if not mat:
+        return []
+    ids = list(mat)
+    m = np.stack([mat[i] for i in ids])
+    m = m / (np.linalg.norm(m, axis=1, keepdims=True) + 1e-9)
+    q = q / (np.linalg.norm(q) + 1e-9)
+    return sorted(zip(ids, (m @ q).tolist()), key=lambda t: -t[1])
+
+
+def _sparse_rank(q: dict[int, float],
+                 docs: dict[str, dict[int, float]]) -> list[tuple[str, float]]:
+    """Score = Σ wq[i]·wd[i] over shared token indices — plain dot product."""
+    if not docs or not q:
+        return []
+    scores = [(cid, sum(q[i] * w for i, w in d.items() if i in q))
+              for cid, d in docs.items()]
+    scores.sort(key=lambda t: -t[1])
+    return scores
+
+
+def retrieve(index: Index, query: str, embedder: LocalEmbedder,
+             reranker: LocalReranker | None = None,
+             sparse: LocalSparseEmbedder | None = None) -> list[dict]:
+    """Dense + sparse + FTS → RRF fuse → rerank → bounded evidence."""
+    chunks = index.active_chunks()
+    if not chunks:
+        return []
+
+    # dense leg
+    qvec = embedder.embed([query])[0]
+    vecs = index.vectors_for([c["chunk_id"] for c in chunks])
+    dense_rank = [cid for cid, _ in _cosine(qvec, vecs)][:TOP_K_RETRIEVE]
+
+    # keyword leg
+    fts_rank = [cid for cid, _ in index.fts_search(query)][:TOP_K_RETRIEVE]
+
+    # sparse leg (optional — learned term weights)
+    sparse_rank: list[str] = []
+    if sparse is not None:
+        q_sparse = sparse.embed([query])[0]
+        doc_sparse = index.sparse_for([c["chunk_id"] for c in chunks])
+        sparse_rank = [cid for cid, _ in
+                       _sparse_rank(q_sparse, doc_sparse)][:TOP_K_RETRIEVE]
+
+    # reciprocal-rank fusion over 2-3 legs
+    rrf: dict[str, float] = {}
+    for leg in (dense_rank, fts_rank, sparse_rank):
+        for rank, cid in enumerate(leg):
+            rrf[cid] = rrf.get(cid, 0.0) + 1.0 / (60 + rank + 1)
+    fused = sorted(rrf, key=lambda c: -rrf[c])[:TOP_K_RETRIEVE]
+    by_id = {c["chunk_id"]: c for c in chunks}
+    candidates = [by_id[c] for c in fused if c in by_id]
+
+    if reranker is not None and candidates:
+        scored = candidates[:RERANK_MAX]
+        scores = reranker.score(query, [c["text"] for c in scored])
+        order = np.argsort(-scores)[:TOP_K_EVIDENCE]
+        evidence = []
+        for i in order:
+            c = dict(scored[i])
+            c["rerank_score"] = float(scores[i])
+            evidence.append(c)
+    else:
+        evidence = [dict(c, rerank_score=None) for c in candidates[:TOP_K_EVIDENCE]]
+
+    pages = index.active_pages()
+    for e in evidence:
+        p = pages.get(e["page_id"], {})
+        e["page_title"], e["page_url"], e["level"] = (
+            p.get("title", ""), p.get("url", ""), p.get("level"))
+    return evidence
+
+
+def check_citations(evidence: list[dict], mappings: list[dict]) -> dict:
+    """Verify claim→chunk citations: chunk exists in this evidence set and the
+    claimed quote is literally present in it. Provenance, not entailment."""
+    by_id = {e["chunk_id"]: e for e in evidence}
+    bad = []
+    for m in mappings:
+        ch = by_id.get(m.get("chunk_id", ""))
+        if ch is None:
+            bad.append({"claim": m.get("claim", "")[:80], "reason": "chunk not in evidence set"})
+            continue
+        quote = m.get("quote", "")
+        if quote and quote not in ch["text"]:
+            bad.append({"claim": m.get("claim", "")[:80],
+                        "reason": "quote not present verbatim in cited chunk"})
+    return {"ok": not bad, "violations": bad}