confluence-knowledge 0.3.1__py3-none-any.whl

confluence_knowledge/extract.py

@@ -0,0 +1,118 @@
+"""Storage-format XHTML → semantic chunks.
+
+Local parse only (no external entity/network resolution), heading-aware block
+splitting with bounded overlap. Page content is data, never instructions.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+from bs4 import BeautifulSoup
+
+MAX_WORDS = 280
+OVERLAP_WORDS = 40
+
+
+@dataclass
+class Chunk:
+    text: str
+    heading_path: str   # "Parent section > Sub"
+    kind: str           # text | table
+    ordinal: int
+    chunk_id: str = ""  # assigned at index time: page_id:hash:ordinal
+
+
+def _table_to_text(tbl) -> str:
+    rows = []
+    for tr in tbl.find_all("tr"):
+        cells = [c.get_text(" ", strip=True) for c in tr.find_all(["th", "td"])]
+        if cells:
+            rows.append(" | ".join(cells))
+    if not rows:
+        return ""
+    header, *body = rows
+    out = [f"Table — columns: {header}"]
+    out += [f"row {i}: {r}" for i, r in enumerate(body, 1)]
+    return "\n".join(out)
+
+
+def extract_chunks(storage_html: str) -> tuple[list[Chunk], list[str]]:
+    """Return (chunks, extraction warnings)."""
+    soup = BeautifulSoup(storage_html, "html.parser")  # stdlib parser; no ext entities
+    warnings: list[str] = []
+
+    # macros that render external/dynamic content — skip entirely
+    SKIP_MACROS = {"jira", "jirachart", "rss", "include", "widget", "gadget",
+                   "livesearch", "recently-updated", "view-file", "gallery",
+                   "content-by-label", "children", "pagetree", "excerpt-include",
+                   "multiexcerpt-include"}
+    skipped_macros: set[str] = set()
+    for tag in soup.find_all(["script", "style", "ac:structured-macro"]):
+        if tag.name.startswith("ac:"):
+            name = tag.get("ac:name", "")
+            if name in SKIP_MACROS:
+                skipped_macros.add(name)
+                tag.decompose()
+                continue
+            # container macros (code, expand, info, panel...) — keep the body:
+            # rich-text bodies are unwrapped so their markup chunks normally;
+            # plain-text bodies (code) become <pre> so the walker keeps order
+            rich = tag.find("ac:rich-text-body")
+            plain = tag.find("ac:plain-text-body")
+            if plain is not None:
+                pre = soup.new_tag("pre")
+                pre.string = plain.get_text(strip=True)
+                tag.replace_with(pre)
+            elif rich is not None:
+                rich.unwrap()
+                tag.unwrap()
+            else:
+                tag.decompose()
+            continue
+        tag.decompose()
+    if skipped_macros:
+        warnings.append(
+            "skipped dynamic macros (content may be incomplete): "
+            + ", ".join(sorted(skipped_macros)))
+
+    chunks: list[Chunk] = []
+    buf: list[str] = []
+    heading_path: list[str] = []
+
+    def flush(kind: str = "text"):
+        text = "\n".join(buf).strip()
+        buf.clear()
+        if not text:
+            return
+        words = text.split()
+        if len(words) <= MAX_WORDS:
+            chunks.append(Chunk(text, " > ".join(heading_path), kind, len(chunks)))
+            return
+        # split on word boundaries with overlap
+        step = MAX_WORDS - OVERLAP_WORDS
+        for i in range(0, len(words), step):
+            seg = " ".join(words[i:i + MAX_WORDS])
+            if seg.strip():
+                chunks.append(Chunk(seg, " > ".join(heading_path), kind, len(chunks)))
+
+    for el in soup.find_all(re.compile(r"^h[1-6]$|^p$|^ul$|^ol$|^table$|^pre$|^blockquote$")):
+        if el.name and el.name.startswith("h"):
+            flush()
+            level = int(el.name[1])
+            title = el.get_text(" ", strip=True)
+            heading_path = heading_path[: level - 1]
+            while len(heading_path) < level - 1:
+                heading_path.append("")
+            heading_path.append(title)
+            buf.append(f"## {title}")
+        elif el.name == "table":
+            flush()
+            t = _table_to_text(el)
+            if t:
+                chunks.append(Chunk(t, " > ".join(heading_path), "table", len(chunks)))
+        else:
+            buf.append(el.get_text(" ", strip=True))
+    flush()
+    return chunks, warnings

confluence_knowledge/index.py

@@ -0,0 +1,193 @@
+"""Local persistent index: SQLite metadata + FTS5 keyword + vector blobs.
+
+One DB per scope (site+account+root). Single-writer transactions; a completed
+crawl is published atomically so a failed refresh never clobbers a good index.
+"""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+import threading
+import time
+from pathlib import Path
+
+import numpy as np
+
+SCHEMA_VERSION = 1
+
+SCHEMA = """
+PRAGMA journal_mode=WAL;
+CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT);
+CREATE TABLE IF NOT EXISTS pages (
+  page_id TEXT PRIMARY KEY, parent_id TEXT, title TEXT, url TEXT,
+  level INTEGER, version TEXT, fetched_at REAL, content_hash TEXT,
+  warnings TEXT, active INTEGER DEFAULT 0);
+CREATE TABLE IF NOT EXISTS chunks (
+  chunk_id TEXT PRIMARY KEY, page_id TEXT, heading TEXT, kind TEXT,
+  ordinal INTEGER, text TEXT, active INTEGER DEFAULT 0);
+CREATE TABLE IF NOT EXISTS vectors (chunk_id TEXT PRIMARY KEY, vec BLOB);
+CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
+  chunk_id UNINDEXED, text);
+CREATE TABLE IF NOT EXISTS jobs (
+  job_id TEXT PRIMARY KEY, root_page TEXT, state TEXT, created_at REAL,
+  queue TEXT, errors TEXT);
+"""
+
+
+class Index:
+    def __init__(self, path: Path):
+        path.parent.mkdir(parents=True, exist_ok=True)
+        # timeout: retry transient locks (AV scan, second process, WAL
+        # checkpoint) instead of failing instantly; check_same_thread: the
+        # crawl worker and tool-call threads share this one connection —
+        # `self.lock` serializes every access so writes never interleave
+        self.db = sqlite3.connect(path, timeout=30, check_same_thread=False)
+        self.lock = threading.RLock()
+        with self.lock:
+            self.db.executescript(SCHEMA)
+            self.db.execute(
+                "INSERT OR IGNORE INTO meta VALUES ('schema_version', ?)",
+                (str(SCHEMA_VERSION),))
+            # an uncommitted INSERT holds the write lock for the connection's
+            # lifetime and deadlocks every other connection (#68)
+            self.db.commit()
+
+    # ---- jobs ----
+    def create_job(self, job_id: str, root_page: str) -> None:
+        with self.lock:
+            self.db.execute(
+                "INSERT OR REPLACE INTO jobs VALUES (?,?,?,?,?,?)",
+                (job_id, root_page, "queued", time.time(),
+                 json.dumps([root_page]), "[]"))
+            self.db.commit()
+
+    def job(self, job_id: str) -> dict | None:
+        with self.lock:
+            row = self.db.execute(
+                "SELECT root_page,state,queue,errors FROM jobs WHERE job_id=?",
+                (job_id,)).fetchone()
+        if not row:
+            return None
+        return {"root_page": row[0], "state": row[1],
+                "queue": json.loads(row[2]), "errors": json.loads(row[3])}
+
+    def abandon_running_jobs(self) -> None:
+        """A new crawl supersedes any unfinished one (e.g. an earlier failed
+        attempt) so it can't leave coverage reported as partial forever."""
+        with self.lock:
+            self.db.execute("UPDATE jobs SET state='abandoned' WHERE state='running'")
+            self.db.commit()
+
+    def latest_job(self) -> dict | None:
+        with self.lock:
+            row = self.db.execute(
+                "SELECT job_id,state,errors FROM jobs ORDER BY created_at DESC LIMIT 1").fetchone()
+        return row and {"job_id": row[0], "state": row[1], "errors": json.loads(row[2])}
+
+    def save_job(self, job_id: str, state: str, queue: list, errors: list) -> None:
+        with self.lock:
+            self.db.execute(
+                "UPDATE jobs SET state=?, queue=?, errors=? WHERE job_id=?",
+                (state, json.dumps(queue), json.dumps(errors), job_id))
+            self.db.commit()
+
+    def meta_get(self, key: str) -> str | None:
+        with self.lock:
+            row = self.db.execute(
+                "SELECT value FROM meta WHERE key=?", (key,)).fetchone()
+        return row[0] if row else None
+
+    def job_progress(self, job_id: str) -> dict:
+        """Snapshot the agent can poll while a background worker crawls."""
+        job = self.job(job_id)
+        if job is None:
+            raise KeyError(job_id)
+        seen = self.meta_get(f"seen:{job_id}")
+        return {"job_id": job_id, "state": job["state"],
+                "remaining": len(job["queue"]),
+                "pages_seen": len(json.loads(seen)) if seen else 0,
+                "errors": job["errors"]}
+
+    # ---- content ----
+    def upsert_page(self, page: dict, chunks: list, vecs: np.ndarray | None) -> None:
+        with self.lock, self.db:
+            self.db.execute(
+                "INSERT OR REPLACE INTO pages VALUES (?,?,?,?,?,?,?,?,?,0)",
+                (page["page_id"], page.get("parent_id"), page["title"],
+                 page["url"], page.get("level", 0), page.get("version"),
+                 time.time(), page.get("content_hash", ""),
+                 json.dumps(page.get("warnings", []))))
+            old = [r[0] for r in self.db.execute(
+                "SELECT chunk_id FROM chunks WHERE page_id=?", (page["page_id"],))]
+            for cid in old:
+                self.db.execute("DELETE FROM chunks_fts WHERE chunk_id=?", (cid,))
+                self.db.execute("DELETE FROM vectors WHERE chunk_id=?", (cid,))
+            self.db.execute("DELETE FROM chunks WHERE page_id=?", (page["page_id"],))
+            for i, c in enumerate(chunks):
+                self.db.execute(
+                    "INSERT INTO chunks VALUES (?,?,?,?,?,?,0)",
+                    (c.chunk_id, page["page_id"], c.heading_path, c.kind,
+                     c.ordinal, c.text))
+                self.db.execute("INSERT INTO chunks_fts VALUES (?,?)", (c.chunk_id, c.text))
+                if vecs is not None and i < len(vecs):
+                    self.db.execute(
+                        "INSERT INTO vectors VALUES (?,?)",
+                        (c.chunk_id, vecs[i].astype(np.float32).tobytes()))
+
+    def publish(self, seen_page_ids: set[str]) -> None:
+        """Atomically make exactly the crawled page set the active snapshot."""
+        with self.lock, self.db:
+            self.db.execute("UPDATE pages SET active=0")
+            self.db.execute("UPDATE chunks SET active=0")
+            if seen_page_ids:
+                q = ",".join("?" * len(seen_page_ids))
+                ids = list(seen_page_ids)
+                self.db.execute(f"UPDATE pages SET active=1 WHERE page_id IN ({q})", ids)
+                self.db.execute(f"UPDATE chunks SET active=1 WHERE page_id IN ({q})", ids)
+
+    # ---- query ----
+    def active_pages(self) -> dict[str, dict]:
+        with self.lock:
+            return {r[0]: {"title": r[1], "url": r[2], "level": r[3]}
+                    for r in self.db.execute(
+                        "SELECT page_id,title,url,level FROM pages WHERE active=1")}
+
+    def active_chunks(self) -> list[dict]:
+        with self.lock:
+            return [{"chunk_id": r[0], "page_id": r[1], "heading": r[2],
+                     "kind": r[3], "text": r[4]}
+                    for r in self.db.execute(
+                        "SELECT chunk_id,page_id,heading,kind,text FROM chunks WHERE active=1")]
+
+    def fts_search(self, query: str, limit: int = 30) -> list[tuple[str, float]]:
+        with self.lock:
+            try:
+                rows = self.db.execute(
+                    "SELECT chunk_id, rank FROM chunks_fts WHERE chunks_fts MATCH ? "
+                    "ORDER BY rank LIMIT ?",
+                    (" OR ".join('"' + w.replace('"', '""') + '"'
+                                 for w in query.split()[:16] if w.strip()) or '""',
+                     limit)).fetchall()
+            except sqlite3.OperationalError:
+                return []
+        active = {c["chunk_id"] for c in self.active_chunks()}
+        return [(r[0], -r[1]) for r in rows if r[0] in active]
+
+    def vectors_for(self, chunk_ids: list[str]) -> dict[str, np.ndarray]:
+        if not chunk_ids:
+            return {}
+        q = ",".join("?" * len(chunk_ids))
+        with self.lock:
+            return {cid: np.frombuffer(b, dtype=np.float32)
+                    for cid, b in self.db.execute(
+                        f"SELECT chunk_id,vec FROM vectors WHERE chunk_id IN ({q})",
+                        chunk_ids)}
+
+    def stats(self) -> dict:
+        with self.lock:
+            p = self.db.execute("SELECT COUNT(*) FROM pages WHERE active=1").fetchone()[0]
+            c = self.db.execute("SELECT COUNT(*) FROM chunks WHERE active=1").fetchone()[0]
+            snap = self.db.execute(
+                "SELECT MAX(fetched_at) FROM pages WHERE active=1").fetchone()[0]
+            return {"pages": p, "chunks": c, "as_of": snap}

confluence_knowledge/models.lock.json

@@ -0,0 +1,15 @@
+{
+  "comment": "Pinned local model artifacts. Embeddings and reranking are local-only at runtime; cloud embedding APIs are never used. Both models are Apache-2.0.",
+  "models": {
+    "embed": {
+      "repo": "sentence-transformers/all-MiniLM-L6-v2",
+      "revision": "e9a7bebe3474e74e8193da1a05abb5b5291dcb6a",
+      "license": "Apache-2.0"
+    },
+    "rerank": {
+      "repo": "cross-encoder/ms-marco-MiniLM-L6-v2",
+      "revision": "233902d25c440f23af6f7d6e94d2946bac0bee0a",
+      "license": "Apache-2.0"
+    }
+  }
+}

confluence_knowledge/registration.py

@@ -0,0 +1,134 @@
+"""IDE MCP registration — writes the config files itself.
+
+The agent can't click IDE settings for the user, but it CAN run this command:
+we write the mcp.json files directly wherever Copilot looks. The command path
+is `sys.executable` — always the venv interpreter that has the package.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import sys
+from pathlib import Path
+
+from .settings import SetupError
+
+SERVER_ID = "confluence-knowledge"
+
+
+def _server_entry(workdir: Path | None) -> dict:
+    entry = {
+        "command": sys.executable,  # the venv python — no PATH guessing
+        "args": ["-m", "confluence_knowledge.server"],
+    }
+    if workdir is not None:
+        entry["env"] = {"CONF_KNOWLEDGE_WORKDIR": str(workdir)}
+    return entry
+
+
+def _platform_appdata() -> Path | None:
+    """Where per-app support data lives, per OS."""
+    if os.name == "nt":
+        base = os.environ.get("LOCALAPPDATA")
+        return Path(base) if base else Path.home() / "AppData" / "Local"
+    if sys.platform == "darwin":
+        return Path.home() / "Library" / "Application Support"
+    return Path(os.environ.get("XDG_DATA_HOME", Path.home() / ".local" / "share"))
+
+
+def _jetbrains_mcp() -> Path:
+    """The user-level config the JetBrains Copilot plugin actually reads —
+    the one we verified on a real Windows machine during field testing."""
+    return _platform_appdata() / "github-copilot" / "intellij" / "mcp.json"
+
+
+def _vscode_user_mcp() -> Path | None:
+    """User-level VS Code mcp.json — only if the profile dir exists."""
+    for code in ("Code", "Code - Insiders", "VSCodium"):
+        d = _platform_appdata() / code / "User"
+        if d.is_dir():
+            return d / "mcp.json"
+    return None
+
+
+def _workspace_mcp(workspace: Path | None) -> Path | None:
+    return (workspace / ".vscode" / "mcp.json") if workspace else None
+
+
+def targets(ide: str = "auto", workspace: Path | None = None) -> list[Path]:
+    """Every config file the chosen IDEs read. `auto` = all that apply."""
+    out = []
+    if ide in ("auto", "all", "jetbrains"):
+        out.append(_jetbrains_mcp())
+    if ide in ("auto", "all", "vscode"):
+        v = _vscode_user_mcp()
+        if v:
+            out.append(v)
+        w = _workspace_mcp(workspace)
+        if w:
+            out.append(w)
+    # de-dupe (user + workspace can resolve to the same file in tests)
+    seen, dedup = set(), []
+    for p in out:
+        if p not in seen:
+            seen.add(p)
+            dedup.append(p)
+    return dedup
+
+
+def _merge(path: Path, entry: dict | None) -> str:
+    """Write `entry` under servers[SERVER_ID], or remove it when None.
+
+    Merges with existing JSON; backs up before overwriting. A file that isn't
+    parseable (JSONC comments etc.) is backed up and replaced — the agent
+    tells the user to re-add their other servers.
+    """
+    path.parent.mkdir(parents=True, exist_ok=True)
+    data: dict = {}
+    replaced = False
+    if path.exists():
+        try:
+            data = json.loads(path.read_text())
+        except (json.JSONDecodeError, UnicodeDecodeError):
+            data = {}
+            replaced = True  # unparseable (JSONC etc.) — back up and rewrite
+        shutil.copy2(path, path.with_suffix(path.suffix + ".bak"))
+    servers = data.setdefault("servers", {})
+    if entry is None:
+        servers.pop(SERVER_ID, None)
+        action = "removed"
+    else:
+        servers[SERVER_ID] = entry
+        action = "backup-replaced" if replaced else "written"
+    tmp = path.with_suffix(".tmp")
+    tmp.write_text(json.dumps(data, indent=2) + "\n")
+    os.replace(tmp, path)
+    return action
+
+
+def register(ide: str = "auto", workspace: Path | None = None,
+             workdir: Path | None = None, remove: bool = False) -> list[tuple[Path, str]]:
+    """Write/remove our server entry in each target. Returns (path, action)."""
+    if ide not in ("auto", "all", "vscode", "jetbrains"):
+        raise SetupError(f"unknown --ide '{ide}' (auto|all|vscode|jetbrains)")
+    files = targets(ide, workspace)
+    if not files:
+        raise SetupError("no IDE config locations found — no Copilot/JetBrains "
+                         "or VS Code profile detected")
+    entry = None if remove else _server_entry(workdir)
+    return [(f, _merge(f, entry)) for f in files]
+
+
+def registered_paths() -> list[Path]:
+    """Existing config files that already contain our server — for doctor."""
+    out = []
+    for p in targets("all"):
+        try:
+            servers = json.loads(p.read_text()).get("servers", {})
+        except (OSError, json.JSONDecodeError, UnicodeDecodeError):
+            continue
+        if p.exists() and SERVER_ID in servers:
+            out.append(p)
+    return out

confluence_knowledge/retrieval.py

@@ -0,0 +1,207 @@
+"""Local hybrid retrieval: vectors + FTS → RRF → local cross-encoder rerank.
+
+Embeddings and reranking run exclusively on this machine. No text ever leaves
+the process for retrieval. Models load local_files_only with trust_remote_code
+disabled; a missing model is a setup error, never a silent fallback.
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+from pathlib import Path
+
+import numpy as np
+
+from .index import Index
+from .settings import SetupError
+
+EMBED_MODEL = "sentence-transformers/all-MiniLM-L6-v2"   # Apache-2.0
+RERANK_MODEL = "cross-encoder/ms-marco-MiniLM-L6-v2"     # Apache-2.0
+
+TOP_K_RETRIEVE = 30
+TOP_K_EVIDENCE = 8
+
+
+def _models_dir(home: Path) -> Path:
+    return home / "models"
+
+
+def _load_sentence_transformer(model_key: str, home: Path):
+    """Local-only model load — fails closed if artifacts aren't prepared."""
+    try:
+        from sentence_transformers import SentenceTransformer
+    except ImportError as e:
+        raise SetupError(
+            "sentence-transformers isn't installed — run "
+            "`confluence-knowledge prepare-models` (see docs/confluence-knowledge.md)"
+        ) from e
+    local = _models_dir(home) / model_key.split("/")[-1]
+    if not local.exists():
+        raise SetupError(
+            f"local model '{model_key}' not prepared — run "
+            "`confluence-knowledge prepare-models`. Cloud embeddings are never used."
+        )
+    os.environ.setdefault("TRANSFORMERS_OFFLINE", "1")
+    os.environ.setdefault("HF_HUB_OFFLINE", "1")
+    os.environ.setdefault("HF_HUB_DISABLE_TELEMETRY", "1")
+    return SentenceTransformer(
+        str(local), local_files_only=True, trust_remote_code=False,
+        device="cpu")
+
+
+class LocalEmbedder:
+    def __init__(self, home: Path):
+        self._m = _load_sentence_transformer(EMBED_MODEL, home)
+
+    def embed(self, texts: list[str]) -> np.ndarray:
+        v = self._m.encode(texts, convert_to_numpy=True, normalize_embeddings=True,
+                           show_progress_bar=False)
+        return np.asarray(v, dtype=np.float32)
+
+
+class LocalReranker:
+    def __init__(self, home: Path):
+        try:
+            from sentence_transformers import CrossEncoder
+        except ImportError as e:
+            raise SetupError(
+                "sentence-transformers isn't installed — run "
+                "`confluence-knowledge prepare-models`"
+            ) from e
+        local = _models_dir(home) / RERANK_MODEL.split("/")[-1]
+        if not local.exists():
+            raise SetupError(
+                f"local reranker '{RERANK_MODEL}' not prepared — run "
+                "`confluence-knowledge prepare-models`.")
+        self._m = CrossEncoder(str(local), device="cpu",
+                               trust_remote_code=False)
+
+    def score(self, query: str, docs: list[str]) -> np.ndarray:
+        return np.asarray(
+            self._m.predict([[query, d] for d in docs], show_progress_bar=False),
+            dtype=np.float32)
+
+
+def prepare_models(home: Path) -> list[str]:
+    """Download pinned models once (public artifacts; no document text sent)."""
+    from huggingface_hub import snapshot_download
+
+    out = []
+    lock = _load_lock()
+    for key, repo in (("embed", EMBED_MODEL), ("rerank", RERANK_MODEL)):
+        rev = lock["models"][key]["revision"]
+        dest = _models_dir(home) / repo.split("/")[-1]
+        if dest.exists():
+            # treat partial/interrupted downloads as not prepared: a finished
+            # dir has config + model weights + tokenizer files
+            complete = (dest / "config.json").exists() and (
+                any(dest.glob("*.safetensors")) or any(dest.rglob("*.safetensors")))
+            if complete:
+                out.append(f"already prepared: {repo}@{rev[:8]}")
+                continue
+            shutil.rmtree(dest)  # incomplete — fetch again
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        snapshot_download(repo_id=repo, revision=rev, local_dir=dest,
+                          # safetensors + tokenizer/config only: the ONNX and
+                          # OpenVINO variants are ~5x the size and never loaded
+                          allow_patterns=["*.json", "*.txt", "*.safetensors",
+                                          "*.md", "LICENSE*",
+                                          "*.model", "*.spm", "modules.json"],
+                          ignore_patterns=["*.bin", "*.msgpack", "*.h5",
+                                           "*.pt", "*.ot"])
+        if not _license_ok(dest):
+            raise SetupError(f"{repo} license artifact missing — refusing to ship")
+        out.append(f"prepared {repo}@{rev[:8]} (Apache-2.0)")
+    return out
+
+
+def _license_ok(dest: Path) -> bool:
+    """LICENSE* file present, or the model card declares a permissive license."""
+    if any(dest.glob("LICENSE*")):
+        return True
+    for md in dest.glob("*.md"):
+        head = md.read_text(errors="ignore")[:6000].lower()
+        if "license: apache-2.0" in head or "license: mit" in head:
+            return True
+    return False
+
+
+def _load_lock() -> dict:
+    import json
+
+    p = Path(__file__).parent / "models.lock.json"  # shipped inside the package
+    if p.exists():
+        return json.loads(p.read_text())
+    # package-data should always be present; refuse rather than float to main
+    raise SetupError("models.lock.json missing from package — reinstall the service")
+
+
+def _cosine(q: np.ndarray, mat: dict[str, np.ndarray]) -> list[tuple[str, float]]:
+    if not mat:
+        return []
+    ids = list(mat)
+    m = np.stack([mat[i] for i in ids])
+    m = m / (np.linalg.norm(m, axis=1, keepdims=True) + 1e-9)
+    q = q / (np.linalg.norm(q) + 1e-9)
+    return sorted(zip(ids, (m @ q).tolist()), key=lambda t: -t[1])
+
+
+def retrieve(index: Index, query: str, embedder: LocalEmbedder,
+             reranker: LocalReranker | None = None) -> list[dict]:
+    """Hybrid retrieve → RRF fuse → rerank → bounded evidence."""
+    chunks = index.active_chunks()
+    if not chunks:
+        return []
+
+    # vector leg
+    qvec = embedder.embed([query])[0]
+    vecs = index.vectors_for([c["chunk_id"] for c in chunks])
+    vec_rank = [cid for cid, _ in _cosine(qvec, vecs)][:TOP_K_RETRIEVE]
+
+    # keyword leg
+    fts_rank = [cid for cid, _ in index.fts_search(query)][:TOP_K_RETRIEVE]
+
+    # reciprocal-rank fusion
+    rrf: dict[str, float] = {}
+    for leg in (vec_rank, fts_rank):
+        for rank, cid in enumerate(leg):
+            rrf[cid] = rrf.get(cid, 0.0) + 1.0 / (60 + rank + 1)
+    fused = sorted(rrf, key=lambda c: -rrf[c])[:TOP_K_RETRIEVE]
+    by_id = {c["chunk_id"]: c for c in chunks}
+    candidates = [by_id[c] for c in fused if c in by_id]
+
+    if reranker is not None and candidates:
+        scores = reranker.score(query, [c["text"] for c in candidates])
+        order = np.argsort(-scores)[:TOP_K_EVIDENCE]
+        evidence = []
+        for i in order:
+            c = dict(candidates[i])
+            c["rerank_score"] = float(scores[i])
+            evidence.append(c)
+    else:
+        evidence = [dict(c, rerank_score=None) for c in candidates[:TOP_K_EVIDENCE]]
+
+    pages = index.active_pages()
+    for e in evidence:
+        p = pages.get(e["page_id"], {})
+        e["page_title"], e["page_url"], e["level"] = (
+            p.get("title", ""), p.get("url", ""), p.get("level"))
+    return evidence
+
+
+def check_citations(evidence: list[dict], mappings: list[dict]) -> dict:
+    """Verify claim→chunk citations: chunk exists in this evidence set and the
+    claimed quote is literally present in it. Provenance, not entailment."""
+    by_id = {e["chunk_id"]: e for e in evidence}
+    bad = []
+    for m in mappings:
+        ch = by_id.get(m.get("chunk_id", ""))
+        if ch is None:
+            bad.append({"claim": m.get("claim", "")[:80], "reason": "chunk not in evidence set"})
+            continue
+        quote = m.get("quote", "")
+        if quote and quote not in ch["text"]:
+            bad.append({"claim": m.get("claim", "")[:80],
+                        "reason": "quote not present verbatim in cited chunk"})
+    return {"ok": not bad, "violations": bad}