confluence-knowledge 0.5.0__tar.gz → 0.5.2__tar.gz

{confluence_knowledge-0.5.0 → confluence_knowledge-0.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: confluence-knowledge
-Version: 0.5.0
+Version: 0.5.2
 Summary: Local, GET-only Confluence subtree indexer + grounded-Q&A MCP service
 License: MIT
 Project-URL: Repository, https://github.com/nkunal1992/agents

{confluence_knowledge-0.5.0 → confluence_knowledge-0.5.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "confluence-knowledge"
-version = "0.5.0"
+version = "0.5.2"
 description = "Local, GET-only Confluence subtree indexer + grounded-Q&A MCP service"
 readme = "README.md"
 requires-python = ">=3.11"

{confluence_knowledge-0.5.0 → confluence_knowledge-0.5.2}/src/confluence_knowledge/cli.py

@@ -98,6 +98,13 @@ def _doctor() -> int:
     try:
         s = status()
         print(json.dumps(s, indent=2))
+        bad = [c for c in s.get("checks", []) if not c["ok"]]
+        if bad:
+            print("\nhealth flags:")
+            for c in bad:
+                print(f"  ✗ {c['flag']}: {c['detail']}")
+                if c.get("remediation"):
+                    print(f"    → {c['remediation']}")
         upd = check("confluence-knowledge")
         if upd["installed"]:
             line = f"version: {upd['installed']}"
@@ -122,7 +129,7 @@ def _doctor() -> int:
             except (AccessError, SetupError) as e:
                 print(f"heartbeat: FAILED — {e}", file=sys.stderr)
                 return 3
-        return 0
+        return 4 if bad else 0
     except SetupError as e:
         print(f"setup error: {e}", file=sys.stderr)
         return 2
@@ -162,6 +169,11 @@ def main() -> None:
         import subprocess
         cmd = [sys.executable, "-m", "pip", "install", "-U",
                "confluence-knowledge[models]"]
+        # a running MCP server holds open files — on Windows pip can't
+        # replace them and on any OS the old code keeps serving until the
+        # IDE restarts. Say it plainly before doing anything.
+        print("note: quit the IDE (or stop the MCP server) before updating —"
+              " a running server can't be replaced cleanly", flush=True)
         if os.name == "nt":
             # this exe is the running process on Windows — pip can't replace
             # a file that's in use; hand the user the equivalent python -m

{confluence_knowledge-0.5.0 → confluence_knowledge-0.5.2}/src/confluence_knowledge/client.py

@@ -114,7 +114,8 @@ class ConfluenceClient:
     def _describe(resp: httpx.Response) -> str:
         s = resp.status_code
         return {
-            401: "authentication failed (401) — check the token and account",
+            401: "authentication failed (401) — re-run `setup` to rotate "
+                 "the credential",
             403: "permission denied (403) — your account can't view this content",
             404: "not found (404) — check the page ID/space and base URL",
             429: "rate limited (429) — retry after the server's Retry-After",

{confluence_knowledge-0.5.0 → confluence_knowledge-0.5.2}/src/confluence_knowledge/crawl.py

@@ -111,7 +111,7 @@ def _fetch_page_dc(client: ConfluenceClient, page_id: str) -> dict:
     }
 
 
-from .retrieval import EMBED_MODEL_ID as _MODEL_ID
+from .retrieval import MODEL_MARKER as _MODEL_ID
 
 
 class Crawler:
@@ -172,6 +172,21 @@ class Crawler:
 
         queue: list[str] = job["queue"]
         errors: list[str] = job["errors"]
+
+        # Mid-crawl model change: already-embedded pages live in the OLD
+        # vector space — finishing this job would publish a mixed-dimension
+        # index. Wipe the scope and fail the job; start_crawl rebuilds from
+        # roots under the new model.
+        stamped = self.index.model_id()
+        if stamped is not None and stamped != _MODEL_ID and queue:
+            self.index.clear_scope()
+            errors = errors + [
+                (f"model changed mid-crawl ({stamped} → {_MODEL_ID}) — "
+                 "index cleared; run start_crawl")]
+            self.index.save_job(job_id, "error", [], errors)
+            return {"job_id": job_id, "state": "error",
+                    "remaining": 0, "errors": errors}
+
         seen_raw = self.index.meta_get(f"seen:{job_id}")
         seen: set[str] = set(json.loads(seen_raw)) if seen_raw else set()
         levels_raw = self.index.meta_get(f"levels:{job_id}")

confluence_knowledge-0.5.2/src/confluence_knowledge/health.py

@@ -0,0 +1,124 @@
+"""Machine-readable health checks for doctor/status — detection only.
+
+Every check is {flag, ok, detail, remediation}; the agent turns a red flag
+into exactly one fix command (see the agent-file remediation table).
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+import sys
+from pathlib import Path
+
+from .settings import SetupError
+
+_DEPS = ("onnxruntime", "tokenizers", "fastembed", "huggingface_hub",
+         "numpy", "mcp")
+
+
+def _dep_ok() -> tuple[bool, list[str]]:
+    import importlib.util
+    missing = [m for m in _DEPS if importlib.util.find_spec(m) is None]
+    return not missing, missing
+
+
+def _models_checks(home: Path, cli: str) -> list[dict]:
+    from .retrieval import models_complete
+    out = []
+    try:
+        mc = models_complete(home)
+    except SetupError as e:
+        return [{"flag": "models_complete", "ok": False, "detail": str(e),
+                 "remediation": f"{cli} prepare-models"}]
+    for key, spec in mc.items():
+        out.append({
+            "flag": f"model_{key}", "ok": spec["ok"],
+            "detail": ("missing: " + ", ".join(spec["missing"])
+                       if spec["missing"] else "present"),
+            "remediation": None if spec["ok"] else f"{cli} prepare-models"})
+    return out
+
+
+def _index_checks(index_root: Path) -> list[dict]:
+    """Per-scope index: integrity + stale-model marker. Detection only."""
+    from .index import Index
+    from .retrieval import MODEL_MARKER as EMBED_MODEL_ID
+    if not index_root.exists():
+        return []
+    out = []
+    for db in sorted(index_root.glob("*.db")):
+        scope = db.stem[:24]
+        idx = None
+        try:
+            idx = Index(db)
+            ok = idx.integrity()
+            stale = idx.model_id() is not None and \
+                idx.model_id() != EMBED_MODEL_ID
+        except Exception:  # noqa: BLE001 — unreadable = corrupt
+            ok, stale = False, False
+        finally:
+            if idx is not None:
+                idx.db.close()  # Windows locks the file while a conn is open
+        out.append({
+            "flag": f"index:{scope}", "ok": ok and not stale,
+            "detail": ("corrupt — delete the index and re-crawl" if not ok
+                       else ("built with an older model — run start_crawl once"
+                             if stale else "ok")),
+            "remediation": None if ok and not stale
+                           else "start_crawl (rebuilds the index)"})
+    return out
+
+
+def health_checks(home: Path, workdir: Path, server_id: str, cli: str,
+                  index_root: Path | None = None) -> list[dict]:
+    """All non-network checks — the machine-readable half of `status()`."""
+    checks = []
+
+    ok, missing = _dep_ok()
+    checks.append({"flag": "deps", "ok": ok,
+                   "detail": "missing: " + ", ".join(missing) if missing
+                             else "all importable",
+                   "remediation": None if ok
+                                else f"pip install -U {cli}[models]"})
+
+    checks += _models_checks(home, cli)
+
+    try:
+        from .registration import registered_paths
+        regs = registered_paths(server_id=server_id, workspace=workdir)
+        ok = bool(regs)
+        checks.append({"flag": "registration", "ok": ok,
+                       "detail": (", ".join(str(p) for p in regs) if regs
+                                  else "no IDE config contains the server"),
+                       "remediation": None if ok
+                                    else f"{cli} register --ide auto"})
+    except Exception as e:  # noqa: BLE001
+        checks.append({"flag": "registration", "ok": False,
+                       "detail": str(e), "remediation": f"{cli} register"})
+
+    if index_root is not None:
+        checks += _index_checks(index_root)
+
+    anchor = home
+    while not anchor.exists() and anchor != anchor.parent:
+        anchor = anchor.parent
+    try:
+        free = shutil.disk_usage(anchor).free // (1024 * 1024)
+    except OSError:
+        free = -1  # can't measure — don't flag
+    checks.append({"flag": "disk_free", "ok": free < 0 or free >= 2000,
+                   "detail": f"{free} MB free",
+                   "remediation": None if free >= 2000
+                                else "free ~2 GB for the model download"})
+
+    checks.append({"flag": "python", "ok": sys.version_info >= (3, 11),
+                   "detail": sys.version.split()[0],
+                   "remediation": None if sys.version_info >= (3, 11)
+                                else "reinstall with Python >=3.11"})
+
+    proxy = os.environ.get("HTTPS_PROXY") or os.environ.get("HTTP_PROXY")
+    checks.append({"flag": "proxy", "ok": True,
+                   "detail": proxy or "none set",
+                   "remediation": None})
+    return checks

{confluence_knowledge-0.5.0 → confluence_knowledge-0.5.2}/src/confluence_knowledge/index.py

@@ -43,7 +43,18 @@ class Index:
         # checkpoint) instead of failing instantly; check_same_thread: the
         # crawl worker and tool-call threads share this one connection —
         # `self.lock` serializes every access so writes never interleave
-        self.db = sqlite3.connect(path, timeout=30, check_same_thread=False)
+        try:
+            self.db = sqlite3.connect(path, timeout=30, check_same_thread=False)
+            self.db.execute("SELECT name FROM sqlite_master LIMIT 1").fetchone()
+        except sqlite3.DatabaseError as e:
+            # close before raising — Windows keeps the file locked otherwise
+            from contextlib import suppress
+            with suppress(Exception):
+                self.db.close()
+            from .settings import SetupError
+            raise SetupError(
+                f"index unreadable at {path} — delete it and re-crawl "
+                "(or run `clean` to drop every index)") from e
         self.lock = threading.RLock()
         with self.lock:
             self.db.executescript(SCHEMA)
@@ -138,7 +149,8 @@ class Index:
                         "INSERT INTO vectors VALUES (?,?)",
                         (c.chunk_id, vecs[i].astype(np.float32).tobytes()))
                 if sparse is not None and i < len(sparse) and sparse[i]:
-                    idx_arr = np.fromiter(sparse[i].keys(), dtype=np.int32)
+                    # bm25 ids are int64 — int32 silently wraps high hashes
+                    idx_arr = np.fromiter(sparse[i].keys(), dtype=np.int64)
                     val_arr = np.fromiter(sparse[i].values(), dtype=np.float32)
                     self.db.execute(
                         "INSERT INTO sparse_vectors VALUES (?,?)",
@@ -154,16 +166,29 @@ class Index:
             for cid, blob in self.db.execute(
                     f"SELECT chunk_id,payload FROM sparse_vectors "
                     f"WHERE chunk_id IN ({q})", chunk_ids):
-                # layout: int32[n] indices + float32[n] values → n = len/8
-                k = len(blob) // 8
-                idx = np.frombuffer(blob[:4 * k], dtype=np.int32)
-                vals = np.frombuffer(blob[4 * k:], dtype=np.float32)
+                # layout: int64[n] indices + float32[n] values → n = len/12
+                k = len(blob) // 12
+                idx = np.frombuffer(blob[:8 * k], dtype=np.int64)
+                vals = np.frombuffer(blob[8 * k:], dtype=np.float32)
                 out[cid] = dict(zip(idx.tolist(), vals.tolist()))
         return out
 
     def model_id(self) -> str | None:
         return self.meta_get("embed_model")
 
+    def clear_scope(self) -> None:
+        """Drop all indexed content (pages/chunks/vectors) + the model
+        marker. Job rows are kept for history. Used when a model change
+        makes the existing vector space invalid."""
+        with self.lock:
+            for t in ("pages", "chunks", "chunks_fts", "vectors",
+                      "sparse_vectors"):
+                self.db.execute(f"DELETE FROM {t}")
+            self.db.execute(
+                "DELETE FROM meta WHERE key='embed_model' OR key LIKE 'seen:%'"
+                " OR key LIKE 'levels:%'")
+            self.db.commit()
+
     def set_model_id(self, model_id: str) -> None:
         with self.lock, self.db:
             self.db.execute("INSERT OR REPLACE INTO meta VALUES ('embed_model', ?)",
@@ -194,7 +219,8 @@ class Index:
                     for r in self.db.execute(
                         "SELECT chunk_id,page_id,heading,kind,text FROM chunks WHERE active=1")]
 
-    def fts_search(self, query: str, limit: int = 30) -> list[tuple[str, float]]:
+    def fts_search(self, query: str, limit: int = 30,
+                   active: set[str] | None = None) -> list[tuple[str, float]]:
         with self.lock:
             try:
                 rows = self.db.execute(
@@ -205,7 +231,8 @@ class Index:
                      limit)).fetchall()
             except sqlite3.OperationalError:
                 return []
-        active = {c["chunk_id"] for c in self.active_chunks()}
+        if active is None:
+            active = {c["chunk_id"] for c in self.active_chunks()}
         return [(r[0], -r[1]) for r in rows if r[0] in active]
 
     def vectors_for(self, chunk_ids: list[str]) -> dict[str, np.ndarray]:
@@ -218,6 +245,16 @@ class Index:
                         f"SELECT chunk_id,vec FROM vectors WHERE chunk_id IN ({q})",
                         chunk_ids)}
 
+    def integrity(self) -> bool:
+        """PRAGMA integrity_check — False means the DB file is damaged and
+        should be deleted + rebuilt (never auto-repaired here)."""
+        try:
+            with self.lock:
+                row = self.db.execute("PRAGMA integrity_check").fetchone()
+            return bool(row and row[0] == "ok")
+        except sqlite3.DatabaseError:
+            return False
+
     def stats(self) -> dict:
         with self.lock:
             p = self.db.execute("SELECT COUNT(*) FROM pages WHERE active=1").fetchone()[0]

{confluence_knowledge-0.5.0 → confluence_knowledge-0.5.2}/src/confluence_knowledge/registration.py

@@ -125,10 +125,18 @@ def register(ide: str = "auto", workspace: Path | None = None,
     return [(f, _merge(f, entry, server_id)) for f in files]
 
 
-def registered_paths(server_id: str = SERVER_ID) -> list[Path]:
-    """Existing config files that already contain our server — for doctor."""
+def registered_paths(server_id: str = SERVER_ID,
+                     workspace: Path | None = None) -> list[Path]:
+    """Existing config files that already contain our server — for doctor.
+    `workspace` also probes workspace-scoped mcp.json files (a
+    `register --ide vscode --workspace <repo>` leaves no user-level trace)."""
     out = []
-    for p in targets("all"):
+    probes = targets("all")
+    if workspace:
+        probes += [_workspace_mcp(p) for p in
+                   {workspace, workspace / ".vscode" if workspace.name != ".vscode"
+                     else workspace.parent}]
+    for p in dict.fromkeys(q for q in probes if q):
         try:
             servers = json.loads(p.read_text()).get("servers", {})
         except (OSError, json.JSONDecodeError, UnicodeDecodeError):

{confluence_knowledge-0.5.0 → confluence_knowledge-0.5.2}/src/confluence_knowledge/retrieval.py

@@ -33,10 +33,14 @@ RERANK_ONNX = "onnx/model_quantized.onnx"
 RERANK_BASE_REPO = "onnx-community/bge-reranker-base-ONNX"  # light override
 
 EMBED_MODEL_ID = "bge-m3-onnx-quantized"
+SPARSE_MODEL_ID = "qdrant-bm25-fastembed"
+MODEL_MARKER = f"{EMBED_MODEL_ID}+{SPARSE_MODEL_ID}"
 
 TOP_K_RETRIEVE = 30
 TOP_K_EVIDENCE = 8
 RERANK_MAX = 25          # bounds cross-encoder latency on CPU
+MIN_RERANK_SCORE = 0.3   # sigmoid floor — below this a hit is "least bad"
+MAX_SEARCHABLE_CHUNKS = 100_000  # whole-index search is O(n) in RAM
 EMBED_MAX_TOKENS = 512   # quantized model input cap (chunks stay under)
 RERANK_MAX_TOKENS = 512
 
@@ -188,13 +192,21 @@ def prepare_models(home: Path) -> list[str]:
         spec = lock["models"][key]
         dest = _models_dir(home) / spec["repo"].split("/")[-1]
         if dest.exists():
-            complete = all((dest / f).exists() for f in spec["files"])
-            if complete:
+            files_ok = all((dest / f).exists() for f in spec["files"])
+            stamp = dest / ".revision"
+            if files_ok and not stamp.exists():
+                # pre-stamp install — verify files, then stamp in place
+                stamp.write_text(spec["revision"])
+            if files_ok and stamp.read_text().strip() == spec["revision"]:
                 out.append(f"already prepared: {spec['repo']}")
                 continue
-            shutil.rmtree(dest)  # partial/interrupted → refetch
+            if stamp.exists() and stamp.read_text().strip() != spec["revision"]:
+                shutil.rmtree(dest)  # wrong revision — untrusted, refetch
+            # else: missing files but right revision — snapshot_download
+            # resumes/skips valid files; only the gaps re-download
         snapshot_download(repo_id=spec["repo"], revision=spec["revision"],
                           local_dir=dest, allow_patterns=spec["files"])
+        (dest / ".revision").write_text(spec["revision"])
         if not _license_ok(dest):
             _license_ok_upstream(dest, spec, snapshot_download)
         if not _license_ok(dest):
@@ -212,6 +224,16 @@ def prepare_models(home: Path) -> list[str]:
         out.append(f"prepared {spec['repo']} ({spec['license']})")
     except Exception as e:
         raise SetupError(f"couldn't fetch sparse model {spec['repo']}: {e}") from e
+
+    # GC: remove model dirs not owned by the current lock — superseded
+    # revisions and legacy models (e.g. the old MiniLM stack) otherwise
+    # accumulate ~600 MB each in the user's home.
+    owned = {lock["models"][k]["repo"].split("/")[-1]
+             for k in ("embed", "rerank", "rerank_light")} | {"fastembed"}
+    for d in models.iterdir():
+        if d.is_dir() and d.name not in owned:
+            shutil.rmtree(d, ignore_errors=True)
+            out.append(f"removed stale model dir: {d.name}")
     return out
 
 
@@ -238,6 +260,27 @@ def _license_ok_upstream(dest: Path, spec: dict, snapshot_download) -> None:
         return
 
 
+def models_complete(home: Path) -> dict:
+    """Per-model completeness vs the lock — a dir that exists but lacks a
+    locked file is a partial download, not a prepared model."""
+    lock = _load_lock()["models"]
+    out = {}
+    for key in ("embed", "rerank", "rerank_light"):
+        spec = lock[key]
+        dest = _models_dir(home) / spec["repo"].split("/")[-1]
+        missing = [f for f in spec["files"] if not (dest / f).exists()]
+        wrong_rev = (dest.exists() and not missing
+                     and (dest / ".revision").exists()
+                     and (dest / ".revision").read_text().strip()
+                         != spec["revision"])
+        out[key] = {"ok": not missing and not wrong_rev,
+                    "missing": missing or (["<wrong revision>"]
+                                            if wrong_rev else [])}
+    fe = _models_dir(home) / "fastembed"
+    out["sparse"] = {"ok": fe.exists() and any(fe.iterdir()), "missing": []}
+    return out
+
+
 def _license_ok(dest: Path) -> bool:
     for md in list(dest.glob("*.md")) + list(dest.glob("LICENSE*")):
         head = md.read_text(errors="ignore")[:6000].lower()
@@ -260,6 +303,12 @@ def _cosine(q: np.ndarray, mat: dict[str, np.ndarray]) -> list[tuple[str, float]
     if not mat:
         return []
     ids = list(mat)
+    # a model swap can leave mismatched-dim blobs — skip, never crash
+    good = [i for i in ids if mat[i].shape == q.shape]
+    if len(good) < len(ids):
+        ids = good
+    if not ids:
+        return []
     m = np.stack([mat[i] for i in ids])
     m = m / (np.linalg.norm(m, axis=1, keepdims=True) + 1e-9)
     q = q / (np.linalg.norm(q) + 1e-9)
@@ -284,14 +333,22 @@ def retrieve(index: Index, query: str, embedder: LocalEmbedder,
     chunks = index.active_chunks()
     if not chunks:
         return []
+    if len(chunks) > MAX_SEARCHABLE_CHUNKS:
+        from .settings import SetupError
+        raise SetupError(
+            f"index holds {len(chunks):,} chunks — searching the whole scope "
+            "at once is unsupported. Connect a narrower scope (or re-index "
+            "with a smaller root).")
 
     # dense leg
     qvec = embedder.embed([query])[0]
     vecs = index.vectors_for([c["chunk_id"] for c in chunks])
     dense_rank = [cid for cid, _ in _cosine(qvec, vecs)][:TOP_K_RETRIEVE]
 
-    # keyword leg
-    fts_rank = [cid for cid, _ in index.fts_search(query)][:TOP_K_RETRIEVE]
+    # keyword leg — reuse the fetched set; fts_search would re-load it
+    active_ids = {c["chunk_id"] for c in chunks}
+    fts_rank = [cid for cid, _ in
+                index.fts_search(query, active=active_ids)][:TOP_K_RETRIEVE]
 
     # sparse leg (optional — learned term weights)
     sparse_rank: list[str] = []
@@ -316,6 +373,10 @@ def retrieve(index: Index, query: str, embedder: LocalEmbedder,
         order = np.argsort(-scores)[:TOP_K_EVIDENCE]
         evidence = []
         for i in order:
+            if scores[i] < MIN_RERANK_SCORE:
+                # below the floor the reranker is only ranking "least bad" —
+                # surfacing it would ground the answer on noise
+                continue
             c = dict(scored[i])
             c["rerank_score"] = float(scores[i])
             evidence.append(c)
@@ -341,7 +402,18 @@ def check_citations(evidence: list[dict], mappings: list[dict]) -> dict:
             bad.append({"claim": m.get("claim", "")[:80], "reason": "chunk not in evidence set"})
             continue
         quote = m.get("quote", "")
-        if quote and quote not in ch["text"]:
+        if quote and _norm(quote) not in _norm(ch["text"]):
             bad.append({"claim": m.get("claim", "")[:80],
                         "reason": "quote not present verbatim in cited chunk"})
     return {"ok": not bad, "violations": bad}
+
+
+def _norm(s: str) -> str:
+    """Typography-insensitive form for citation matching — docx extraction
+    leaves smart quotes/NBSP/dashes that shouldn't fail a true quote."""
+    import unicodedata
+    s = unicodedata.normalize("NFKC", s)
+    for a, b in (("‘", "'"), ("’", "'"), ("“", '"'), ("”", '"'),
+                 ("–", "-"), ("—", "-"), (" ", " "), ("−", "-")):
+        s = s.replace(a, b)
+    return " ".join(s.split())

{confluence_knowledge-0.5.0 → confluence_knowledge-0.5.2}/src/confluence_knowledge/server.py

@@ -18,7 +18,9 @@ from .client import AccessError, ConfluenceClient, SecurityException, make_clien
 from .crawl import Crawler
 from .index import Index
 from .retrieval import (
-    EMBED_MODEL_ID,
+    MODEL_MARKER as EMBED_MODEL_ID,
+)
+from .retrieval import (
     LocalEmbedder,
     LocalReranker,
     LocalSparseEmbedder,
@@ -156,16 +158,31 @@ def status() -> dict:
         cfg = {"env": profile.env, "site": profile.base_url}
     except SetupError as e:
         cfg = {"error": str(e)}
-    models = home / "models"
+    from .health import health_checks
+    from .registration import SERVER_ID
+    idx_root = workspace_index_root(_workdir())
+    checks = health_checks(home, _workdir(), SERVER_ID, "confluence-knowledge",
+                           index_root=idx_root)
+    if "error" not in cfg:
+        # cheap liveness: a 401 here means the saved token died — point at
+        # setup, not at the docs
+        try:
+            client = _client_for(profile)
+            ep = ("wiki/rest/api/user/current" if profile.env == "cloud"
+                  else "rest/api/user/current")
+            client.get(ep)
+            ok, detail = True, "token valid"
+        except (AccessError, SetupError) as e:
+            ok, detail = False, str(e)
+        checks.append({"flag": "creds_valid", "ok": ok, "detail": detail,
+                       "remediation": None if ok else
+                       "confluence-knowledge setup (rotate the token)"})
     return {
         "configured": "error" not in cfg,
         "profile": cfg,
-        "embedding_model_ready": (models / "bge-m3").exists(),
-        "reranker_model_ready": (models / "bge-reranker-v2-m3-ONNX").exists()
-            or (models / "bge-reranker-base-ONNX").exists(),
-        "sparse_model_ready": (models / "fastembed").exists(),
+        "checks": checks,
         "models_loaded": _models_ready.is_set() and _EMBEDDER is not None,
-        "index_root": str(workspace_index_root(_workdir())),
+        "index_root": str(idx_root),
         "live_scopes": sorted(_scopes),
     }
 

{confluence_knowledge-0.5.0 → confluence_knowledge-0.5.2}/src/confluence_knowledge.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: confluence-knowledge
-Version: 0.5.0
+Version: 0.5.2
 Summary: Local, GET-only Confluence subtree indexer + grounded-Q&A MCP service
 License: MIT
 Project-URL: Repository, https://github.com/nkunal1992/agents

{confluence_knowledge-0.5.0 → confluence_knowledge-0.5.2}/src/confluence_knowledge.egg-info/SOURCES.txt

@@ -7,6 +7,7 @@ src/confluence_knowledge/client.py
 src/confluence_knowledge/crawl.py
 src/confluence_knowledge/extract.py
 src/confluence_knowledge/extract_files.py
+src/confluence_knowledge/health.py
 src/confluence_knowledge/index.py
 src/confluence_knowledge/models.lock.json
 src/confluence_knowledge/registration.py
@@ -24,6 +25,7 @@ tests/test_agent_doc_consistency.py
 tests/test_client_get_only.py
 tests/test_crawl.py
 tests/test_extract.py
+tests/test_health_checks.py
 tests/test_index.py
 tests/test_prepare_models.py
 tests/test_registration.py

confluence_knowledge-0.5.2/tests/test_health_checks.py

@@ -0,0 +1,145 @@
+"""health_checks flags: each failure mode → exactly one remediation."""
+
+
+from pathlib import Path
+
+import pytest
+
+from confluence_knowledge.health import health_checks
+from confluence_knowledge.index import Index
+from confluence_knowledge.retrieval import MODEL_MARKER
+from confluence_knowledge.settings import SetupError
+
+
+def _flag(checks, name):
+    return {c["flag"]: c for c in checks}[name]
+
+
+def test_models_complete_flags_partial_download(tmp_path, monkeypatch):
+    home = tmp_path / "home"
+    # partial bge-m3: dir exists, onnx missing — the old shallow check
+    # reported this as "ready"
+    (home / "models" / "bge-m3").mkdir(parents=True)
+    (home / "models" / "bge-m3" / "tokenizer.json").write_text("{}")
+    checks = health_checks(home, tmp_path / "work", "sid", "ck")
+    emb = _flag(checks, "model_embed")
+    assert emb["ok"] is False and "prepare-models" in emb["remediation"]
+
+
+def test_models_complete_green_when_full(tmp_path):
+    home = tmp_path / "home"
+    for name, files in (
+            ("bge-m3", ["onnx/sentence_transformers_quantized.onnx",
+                        "tokenizer.json", "tokenizer_config.json",
+                        "config.json", "sentencepiece.bpe.model",
+                        "special_tokens_map.json", "README.md"]),
+            ("bge-reranker-v2-m3-ONNX", ["onnx/model_quantized.onnx",
+                                         "tokenizer.json",
+                                         "tokenizer_config.json",
+                                         "config.json",
+                                         "special_tokens_map.json",
+                                         "quantize_config.json", "README.md"]),
+            ("bge-reranker-base-ONNX", ["onnx/model_quantized.onnx",
+                                        "tokenizer.json",
+                                        "tokenizer_config.json",
+                                        "config.json",
+                                        "special_tokens_map.json",
+                                        "quantize_config.json", "README.md"])):
+        for f in files:
+            p = home / "models" / name / f
+            p.parent.mkdir(parents=True, exist_ok=True)
+            p.write_text("x")
+    (home / "models" / "fastembed" / "x").mkdir(parents=True)
+    checks = health_checks(home, tmp_path / "work", "sid", "ck")
+    for f in ("model_embed", "model_rerank", "model_rerank_light",
+              "model_sparse"):
+        assert _flag(checks, f)["ok"] is True, f
+
+
+def test_index_corrupt_flagged(tmp_path):
+    idx_root = tmp_path / "idx"
+    idx_root.mkdir()
+    (idx_root / "scope1.db").write_bytes(b"not a sqlite file" * 100)
+    checks = health_checks(tmp_path / "home", tmp_path / "work", "sid", "ck",
+                           index_root=idx_root)
+    f = [c for c in checks if c["flag"].startswith("index:")]
+    assert f and f[0]["ok"] is False and "corrupt" in f[0]["detail"]
+
+
+def test_index_stale_model_flagged(tmp_path):
+    idx_root = tmp_path / "idx"
+    idx_root.mkdir()
+    idx = Index(idx_root / "scope1.db")
+    idx.set_model_id("old-model-xyz")
+    checks = health_checks(tmp_path / "home", tmp_path / "work", "sid", "ck",
+                           index_root=idx_root)
+    f = [c for c in checks if c["flag"].startswith("index:")]
+    assert f and f[0]["ok"] is False and "start_crawl" in f[0]["remediation"]
+
+
+def test_index_healthy_passes(tmp_path):
+    idx_root = tmp_path / "idx"
+    idx_root.mkdir()
+    idx = Index(idx_root / "scope1.db")
+    idx.set_model_id(MODEL_MARKER)
+    checks = health_checks(tmp_path / "home", tmp_path / "work", "sid", "ck",
+                           index_root=idx_root)
+    f = [c for c in checks if c["flag"].startswith("index:")]
+    assert f and f[0]["ok"] is True
+
+
+def test_corrupt_db_raises_setup_error_not_sqlite(tmp_path):
+    db = tmp_path / "bad.db"
+    db.write_bytes(b"garbage" * 1000)
+    with pytest.raises(SetupError, match="index unreadable"):
+        Index(db)
+
+
+def test_registration_missing_flagged(tmp_path, monkeypatch):
+    monkeypatch.setattr("confluence_knowledge.registration.targets",
+                        lambda ide, workspace=None: [])
+    checks = health_checks(tmp_path / "home", tmp_path / "work", "sid", "ck")
+    reg = _flag(checks, "registration")
+    assert reg["ok"] is False and "register" in reg["remediation"]
+
+
+def test_deps_flag_when_missing(tmp_path, monkeypatch):
+    import importlib.util
+    real = importlib.util.find_spec
+    monkeypatch.setattr(importlib.util, "find_spec",
+                        lambda m: None if m == "fastembed" else real(m))
+    checks = health_checks(tmp_path / "home", tmp_path / "work", "sid", "ck")
+    assert _flag(checks, "deps")["ok"] is False
+
+
+def test_every_red_flag_has_a_remediation(tmp_path, monkeypatch):
+    """The agent's contract: a red flag never lands without a fix command."""
+    monkeypatch.setattr("confluence_knowledge.registration.targets",
+                        lambda ide, workspace=None: [])
+    checks = health_checks(tmp_path / "home", tmp_path / "work", "sid", "ck")
+    red = [c for c in checks if c["ok"] is False and c["flag"] != "proxy"]
+    assert red  # some flags SHOULD be red on a bare tmp home
+    for c in red:
+        assert c["remediation"], f"red flag {c['flag']} has no remediation"
+
+
+def test_documented_flags_exist(tmp_path, monkeypatch):
+    """Every flag named in the agent's remediation table must be a flag
+    doctor can actually emit — drift silently strands users (hunt F)."""
+    import re
+    monkeypatch.setenv("CONF_KNOWLEDGE_HOME", str(tmp_path))
+    monkeypatch.setenv("CONF_KNOWLEDGE_WORKDIR", str(tmp_path / "w"))
+    idx_root = tmp_path / "idx"
+    idx_root.mkdir()
+    Index(idx_root / "scope.db")  # so index:* flags are emitted
+    checks = health_checks(tmp_path / "home", tmp_path / "w",
+                           "confluence-knowledge", "confluence-knowledge",
+                           index_root=idx_root)
+    emitted = {c["flag"].split(":")[0] for c in checks}
+    emitted.add("creds_valid")  # network flag — status() level, not here
+    agent = (Path(__file__).parents[3] / "agents"
+             / "confluence-knowledge.agent.md").read_text()
+    documented = set(re.findall(r"`(deps|model_\w+|registration|index|"
+                              r"disk_free|python|proxy|creds_valid)", agent))
+    missing = documented - emitted
+    assert not missing, f"agent documents flags doctor never emits: {missing}"

{confluence_knowledge-0.5.0 → confluence_knowledge-0.5.2}/tests/test_server_flow.py

@@ -100,7 +100,8 @@ class _FakeEmbedder:
 class _FakeReranker:
     def score(self, query, docs):
         import numpy as np
-        return np.arange(len(docs), dtype=np.float32)
+        # descending 0.9, 0.8, … — above the 0.3 evidence floor
+        return np.linspace(0.9, 0.5, len(docs), dtype=np.float32)
 
 
 def test_full_agent_flow_with_models(wired, monkeypatch):
@@ -127,9 +128,11 @@ def test_full_agent_flow_with_models(wired, monkeypatch):
 
 def test_search_during_crawl_says_crawling(wired, monkeypatch):
     # keep the worker parked on the model-warmup gate so the job stays
-    # "running" deterministically — a 1-page crawl finishes before the race
-    # could otherwise resolve
+    # "running" deterministically: a never-set event AND warmup disabled —
+    # otherwise the warmup thread sets the event, the worker finishes the
+    # 1-page crawl, and search reaches retrieve instead of the early return
     monkeypatch.setattr(server, "_models_ready", threading.Event())
+    monkeypatch.setattr(server, "_start_warmup", lambda: None)
     c = server.connect("1")
     server.start_crawl(c["scope_handle"])
     res = server.search(c["scope_handle"], "text")

{confluence_knowledge-0.5.0 → confluence_knowledge-0.5.2}/tests/test_sparse_retrieval.py

@@ -4,7 +4,9 @@ import numpy as np
 import pytest
 
 from confluence_knowledge import retrieval
+from confluence_knowledge.extract import Chunk
 from confluence_knowledge.index import Index
+from confluence_knowledge.retrieval import MODEL_MARKER
 
 
 class _C:
@@ -74,5 +76,55 @@ def test_sparse_leg_optional(tmp_path):
 def test_model_marker(tmp_path):
     idx = Index(tmp_path / "i.db")
     assert idx.model_id() is None           # pre-0.5 index → no marker
-    idx.set_model_id("bge-m3-onnx-quantized")
-    assert idx.model_id() == "bge-m3-onnx-quantized"
+    idx.set_model_id(MODEL_MARKER)
+    assert idx.model_id() == MODEL_MARKER
+
+
+def test_sparse_id_int64_range(tmp_path):
+    """fastembed Qdrant/bm25 emits int64 hashed ids — int32 packing would
+    silently wrap ids >2^31 and kill the sparse leg (C-4 hunt bug)."""
+    idx = Index(tmp_path / "i.db")
+    huge = 2**31 + 12345      # > int32 range — wraps negative if packed as i4
+    c = Chunk(text="word", heading_path="", kind="text", ordinal=0,
+              chunk_id="c1")
+    p = {"page_id": "p", "parent_id": None, "title": "t", "url": "u",
+         "level": 0, "version": "", "warnings": ""}
+    idx.upsert_page(p, [c], vecs=None, sparse=[{huge: 1.5}])
+    idx.publish({"p"})
+    got = idx.sparse_for(["c1"])
+    assert huge in got["c1"], "sparse id >2^31 must round-trip intact"
+    assert got["c1"][huge] == pytest.approx(1.5)
+
+
+def test_cosine_mixed_dims_skips(tmp_path):
+    """A model swap can leave mismatched-dim blobs — retrieval must skip
+    them, not crash np.stack (C-1 hunt bug)."""
+    q = np.ones(8, dtype=np.float32)
+    mat = {"ok": np.ones(8, dtype=np.float32),
+           "stale": np.ones(4, dtype=np.float32)}
+    ranked = retrieval._cosine(q, mat)
+    assert [cid for cid, _ in ranked] == ["ok"]
+
+
+def test_step_aborts_on_stale_model_mid_crawl(tmp_path):
+    """Job checkpointed under an old model must not resume mixing dims —
+    scope is wiped and the job errors so start_crawl rebuilds (C-1)."""
+    idx = Index(tmp_path / "i.db")
+    idx.set_model_id("old-model-marker")
+    # a page already indexed under the old model
+    c = Chunk(text="stale", heading_path="", kind="text", ordinal=0,
+              chunk_id="c1")
+    idx.upsert_page({"page_id": "p1", "parent_id": None, "title": "t",
+                     "url": "u", "level": 0, "version": "", "warnings": ""},
+                    [c], vecs=None, sparse=None)
+    idx.create_job("j1", "root")
+    idx.save_job("j1", "running", ["page:2", "page:3"], [])
+    idx.db.execute("INSERT OR REPLACE INTO meta VALUES (?,?)", ("seen:j1", '["p1"]')); idx.db.commit()
+
+    from confluence_knowledge.crawl import Crawler
+    cr = Crawler(client=None, index=idx, env="datacenter")
+    r = cr.step("j1", batch=None, budget_s=None)
+    assert r["state"] == "error"
+    assert "model changed" in r["errors"][0]
+    assert idx.model_id() is None            # marker wiped
+    assert idx.active_chunks() == []         # stale content cleared