confluence-knowledge 0.5.1__tar.gz → 0.5.2__tar.gz

{confluence_knowledge-0.5.1 → confluence_knowledge-0.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: confluence-knowledge
-Version: 0.5.1
+Version: 0.5.2
 Summary: Local, GET-only Confluence subtree indexer + grounded-Q&A MCP service
 License: MIT
 Project-URL: Repository, https://github.com/nkunal1992/agents

{confluence_knowledge-0.5.1 → confluence_knowledge-0.5.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "confluence-knowledge"
-version = "0.5.1"
+version = "0.5.2"
 description = "Local, GET-only Confluence subtree indexer + grounded-Q&A MCP service"
 readme = "README.md"
 requires-python = ">=3.11"

{confluence_knowledge-0.5.1 → confluence_knowledge-0.5.2}/src/confluence_knowledge/cli.py

@@ -169,6 +169,11 @@ def main() -> None:
         import subprocess
         cmd = [sys.executable, "-m", "pip", "install", "-U",
                "confluence-knowledge[models]"]
+        # a running MCP server holds open files — on Windows pip can't
+        # replace them and on any OS the old code keeps serving until the
+        # IDE restarts. Say it plainly before doing anything.
+        print("note: quit the IDE (or stop the MCP server) before updating —"
+              " a running server can't be replaced cleanly", flush=True)
         if os.name == "nt":
             # this exe is the running process on Windows — pip can't replace
             # a file that's in use; hand the user the equivalent python -m

{confluence_knowledge-0.5.1 → confluence_knowledge-0.5.2}/src/confluence_knowledge/crawl.py

@@ -111,7 +111,7 @@ def _fetch_page_dc(client: ConfluenceClient, page_id: str) -> dict:
     }
 
 
-from .retrieval import EMBED_MODEL_ID as _MODEL_ID
+from .retrieval import MODEL_MARKER as _MODEL_ID
 
 
 class Crawler:
@@ -172,6 +172,21 @@ class Crawler:
 
         queue: list[str] = job["queue"]
         errors: list[str] = job["errors"]
+
+        # Mid-crawl model change: already-embedded pages live in the OLD
+        # vector space — finishing this job would publish a mixed-dimension
+        # index. Wipe the scope and fail the job; start_crawl rebuilds from
+        # roots under the new model.
+        stamped = self.index.model_id()
+        if stamped is not None and stamped != _MODEL_ID and queue:
+            self.index.clear_scope()
+            errors = errors + [
+                (f"model changed mid-crawl ({stamped} → {_MODEL_ID}) — "
+                 "index cleared; run start_crawl")]
+            self.index.save_job(job_id, "error", [], errors)
+            return {"job_id": job_id, "state": "error",
+                    "remaining": 0, "errors": errors}
+
         seen_raw = self.index.meta_get(f"seen:{job_id}")
         seen: set[str] = set(json.loads(seen_raw)) if seen_raw else set()
         levels_raw = self.index.meta_get(f"levels:{job_id}")

{confluence_knowledge-0.5.1 → confluence_knowledge-0.5.2}/src/confluence_knowledge/health.py

@@ -43,12 +43,13 @@ def _models_checks(home: Path, cli: str) -> list[dict]:
 def _index_checks(index_root: Path) -> list[dict]:
     """Per-scope index: integrity + stale-model marker. Detection only."""
     from .index import Index
-    from .retrieval import EMBED_MODEL_ID
+    from .retrieval import MODEL_MARKER as EMBED_MODEL_ID
     if not index_root.exists():
         return []
     out = []
     for db in sorted(index_root.glob("*.db")):
         scope = db.stem[:24]
+        idx = None
         try:
             idx = Index(db)
             ok = idx.integrity()
@@ -56,6 +57,9 @@ def _index_checks(index_root: Path) -> list[dict]:
                 idx.model_id() != EMBED_MODEL_ID
         except Exception:  # noqa: BLE001 — unreadable = corrupt
             ok, stale = False, False
+        finally:
+            if idx is not None:
+                idx.db.close()  # Windows locks the file while a conn is open
         out.append({
             "flag": f"index:{scope}", "ok": ok and not stale,
             "detail": ("corrupt — delete the index and re-crawl" if not ok
@@ -82,7 +86,7 @@ def health_checks(home: Path, workdir: Path, server_id: str, cli: str,
 
     try:
         from .registration import registered_paths
-        regs = registered_paths(server_id=server_id)
+        regs = registered_paths(server_id=server_id, workspace=workdir)
         ok = bool(regs)
         checks.append({"flag": "registration", "ok": ok,
                        "detail": (", ".join(str(p) for p in regs) if regs

{confluence_knowledge-0.5.1 → confluence_knowledge-0.5.2}/src/confluence_knowledge/index.py

@@ -47,6 +47,10 @@ class Index:
             self.db = sqlite3.connect(path, timeout=30, check_same_thread=False)
             self.db.execute("SELECT name FROM sqlite_master LIMIT 1").fetchone()
         except sqlite3.DatabaseError as e:
+            # close before raising — Windows keeps the file locked otherwise
+            from contextlib import suppress
+            with suppress(Exception):
+                self.db.close()
             from .settings import SetupError
             raise SetupError(
                 f"index unreadable at {path} — delete it and re-crawl "
@@ -145,7 +149,8 @@ class Index:
                         "INSERT INTO vectors VALUES (?,?)",
                         (c.chunk_id, vecs[i].astype(np.float32).tobytes()))
                 if sparse is not None and i < len(sparse) and sparse[i]:
-                    idx_arr = np.fromiter(sparse[i].keys(), dtype=np.int32)
+                    # bm25 ids are int64 — int32 silently wraps high hashes
+                    idx_arr = np.fromiter(sparse[i].keys(), dtype=np.int64)
                     val_arr = np.fromiter(sparse[i].values(), dtype=np.float32)
                     self.db.execute(
                         "INSERT INTO sparse_vectors VALUES (?,?)",
@@ -161,16 +166,29 @@ class Index:
             for cid, blob in self.db.execute(
                     f"SELECT chunk_id,payload FROM sparse_vectors "
                     f"WHERE chunk_id IN ({q})", chunk_ids):
-                # layout: int32[n] indices + float32[n] values → n = len/8
-                k = len(blob) // 8
-                idx = np.frombuffer(blob[:4 * k], dtype=np.int32)
-                vals = np.frombuffer(blob[4 * k:], dtype=np.float32)
+                # layout: int64[n] indices + float32[n] values → n = len/12
+                k = len(blob) // 12
+                idx = np.frombuffer(blob[:8 * k], dtype=np.int64)
+                vals = np.frombuffer(blob[8 * k:], dtype=np.float32)
                 out[cid] = dict(zip(idx.tolist(), vals.tolist()))
         return out
 
     def model_id(self) -> str | None:
         return self.meta_get("embed_model")
 
+    def clear_scope(self) -> None:
+        """Drop all indexed content (pages/chunks/vectors) + the model
+        marker. Job rows are kept for history. Used when a model change
+        makes the existing vector space invalid."""
+        with self.lock:
+            for t in ("pages", "chunks", "chunks_fts", "vectors",
+                      "sparse_vectors"):
+                self.db.execute(f"DELETE FROM {t}")
+            self.db.execute(
+                "DELETE FROM meta WHERE key='embed_model' OR key LIKE 'seen:%'"
+                " OR key LIKE 'levels:%'")
+            self.db.commit()
+
     def set_model_id(self, model_id: str) -> None:
         with self.lock, self.db:
             self.db.execute("INSERT OR REPLACE INTO meta VALUES ('embed_model', ?)",
@@ -201,7 +219,8 @@ class Index:
                     for r in self.db.execute(
                         "SELECT chunk_id,page_id,heading,kind,text FROM chunks WHERE active=1")]
 
-    def fts_search(self, query: str, limit: int = 30) -> list[tuple[str, float]]:
+    def fts_search(self, query: str, limit: int = 30,
+                   active: set[str] | None = None) -> list[tuple[str, float]]:
         with self.lock:
             try:
                 rows = self.db.execute(
@@ -212,7 +231,8 @@ class Index:
                      limit)).fetchall()
             except sqlite3.OperationalError:
                 return []
-        active = {c["chunk_id"] for c in self.active_chunks()}
+        if active is None:
+            active = {c["chunk_id"] for c in self.active_chunks()}
         return [(r[0], -r[1]) for r in rows if r[0] in active]
 
     def vectors_for(self, chunk_ids: list[str]) -> dict[str, np.ndarray]:

{confluence_knowledge-0.5.1 → confluence_knowledge-0.5.2}/src/confluence_knowledge/registration.py

@@ -125,10 +125,18 @@ def register(ide: str = "auto", workspace: Path | None = None,
     return [(f, _merge(f, entry, server_id)) for f in files]
 
 
-def registered_paths(server_id: str = SERVER_ID) -> list[Path]:
-    """Existing config files that already contain our server — for doctor."""
+def registered_paths(server_id: str = SERVER_ID,
+                     workspace: Path | None = None) -> list[Path]:
+    """Existing config files that already contain our server — for doctor.
+    `workspace` also probes workspace-scoped mcp.json files (a
+    `register --ide vscode --workspace <repo>` leaves no user-level trace)."""
     out = []
-    for p in targets("all"):
+    probes = targets("all")
+    if workspace:
+        probes += [_workspace_mcp(p) for p in
+                   {workspace, workspace / ".vscode" if workspace.name != ".vscode"
+                     else workspace.parent}]
+    for p in dict.fromkeys(q for q in probes if q):
         try:
             servers = json.loads(p.read_text()).get("servers", {})
         except (OSError, json.JSONDecodeError, UnicodeDecodeError):

{confluence_knowledge-0.5.1 → confluence_knowledge-0.5.2}/src/confluence_knowledge/retrieval.py

@@ -33,10 +33,14 @@ RERANK_ONNX = "onnx/model_quantized.onnx"
 RERANK_BASE_REPO = "onnx-community/bge-reranker-base-ONNX"  # light override
 
 EMBED_MODEL_ID = "bge-m3-onnx-quantized"
+SPARSE_MODEL_ID = "qdrant-bm25-fastembed"
+MODEL_MARKER = f"{EMBED_MODEL_ID}+{SPARSE_MODEL_ID}"
 
 TOP_K_RETRIEVE = 30
 TOP_K_EVIDENCE = 8
 RERANK_MAX = 25          # bounds cross-encoder latency on CPU
+MIN_RERANK_SCORE = 0.3   # sigmoid floor — below this a hit is "least bad"
+MAX_SEARCHABLE_CHUNKS = 100_000  # whole-index search is O(n) in RAM
 EMBED_MAX_TOKENS = 512   # quantized model input cap (chunks stay under)
 RERANK_MAX_TOKENS = 512
 
@@ -188,13 +192,21 @@ def prepare_models(home: Path) -> list[str]:
         spec = lock["models"][key]
         dest = _models_dir(home) / spec["repo"].split("/")[-1]
         if dest.exists():
-            complete = all((dest / f).exists() for f in spec["files"])
-            if complete:
+            files_ok = all((dest / f).exists() for f in spec["files"])
+            stamp = dest / ".revision"
+            if files_ok and not stamp.exists():
+                # pre-stamp install — verify files, then stamp in place
+                stamp.write_text(spec["revision"])
+            if files_ok and stamp.read_text().strip() == spec["revision"]:
                 out.append(f"already prepared: {spec['repo']}")
                 continue
-            shutil.rmtree(dest)  # partial/interrupted → refetch
+            if stamp.exists() and stamp.read_text().strip() != spec["revision"]:
+                shutil.rmtree(dest)  # wrong revision — untrusted, refetch
+            # else: missing files but right revision — snapshot_download
+            # resumes/skips valid files; only the gaps re-download
         snapshot_download(repo_id=spec["repo"], revision=spec["revision"],
                           local_dir=dest, allow_patterns=spec["files"])
+        (dest / ".revision").write_text(spec["revision"])
         if not _license_ok(dest):
             _license_ok_upstream(dest, spec, snapshot_download)
         if not _license_ok(dest):
@@ -212,6 +224,16 @@ def prepare_models(home: Path) -> list[str]:
         out.append(f"prepared {spec['repo']} ({spec['license']})")
     except Exception as e:
         raise SetupError(f"couldn't fetch sparse model {spec['repo']}: {e}") from e
+
+    # GC: remove model dirs not owned by the current lock — superseded
+    # revisions and legacy models (e.g. the old MiniLM stack) otherwise
+    # accumulate ~600 MB each in the user's home.
+    owned = {lock["models"][k]["repo"].split("/")[-1]
+             for k in ("embed", "rerank", "rerank_light")} | {"fastembed"}
+    for d in models.iterdir():
+        if d.is_dir() and d.name not in owned:
+            shutil.rmtree(d, ignore_errors=True)
+            out.append(f"removed stale model dir: {d.name}")
     return out
 
 
@@ -247,7 +269,13 @@ def models_complete(home: Path) -> dict:
         spec = lock[key]
         dest = _models_dir(home) / spec["repo"].split("/")[-1]
         missing = [f for f in spec["files"] if not (dest / f).exists()]
-        out[key] = {"ok": not missing, "missing": missing}
+        wrong_rev = (dest.exists() and not missing
+                     and (dest / ".revision").exists()
+                     and (dest / ".revision").read_text().strip()
+                         != spec["revision"])
+        out[key] = {"ok": not missing and not wrong_rev,
+                    "missing": missing or (["<wrong revision>"]
+                                            if wrong_rev else [])}
     fe = _models_dir(home) / "fastembed"
     out["sparse"] = {"ok": fe.exists() and any(fe.iterdir()), "missing": []}
     return out
@@ -275,6 +303,12 @@ def _cosine(q: np.ndarray, mat: dict[str, np.ndarray]) -> list[tuple[str, float]
     if not mat:
         return []
     ids = list(mat)
+    # a model swap can leave mismatched-dim blobs — skip, never crash
+    good = [i for i in ids if mat[i].shape == q.shape]
+    if len(good) < len(ids):
+        ids = good
+    if not ids:
+        return []
     m = np.stack([mat[i] for i in ids])
     m = m / (np.linalg.norm(m, axis=1, keepdims=True) + 1e-9)
     q = q / (np.linalg.norm(q) + 1e-9)
@@ -299,14 +333,22 @@ def retrieve(index: Index, query: str, embedder: LocalEmbedder,
     chunks = index.active_chunks()
     if not chunks:
         return []
+    if len(chunks) > MAX_SEARCHABLE_CHUNKS:
+        from .settings import SetupError
+        raise SetupError(
+            f"index holds {len(chunks):,} chunks — searching the whole scope "
+            "at once is unsupported. Connect a narrower scope (or re-index "
+            "with a smaller root).")
 
     # dense leg
     qvec = embedder.embed([query])[0]
     vecs = index.vectors_for([c["chunk_id"] for c in chunks])
     dense_rank = [cid for cid, _ in _cosine(qvec, vecs)][:TOP_K_RETRIEVE]
 
-    # keyword leg
-    fts_rank = [cid for cid, _ in index.fts_search(query)][:TOP_K_RETRIEVE]
+    # keyword leg — reuse the fetched set; fts_search would re-load it
+    active_ids = {c["chunk_id"] for c in chunks}
+    fts_rank = [cid for cid, _ in
+                index.fts_search(query, active=active_ids)][:TOP_K_RETRIEVE]
 
     # sparse leg (optional — learned term weights)
     sparse_rank: list[str] = []
@@ -331,6 +373,10 @@ def retrieve(index: Index, query: str, embedder: LocalEmbedder,
         order = np.argsort(-scores)[:TOP_K_EVIDENCE]
         evidence = []
         for i in order:
+            if scores[i] < MIN_RERANK_SCORE:
+                # below the floor the reranker is only ranking "least bad" —
+                # surfacing it would ground the answer on noise
+                continue
             c = dict(scored[i])
             c["rerank_score"] = float(scores[i])
             evidence.append(c)
@@ -356,7 +402,18 @@ def check_citations(evidence: list[dict], mappings: list[dict]) -> dict:
             bad.append({"claim": m.get("claim", "")[:80], "reason": "chunk not in evidence set"})
             continue
         quote = m.get("quote", "")
-        if quote and quote not in ch["text"]:
+        if quote and _norm(quote) not in _norm(ch["text"]):
             bad.append({"claim": m.get("claim", "")[:80],
                         "reason": "quote not present verbatim in cited chunk"})
     return {"ok": not bad, "violations": bad}
+
+
+def _norm(s: str) -> str:
+    """Typography-insensitive form for citation matching — docx extraction
+    leaves smart quotes/NBSP/dashes that shouldn't fail a true quote."""
+    import unicodedata
+    s = unicodedata.normalize("NFKC", s)
+    for a, b in (("‘", "'"), ("’", "'"), ("“", '"'), ("”", '"'),
+                 ("–", "-"), ("—", "-"), (" ", " "), ("−", "-")):
+        s = s.replace(a, b)
+    return " ".join(s.split())

{confluence_knowledge-0.5.1 → confluence_knowledge-0.5.2}/src/confluence_knowledge/server.py

@@ -18,7 +18,9 @@ from .client import AccessError, ConfluenceClient, SecurityException, make_clien
 from .crawl import Crawler
 from .index import Index
 from .retrieval import (
-    EMBED_MODEL_ID,
+    MODEL_MARKER as EMBED_MODEL_ID,
+)
+from .retrieval import (
     LocalEmbedder,
     LocalReranker,
     LocalSparseEmbedder,

{confluence_knowledge-0.5.1 → confluence_knowledge-0.5.2}/src/confluence_knowledge.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: confluence-knowledge
-Version: 0.5.1
+Version: 0.5.2
 Summary: Local, GET-only Confluence subtree indexer + grounded-Q&A MCP service
 License: MIT
 Project-URL: Repository, https://github.com/nkunal1992/agents

{confluence_knowledge-0.5.1 → confluence_knowledge-0.5.2}/tests/test_health_checks.py

@@ -1,10 +1,13 @@
 """health_checks flags: each failure mode → exactly one remediation."""
 
 
+from pathlib import Path
+
 import pytest
 
 from confluence_knowledge.health import health_checks
 from confluence_knowledge.index import Index
+from confluence_knowledge.retrieval import MODEL_MARKER
 from confluence_knowledge.settings import SetupError
 
 
@@ -78,7 +81,7 @@ def test_index_healthy_passes(tmp_path):
     idx_root = tmp_path / "idx"
     idx_root.mkdir()
     idx = Index(idx_root / "scope1.db")
-    idx.set_model_id("bge-m3-onnx-quantized")
+    idx.set_model_id(MODEL_MARKER)
     checks = health_checks(tmp_path / "home", tmp_path / "work", "sid", "ck",
                            index_root=idx_root)
     f = [c for c in checks if c["flag"].startswith("index:")]
@@ -118,3 +121,25 @@ def test_every_red_flag_has_a_remediation(tmp_path, monkeypatch):
     assert red  # some flags SHOULD be red on a bare tmp home
     for c in red:
         assert c["remediation"], f"red flag {c['flag']} has no remediation"
+
+
+def test_documented_flags_exist(tmp_path, monkeypatch):
+    """Every flag named in the agent's remediation table must be a flag
+    doctor can actually emit — drift silently strands users (hunt F)."""
+    import re
+    monkeypatch.setenv("CONF_KNOWLEDGE_HOME", str(tmp_path))
+    monkeypatch.setenv("CONF_KNOWLEDGE_WORKDIR", str(tmp_path / "w"))
+    idx_root = tmp_path / "idx"
+    idx_root.mkdir()
+    Index(idx_root / "scope.db")  # so index:* flags are emitted
+    checks = health_checks(tmp_path / "home", tmp_path / "w",
+                           "confluence-knowledge", "confluence-knowledge",
+                           index_root=idx_root)
+    emitted = {c["flag"].split(":")[0] for c in checks}
+    emitted.add("creds_valid")  # network flag — status() level, not here
+    agent = (Path(__file__).parents[3] / "agents"
+             / "confluence-knowledge.agent.md").read_text()
+    documented = set(re.findall(r"`(deps|model_\w+|registration|index|"
+                              r"disk_free|python|proxy|creds_valid)", agent))
+    missing = documented - emitted
+    assert not missing, f"agent documents flags doctor never emits: {missing}"

{confluence_knowledge-0.5.1 → confluence_knowledge-0.5.2}/tests/test_server_flow.py

@@ -100,7 +100,8 @@ class _FakeEmbedder:
 class _FakeReranker:
     def score(self, query, docs):
         import numpy as np
-        return np.arange(len(docs), dtype=np.float32)
+        # descending 0.9, 0.8, … — above the 0.3 evidence floor
+        return np.linspace(0.9, 0.5, len(docs), dtype=np.float32)
 
 
 def test_full_agent_flow_with_models(wired, monkeypatch):

{confluence_knowledge-0.5.1 → confluence_knowledge-0.5.2}/tests/test_sparse_retrieval.py

@@ -4,7 +4,9 @@ import numpy as np
 import pytest
 
 from confluence_knowledge import retrieval
+from confluence_knowledge.extract import Chunk
 from confluence_knowledge.index import Index
+from confluence_knowledge.retrieval import MODEL_MARKER
 
 
 class _C:
@@ -74,5 +76,55 @@ def test_sparse_leg_optional(tmp_path):
 def test_model_marker(tmp_path):
     idx = Index(tmp_path / "i.db")
     assert idx.model_id() is None           # pre-0.5 index → no marker
-    idx.set_model_id("bge-m3-onnx-quantized")
-    assert idx.model_id() == "bge-m3-onnx-quantized"
+    idx.set_model_id(MODEL_MARKER)
+    assert idx.model_id() == MODEL_MARKER
+
+
+def test_sparse_id_int64_range(tmp_path):
+    """fastembed Qdrant/bm25 emits int64 hashed ids — int32 packing would
+    silently wrap ids >2^31 and kill the sparse leg (C-4 hunt bug)."""
+    idx = Index(tmp_path / "i.db")
+    huge = 2**31 + 12345      # > int32 range — wraps negative if packed as i4
+    c = Chunk(text="word", heading_path="", kind="text", ordinal=0,
+              chunk_id="c1")
+    p = {"page_id": "p", "parent_id": None, "title": "t", "url": "u",
+         "level": 0, "version": "", "warnings": ""}
+    idx.upsert_page(p, [c], vecs=None, sparse=[{huge: 1.5}])
+    idx.publish({"p"})
+    got = idx.sparse_for(["c1"])
+    assert huge in got["c1"], "sparse id >2^31 must round-trip intact"
+    assert got["c1"][huge] == pytest.approx(1.5)
+
+
+def test_cosine_mixed_dims_skips(tmp_path):
+    """A model swap can leave mismatched-dim blobs — retrieval must skip
+    them, not crash np.stack (C-1 hunt bug)."""
+    q = np.ones(8, dtype=np.float32)
+    mat = {"ok": np.ones(8, dtype=np.float32),
+           "stale": np.ones(4, dtype=np.float32)}
+    ranked = retrieval._cosine(q, mat)
+    assert [cid for cid, _ in ranked] == ["ok"]
+
+
+def test_step_aborts_on_stale_model_mid_crawl(tmp_path):
+    """Job checkpointed under an old model must not resume mixing dims —
+    scope is wiped and the job errors so start_crawl rebuilds (C-1)."""
+    idx = Index(tmp_path / "i.db")
+    idx.set_model_id("old-model-marker")
+    # a page already indexed under the old model
+    c = Chunk(text="stale", heading_path="", kind="text", ordinal=0,
+              chunk_id="c1")
+    idx.upsert_page({"page_id": "p1", "parent_id": None, "title": "t",
+                     "url": "u", "level": 0, "version": "", "warnings": ""},
+                    [c], vecs=None, sparse=None)
+    idx.create_job("j1", "root")
+    idx.save_job("j1", "running", ["page:2", "page:3"], [])
+    idx.db.execute("INSERT OR REPLACE INTO meta VALUES (?,?)", ("seen:j1", '["p1"]')); idx.db.commit()
+
+    from confluence_knowledge.crawl import Crawler
+    cr = Crawler(client=None, index=idx, env="datacenter")
+    r = cr.step("j1", batch=None, budget_s=None)
+    assert r["state"] == "error"
+    assert "model changed" in r["errors"][0]
+    assert idx.model_id() is None            # marker wiped
+    assert idx.active_chunks() == []         # stale content cleared