confluence-knowledge 0.5.2__tar.gz → 0.5.3__tar.gz

{confluence_knowledge-0.5.2 → confluence_knowledge-0.5.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: confluence-knowledge
-Version: 0.5.2
+Version: 0.5.3
 Summary: Local, GET-only Confluence subtree indexer + grounded-Q&A MCP service
 License: MIT
 Project-URL: Repository, https://github.com/nkunal1992/agents

{confluence_knowledge-0.5.2 → confluence_knowledge-0.5.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "confluence-knowledge"
-version = "0.5.2"
+version = "0.5.3"
 description = "Local, GET-only Confluence subtree indexer + grounded-Q&A MCP service"
 readme = "README.md"
 requires-python = ">=3.11"

{confluence_knowledge-0.5.2 → confluence_knowledge-0.5.3}/src/confluence_knowledge/crawl.py

@@ -172,6 +172,9 @@ class Crawler:
 
         queue: list[str] = job["queue"]
         errors: list[str] = job["errors"]
+        # pages that fail with deny/gone errors are meant to drop out of the
+        # snapshot; transient failures keep their previously-indexed content
+        keep_failed: set[str] = set()
 
         # Mid-crawl model change: already-embedded pages live in the OLD
         # vector space — finishing this job would publish a mixed-dimension
@@ -216,6 +219,9 @@ class Crawler:
                 page = self._page(pid)
             except AccessError as e:
                 errors.append(f"page {pid}: {e}")
+                if not any(m in str(e) for m in
+                           ("401", "403", "404", "permission", "not found")):
+                    keep_failed.add(pid)
                 continue
             if pid != job["root_page"]:
                 anc = page.get("ancestors") or []
@@ -250,7 +256,7 @@ class Crawler:
             self.index.save_job(job_id, "running", queue, errors)
             state = "running"
         else:
-            self.index.publish(seen)
+            self.index.publish(seen | keep_failed)
             self.index.set_model_id(_MODEL_ID)
             self.index.save_job(job_id, "done", [], errors)
             state = "done"

{confluence_knowledge-0.5.2 → confluence_knowledge-0.5.3}/src/confluence_knowledge/extract.py

@@ -38,6 +38,28 @@ def _table_to_text(tbl) -> str:
     return "\n".join(out)
 
 
+def split_table_text(t: str) -> list[str]:
+    """An oversized table becomes several ≤MAX_WORDS chunks, each carrying
+    the header line — one unbounded chunk would exceed the embedder and the
+    reranker's token limits and silently lose its tail rows."""
+    if len(t.split()) <= MAX_WORDS:
+        return [t]
+    lines = t.split("\n")
+    header, rows = lines[0], lines[1:]
+    hw = len(header.split())
+    segs, buf, w = [], [], hw + 2
+    for r in rows:
+        rw = len(r.split())
+        if buf and w + rw > MAX_WORDS:
+            segs.append(header + "\n" + "\n".join(buf))
+            buf, w = [], hw + 2
+        buf.append(r)
+        w += rw
+    if buf:
+        segs.append(header + "\n" + "\n".join(buf))
+    return segs
+
+
 def extract_chunks(storage_html: str) -> tuple[list[Chunk], list[str]]:
     """Return (chunks, extraction warnings)."""
     soup = BeautifulSoup(storage_html, "html.parser")  # stdlib parser; no ext entities
@@ -110,8 +132,9 @@ def extract_chunks(storage_html: str) -> tuple[list[Chunk], list[str]]:
         elif el.name == "table":
             flush()
             t = _table_to_text(el)
-            if t:
-                chunks.append(Chunk(t, " > ".join(heading_path), "table", len(chunks)))
+            for seg in split_table_text(t):
+                chunks.append(Chunk(seg, " > ".join(heading_path), "table",
+                                    len(chunks)))
         else:
             buf.append(el.get_text(" ", strip=True))
     flush()

{confluence_knowledge-0.5.2 → confluence_knowledge-0.5.3}/src/confluence_knowledge/extract_files.py

@@ -12,7 +12,7 @@ import csv
 import io
 import zipfile
 
-from .extract import Chunk, extract_chunks
+from .extract import Chunk, extract_chunks, split_table_text
 from .settings import SetupError
 
 MAX_FILE_WORDS = 200_000   # ~1MB of text — beyond that, index metadata only
@@ -111,7 +111,9 @@ def _docx_chunks(data: bytes) -> list[Chunk]:
                 header, *body = rows
                 t = "\n".join([f"Table — columns: {header}"]
                               + [f"row {i}: {r}" for i, r in enumerate(body, 1)])
-                chunks.extend(_text_chunks(t, " > ".join(heading), "table"))
+                for seg in split_table_text(t):
+                    chunks.append(Chunk(seg, " > ".join(heading), "table",
+                                        len(chunks)))
     flush()
     return chunks
 
@@ -157,7 +159,9 @@ def _xlsx_chunks(data: bytes) -> list[Chunk]:
         header, *body = rows
         text = "\n".join([f"Table — columns: {header}"]
                          + [f"row {i}: {r}" for i, r in enumerate(body, 1)])
-        chunks.extend(_text_chunks(text, f"Sheet: {ws.title}", "table"))
+        for seg in split_table_text(text):
+            chunks.append(Chunk(seg, f"Sheet: {ws.title}", "table",
+                                len(chunks)))
     wb.close()
     return chunks
 
@@ -191,7 +195,8 @@ def _csv_chunks(data: bytes) -> list[Chunk]:
     header, *body = rows
     out = "\n".join([f"Table — columns: {' | '.join(header)}"]
                     + [f"row {i}: {' | '.join(r)}" for i, r in enumerate(body, 1)])
-    return _text_chunks(out, "", "table")
+    return [Chunk(seg, "", "table", i)
+            for i, seg in enumerate(split_table_text(out))]
 
 
 def extract_file_chunks(filename: str, data: bytes) -> tuple[list[Chunk], list[str]]:

{confluence_knowledge-0.5.2 → confluence_knowledge-0.5.3}/src/confluence_knowledge/index.py

@@ -195,7 +195,9 @@ class Index:
                             (model_id,))
 
     def publish(self, seen_page_ids: set[str]) -> None:
-        """Atomically make exactly the crawled page set the active snapshot."""
+        """Atomically make exactly the crawled page set the active snapshot.
+        Content belonging to pages no longer seen is DELETED — inactive rows
+        are unreachable forever anyway and would otherwise bloat every table."""
         with self.lock, self.db:
             self.db.execute("UPDATE pages SET active=0")
             self.db.execute("UPDATE chunks SET active=0")
@@ -204,6 +206,20 @@ class Index:
                 ids = list(seen_page_ids)
                 self.db.execute(f"UPDATE pages SET active=1 WHERE page_id IN ({q})", ids)
                 self.db.execute(f"UPDATE chunks SET active=1 WHERE page_id IN ({q})", ids)
+            # gc: drop chunks (and their retrieval rows) of inactive pages,
+            # then the inactive page rows themselves
+            dead_chunks = [r[0] for r in self.db.execute(
+                "SELECT chunk_id FROM chunks WHERE page_id IN "
+                "(SELECT page_id FROM pages WHERE active=0)")]
+            if dead_chunks:
+                q = ",".join("?" * len(dead_chunks))
+                for t in ("chunks_fts", "vectors", "sparse_vectors"):
+                    self.db.execute(
+                        f"DELETE FROM {t} WHERE chunk_id IN ({q})",
+                        dead_chunks)
+                self.db.execute(f"DELETE FROM chunks WHERE chunk_id IN ({q})",
+                                dead_chunks)
+            self.db.execute("DELETE FROM pages WHERE active=0")
 
     # ---- query ----
     def active_pages(self) -> dict[str, dict]:

{confluence_knowledge-0.5.2 → confluence_knowledge-0.5.3}/src/confluence_knowledge/retrieval.py

@@ -39,7 +39,8 @@ MODEL_MARKER = f"{EMBED_MODEL_ID}+{SPARSE_MODEL_ID}"
 TOP_K_RETRIEVE = 30
 TOP_K_EVIDENCE = 8
 RERANK_MAX = 25          # bounds cross-encoder latency on CPU
-MIN_RERANK_SCORE = 0.3   # sigmoid floor — below this a hit is "least bad"
+MIN_RERANK_SCORE = 0.02  # noise floor — real reranker zeros junk at ~0.00
+LOW_CONFIDENCE_SCORE = 0.3  # below this the hit is surfaced but flagged
 MAX_SEARCHABLE_CHUNKS = 100_000  # whole-index search is O(n) in RAM
 EMBED_MAX_TOKENS = 512   # quantized model input cap (chunks stay under)
 RERANK_MAX_TOKENS = 512
@@ -374,11 +375,14 @@ def retrieve(index: Index, query: str, embedder: LocalEmbedder,
         evidence = []
         for i in order:
             if scores[i] < MIN_RERANK_SCORE:
-                # below the floor the reranker is only ranking "least bad" —
-                # surfacing it would ground the answer on noise
+                # below the noise floor the reranker is only ranking "least
+                # bad" — surfacing it would ground the answer on junk
                 continue
             c = dict(scored[i])
             c["rerank_score"] = float(scores[i])
+            # borderline matches are surfaced but flagged — the agent can
+            # caveat instead of answering on silently-weak evidence
+            c["low_confidence"] = bool(scores[i] < LOW_CONFIDENCE_SCORE)
             evidence.append(c)
     else:
         evidence = [dict(c, rerank_score=None) for c in candidates[:TOP_K_EVIDENCE]]

{confluence_knowledge-0.5.2 → confluence_knowledge-0.5.3}/src/confluence_knowledge/server.py

@@ -40,7 +40,7 @@ mcp = FastMCP("confluence-knowledge")
 
 # in-process caches (per server lifetime)
 _scopes: dict[str, dict] = {}
-_evidence_sets: dict[str, list[dict]] = {}
+_evidence_sets: dict[str, dict] = {}  # set_id → {scope, ev}
 _indexes: dict[str, Index] = {}  # one connection per DB for the server lifetime
 _clients: dict[str, ConfluenceClient] = {}  # one HTTP pool per credential
 _EMBEDDER = None
@@ -333,7 +333,10 @@ def search(scope_handle: str, question: str) -> dict:
                         "— run start_crawl once to rebuild it, then search again"}
     ev = retrieve(idx, question, _embedder(), _reranker(), sparse=_sparse())
     set_id = f"ev:{len(_evidence_sets)}:{s['key'][:8]}"
-    _evidence_sets[set_id] = ev
+    # bound the session cache — sets are never reused after the turn ends
+    while len(_evidence_sets) >= 32:
+        _evidence_sets.pop(next(iter(_evidence_sets)))
+    _evidence_sets[set_id] = {"scope": s["key"], "ev": ev}
     return {
         "evidence_set": set_id,
         "coverage": _coverage(idx),
@@ -341,7 +344,9 @@ def search(scope_handle: str, question: str) -> dict:
         "evidence": [
             {"chunk_id": e["chunk_id"], "page_title": e["page_title"],
              "page_url": e["page_url"], "heading": e["heading"],
-             "score": e.get("rerank_score"), "text": e["text"][:2000]}
+             "score": e.get("rerank_score"), "text": e["text"][:12000],
+             "low_confidence": e.get("low_confidence", False),
+             "truncated": len(e["text"]) > 12000}
             for e in ev
         ],
     }
@@ -367,12 +372,13 @@ def get_evidence(scope_handle: str, chunk_ids: list[str]) -> dict:
 @mcp.tool()
 def verify_citations(scope_handle: str, evidence_set: str,
                      mappings: list[dict]) -> dict:
-    _scope(scope_handle)
-    ev = _evidence_sets.get(evidence_set)
-    if ev is None or not evidence_set.endswith(_scopes[scope_handle]["key"][:8]):
+    s = _scope(scope_handle)  # binds or SetupError — never KeyError below
+    rec = _evidence_sets.get(evidence_set)
+    # full 24-char scope key — the key[:8] suffix in the id is cosmetic
+    if rec is None or rec["scope"] != s["key"]:
         raise SecurityException(
             "unknown or cross-scope evidence set — run search again and cite its evidence_set")
-    return check_citations(ev, mappings)
+    return check_citations(rec["ev"], mappings)
 
 
 def main() -> None:

{confluence_knowledge-0.5.2 → confluence_knowledge-0.5.3}/src/confluence_knowledge.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: confluence-knowledge
-Version: 0.5.2
+Version: 0.5.3
 Summary: Local, GET-only Confluence subtree indexer + grounded-Q&A MCP service
 License: MIT
 Project-URL: Repository, https://github.com/nkunal1992/agents