sharepoint-knowledge 0.1.0__tar.gz → 0.2.0__tar.gz

{sharepoint_knowledge-0.1.0 → sharepoint_knowledge-0.2.0}/PKG-INFO

@@ -1,17 +1,17 @@
 Metadata-Version: 2.4
 Name: sharepoint-knowledge
-Version: 0.1.0
+Version: 0.2.0
 Summary: SharePoint connector for the grounded-Q&A engine — site pages + document libraries, Entra device-code sign-in, local models, cited answers
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/nkunal1992/agents
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
-Requires-Dist: confluence-knowledge[files]>=0.4.1
+Requires-Dist: confluence-knowledge[files]>=0.5.0
 Requires-Dist: msal>=1.31
 Requires-Dist: httpx>=0.27
 Requires-Dist: mcp>=1.8
 Provides-Extra: models
-Requires-Dist: confluence-knowledge[models]>=0.4.1; extra == "models"
+Requires-Dist: confluence-knowledge[models]>=0.5.0; extra == "models"
 Provides-Extra: dev
 Requires-Dist: pytest>=8.0; extra == "dev"
 Requires-Dist: ruff>=0.5; extra == "dev"

{sharepoint_knowledge-0.1.0 → sharepoint_knowledge-0.2.0}/pyproject.toml

@@ -4,13 +4,13 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "sharepoint-knowledge"
-version = "0.1.0"
+version = "0.2.0"
 description = "SharePoint connector for the grounded-Q&A engine — site pages + document libraries, Entra device-code sign-in, local models, cited answers"
 readme = "README.md"
 requires-python = ">=3.11"
 license = {text = "Apache-2.0"}
 dependencies = [
-    "confluence-knowledge[files]>=0.4.1",
+    "confluence-knowledge[files]>=0.5.0",
     "msal>=1.31",
     "httpx>=0.27",
     "mcp>=1.8",
@@ -18,7 +18,7 @@ dependencies = [
 
 [project.optional-dependencies]
 models = [
-    "confluence-knowledge[models]>=0.4.1",
+    "confluence-knowledge[models]>=0.5.0",
 ]
 dev = [
     "pytest>=8.0",

{sharepoint_knowledge-0.1.0 → sharepoint_knowledge-0.2.0}/src/sharepoint_knowledge/cli_sp.py

@@ -84,7 +84,7 @@ def _setup() -> int:
     ensure_gitignore(workdir(), [AUTH_FILE_NAME,
                                  f"{INDEX_DIR_NAME}/"])
 
-    if input("\nDownload the two local models now (~175 MB, one-time)? "
+    if input("\nDownload the local models now (~1.3 GB, one-time)? "
              "[Y/n] ").strip().lower() not in ("n", "no"):
         for line in prepare_models(home()):
             print(line)
@@ -152,8 +152,9 @@ def main() -> None:
             return
         r = subprocess.run(cmd, check=False)
         if r.returncode == 0:
-            print("\nupdated — models are pinned and unchanged; "
-                  "run `sharepoint-knowledge doctor` to confirm")
+            print("\nupdated — run `sharepoint-knowledge prepare-models` "
+                  "(new releases may pin new model artifacts) then "
+                  "`sharepoint-knowledge doctor`")
         sys.exit(r.returncode)
     if args.cmd == "register":
         results = _register(

{sharepoint_knowledge-0.1.0 → sharepoint_knowledge-0.2.0}/src/sharepoint_knowledge/crawl_sp.py

@@ -23,6 +23,8 @@ from confluence_knowledge.settings import SetupError
 MAX_FILE_BYTES = 25 * 1024 * 1024
 MAX_FILES = 20_000
 
+from confluence_knowledge.retrieval import EMBED_MODEL_ID as _MODEL_ID
+
 
 def resolve_root(client: ConfluenceClient, url: str) -> tuple[str, str]:
     """SharePoint URL → (root queue entry, site webUrl for citations).
@@ -77,10 +79,11 @@ def _paged(client: ConfluenceClient, path: str, params: dict | None = None):
 
 class SPCrawler:
     def __init__(self, client: ConfluenceClient, index: Index, embedder=None,
-                 downloader=None):
+                 downloader=None, sparse=None):
         self.client = client
         self.index = index
         self.embedder = embedder
+        self.sparse = sparse
         self.downloader = downloader  # injected for tests; real one in server
 
     # ---- expansions ----
@@ -255,20 +258,23 @@ class SPCrawler:
                 continue
             for c in chunks:
                 c.chunk_id = f"{pid}:{page['content_hash'][:8]}:{c.ordinal}"
-            vecs = None
+            vecs, sparse_vecs = None, None
             if self.embedder is not None and chunks:
-                vecs = self.embedder.embed(
-                    [f"{c.heading_path}\n{c.text}" for c in chunks])
+                texts = [f"{c.heading_path}\n{c.text}" for c in chunks]
+                vecs = self.embedder.embed(texts)
+                if self.sparse is not None:
+                    sparse_vecs = self.sparse.embed(texts)
             self.index.upsert_page({
                 "page_id": pid, "parent_id": None, "title": page["title"],
                 "url": page["url"], "level": levels.get(pid, 0),
                 "version": page["version"],
                 "content_hash": page["content_hash"],
-                "warnings": page["warnings"]}, chunks, vecs)
+                "warnings": page["warnings"]}, chunks, vecs, sparse_vecs)
             seen.add(pid)
 
         self._checkpoint(job_id, queue, errors, seen, levels, warns)
         self.index.publish(seen)
+        self.index.set_model_id(_MODEL_ID)
         self.index.save_job(job_id, "done" if not queue else "running",
                             queue, errors)
         return {"job_id": job_id,

{sharepoint_knowledge-0.1.0 → sharepoint_knowledge-0.2.0}/src/sharepoint_knowledge/server_sp.py

@@ -10,8 +10,10 @@ import threading
 from confluence_knowledge.client import AccessError, SecurityException
 from confluence_knowledge.index import Index
 from confluence_knowledge.retrieval import (
+    EMBED_MODEL_ID,
     LocalEmbedder,
     LocalReranker,
+    LocalSparseEmbedder,
     check_citations,
     retrieve,
 )
@@ -31,6 +33,7 @@ _indexes: dict[str, Index] = {}
 _clients: dict[str, object] = {}
 _EMBEDDER = None
 _RERANKER = None
+_SPARSE = None
 _model_lock = threading.Lock()
 _models_ready = threading.Event()
 _warm_lock = threading.Lock()
@@ -62,10 +65,19 @@ def _reranker():
     return _RERANKER
 
 
+def _sparse():
+    global _SPARSE
+    with _model_lock:
+        if _SPARSE is None:
+            _SPARSE = LocalSparseEmbedder(home(), APP_NAME)
+    return _SPARSE
+
+
 def _warm_models() -> None:
     try:
         _embedder()
         _reranker()
+        _sparse()
     except SetupError:
         pass
     finally:
@@ -140,8 +152,10 @@ def status() -> dict:
     return {
         "configured": configured,
         "profile": cfg,
-        "embedding_model_ready": (models / "all-MiniLM-L6-v2").exists(),
-        "reranker_model_ready": (models / "ms-marco-MiniLM-L6-v2").exists(),
+        "embedding_model_ready": (models / "bge-m3").exists(),
+        "reranker_model_ready": (models / "bge-reranker-v2-m3-ONNX").exists()
+            or (models / "bge-reranker-base-ONNX").exists(),
+        "sparse_model_ready": (models / "fastembed").exists(),
         "models_loaded": _models_ready.is_set() and _EMBEDDER is not None,
         "index_root": str(index_root()),
         "live_scopes": sorted(_scopes),
@@ -166,6 +180,7 @@ def connect(root: str) -> dict:
     _scopes[handle] = {"key": scope_key, "root_entry": root_entry,
                        "site_web": site_web, "client": cli}
     _start_warmup()
+    stale = idx.model_id() is not None and idx.model_id() != EMBED_MODEL_ID
     return {
         "scope_handle": handle,
         "root": root_entry,
@@ -173,6 +188,9 @@ def connect(root: str) -> dict:
         "cached": st["pages"] > 0,
         "cached_pages": st["pages"],
         "cached_as_of": st["as_of"],
+        "stale_model": stale,
+        "note": ("index was built with an older model — run start_crawl once "
+                 "to rebuild it with the current models" if stale else None),
     }
 
 
@@ -186,7 +204,7 @@ def _worker_loop(s: dict, job_id: str) -> None:
     try:
         _models_ready.wait()
         c = SPCrawler(s["client"], idx, embedder=_embedder(),
-                      downloader=download_file)
+                      downloader=download_file, sparse=_sparse())
         r = c.step(job_id)
         while r["state"] == "running":
             r = c.step(job_id)
@@ -280,7 +298,11 @@ def search(scope_handle: str, question: str) -> dict:
                 "note": "nothing indexed — run a crawl first"}
     if not _models_loaded(10.0):
         return {"evidence": [], "coverage": "loading", "note": LOADING_NOTE}
-    ev = retrieve(idx, question, _embedder(), _reranker())
+    if idx.model_id() is not None and idx.model_id() != EMBED_MODEL_ID:
+        return {"evidence": [], "coverage": "stale_model",
+                "note": "this index was built with an older embedding model "
+                        "— run start_crawl once to rebuild it, then search again"}
+    ev = retrieve(idx, question, _embedder(), _reranker(), sparse=_sparse())
     set_id = f"ev:{len(_evidence_sets)}:{s['key'][:8]}"
     _evidence_sets[set_id] = ev
     return {

{sharepoint_knowledge-0.1.0 → sharepoint_knowledge-0.2.0}/src/sharepoint_knowledge.egg-info/PKG-INFO

@@ -1,17 +1,17 @@
 Metadata-Version: 2.4
 Name: sharepoint-knowledge
-Version: 0.1.0
+Version: 0.2.0
 Summary: SharePoint connector for the grounded-Q&A engine — site pages + document libraries, Entra device-code sign-in, local models, cited answers
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/nkunal1992/agents
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
-Requires-Dist: confluence-knowledge[files]>=0.4.1
+Requires-Dist: confluence-knowledge[files]>=0.5.0
 Requires-Dist: msal>=1.31
 Requires-Dist: httpx>=0.27
 Requires-Dist: mcp>=1.8
 Provides-Extra: models
-Requires-Dist: confluence-knowledge[models]>=0.4.1; extra == "models"
+Requires-Dist: confluence-knowledge[models]>=0.5.0; extra == "models"
 Provides-Extra: dev
 Requires-Dist: pytest>=8.0; extra == "dev"
 Requires-Dist: ruff>=0.5; extra == "dev"

{sharepoint_knowledge-0.1.0 → sharepoint_knowledge-0.2.0}/src/sharepoint_knowledge.egg-info/requires.txt

@@ -1,4 +1,4 @@
-confluence-knowledge[files]>=0.4.1
+confluence-knowledge[files]>=0.5.0
 msal>=1.31
 httpx>=0.27
 mcp>=1.8
@@ -12,4 +12,4 @@ openpyxl>=3.1
 pypdf>=5.0
 
 [models]
-confluence-knowledge[models]>=0.4.1
+confluence-knowledge[models]>=0.5.0

{sharepoint_knowledge-0.1.0 → sharepoint_knowledge-0.2.0}/tests/test_client_sp.py

@@ -3,7 +3,6 @@ from __future__ import annotations
 
 import pytest
 from confluence_knowledge.client import SecurityException
-
 from sharepoint_knowledge.client_sp import GRAPH_RULES, download_file, make_client
 
 

{sharepoint_knowledge-0.1.0 → sharepoint_knowledge-0.2.0}/tests/test_crawl_sp.py

@@ -11,7 +11,6 @@ import pytest
 sys.path.insert(0, str(Path(__file__).parent))
 from _fake_graph import BLOBS, DRIVE_ID, LIST_ID, SITE_ID, FakeGraph, site_routes
 from confluence_knowledge.index import Index
-
 from sharepoint_knowledge.crawl_sp import SPCrawler
 
 

{sharepoint_knowledge-0.1.0 → sharepoint_knowledge-0.2.0}/tests/test_server_flow_sp.py

@@ -25,6 +25,11 @@ class FakeReranker:
         return np.arange(len(docs), dtype=np.float32)[::-1]
 
 
+class FakeSparse:
+    def embed(self, texts):
+        return [{} for _ in texts]
+
+
 @pytest.fixture()
 def sp_env(tmp_path, monkeypatch):
     monkeypatch.setenv("SP_KNOWLEDGE_HOME", str(tmp_path / "home"))
@@ -40,6 +45,7 @@ def sp_env(tmp_path, monkeypatch):
 
     server_sp._EMBEDDER = FakeEmbedder()
     server_sp._RERANKER = FakeReranker()
+    server_sp._SPARSE = FakeSparse()
     server_sp._models_ready.set()
 
     # the real client goes through httpx — replace it with the fake