local-knowledge 0.1.0__tar.gz → 0.2.0__tar.gz

{local_knowledge-0.1.0 → local_knowledge-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: local-knowledge
-Version: 0.1.0
+Version: 0.2.0
 Summary: Local file/folder indexer + grounded-Q&A MCP service — same engine as confluence-knowledge, no credentials needed
 License: MIT
 Project-URL: Repository, https://github.com/nkunal1992/agents
@@ -12,7 +12,7 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
-Requires-Dist: confluence-knowledge>=0.4.0
+Requires-Dist: confluence-knowledge>=0.5.0
 Requires-Dist: python-docx>=1.1
 Requires-Dist: pypdf>=4.0
 Requires-Dist: python-pptx>=1.0
@@ -20,7 +20,7 @@ Requires-Dist: openpyxl>=3.1
 Requires-Dist: mcp<2,>=1.6
 Requires-Dist: pip-system-certs>=5.0
 Provides-Extra: models
-Requires-Dist: confluence-knowledge[models]>=0.4.0; extra == "models"
+Requires-Dist: confluence-knowledge[models]>=0.5.0; extra == "models"
 Provides-Extra: dev
 Requires-Dist: pytest>=8.0; extra == "dev"
 Requires-Dist: pytest-cov>=5.0; extra == "dev"

{local_knowledge-0.1.0 → local_knowledge-0.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "local-knowledge"
-version = "0.1.0"
+version = "0.2.0"
 description = "Local file/folder indexer + grounded-Q&A MCP service — same engine as confluence-knowledge, no credentials needed"
 readme = "README.md"
 requires-python = ">=3.11"
@@ -16,7 +16,7 @@ classifiers = [
     "Programming Language :: Python :: 3 :: Only",
 ]
 dependencies = [
-    "confluence-knowledge>=0.4.0",
+    "confluence-knowledge>=0.5.0",
     "python-docx>=1.1",
     "pypdf>=4.0",
     "python-pptx>=1.0",
@@ -27,7 +27,7 @@ dependencies = [
 
 [project.optional-dependencies]
 models = [
-    "confluence-knowledge[models]>=0.4.0",
+    "confluence-knowledge[models]>=0.5.0",
 ]
 dev = [
     "pytest>=8.0",

{local_knowledge-0.1.0 → local_knowledge-0.2.0}/src/local_knowledge/cli_local.py

@@ -25,9 +25,22 @@ WORKDIR_ENV = "LOCAL_KNOWLEDGE_WORKDIR"
 
 
 def _doctor() -> int:
+    from confluence_knowledge.update import check
+
     from .server_local import status
     s = status()
     print(json.dumps(s, indent=2))
+    upd = check("local-knowledge")
+    if upd["installed"]:
+        line = f"version: {upd['installed']}"
+        if upd["latest"]:
+            line += f" (latest: {upd['latest']})"
+        elif upd["latest"] is None:
+            line += " (latest: unchecked — offline or internal mirror)"
+        print(line)
+    if upd["update_available"]:
+        print("update available: run `local-knowledge update` "
+              "(or pip install -U local-knowledge[models])")
     return 0
 
 
@@ -36,6 +49,7 @@ def main() -> None:
     sub = ap.add_subparsers(dest="cmd", required=True)
     sub.add_parser("doctor", help="verify install + models")
     sub.add_parser("prepare-models", help="download the pinned local models")
+    sub.add_parser("update", help="upgrade this package via pip -U")
     reg = sub.add_parser(
         "register",
         help="write this server into the IDE's MCP config (no manual paste)")
@@ -54,6 +68,22 @@ def main() -> None:
         for line in prepare_models(home()):
             print(line)
         return
+    if args.cmd == "update":
+        import subprocess
+        cmd = [sys.executable, "-m", "pip", "install", "-U",
+               "local-knowledge[models]"]
+        if os.name == "nt":
+            # this exe is the running process on Windows — pip can't replace
+            # a file that's in use; hand the user the equivalent command
+            print("on Windows, upgrade via the venv python directly:\n  "
+                  + " ".join(f'"{c}"' if " " in c else c for c in cmd))
+            return
+        r = subprocess.run(cmd, check=False)
+        if r.returncode == 0:
+            print("\nupdated — run `local-knowledge prepare-models` "
+                  "(new releases may pin new model artifacts) then "
+                  "`local-knowledge doctor`")
+        sys.exit(r.returncode)
     if args.cmd == "register":
         from confluence_knowledge.registration import register
         try:

{local_knowledge-0.1.0 → local_knowledge-0.2.0}/src/local_knowledge/crawl_local.py

@@ -32,11 +32,15 @@ def resolve_root(path: str) -> Path:
     return p.resolve()
 
 
+from confluence_knowledge.retrieval import EMBED_MODEL_ID as _MODEL_ID
+
+
 class LocalCrawler:
-    def __init__(self, index: Index, root: Path, embedder=None):
+    def __init__(self, index: Index, root: Path, embedder=None, sparse=None):
         self.index = index
         self.root = root      # realpath — everything indexed must live under it
         self.embedder = embedder
+        self.sparse = sparse
 
     def start(self) -> str:
         job_id = uuid.uuid4().hex[:12]
@@ -176,21 +180,24 @@ class LocalCrawler:
                 continue
             for c in chunks:
                 c.chunk_id = f"{rel}:{page['content_hash'][:8]}:{c.ordinal}"
-            vecs = None
+            vecs, sparse_vecs = None, None
             if self.embedder is not None and chunks:
-                vecs = self.embedder.embed(
-                    [f"{c.heading_path}\n{c.text}" for c in chunks])
+                texts = [f"{c.heading_path}\n{c.text}" for c in chunks]
+                vecs = self.embedder.embed(texts)
+                if self.sparse is not None:
+                    sparse_vecs = self.sparse.embed(texts)
             self.index.upsert_page({
                 "page_id": rel, "parent_id": None, "title": page["title"],
                 "url": page["url"], "level": levels.get(rel, path_s.count("/")),
                 "version": page["version"], "content_hash": page["content_hash"],
-                "warnings": page["warnings"]}, chunks, vecs)
+                "warnings": page["warnings"]}, chunks, vecs, sparse_vecs)
             seen.add(rel)
 
         # persist final seen/levels/warns too — the in-loop checkpoint runs
         # before each item is processed, so the last item would undercount
         self._checkpoint(job_id, queue, errors, seen, levels, warns)
         self.index.publish(seen)
+        self.index.set_model_id(_MODEL_ID)
         self.index.save_job(job_id, "done" if not queue else "running",
                             queue, errors)
         return {"job_id": job_id,

{local_knowledge-0.1.0 → local_knowledge-0.2.0}/src/local_knowledge/server_local.py

@@ -14,8 +14,10 @@ import threading
 from confluence_knowledge.client import AccessError, SecurityException
 from confluence_knowledge.index import Index
 from confluence_knowledge.retrieval import (
+    EMBED_MODEL_ID,
     LocalEmbedder,
     LocalReranker,
+    LocalSparseEmbedder,
     check_citations,
     retrieve,
 )
@@ -32,6 +34,7 @@ _evidence_sets: dict[str, list[dict]] = {}
 _indexes: dict[str, Index] = {}
 _EMBEDDER = None
 _RERANKER = None
+_SPARSE = None
 _model_lock = threading.Lock()
 _models_ready = threading.Event()
 _warm_lock = threading.Lock()
@@ -63,10 +66,19 @@ def _reranker():
     return _RERANKER
 
 
+def _sparse():
+    global _SPARSE
+    with _model_lock:
+        if _SPARSE is None:
+            _SPARSE = LocalSparseEmbedder(home(), APP_NAME)
+    return _SPARSE
+
+
 def _warm_models() -> None:
     try:
         _embedder()
         _reranker()
+        _sparse()
     except SetupError:
         pass
     finally:
@@ -110,8 +122,10 @@ def status() -> dict:
     return {
         "configured": "error" not in cfg,
         "profile": cfg,
-        "embedding_model_ready": (models / "all-MiniLM-L6-v2").exists(),
-        "reranker_model_ready": (models / "ms-marco-MiniLM-L6-v2").exists(),
+        "embedding_model_ready": (models / "bge-m3").exists(),
+        "reranker_model_ready": (models / "bge-reranker-v2-m3-ONNX").exists()
+            or (models / "bge-reranker-base-ONNX").exists(),
+        "sparse_model_ready": (models / "fastembed").exists(),
         "models_loaded": _models_ready.is_set() and _EMBEDDER is not None,
         "index_root": str(index_root()),
         "live_scopes": sorted(_scopes),
@@ -131,6 +145,7 @@ def connect(root: str) -> dict:
     handle = f"scope:{scope_key}:{path}"
     _scopes[handle] = {"key": scope_key, "root": path}
     _start_warmup()
+    stale = idx.model_id() is not None and idx.model_id() != EMBED_MODEL_ID
     return {
         "scope_handle": handle,
         "root": str(path),
@@ -138,6 +153,9 @@ def connect(root: str) -> dict:
         "cached": st["pages"] > 0,
         "cached_pages": st["pages"],
         "cached_as_of": st["as_of"],
+        "stale_model": stale,
+        "note": ("index was built with an older model — run start_crawl once "
+                 "to rebuild it with the current models" if stale else None),
     }
 
 
@@ -150,7 +168,8 @@ def _worker_loop(s: dict, job_id: str) -> None:
     idx = _index_for(s["key"])
     try:
         _models_ready.wait()
-        c = LocalCrawler(idx, s["root"], embedder=_embedder())
+        c = LocalCrawler(idx, s["root"], embedder=_embedder(),
+                         sparse=_sparse())
         r = c.step(job_id)
         while r["state"] == "running":
             r = c.step(job_id)
@@ -244,7 +263,11 @@ def search(scope_handle: str, question: str) -> dict:
                 "note": "no files indexed — run a crawl first"}
     if not _models_loaded(10.0):
         return {"evidence": [], "coverage": "loading", "note": LOADING_NOTE}
-    ev = retrieve(idx, question, _embedder(), _reranker())
+    if idx.model_id() is not None and idx.model_id() != EMBED_MODEL_ID:
+        return {"evidence": [], "coverage": "stale_model",
+                "note": "this index was built with an older embedding model "
+                        "— run start_crawl once to rebuild it, then search again"}
+    ev = retrieve(idx, question, _embedder(), _reranker(), sparse=_sparse())
     set_id = f"ev:{len(_evidence_sets)}:{s['key'][:8]}"
     _evidence_sets[set_id] = ev
     return {

{local_knowledge-0.1.0 → local_knowledge-0.2.0}/src/local_knowledge.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: local-knowledge
-Version: 0.1.0
+Version: 0.2.0
 Summary: Local file/folder indexer + grounded-Q&A MCP service — same engine as confluence-knowledge, no credentials needed
 License: MIT
 Project-URL: Repository, https://github.com/nkunal1992/agents
@@ -12,7 +12,7 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
-Requires-Dist: confluence-knowledge>=0.4.0
+Requires-Dist: confluence-knowledge>=0.5.0
 Requires-Dist: python-docx>=1.1
 Requires-Dist: pypdf>=4.0
 Requires-Dist: python-pptx>=1.0
@@ -20,7 +20,7 @@ Requires-Dist: openpyxl>=3.1
 Requires-Dist: mcp<2,>=1.6
 Requires-Dist: pip-system-certs>=5.0
 Provides-Extra: models
-Requires-Dist: confluence-knowledge[models]>=0.4.0; extra == "models"
+Requires-Dist: confluence-knowledge[models]>=0.5.0; extra == "models"
 Provides-Extra: dev
 Requires-Dist: pytest>=8.0; extra == "dev"
 Requires-Dist: pytest-cov>=5.0; extra == "dev"

{local_knowledge-0.1.0 → local_knowledge-0.2.0}/src/local_knowledge.egg-info/requires.txt

@@ -1,4 +1,4 @@
-confluence-knowledge>=0.4.0
+confluence-knowledge>=0.5.0
 python-docx>=1.1
 pypdf>=4.0
 python-pptx>=1.0
@@ -12,4 +12,4 @@ pytest-cov>=5.0
 ruff>=0.6
 
 [models]
-confluence-knowledge[models]>=0.4.0
+confluence-knowledge[models]>=0.5.0

{local_knowledge-0.1.0 → local_knowledge-0.2.0}/tests/test_server_flow_local.py

@@ -24,12 +24,18 @@ class FakeReranker:
         return np.arange(len(docs), dtype=np.float32)[::-1]
 
 
+class FakeSparse:
+    def embed(self, texts):
+        return [{} for _ in texts]
+
+
 @pytest.fixture(autouse=True)
 def isolated(tmp_path, monkeypatch):
     monkeypatch.setenv("LOCAL_KNOWLEDGE_HOME", str(tmp_path / "home"))
     monkeypatch.setenv("LOCAL_KNOWLEDGE_WORKDIR", str(tmp_path / "work"))
     monkeypatch.setattr(server_local, "_EMBEDDER", FakeEmbedder())
     monkeypatch.setattr(server_local, "_RERANKER", FakeReranker())
+    monkeypatch.setattr(server_local, "_SPARSE", FakeSparse())
     server_local._models_ready.set()
     yield
     server_local._scopes.clear()