local-knowledge 0.1.0__tar.gz

local_knowledge-0.1.0/PKG-INFO

@@ -0,0 +1,45 @@
+Metadata-Version: 2.4
+Name: local-knowledge
+Version: 0.1.0
+Summary: Local file/folder indexer + grounded-Q&A MCP service — same engine as confluence-knowledge, no credentials needed
+License: MIT
+Project-URL: Repository, https://github.com/nkunal1992/agents
+Project-URL: Documentation, https://github.com/nkunal1992/agents/blob/main/docs/local-knowledge.md
+Project-URL: Issues, https://github.com/nkunal1992/agents/issues
+Classifier: Operating System :: OS Independent
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3 :: Only
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: confluence-knowledge>=0.4.0
+Requires-Dist: python-docx>=1.1
+Requires-Dist: pypdf>=4.0
+Requires-Dist: python-pptx>=1.0
+Requires-Dist: openpyxl>=3.1
+Requires-Dist: mcp<2,>=1.6
+Requires-Dist: pip-system-certs>=5.0
+Provides-Extra: models
+Requires-Dist: confluence-knowledge[models]>=0.4.0; extra == "models"
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: pytest-cov>=5.0; extra == "dev"
+Requires-Dist: ruff>=0.6; extra == "dev"
+
+# local-knowledge
+
+Local file/folder indexer + grounded-Q&A MCP service — the same engine as
+[`confluence-knowledge`](../confluence-knowledge), pointed at the filesystem
+instead of Confluence. No credentials, no HTTP.
+
+- `local_knowledge/crawl_local.py` — durable filesystem BFS (secret-file
+  exclusions, symlink-escape guards, checkpointed resume)
+- `local_knowledge/server_local.py` — the `local-knowledge/*` MCP tools
+- `local_knowledge/cli_local.py` — `local-knowledge` CLI (no `setup`; there's
+  nothing to authenticate)
+
+Shares `confluence_knowledge`'s index, extraction (`extract` +
+`extract_files`), retrieval, registration and auth plumbing.
+
+See `docs/local-knowledge.md` in the repo root for the full guide and
+`bundle/local-README.md` for the store-listing copy.

local_knowledge-0.1.0/README.md

@@ -0,0 +1,17 @@
+# local-knowledge
+
+Local file/folder indexer + grounded-Q&A MCP service — the same engine as
+[`confluence-knowledge`](../confluence-knowledge), pointed at the filesystem
+instead of Confluence. No credentials, no HTTP.
+
+- `local_knowledge/crawl_local.py` — durable filesystem BFS (secret-file
+  exclusions, symlink-escape guards, checkpointed resume)
+- `local_knowledge/server_local.py` — the `local-knowledge/*` MCP tools
+- `local_knowledge/cli_local.py` — `local-knowledge` CLI (no `setup`; there's
+  nothing to authenticate)
+
+Shares `confluence_knowledge`'s index, extraction (`extract` +
+`extract_files`), retrieval, registration and auth plumbing.
+
+See `docs/local-knowledge.md` in the repo root for the full guide and
+`bundle/local-README.md` for the store-listing copy.

local_knowledge-0.1.0/pyproject.toml

@@ -0,0 +1,53 @@
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "local-knowledge"
+version = "0.1.0"
+description = "Local file/folder indexer + grounded-Q&A MCP service — same engine as confluence-knowledge, no credentials needed"
+readme = "README.md"
+requires-python = ">=3.11"
+license = { text = "MIT" }
+classifiers = [
+    "Operating System :: OS Independent",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3 :: Only",
+]
+dependencies = [
+    "confluence-knowledge>=0.4.0",
+    "python-docx>=1.1",
+    "pypdf>=4.0",
+    "python-pptx>=1.0",
+    "openpyxl>=3.1",
+    "mcp>=1.6,<2",
+    "pip-system-certs>=5.0",
+]
+
+[project.optional-dependencies]
+models = [
+    "confluence-knowledge[models]>=0.4.0",
+]
+dev = [
+    "pytest>=8.0",
+    "pytest-cov>=5.0",
+    "ruff>=0.6",
+]
+
+[project.urls]
+Repository = "https://github.com/nkunal1992/agents"
+Documentation = "https://github.com/nkunal1992/agents/blob/main/docs/local-knowledge.md"
+Issues = "https://github.com/nkunal1992/agents/issues"
+
+[project.scripts]
+local-knowledge = "local_knowledge.cli_local:main"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+[tool.ruff]
+line-length = 110

local_knowledge-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

local_knowledge-0.1.0/src/local_knowledge/__init__.py

@@ -0,0 +1 @@
+"""local-knowledge — local file/folder indexing + grounded Q&A MCP service."""

local_knowledge-0.1.0/src/local_knowledge/cli_local.py

@@ -0,0 +1,92 @@
+"""local-knowledge CLI: doctor / prepare-models / register / serve / clean.
+
+No `setup` — there are no credentials to collect. connect() takes the path.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+
+from confluence_knowledge.retrieval import prepare_models
+from confluence_knowledge.settings import SetupError
+
+from .settings_local import (
+    APP_NAME,
+    INDEX_DIR_NAME,
+    home,
+    workdir,
+)
+
+WORKDIR_ENV = "LOCAL_KNOWLEDGE_WORKDIR"
+
+
+def _doctor() -> int:
+    from .server_local import status
+    s = status()
+    print(json.dumps(s, indent=2))
+    return 0
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser(prog=APP_NAME)
+    sub = ap.add_subparsers(dest="cmd", required=True)
+    sub.add_parser("doctor", help="verify install + models")
+    sub.add_parser("prepare-models", help="download the pinned local models")
+    reg = sub.add_parser(
+        "register",
+        help="write this server into the IDE's MCP config (no manual paste)")
+    reg.add_argument("--ide", default="auto",
+                     choices=["auto", "all", "vscode", "jetbrains"])
+    reg.add_argument("--workspace", type=Path, default=None)
+    reg.add_argument("--remove", action="store_true")
+    sub.add_parser("serve", help="run the stdio MCP server")
+    clean = sub.add_parser("clean", help="delete the local index")
+    clean.add_argument("--yes", action="store_true")
+    args = ap.parse_args()
+
+    if args.cmd == "doctor":
+        sys.exit(_doctor())
+    if args.cmd == "prepare-models":
+        for line in prepare_models(home()):
+            print(line)
+        return
+    if args.cmd == "register":
+        from confluence_knowledge.registration import register
+        try:
+            wd = (Path(os.environ[WORKDIR_ENV]).expanduser().resolve()
+                  if os.environ.get(WORKDIR_ENV) else None)
+            results = register(ide=args.ide, workspace=args.workspace,
+                               workdir=wd, remove=args.remove,
+                               server_id=APP_NAME,
+                               module="local_knowledge.server_local",
+                               env_var=WORKDIR_ENV)
+        except SetupError as e:
+            print(f"error: {e}", file=sys.stderr)
+            sys.exit(2)
+        for path, action in results:
+            print(f"{action}: {path}")
+        if not args.remove:
+            print("\nReload your IDE so Copilot picks up the server, "
+                  "then re-ask the agent.")
+        return
+    if args.cmd == "serve":
+        from .server_local import main as serve
+        serve()
+        return
+    if args.cmd == "clean":
+        import shutil
+
+        target = workdir() / INDEX_DIR_NAME
+        if target.exists() and (args.yes or input(
+                f"delete {target}? [y/N] ").strip().lower() == "y"):
+            shutil.rmtree(target)
+            print("index deleted")
+        return
+
+
+if __name__ == "__main__":
+    main()

local_knowledge-0.1.0/src/local_knowledge/crawl_local.py

@@ -0,0 +1,199 @@
+"""Durable, resumable filesystem crawl — one path (file or folder) → index.
+
+Same durable-queue/checkpoint contract as the Confluence crawler: progress
+survives interruption, the background worker runs to completion, and only a
+completed crawl publishes (a partial refresh never clobbers a good snapshot).
+Reads only — never writes, follows no symlinks outside the root.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import uuid
+from pathlib import Path
+
+from confluence_knowledge.extract_files import extract_file_chunks, is_secret_name
+from confluence_knowledge.index import Index
+from confluence_knowledge.settings import SetupError
+
+# dirs never worth indexing — noise or actively harmful to read
+SKIP_DIRS = {".git", ".hg", ".svn", "node_modules", "__pycache__", ".venv",
+             "venv", ".idea", ".vscode", "target", "dist", "build", ".tox"}
+MAX_FILE_BYTES = 25 * 1024 * 1024  # 25MB — bigger files get a warning, not a crash
+MAX_FILES = 20_000                  # guard against "pointed at $HOME" accidents
+
+
+def resolve_root(path: str) -> Path:
+    """Validate + canonicalize a user-supplied path (file or folder)."""
+    p = Path(path).expanduser()
+    if not p.exists():
+        raise SetupError(f"path does not exist: {path}")
+    return p.resolve()
+
+
+class LocalCrawler:
+    def __init__(self, index: Index, root: Path, embedder=None):
+        self.index = index
+        self.root = root      # realpath — everything indexed must live under it
+        self.embedder = embedder
+
+    def start(self) -> str:
+        job_id = uuid.uuid4().hex[:12]
+        self.index.abandon_running_jobs()
+        self.index.create_job(job_id, str(self.root))
+        first = f"dir:{self.root}" if self.root.is_dir() else f"file:{self.root}"
+        self.index.save_job(job_id, "running", [first], [])
+        return job_id
+
+    def _rel(self, path: Path) -> str:
+        try:
+            return str(path.relative_to(self.root.parent)).replace("\\", "/")
+        except ValueError:
+            return path.name
+
+    def _children(self, d: Path) -> tuple[list[str], list[str]]:
+        """(new queue entries, warnings) — guarded, secret-aware listing."""
+        out, warns = [], []
+        try:
+            entries = sorted(d.iterdir())
+        except (PermissionError, OSError) as e:
+            return out, [f"{self._rel(d)}: unreadable — {e}"]
+        for e in entries:
+            if e.is_symlink():
+                # resolve and refuse escape — a symlink inside the root that
+                # points outside is disclosed, never followed
+                try:
+                    rp = e.resolve()
+                except OSError:
+                    warns.append(f"{self._rel(e)}: broken symlink — skipped")
+                    continue
+                if self.root.is_dir() and not rp.is_relative_to(self.root):
+                    warns.append(f"{self._rel(e)}: symlink escapes the root — skipped")
+                    continue
+                e = rp
+            if e.is_dir():
+                if e.name in SKIP_DIRS or e.name.startswith(".git"):
+                    continue
+                out.append(f"dir:{e}")
+            elif e.is_file():
+                if is_secret_name(e.name):
+                    warns.append(f"{self._rel(e)}: looks like a secret — skipped")
+                    continue
+                out.append(f"file:{e}")
+        return out, warns
+
+    def _page(self, f: Path) -> tuple[dict, list]:
+        try:
+            st = f.stat()
+            if st.st_size > MAX_FILE_BYTES:
+                return ({"page_id": self._rel(f), "title": f.name,
+                         "url": f"file://{f}", "version": f"{st.st_mtime_ns}:{st.st_size}",
+                         "content_hash": "",
+                         "warnings": [f"file too large ({st.st_size} bytes) — skipped"]},
+                        [])
+            data = f.read_bytes()
+        except OSError as e:
+            raise SetupError(f"{self._rel(f)}: unreadable — {e}") from e
+        chunks, warns = extract_file_chunks(f.name, data)
+        content_hash = hashlib.sha256(data).hexdigest()
+        return ({"page_id": self._rel(f), "title": f.name,
+                 "url": f"file://{f}",
+                 "version": f"{st.st_mtime_ns}:{st.st_size}",
+                 "content_hash": content_hash,
+                 "warnings": warns},
+                chunks)
+
+    def _checkpoint(self, job_id, queue, errors, seen, levels, warns) -> None:
+        with self.index.lock, self.index.db:
+            self.index.db.execute("INSERT OR REPLACE INTO meta VALUES (?,?)",
+                                  (f"seen:{job_id}", json.dumps(sorted(seen))))
+            self.index.db.execute("INSERT OR REPLACE INTO meta VALUES (?,?)",
+                                  (f"levels:{job_id}", json.dumps(levels)))
+            self.index.db.execute("INSERT OR REPLACE INTO meta VALUES (?,?)",
+                                  (f"warns:{job_id}", json.dumps(warns)))
+            self.index.db.execute(
+                "UPDATE jobs SET queue=?, errors=? WHERE job_id=?",
+                (json.dumps(queue), json.dumps(errors), job_id))
+
+    def step(self, job_id: str) -> dict:
+        """Process the durable queue to completion — files are cheap; the
+        time-budget concern that drove Confluence's per-step cap doesn't apply
+        to filesystem reads, and the worker calls this unbounded anyway."""
+        job = self.index.job(job_id)
+        if job is None:
+            raise SetupError(f"unknown job {job_id}")
+        if job["state"] != "running":
+            return {"job_id": job_id, "state": job["state"],
+                    "remaining": len(job["queue"]), "errors": job["errors"]}
+
+        queue: list[str] = job["queue"]
+        errors: list[str] = job["errors"]
+        seen_raw = self.index.meta_get(f"seen:{job_id}")
+        seen: set[str] = set(json.loads(seen_raw)) if seen_raw else set()
+        levels_raw = self.index.meta_get(f"levels:{job_id}")
+        levels: dict[str, int] = (json.loads(levels_raw) if levels_raw else {})
+        warns_raw = self.index.meta_get(f"warns:{job_id}")
+        # deliberate skips (secrets, escaping symlinks, unreadable dirs) are
+        # disclosures — reported to the agent but not coverage-degrading errors
+        warns: list[str] = json.loads(warns_raw) if warns_raw else []
+        visited_dirs: set[str] = set()   # guards symlink/dir-graph cycles
+
+        processed = 0
+        while queue:
+            if len(seen) >= MAX_FILES:
+                errors.append(f"file cap reached ({MAX_FILES}) — remaining skipped")
+                queue.clear()
+                break
+            if processed:
+                self._checkpoint(job_id, queue, errors, seen, levels, warns)
+            entry = queue.pop(0)
+            # entries are "dir:<path>" / "file:<path>" — split on the PREFIX,
+            # not ':' (Windows paths contain their own drive colon)
+            is_dir_entry = entry.startswith("dir:")
+            path_s = entry[4:] if is_dir_entry else entry[5:]
+            path = Path(path_s)
+            if path_s in seen:
+                continue
+            processed += 1
+            if is_dir_entry:
+                real = str(path.resolve())
+                if real in visited_dirs:
+                    continue  # symlink/loop — directories aren't pages
+                visited_dirs.add(real)
+                kids, warns2 = self._children(path)
+                warns.extend(warns2)
+                queue.extend(k for k in kids
+                             if (k[4:] if k.startswith("dir:") else k[5:]) not in seen)
+                continue
+            try:
+                page, chunks = self._page(path)
+            except SetupError as e:
+                errors.append(str(e))
+                continue
+            rel = page["page_id"]
+            if rel in seen:
+                continue
+            for c in chunks:
+                c.chunk_id = f"{rel}:{page['content_hash'][:8]}:{c.ordinal}"
+            vecs = None
+            if self.embedder is not None and chunks:
+                vecs = self.embedder.embed(
+                    [f"{c.heading_path}\n{c.text}" for c in chunks])
+            self.index.upsert_page({
+                "page_id": rel, "parent_id": None, "title": page["title"],
+                "url": page["url"], "level": levels.get(rel, path_s.count("/")),
+                "version": page["version"], "content_hash": page["content_hash"],
+                "warnings": page["warnings"]}, chunks, vecs)
+            seen.add(rel)
+
+        # persist final seen/levels/warns too — the in-loop checkpoint runs
+        # before each item is processed, so the last item would undercount
+        self._checkpoint(job_id, queue, errors, seen, levels, warns)
+        self.index.publish(seen)
+        self.index.save_job(job_id, "done" if not queue else "running",
+                            queue, errors)
+        return {"job_id": job_id,
+                "state": "done" if not queue else "running",
+                "remaining": len(queue), "pages_seen": len(seen),
+                "errors": errors, "warnings": warns}

local_knowledge-0.1.0/src/local_knowledge/server_local.py

@@ -0,0 +1,298 @@
+"""stdio MCP server for local-knowledge — the only surface the agent reaches.
+
+Same tool surface as confluence-knowledge minus the credential machinery:
+there is nothing to authenticate. Scope handles are opaque paths bound to the
+canonical realpath partition.
+"""
+
+from __future__ import annotations
+
+import logging
+import socket
+import threading
+
+from confluence_knowledge.client import AccessError, SecurityException
+from confluence_knowledge.index import Index
+from confluence_knowledge.retrieval import (
+    LocalEmbedder,
+    LocalReranker,
+    check_citations,
+    retrieve,
+)
+from confluence_knowledge.settings import SetupError
+from mcp.server.fastmcp import FastMCP
+
+from .crawl_local import LocalCrawler, resolve_root
+from .settings_local import APP_NAME, LocalProfile, home, index_root
+
+mcp = FastMCP("local-knowledge")
+
+_scopes: dict[str, dict] = {}
+_evidence_sets: dict[str, list[dict]] = {}
+_indexes: dict[str, Index] = {}
+_EMBEDDER = None
+_RERANKER = None
+_model_lock = threading.Lock()
+_models_ready = threading.Event()
+_warm_lock = threading.Lock()
+_warm_started = False
+
+LOADING_NOTE = "loading local models — call this tool again"
+
+
+def _index_for(scope_key: str) -> Index:
+    idx = _indexes.get(scope_key)
+    if idx is None:
+        idx = _indexes[scope_key] = Index(index_root() / f"{scope_key}.db")
+    return idx
+
+
+def _embedder():
+    global _EMBEDDER
+    with _model_lock:
+        if _EMBEDDER is None:
+            _EMBEDDER = LocalEmbedder(home(), APP_NAME)
+    return _EMBEDDER
+
+
+def _reranker():
+    global _RERANKER
+    with _model_lock:
+        if _RERANKER is None:
+            _RERANKER = LocalReranker(home(), APP_NAME)
+    return _RERANKER
+
+
+def _warm_models() -> None:
+    try:
+        _embedder()
+        _reranker()
+    except SetupError:
+        pass
+    finally:
+        _models_ready.set()
+
+
+def _start_warmup() -> None:
+    global _warm_started
+    with _warm_lock:
+        if _warm_started:
+            return
+        _warm_started = True
+    threading.Thread(target=_warm_models, name="model-warmup", daemon=True).start()
+
+
+def _models_loaded(wait: float) -> bool:
+    _start_warmup()
+    return _models_ready.wait(wait)
+
+
+def _scope(scope_handle: str) -> dict:
+    s = _scopes.get(scope_handle)
+    if s is None:
+        parts = scope_handle.split(":", 2)
+        if len(parts) == 3 and parts[0] == "scope" and \
+                connect(parts[2])["scope_handle"] == scope_handle:
+            return _scopes[scope_handle]
+        raise SetupError("unknown scope — call connect first")
+    return s
+
+
+@mcp.tool()
+def status() -> dict:
+    """Report setup readiness — no credentials exist for this agent."""
+    models = home() / "models"
+    try:
+        profile = LocalProfile.load()
+        cfg = {"root": profile.root_path, "device": profile.device}
+    except SetupError as e:
+        cfg = {"error": str(e)}
+    return {
+        "configured": "error" not in cfg,
+        "profile": cfg,
+        "embedding_model_ready": (models / "all-MiniLM-L6-v2").exists(),
+        "reranker_model_ready": (models / "ms-marco-MiniLM-L6-v2").exists(),
+        "models_loaded": _models_ready.is_set() and _EMBEDDER is not None,
+        "index_root": str(index_root()),
+        "live_scopes": sorted(_scopes),
+    }
+
+
+@mcp.tool()
+def connect(root: str) -> dict:
+    """Bind a scope to a file or folder path. No credentials needed — the
+    filesystem itself is the ACL (we can only read what the user can)."""
+    path = resolve_root(root)
+    profile = LocalProfile(str(path), socket.gethostname())
+    profile.save()          # remember last-used root for status/doctor
+    scope_key = profile.scope_key()
+    idx = _index_for(scope_key)
+    st = idx.stats()
+    handle = f"scope:{scope_key}:{path}"
+    _scopes[handle] = {"key": scope_key, "root": path}
+    _start_warmup()
+    return {
+        "scope_handle": handle,
+        "root": str(path),
+        "kind": "folder" if path.is_dir() else "file",
+        "cached": st["pages"] > 0,
+        "cached_pages": st["pages"],
+        "cached_as_of": st["as_of"],
+    }
+
+
+_workers: dict[str, threading.Thread] = {}
+_worker_errors: dict[str, str] = {}
+_workers_lock = threading.Lock()
+
+
+def _worker_loop(s: dict, job_id: str) -> None:
+    idx = _index_for(s["key"])
+    try:
+        _models_ready.wait()
+        c = LocalCrawler(idx, s["root"], embedder=_embedder())
+        r = c.step(job_id)
+        while r["state"] == "running":
+            r = c.step(job_id)
+    except Exception as e:  # noqa: BLE001 — surfaced via job state
+        err = f"{type(e).__name__}: {e}"
+        _worker_errors[f"{s['key']}:{job_id}"] = err
+        job = idx.job(job_id)
+        if job and job["state"] == "running":
+            idx.save_job(job_id, "error", job["queue"], job["errors"] + [err])
+
+
+def _ensure_worker(s: dict, job_id: str) -> None:
+    key = f"{s['key']}:{job_id}"
+    with _workers_lock:
+        t = _workers.get(key)
+        if t is not None and t.is_alive():
+            return
+        _workers[key] = threading.Thread(
+            target=_worker_loop, args=(s, job_id),
+            name=f"crawl-{job_id}", daemon=True)
+        _workers[key].start()
+
+
+@mcp.tool()
+def start_crawl(scope_handle: str, refresh: bool = False) -> dict:
+    """Begin indexing the root (file or folder, recursive) in the background."""
+    s = _scope(scope_handle)
+    idx = _index_for(s["key"])
+    c = LocalCrawler(idx, s["root"])
+    job_id = c.start()
+    _ensure_worker(s, job_id)
+    return {"job_id": job_id, "state": "running",
+            "note": "crawl runs in the background — call crawl_step to poll"}
+
+
+@mcp.tool()
+def crawl_step(scope_handle: str, job_id: str) -> dict:
+    """Read crawl progress instantly; resumes a dead worker automatically."""
+    s = _scope(scope_handle)
+    idx = _index_for(s["key"])
+    job = idx.job(job_id)
+    if job is None:
+        raise AccessError(f"unknown job {job_id}")
+    def progress() -> dict:
+        p = idx.job_progress(job_id)
+        import json
+        warns = idx.meta_get(f"warns:{job_id}")
+        if warns:
+            p["warnings"] = json.loads(warns)
+        return p
+
+    if job["state"] == "running":
+        if not _models_ready.is_set():
+            _start_warmup()
+            note = LOADING_NOTE
+        else:
+            note = None
+        _ensure_worker(s, job_id)
+        p = progress()
+        if note:
+            p["note"] = note
+        return p
+    p = progress()
+    if p["state"] == "done":
+        p["note"] = "no further calls needed — the index is published"
+    return p
+
+
+def _coverage(idx: Index) -> str:
+    job = idx.latest_job()
+    if job is None or job["state"] == "abandoned":
+        return "unknown"
+    if job["state"] == "running":
+        return "refresh in progress — answering from the previous snapshot"
+    return "partial" if job["errors"] else "complete"
+
+
+@mcp.tool()
+def search(scope_handle: str, question: str) -> dict:
+    """Hybrid retrieve + local rerank; returns bounded, cited evidence."""
+    s = _scope(scope_handle)
+    idx = _index_for(s["key"])
+    st = idx.stats()
+    if st["pages"] == 0:
+        job = idx.latest_job()
+        if job and job["state"] == "running":
+            return {"evidence": [], "coverage": "crawling",
+                    "note": f"crawl {job['job_id']} still running — keep calling crawl_step "
+                            "until state is done, then search"}
+        return {"evidence": [], "coverage": "empty",
+                "note": "no files indexed — run a crawl first"}
+    if not _models_loaded(10.0):
+        return {"evidence": [], "coverage": "loading", "note": LOADING_NOTE}
+    ev = retrieve(idx, question, _embedder(), _reranker())
+    set_id = f"ev:{len(_evidence_sets)}:{s['key'][:8]}"
+    _evidence_sets[set_id] = ev
+    return {
+        "evidence_set": set_id,
+        "coverage": _coverage(idx),
+        "as_of": st["as_of"],
+        "evidence": [
+            {"chunk_id": e["chunk_id"], "page_title": e["page_title"],
+             "page_url": e["page_url"], "heading": e["heading"],
+             "score": e.get("rerank_score"), "text": e["text"][:2000]}
+            for e in ev
+        ],
+    }
+
+
+@mcp.tool()
+def get_evidence(scope_handle: str, chunk_ids: list[str]) -> dict:
+    """Return full text of previously returned chunks — same scope only."""
+    s = _scope(scope_handle)
+    idx = _index_for(s["key"])
+    all_chunks = {c["chunk_id"]: c for c in idx.active_chunks()}
+    out = []
+    for cid in chunk_ids[:16]:
+        c = all_chunks.get(cid)
+        if c is None:
+            raise SecurityException(f"chunk {cid} not in this scope's index")
+        p = idx.active_pages().get(c["page_id"], {})
+        out.append({"chunk_id": cid, "text": c["text"],
+                    "page_title": p.get("title", ""), "page_url": p.get("url", "")})
+    return {"chunks": out}
+
+
+@mcp.tool()
+def verify_citations(scope_handle: str, evidence_set: str,
+                     mappings: list[dict]) -> dict:
+    _scope(scope_handle)
+    ev = _evidence_sets.get(evidence_set)
+    if ev is None or not evidence_set.endswith(_scopes[scope_handle]["key"][:8]):
+        raise SecurityException(
+            "unknown or cross-scope evidence set — run search again and cite its evidence_set")
+    return check_citations(ev, mappings)
+
+
+def main() -> None:
+    logging.getLogger("httpx").setLevel(logging.WARNING)
+    _start_warmup()
+    mcp.run()
+
+
+if __name__ == "__main__":
+    main()

local_knowledge-0.1.0/src/local_knowledge/settings_local.py

@@ -0,0 +1,67 @@
+"""Filesystem layout for local-knowledge — ~/.local-knowledge, LOCAL_KNOWLEDGE_*.
+
+The engine is confluence_knowledge's settings machinery parametrized for the
+local connector; no credentials exist (connect reads only paths).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+from confluence_knowledge.settings import (
+    SetupError,
+    default_home,
+    resolve_workdir,
+)
+
+APP_NAME = "local-knowledge"
+INDEX_DIR_NAME = ".local_knowledge_index"
+PROFILE_NAME = "profile.json"
+
+
+def home() -> Path:
+    return default_home("LOCAL_KNOWLEDGE_HOME", ".local-knowledge")
+
+
+def workdir() -> Path:
+    return resolve_workdir("LOCAL_KNOWLEDGE_WORKDIR",
+                           "LOCAL_KNOWLEDGE_HOME", ".local-knowledge")
+
+
+def index_root() -> Path:
+    return workdir() / INDEX_DIR_NAME
+
+
+@dataclass(frozen=True)
+class LocalProfile:
+    """A non-secret connect profile — just the root path that was indexed."""
+
+    root_path: str      # canonical realpath of the file/folder
+    device: str         # machine the path lives on (informational)
+
+    def scope_key(self) -> str:
+        raw = f"{self.root_path}|{self.device}"
+        return hashlib.sha256(raw.encode()).hexdigest()[:24]
+
+    def save(self) -> Path:
+        p = home() / PROFILE_NAME
+        p.parent.mkdir(parents=True, exist_ok=True)
+        tmp = p.with_suffix(".tmp")
+        tmp.write_text(json.dumps({
+            "root_path": self.root_path, "device": self.device,
+        }, indent=2))
+        os.replace(tmp, p)
+        return p
+
+    @staticmethod
+    def load() -> LocalProfile:
+        p = home() / PROFILE_NAME
+        if not p.exists():
+            raise SetupError("no local profile — run `local-knowledge connect` "
+                             "with a path via the agent, or `local-knowledge doctor`")
+        d = json.loads(p.read_text())
+        return LocalProfile(d["root_path"], d.get("device", "local"))

local_knowledge-0.1.0/src/local_knowledge.egg-info/PKG-INFO

@@ -0,0 +1,45 @@
+Metadata-Version: 2.4
+Name: local-knowledge
+Version: 0.1.0
+Summary: Local file/folder indexer + grounded-Q&A MCP service — same engine as confluence-knowledge, no credentials needed
+License: MIT
+Project-URL: Repository, https://github.com/nkunal1992/agents
+Project-URL: Documentation, https://github.com/nkunal1992/agents/blob/main/docs/local-knowledge.md
+Project-URL: Issues, https://github.com/nkunal1992/agents/issues
+Classifier: Operating System :: OS Independent
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3 :: Only
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: confluence-knowledge>=0.4.0
+Requires-Dist: python-docx>=1.1
+Requires-Dist: pypdf>=4.0
+Requires-Dist: python-pptx>=1.0
+Requires-Dist: openpyxl>=3.1
+Requires-Dist: mcp<2,>=1.6
+Requires-Dist: pip-system-certs>=5.0
+Provides-Extra: models
+Requires-Dist: confluence-knowledge[models]>=0.4.0; extra == "models"
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: pytest-cov>=5.0; extra == "dev"
+Requires-Dist: ruff>=0.6; extra == "dev"
+
+# local-knowledge
+
+Local file/folder indexer + grounded-Q&A MCP service — the same engine as
+[`confluence-knowledge`](../confluence-knowledge), pointed at the filesystem
+instead of Confluence. No credentials, no HTTP.
+
+- `local_knowledge/crawl_local.py` — durable filesystem BFS (secret-file
+  exclusions, symlink-escape guards, checkpointed resume)
+- `local_knowledge/server_local.py` — the `local-knowledge/*` MCP tools
+- `local_knowledge/cli_local.py` — `local-knowledge` CLI (no `setup`; there's
+  nothing to authenticate)
+
+Shares `confluence_knowledge`'s index, extraction (`extract` +
+`extract_files`), retrieval, registration and auth plumbing.
+
+See `docs/local-knowledge.md` in the repo root for the full guide and
+`bundle/local-README.md` for the store-listing copy.

local_knowledge-0.1.0/src/local_knowledge.egg-info/SOURCES.txt

@@ -0,0 +1,17 @@
+README.md
+pyproject.toml
+src/local_knowledge/__init__.py
+src/local_knowledge/cli_local.py
+src/local_knowledge/crawl_local.py
+src/local_knowledge/server_local.py
+src/local_knowledge/settings_local.py
+src/local_knowledge.egg-info/PKG-INFO
+src/local_knowledge.egg-info/SOURCES.txt
+src/local_knowledge.egg-info/dependency_links.txt
+src/local_knowledge.egg-info/entry_points.txt
+src/local_knowledge.egg-info/requires.txt
+src/local_knowledge.egg-info/top_level.txt
+tests/test_agent_doc_consistency.py
+tests/test_crawl_local.py
+tests/test_extract_files.py
+tests/test_server_flow_local.py

local_knowledge-0.1.0/src/local_knowledge.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

local_knowledge-0.1.0/src/local_knowledge.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+local-knowledge = local_knowledge.cli_local:main

local_knowledge-0.1.0/src/local_knowledge.egg-info/requires.txt

@@ -0,0 +1,15 @@
+confluence-knowledge>=0.4.0
+python-docx>=1.1
+pypdf>=4.0
+python-pptx>=1.0
+openpyxl>=3.1
+mcp<2,>=1.6
+pip-system-certs>=5.0
+
+[dev]
+pytest>=8.0
+pytest-cov>=5.0
+ruff>=0.6
+
+[models]
+confluence-knowledge[models]>=0.4.0

local_knowledge-0.1.0/src/local_knowledge.egg-info/top_level.txt

@@ -0,0 +1 @@
+local_knowledge

local_knowledge-0.1.0/tests/test_agent_doc_consistency.py

@@ -0,0 +1,31 @@
+"""The local-knowledge agent file must only reference CLI commands that exist."""
+
+import re
+from pathlib import Path
+
+REPO = Path(__file__).resolve().parents[3]
+AGENT = REPO / "agents" / "local-knowledge.agent.md"
+
+
+def test_agent_references_only_real_subcommands():
+    if not AGENT.exists():
+        import pytest
+        pytest.skip("agent file not in this checkout layout")
+    src = (Path(__file__).parents[1] / "src" /
+           "local_knowledge" / "cli_local.py").read_text()
+    real = set(re.findall(r'add_parser\(\s*"([a-z-]+)"', src))
+    doc = AGENT.read_text()
+    # prose like "the local-knowledge service isn't responding" is not a
+    # subcommand — the excluded words below are nouns, not commands
+    referenced = set(re.findall(r"local-knowledge[` ]+([a-z-]+)", doc)) - \
+        {"doctor", "-", "exe", "service", "agent", "knowledge", "bundle"}
+    for cmd in referenced:
+        assert cmd in real, \
+            f"agent references `local-knowledge {cmd}` — not a real subcommand"
+
+
+def test_agent_references_repo_files_that_exist():
+    doc = AGENT.read_text()
+    for m in re.findall(r"prompts/([\w.-]+)", doc):
+        assert (REPO / "prompts" / m).exists(), \
+            f"agent references missing prompts/{m}"

local_knowledge-0.1.0/tests/test_crawl_local.py

@@ -0,0 +1,118 @@
+"""LocalCrawler: file/folder scope, secret exclusion, symlink guards, resume."""
+from __future__ import annotations
+
+import json
+
+import pytest
+from confluence_knowledge.index import Index
+from confluence_knowledge.settings import SetupError
+
+from local_knowledge.crawl_local import LocalCrawler, resolve_root
+
+
+@pytest.fixture
+def tree(tmp_path):
+    root = tmp_path / "docs"
+    (root / "sub").mkdir(parents=True)
+    (root / "guide.md").write_text("the gateway restart procedure")
+    (root / "sub" / "deep.txt").write_text("nested content here")
+    (root / "data.csv").write_text("k,v\na,1\n")
+    (root / ".env").write_text("SECRET=x")
+    (root / "id_rsa").write_text("PRIVATE KEY")
+    (root / ".git").mkdir()
+    (root / ".git" / "config").write_text("should be skipped")
+    (root / "weird.bin").write_bytes(b"\x00\x01\x02")
+    return root
+
+
+@pytest.fixture
+def idx(tmp_path):
+    return Index(tmp_path / "i.db")
+
+
+def run_all(c: LocalCrawler, job: str) -> dict:
+    r = c.step(job)
+    return r
+
+
+def test_crawl_folder(tree, idx):
+    c = LocalCrawler(idx, tree)
+    r = run_all(c, c.start())
+    assert r["state"] == "done"
+    pages = idx.active_pages()
+    titles = {p["title"] for p in pages.values()}
+    assert {"guide.md", "deep.txt", "data.csv"} <= titles
+    # secret files + .git contents never indexed
+    assert "SECRET=x" not in " ".join(p["title"] for p in pages.values())
+    assert ".env" not in titles and "id_rsa" not in titles
+    assert "config" not in titles
+    # excluded + unsupported disclosed — secrets land in job warnings (not
+    # coverage-degrading errors), parse issues in that page's warnings
+    assert "secret" in " ".join(r["warnings"])
+    warn_pages = [w for p in idx.db.execute(
+        "SELECT warnings FROM pages").fetchall() for w in json.loads(p[0])]
+    assert any("unsupported" in w for w in warn_pages)
+
+
+def test_crawl_single_file(tmp_path, idx):
+    f = tmp_path / "one.txt"
+    f.write_text("solo file content")
+    c = LocalCrawler(idx, f)
+    r = run_all(c, c.start())
+    assert r["state"] == "done" and r["pages_seen"] == 1
+    assert idx.active_chunks()
+
+
+def test_secret_file_directly_is_indexed(tmp_path, idx):
+    """An explicit single-file root is indexed even if it looks like a secret
+    — the user pointed at it directly; disclosure only applies to listings."""
+    f = tmp_path / ".env"
+    f.write_text("USER_SECRET=abc")
+    c = LocalCrawler(idx, f)
+    r = run_all(c, c.start())
+    assert r["pages_seen"] == 1  # indexed — explicit user choice
+
+
+def test_symlink_escape_disclosed(tmp_path, idx):
+    root = tmp_path / "docs"
+    root.mkdir()
+    (root / "a.txt").write_text("inside")
+    outside = tmp_path / "outside.txt"
+    outside.write_text("escaped content")
+    link = root / "escape.txt"
+    link.symlink_to(outside)
+    c = LocalCrawler(idx, root)
+    r = run_all(c, c.start())
+    assert "escape" in " ".join(r["warnings"])
+    titles = {p["title"] for p in idx.active_pages().values()}
+    assert "escape.txt" not in titles and "a.txt" in titles
+
+
+def test_dir_cycle_terminates(tmp_path, idx):
+    root = tmp_path / "docs"
+    (root / "sub").mkdir(parents=True)
+    (root / "a.txt").write_text("x")
+    (root / "sub" / "loop").symlink_to(root, target_is_directory=True)
+    c = LocalCrawler(idx, root)
+    r = run_all(c, c.start())
+    assert r["state"] == "done"  # terminates — no infinite queue
+
+
+def test_resolve_root(tmp_path):
+    assert resolve_root(str(tmp_path)) == tmp_path.resolve()
+    with pytest.raises(SetupError):
+        resolve_root(str(tmp_path / "nope"))
+
+
+def test_publish_is_atomic(tree, idx, tmp_path):
+    """A crawl that later sees an empty folder publishes only the new set."""
+    c = LocalCrawler(idx, tree)
+    run_all(c, c.start())
+    assert idx.stats()["pages"] >= 3
+    # delete a file and re-crawl — the index drops it from active
+    (tree / "sub" / "deep.txt").unlink()
+    c2 = LocalCrawler(idx, tree)
+    r2 = run_all(c2, c2.start())
+    assert r2["state"] == "done"
+    titles = {p["title"] for p in idx.active_pages().values()}
+    assert "deep.txt" not in titles

local_knowledge-0.1.0/tests/test_extract_files.py

@@ -0,0 +1,132 @@
+"""extract_files: real bytes → chunks per format; warnings, never crashes."""
+from __future__ import annotations
+
+import io
+
+import pytest
+from confluence_knowledge.extract_files import (
+    extract_file_chunks,
+    is_secret_name,
+)
+
+pytest.importorskip("docx")
+pytest.importorskip("openpyxl")
+pytest.importorskip("pptx")
+pytest.importorskip("pypdf")
+
+
+def _docx_bytes(text: str = "The deploy runbook says restart the gateway first.") -> bytes:
+    import docx
+    d = docx.Document()
+    d.add_heading("Deploy Runbook", level=1)
+    d.add_paragraph(text)
+    buf = io.BytesIO()
+    d.save(buf)
+    return buf.getvalue()
+
+
+def _xlsx_bytes() -> bytes:
+    import openpyxl
+    wb = openpyxl.Workbook()
+    ws = wb.active
+    ws.title = "Owners"
+    ws.append(["service", "owner"])
+    ws.append(["gateway", "narayan"])
+    buf = io.BytesIO()
+    wb.save(buf)
+    return buf.getvalue()
+
+
+def _pptx_bytes() -> bytes:
+    from pptx import Presentation
+    prs = Presentation()
+    slide = prs.slides.add_slide(prs.slide_layouts[1])  # title + content
+    slide.shapes.title.text = "Migration Plan"
+    body = slide.placeholders[1]
+    body.text = "Phase one: move the database."
+    buf = io.BytesIO()
+    prs.save(buf)
+    return buf.getvalue()
+
+
+def test_txt_chunks():
+    chunks, warns = extract_file_chunks("notes.txt", b"hello world " * 100)
+    assert chunks and not warns
+    assert "hello world" in chunks[0].text
+
+
+def test_md_chunks():
+    chunks, _ = extract_file_chunks("r.md", b"# Title\n\nsome content")
+    assert chunks and "content" in chunks[0].text
+
+
+def test_csv_chunks():
+    chunks, _ = extract_file_chunks("t.csv", b"a,b\n1,2\n3,4\n")
+    assert chunks[0].kind == "table" and "row 1: 1 | 2" in chunks[0].text
+
+
+def test_html_chunks():
+    chunks, _ = extract_file_chunks("p.html", b"<h1>H</h1><p>body text</p>")
+    assert chunks and "body text" in chunks[0].text
+
+
+def test_docx_chunks():
+    chunks, warns = extract_file_chunks("runbook.docx", _docx_bytes())
+    assert chunks and not warns
+    assert "gateway" in " ".join(c.text for c in chunks)
+
+
+def test_docx_heading_path():
+    chunks, _ = extract_file_chunks("r.docx", _docx_bytes())
+    assert any("Deploy Runbook" in c.heading_path for c in chunks)
+
+
+def test_xlsx_chunks():
+    chunks, _ = extract_file_chunks("o.xlsx", _xlsx_bytes())
+    text = " ".join(c.text for c in chunks)
+    assert "gateway" in text and "narayan" in text
+    assert any("Sheet: Owners" in c.heading_path for c in chunks)
+
+
+def test_pptx_chunks():
+    chunks, _ = extract_file_chunks("p.pptx", _pptx_bytes())
+    text = " ".join(c.text for c in chunks)
+    assert "Migration Plan" in text and "database" in text
+
+
+def test_unknown_type_warns():
+    chunks, warns = extract_file_chunks("x.bin", b"\x00\x01")
+    assert not chunks and warns and "unsupported" in warns[0]
+
+
+def test_corrupt_office_warns_not_crashes():
+    chunks, warns = extract_file_chunks("bad.docx", b"not a zip")
+    assert not chunks and warns
+
+
+def test_secret_names():
+    assert is_secret_name(".env") and is_secret_name("id_rsa")
+    assert is_secret_name("cert.pem") and is_secret_name("credentials.json")
+    assert is_secret_name(".env.production")
+    assert not is_secret_name("deploy.docx") and not is_secret_name("notes.txt")
+
+
+def test_pdf_minimal():
+    """A real (hand-built) one-page PDF with extractable text."""
+    pdf = b"""%PDF-1.4
+1 0 obj<</Type/Catalog/Pages 2 0 R>>endobj
+2 0 obj<</Type/Pages/Kids[3 0 R]/Count 1>>endobj
+3 0 obj<</Type/Page/Parent 2 0 R/MediaBox[0 0 612 792]/Contents 4 0 R/Resources<</Font<</F1 5 0 R>>>>>>endobj
+4 0 obj<</Length 70>>stream
+BT /F1 24 Tf 100 700 Td (deploy steps live here) Tj ET
+endstream
+endobj
+5 0 obj<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>endobj
+trailer<</Root 1 0 R>>
+"""
+    chunks, warns = extract_file_chunks("doc.pdf", pdf)
+    # pypdf extracts the text; if a version can't parse the hand-rolled xref,
+    # it must warn — never crash
+    assert chunks or warns
+    if chunks:
+        assert "deploy steps" in " ".join(c.text for c in chunks)

local_knowledge-0.1.0/tests/test_server_flow_local.py

@@ -0,0 +1,92 @@
+"""local-knowledge server flow: connect → crawl → search → verify citations."""
+from __future__ import annotations
+
+import numpy as np
+import pytest
+
+from local_knowledge import server_local
+from local_knowledge.settings_local import LocalProfile
+
+
+class FakeEmbedder:
+    def __init__(self, _home=None, _cli=None):
+        pass
+
+    def embed(self, texts):
+        return np.random.rand(len(texts), 8).astype(np.float32)
+
+
+class FakeReranker:
+    def __init__(self, _home=None, _cli=None):
+        pass
+
+    def score(self, query, docs):
+        return np.arange(len(docs), dtype=np.float32)[::-1]
+
+
+@pytest.fixture(autouse=True)
+def isolated(tmp_path, monkeypatch):
+    monkeypatch.setenv("LOCAL_KNOWLEDGE_HOME", str(tmp_path / "home"))
+    monkeypatch.setenv("LOCAL_KNOWLEDGE_WORKDIR", str(tmp_path / "work"))
+    monkeypatch.setattr(server_local, "_EMBEDDER", FakeEmbedder())
+    monkeypatch.setattr(server_local, "_RERANKER", FakeReranker())
+    server_local._models_ready.set()
+    yield
+    server_local._scopes.clear()
+    server_local._indexes.clear()
+    server_local._evidence_sets.clear()
+    server_local._models_ready.clear()
+
+
+def test_full_flow(tmp_path):
+    root = tmp_path / "docs"
+    root.mkdir()
+    (root / "guide.md").write_text("restart the gateway before the deploy")
+    (root / "data.csv").write_text("service,owner\ngateway,ops\n")
+    (root / ".env").write_text("SECRET=x")
+
+    c = server_local.connect(str(root))
+    assert c["kind"] == "folder"
+    job = server_local.start_crawl(c["scope_handle"])
+    r = server_local.crawl_step(c["scope_handle"], job["job_id"])
+    # worker may still be running — poll until done
+    while r["state"] == "running":
+        r = server_local.crawl_step(c["scope_handle"], job["job_id"])
+    assert r["pages_seen"] >= 2
+
+    res = server_local.search(c["scope_handle"], "who owns the gateway?")
+    assert res["coverage"] == "complete"
+    assert res["evidence"]
+    first = res["evidence"][0]
+    assert first["page_url"].startswith("file://")
+
+    v = server_local.verify_citations(
+        c["scope_handle"], res["evidence_set"],
+        [{"claim": "gateway is owned by ops",
+          "quotes": [res["evidence"][0]["text"][:60]]}])
+    assert "results" in v or isinstance(v, dict)
+
+
+def test_search_while_crawling(tmp_path):
+    root = tmp_path / "docs"
+    root.mkdir()
+    (root / "a.txt").write_text("x")
+    c = server_local.connect(str(root))
+    # search before any crawl → empty + told to crawl
+    res = server_local.search(c["scope_handle"], "anything")
+    assert res["coverage"] == "empty"
+
+
+def test_unknown_scope():
+    from confluence_knowledge.settings import SetupError
+    with pytest.raises(SetupError):
+        server_local.search("scope:bad:handle", "q")
+
+
+def test_profile_saved(tmp_path):
+    root = tmp_path / "docs"
+    root.mkdir()
+    (root / "a.txt").write_text("x")
+    server_local.connect(str(root))
+    p = LocalProfile.load()
+    assert p.root_path == str(root.resolve())