local-knowledge 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- local_knowledge/__init__.py +1 -0
- local_knowledge/cli_local.py +92 -0
- local_knowledge/crawl_local.py +199 -0
- local_knowledge/server_local.py +298 -0
- local_knowledge/settings_local.py +67 -0
- local_knowledge-0.1.0.dist-info/METADATA +45 -0
- local_knowledge-0.1.0.dist-info/RECORD +10 -0
- local_knowledge-0.1.0.dist-info/WHEEL +5 -0
- local_knowledge-0.1.0.dist-info/entry_points.txt +2 -0
- local_knowledge-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""local-knowledge — local file/folder indexing + grounded Q&A MCP service."""
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
"""local-knowledge CLI: doctor / prepare-models / register / serve / clean.
|
|
2
|
+
|
|
3
|
+
No `setup` — there are no credentials to collect. connect() takes the path.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import argparse
|
|
9
|
+
import json
|
|
10
|
+
import os
|
|
11
|
+
import sys
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
from confluence_knowledge.retrieval import prepare_models
|
|
15
|
+
from confluence_knowledge.settings import SetupError
|
|
16
|
+
|
|
17
|
+
from .settings_local import (
|
|
18
|
+
APP_NAME,
|
|
19
|
+
INDEX_DIR_NAME,
|
|
20
|
+
home,
|
|
21
|
+
workdir,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
WORKDIR_ENV = "LOCAL_KNOWLEDGE_WORKDIR"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _doctor() -> int:
|
|
28
|
+
from .server_local import status
|
|
29
|
+
s = status()
|
|
30
|
+
print(json.dumps(s, indent=2))
|
|
31
|
+
return 0
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def main() -> None:
|
|
35
|
+
ap = argparse.ArgumentParser(prog=APP_NAME)
|
|
36
|
+
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
37
|
+
sub.add_parser("doctor", help="verify install + models")
|
|
38
|
+
sub.add_parser("prepare-models", help="download the pinned local models")
|
|
39
|
+
reg = sub.add_parser(
|
|
40
|
+
"register",
|
|
41
|
+
help="write this server into the IDE's MCP config (no manual paste)")
|
|
42
|
+
reg.add_argument("--ide", default="auto",
|
|
43
|
+
choices=["auto", "all", "vscode", "jetbrains"])
|
|
44
|
+
reg.add_argument("--workspace", type=Path, default=None)
|
|
45
|
+
reg.add_argument("--remove", action="store_true")
|
|
46
|
+
sub.add_parser("serve", help="run the stdio MCP server")
|
|
47
|
+
clean = sub.add_parser("clean", help="delete the local index")
|
|
48
|
+
clean.add_argument("--yes", action="store_true")
|
|
49
|
+
args = ap.parse_args()
|
|
50
|
+
|
|
51
|
+
if args.cmd == "doctor":
|
|
52
|
+
sys.exit(_doctor())
|
|
53
|
+
if args.cmd == "prepare-models":
|
|
54
|
+
for line in prepare_models(home()):
|
|
55
|
+
print(line)
|
|
56
|
+
return
|
|
57
|
+
if args.cmd == "register":
|
|
58
|
+
from confluence_knowledge.registration import register
|
|
59
|
+
try:
|
|
60
|
+
wd = (Path(os.environ[WORKDIR_ENV]).expanduser().resolve()
|
|
61
|
+
if os.environ.get(WORKDIR_ENV) else None)
|
|
62
|
+
results = register(ide=args.ide, workspace=args.workspace,
|
|
63
|
+
workdir=wd, remove=args.remove,
|
|
64
|
+
server_id=APP_NAME,
|
|
65
|
+
module="local_knowledge.server_local",
|
|
66
|
+
env_var=WORKDIR_ENV)
|
|
67
|
+
except SetupError as e:
|
|
68
|
+
print(f"error: {e}", file=sys.stderr)
|
|
69
|
+
sys.exit(2)
|
|
70
|
+
for path, action in results:
|
|
71
|
+
print(f"{action}: {path}")
|
|
72
|
+
if not args.remove:
|
|
73
|
+
print("\nReload your IDE so Copilot picks up the server, "
|
|
74
|
+
"then re-ask the agent.")
|
|
75
|
+
return
|
|
76
|
+
if args.cmd == "serve":
|
|
77
|
+
from .server_local import main as serve
|
|
78
|
+
serve()
|
|
79
|
+
return
|
|
80
|
+
if args.cmd == "clean":
|
|
81
|
+
import shutil
|
|
82
|
+
|
|
83
|
+
target = workdir() / INDEX_DIR_NAME
|
|
84
|
+
if target.exists() and (args.yes or input(
|
|
85
|
+
f"delete {target}? [y/N] ").strip().lower() == "y"):
|
|
86
|
+
shutil.rmtree(target)
|
|
87
|
+
print("index deleted")
|
|
88
|
+
return
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
if __name__ == "__main__":
|
|
92
|
+
main()
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
"""Durable, resumable filesystem crawl — one path (file or folder) → index.
|
|
2
|
+
|
|
3
|
+
Same durable-queue/checkpoint contract as the Confluence crawler: progress
|
|
4
|
+
survives interruption, the background worker runs to completion, and only a
|
|
5
|
+
completed crawl publishes (a partial refresh never clobbers a good snapshot).
|
|
6
|
+
Reads only — never writes, follows no symlinks outside the root.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import hashlib
|
|
12
|
+
import json
|
|
13
|
+
import uuid
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
from confluence_knowledge.extract_files import extract_file_chunks, is_secret_name
|
|
17
|
+
from confluence_knowledge.index import Index
|
|
18
|
+
from confluence_knowledge.settings import SetupError
|
|
19
|
+
|
|
20
|
+
# dirs never worth indexing — noise or actively harmful to read
|
|
21
|
+
SKIP_DIRS = {".git", ".hg", ".svn", "node_modules", "__pycache__", ".venv",
|
|
22
|
+
"venv", ".idea", ".vscode", "target", "dist", "build", ".tox"}
|
|
23
|
+
MAX_FILE_BYTES = 25 * 1024 * 1024 # 25MB — bigger files get a warning, not a crash
|
|
24
|
+
MAX_FILES = 20_000 # guard against "pointed at $HOME" accidents
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def resolve_root(path: str) -> Path:
|
|
28
|
+
"""Validate + canonicalize a user-supplied path (file or folder)."""
|
|
29
|
+
p = Path(path).expanduser()
|
|
30
|
+
if not p.exists():
|
|
31
|
+
raise SetupError(f"path does not exist: {path}")
|
|
32
|
+
return p.resolve()
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class LocalCrawler:
|
|
36
|
+
def __init__(self, index: Index, root: Path, embedder=None):
|
|
37
|
+
self.index = index
|
|
38
|
+
self.root = root # realpath — everything indexed must live under it
|
|
39
|
+
self.embedder = embedder
|
|
40
|
+
|
|
41
|
+
def start(self) -> str:
|
|
42
|
+
job_id = uuid.uuid4().hex[:12]
|
|
43
|
+
self.index.abandon_running_jobs()
|
|
44
|
+
self.index.create_job(job_id, str(self.root))
|
|
45
|
+
first = f"dir:{self.root}" if self.root.is_dir() else f"file:{self.root}"
|
|
46
|
+
self.index.save_job(job_id, "running", [first], [])
|
|
47
|
+
return job_id
|
|
48
|
+
|
|
49
|
+
def _rel(self, path: Path) -> str:
|
|
50
|
+
try:
|
|
51
|
+
return str(path.relative_to(self.root.parent)).replace("\\", "/")
|
|
52
|
+
except ValueError:
|
|
53
|
+
return path.name
|
|
54
|
+
|
|
55
|
+
def _children(self, d: Path) -> tuple[list[str], list[str]]:
|
|
56
|
+
"""(new queue entries, warnings) — guarded, secret-aware listing."""
|
|
57
|
+
out, warns = [], []
|
|
58
|
+
try:
|
|
59
|
+
entries = sorted(d.iterdir())
|
|
60
|
+
except (PermissionError, OSError) as e:
|
|
61
|
+
return out, [f"{self._rel(d)}: unreadable — {e}"]
|
|
62
|
+
for e in entries:
|
|
63
|
+
if e.is_symlink():
|
|
64
|
+
# resolve and refuse escape — a symlink inside the root that
|
|
65
|
+
# points outside is disclosed, never followed
|
|
66
|
+
try:
|
|
67
|
+
rp = e.resolve()
|
|
68
|
+
except OSError:
|
|
69
|
+
warns.append(f"{self._rel(e)}: broken symlink — skipped")
|
|
70
|
+
continue
|
|
71
|
+
if self.root.is_dir() and not rp.is_relative_to(self.root):
|
|
72
|
+
warns.append(f"{self._rel(e)}: symlink escapes the root — skipped")
|
|
73
|
+
continue
|
|
74
|
+
e = rp
|
|
75
|
+
if e.is_dir():
|
|
76
|
+
if e.name in SKIP_DIRS or e.name.startswith(".git"):
|
|
77
|
+
continue
|
|
78
|
+
out.append(f"dir:{e}")
|
|
79
|
+
elif e.is_file():
|
|
80
|
+
if is_secret_name(e.name):
|
|
81
|
+
warns.append(f"{self._rel(e)}: looks like a secret — skipped")
|
|
82
|
+
continue
|
|
83
|
+
out.append(f"file:{e}")
|
|
84
|
+
return out, warns
|
|
85
|
+
|
|
86
|
+
def _page(self, f: Path) -> tuple[dict, list]:
|
|
87
|
+
try:
|
|
88
|
+
st = f.stat()
|
|
89
|
+
if st.st_size > MAX_FILE_BYTES:
|
|
90
|
+
return ({"page_id": self._rel(f), "title": f.name,
|
|
91
|
+
"url": f"file://{f}", "version": f"{st.st_mtime_ns}:{st.st_size}",
|
|
92
|
+
"content_hash": "",
|
|
93
|
+
"warnings": [f"file too large ({st.st_size} bytes) — skipped"]},
|
|
94
|
+
[])
|
|
95
|
+
data = f.read_bytes()
|
|
96
|
+
except OSError as e:
|
|
97
|
+
raise SetupError(f"{self._rel(f)}: unreadable — {e}") from e
|
|
98
|
+
chunks, warns = extract_file_chunks(f.name, data)
|
|
99
|
+
content_hash = hashlib.sha256(data).hexdigest()
|
|
100
|
+
return ({"page_id": self._rel(f), "title": f.name,
|
|
101
|
+
"url": f"file://{f}",
|
|
102
|
+
"version": f"{st.st_mtime_ns}:{st.st_size}",
|
|
103
|
+
"content_hash": content_hash,
|
|
104
|
+
"warnings": warns},
|
|
105
|
+
chunks)
|
|
106
|
+
|
|
107
|
+
def _checkpoint(self, job_id, queue, errors, seen, levels, warns) -> None:
|
|
108
|
+
with self.index.lock, self.index.db:
|
|
109
|
+
self.index.db.execute("INSERT OR REPLACE INTO meta VALUES (?,?)",
|
|
110
|
+
(f"seen:{job_id}", json.dumps(sorted(seen))))
|
|
111
|
+
self.index.db.execute("INSERT OR REPLACE INTO meta VALUES (?,?)",
|
|
112
|
+
(f"levels:{job_id}", json.dumps(levels)))
|
|
113
|
+
self.index.db.execute("INSERT OR REPLACE INTO meta VALUES (?,?)",
|
|
114
|
+
(f"warns:{job_id}", json.dumps(warns)))
|
|
115
|
+
self.index.db.execute(
|
|
116
|
+
"UPDATE jobs SET queue=?, errors=? WHERE job_id=?",
|
|
117
|
+
(json.dumps(queue), json.dumps(errors), job_id))
|
|
118
|
+
|
|
119
|
+
def step(self, job_id: str) -> dict:
|
|
120
|
+
"""Process the durable queue to completion — files are cheap; the
|
|
121
|
+
time-budget concern that drove Confluence's per-step cap doesn't apply
|
|
122
|
+
to filesystem reads, and the worker calls this unbounded anyway."""
|
|
123
|
+
job = self.index.job(job_id)
|
|
124
|
+
if job is None:
|
|
125
|
+
raise SetupError(f"unknown job {job_id}")
|
|
126
|
+
if job["state"] != "running":
|
|
127
|
+
return {"job_id": job_id, "state": job["state"],
|
|
128
|
+
"remaining": len(job["queue"]), "errors": job["errors"]}
|
|
129
|
+
|
|
130
|
+
queue: list[str] = job["queue"]
|
|
131
|
+
errors: list[str] = job["errors"]
|
|
132
|
+
seen_raw = self.index.meta_get(f"seen:{job_id}")
|
|
133
|
+
seen: set[str] = set(json.loads(seen_raw)) if seen_raw else set()
|
|
134
|
+
levels_raw = self.index.meta_get(f"levels:{job_id}")
|
|
135
|
+
levels: dict[str, int] = (json.loads(levels_raw) if levels_raw else {})
|
|
136
|
+
warns_raw = self.index.meta_get(f"warns:{job_id}")
|
|
137
|
+
# deliberate skips (secrets, escaping symlinks, unreadable dirs) are
|
|
138
|
+
# disclosures — reported to the agent but not coverage-degrading errors
|
|
139
|
+
warns: list[str] = json.loads(warns_raw) if warns_raw else []
|
|
140
|
+
visited_dirs: set[str] = set() # guards symlink/dir-graph cycles
|
|
141
|
+
|
|
142
|
+
processed = 0
|
|
143
|
+
while queue:
|
|
144
|
+
if len(seen) >= MAX_FILES:
|
|
145
|
+
errors.append(f"file cap reached ({MAX_FILES}) — remaining skipped")
|
|
146
|
+
queue.clear()
|
|
147
|
+
break
|
|
148
|
+
if processed:
|
|
149
|
+
self._checkpoint(job_id, queue, errors, seen, levels, warns)
|
|
150
|
+
entry = queue.pop(0)
|
|
151
|
+
# entries are "dir:<path>" / "file:<path>" — split on the PREFIX,
|
|
152
|
+
# not ':' (Windows paths contain their own drive colon)
|
|
153
|
+
is_dir_entry = entry.startswith("dir:")
|
|
154
|
+
path_s = entry[4:] if is_dir_entry else entry[5:]
|
|
155
|
+
path = Path(path_s)
|
|
156
|
+
if path_s in seen:
|
|
157
|
+
continue
|
|
158
|
+
processed += 1
|
|
159
|
+
if is_dir_entry:
|
|
160
|
+
real = str(path.resolve())
|
|
161
|
+
if real in visited_dirs:
|
|
162
|
+
continue # symlink/loop — directories aren't pages
|
|
163
|
+
visited_dirs.add(real)
|
|
164
|
+
kids, warns2 = self._children(path)
|
|
165
|
+
warns.extend(warns2)
|
|
166
|
+
queue.extend(k for k in kids
|
|
167
|
+
if (k[4:] if k.startswith("dir:") else k[5:]) not in seen)
|
|
168
|
+
continue
|
|
169
|
+
try:
|
|
170
|
+
page, chunks = self._page(path)
|
|
171
|
+
except SetupError as e:
|
|
172
|
+
errors.append(str(e))
|
|
173
|
+
continue
|
|
174
|
+
rel = page["page_id"]
|
|
175
|
+
if rel in seen:
|
|
176
|
+
continue
|
|
177
|
+
for c in chunks:
|
|
178
|
+
c.chunk_id = f"{rel}:{page['content_hash'][:8]}:{c.ordinal}"
|
|
179
|
+
vecs = None
|
|
180
|
+
if self.embedder is not None and chunks:
|
|
181
|
+
vecs = self.embedder.embed(
|
|
182
|
+
[f"{c.heading_path}\n{c.text}" for c in chunks])
|
|
183
|
+
self.index.upsert_page({
|
|
184
|
+
"page_id": rel, "parent_id": None, "title": page["title"],
|
|
185
|
+
"url": page["url"], "level": levels.get(rel, path_s.count("/")),
|
|
186
|
+
"version": page["version"], "content_hash": page["content_hash"],
|
|
187
|
+
"warnings": page["warnings"]}, chunks, vecs)
|
|
188
|
+
seen.add(rel)
|
|
189
|
+
|
|
190
|
+
# persist final seen/levels/warns too — the in-loop checkpoint runs
|
|
191
|
+
# before each item is processed, so the last item would undercount
|
|
192
|
+
self._checkpoint(job_id, queue, errors, seen, levels, warns)
|
|
193
|
+
self.index.publish(seen)
|
|
194
|
+
self.index.save_job(job_id, "done" if not queue else "running",
|
|
195
|
+
queue, errors)
|
|
196
|
+
return {"job_id": job_id,
|
|
197
|
+
"state": "done" if not queue else "running",
|
|
198
|
+
"remaining": len(queue), "pages_seen": len(seen),
|
|
199
|
+
"errors": errors, "warnings": warns}
|
|
@@ -0,0 +1,298 @@
|
|
|
1
|
+
"""stdio MCP server for local-knowledge — the only surface the agent reaches.
|
|
2
|
+
|
|
3
|
+
Same tool surface as confluence-knowledge minus the credential machinery:
|
|
4
|
+
there is nothing to authenticate. Scope handles are opaque paths bound to the
|
|
5
|
+
canonical realpath partition.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import logging
|
|
11
|
+
import socket
|
|
12
|
+
import threading
|
|
13
|
+
|
|
14
|
+
from confluence_knowledge.client import AccessError, SecurityException
|
|
15
|
+
from confluence_knowledge.index import Index
|
|
16
|
+
from confluence_knowledge.retrieval import (
|
|
17
|
+
LocalEmbedder,
|
|
18
|
+
LocalReranker,
|
|
19
|
+
check_citations,
|
|
20
|
+
retrieve,
|
|
21
|
+
)
|
|
22
|
+
from confluence_knowledge.settings import SetupError
|
|
23
|
+
from mcp.server.fastmcp import FastMCP
|
|
24
|
+
|
|
25
|
+
from .crawl_local import LocalCrawler, resolve_root
|
|
26
|
+
from .settings_local import APP_NAME, LocalProfile, home, index_root
|
|
27
|
+
|
|
28
|
+
mcp = FastMCP("local-knowledge")
|
|
29
|
+
|
|
30
|
+
_scopes: dict[str, dict] = {}
|
|
31
|
+
_evidence_sets: dict[str, list[dict]] = {}
|
|
32
|
+
_indexes: dict[str, Index] = {}
|
|
33
|
+
_EMBEDDER = None
|
|
34
|
+
_RERANKER = None
|
|
35
|
+
_model_lock = threading.Lock()
|
|
36
|
+
_models_ready = threading.Event()
|
|
37
|
+
_warm_lock = threading.Lock()
|
|
38
|
+
_warm_started = False
|
|
39
|
+
|
|
40
|
+
LOADING_NOTE = "loading local models — call this tool again"
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _index_for(scope_key: str) -> Index:
|
|
44
|
+
idx = _indexes.get(scope_key)
|
|
45
|
+
if idx is None:
|
|
46
|
+
idx = _indexes[scope_key] = Index(index_root() / f"{scope_key}.db")
|
|
47
|
+
return idx
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _embedder():
|
|
51
|
+
global _EMBEDDER
|
|
52
|
+
with _model_lock:
|
|
53
|
+
if _EMBEDDER is None:
|
|
54
|
+
_EMBEDDER = LocalEmbedder(home(), APP_NAME)
|
|
55
|
+
return _EMBEDDER
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _reranker():
|
|
59
|
+
global _RERANKER
|
|
60
|
+
with _model_lock:
|
|
61
|
+
if _RERANKER is None:
|
|
62
|
+
_RERANKER = LocalReranker(home(), APP_NAME)
|
|
63
|
+
return _RERANKER
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _warm_models() -> None:
|
|
67
|
+
try:
|
|
68
|
+
_embedder()
|
|
69
|
+
_reranker()
|
|
70
|
+
except SetupError:
|
|
71
|
+
pass
|
|
72
|
+
finally:
|
|
73
|
+
_models_ready.set()
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _start_warmup() -> None:
|
|
77
|
+
global _warm_started
|
|
78
|
+
with _warm_lock:
|
|
79
|
+
if _warm_started:
|
|
80
|
+
return
|
|
81
|
+
_warm_started = True
|
|
82
|
+
threading.Thread(target=_warm_models, name="model-warmup", daemon=True).start()
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _models_loaded(wait: float) -> bool:
|
|
86
|
+
_start_warmup()
|
|
87
|
+
return _models_ready.wait(wait)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _scope(scope_handle: str) -> dict:
|
|
91
|
+
s = _scopes.get(scope_handle)
|
|
92
|
+
if s is None:
|
|
93
|
+
parts = scope_handle.split(":", 2)
|
|
94
|
+
if len(parts) == 3 and parts[0] == "scope" and \
|
|
95
|
+
connect(parts[2])["scope_handle"] == scope_handle:
|
|
96
|
+
return _scopes[scope_handle]
|
|
97
|
+
raise SetupError("unknown scope — call connect first")
|
|
98
|
+
return s
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
@mcp.tool()
|
|
102
|
+
def status() -> dict:
|
|
103
|
+
"""Report setup readiness — no credentials exist for this agent."""
|
|
104
|
+
models = home() / "models"
|
|
105
|
+
try:
|
|
106
|
+
profile = LocalProfile.load()
|
|
107
|
+
cfg = {"root": profile.root_path, "device": profile.device}
|
|
108
|
+
except SetupError as e:
|
|
109
|
+
cfg = {"error": str(e)}
|
|
110
|
+
return {
|
|
111
|
+
"configured": "error" not in cfg,
|
|
112
|
+
"profile": cfg,
|
|
113
|
+
"embedding_model_ready": (models / "all-MiniLM-L6-v2").exists(),
|
|
114
|
+
"reranker_model_ready": (models / "ms-marco-MiniLM-L6-v2").exists(),
|
|
115
|
+
"models_loaded": _models_ready.is_set() and _EMBEDDER is not None,
|
|
116
|
+
"index_root": str(index_root()),
|
|
117
|
+
"live_scopes": sorted(_scopes),
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
@mcp.tool()
|
|
122
|
+
def connect(root: str) -> dict:
|
|
123
|
+
"""Bind a scope to a file or folder path. No credentials needed — the
|
|
124
|
+
filesystem itself is the ACL (we can only read what the user can)."""
|
|
125
|
+
path = resolve_root(root)
|
|
126
|
+
profile = LocalProfile(str(path), socket.gethostname())
|
|
127
|
+
profile.save() # remember last-used root for status/doctor
|
|
128
|
+
scope_key = profile.scope_key()
|
|
129
|
+
idx = _index_for(scope_key)
|
|
130
|
+
st = idx.stats()
|
|
131
|
+
handle = f"scope:{scope_key}:{path}"
|
|
132
|
+
_scopes[handle] = {"key": scope_key, "root": path}
|
|
133
|
+
_start_warmup()
|
|
134
|
+
return {
|
|
135
|
+
"scope_handle": handle,
|
|
136
|
+
"root": str(path),
|
|
137
|
+
"kind": "folder" if path.is_dir() else "file",
|
|
138
|
+
"cached": st["pages"] > 0,
|
|
139
|
+
"cached_pages": st["pages"],
|
|
140
|
+
"cached_as_of": st["as_of"],
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
_workers: dict[str, threading.Thread] = {}
|
|
145
|
+
_worker_errors: dict[str, str] = {}
|
|
146
|
+
_workers_lock = threading.Lock()
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _worker_loop(s: dict, job_id: str) -> None:
|
|
150
|
+
idx = _index_for(s["key"])
|
|
151
|
+
try:
|
|
152
|
+
_models_ready.wait()
|
|
153
|
+
c = LocalCrawler(idx, s["root"], embedder=_embedder())
|
|
154
|
+
r = c.step(job_id)
|
|
155
|
+
while r["state"] == "running":
|
|
156
|
+
r = c.step(job_id)
|
|
157
|
+
except Exception as e: # noqa: BLE001 — surfaced via job state
|
|
158
|
+
err = f"{type(e).__name__}: {e}"
|
|
159
|
+
_worker_errors[f"{s['key']}:{job_id}"] = err
|
|
160
|
+
job = idx.job(job_id)
|
|
161
|
+
if job and job["state"] == "running":
|
|
162
|
+
idx.save_job(job_id, "error", job["queue"], job["errors"] + [err])
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _ensure_worker(s: dict, job_id: str) -> None:
|
|
166
|
+
key = f"{s['key']}:{job_id}"
|
|
167
|
+
with _workers_lock:
|
|
168
|
+
t = _workers.get(key)
|
|
169
|
+
if t is not None and t.is_alive():
|
|
170
|
+
return
|
|
171
|
+
_workers[key] = threading.Thread(
|
|
172
|
+
target=_worker_loop, args=(s, job_id),
|
|
173
|
+
name=f"crawl-{job_id}", daemon=True)
|
|
174
|
+
_workers[key].start()
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
@mcp.tool()
|
|
178
|
+
def start_crawl(scope_handle: str, refresh: bool = False) -> dict:
|
|
179
|
+
"""Begin indexing the root (file or folder, recursive) in the background."""
|
|
180
|
+
s = _scope(scope_handle)
|
|
181
|
+
idx = _index_for(s["key"])
|
|
182
|
+
c = LocalCrawler(idx, s["root"])
|
|
183
|
+
job_id = c.start()
|
|
184
|
+
_ensure_worker(s, job_id)
|
|
185
|
+
return {"job_id": job_id, "state": "running",
|
|
186
|
+
"note": "crawl runs in the background — call crawl_step to poll"}
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
@mcp.tool()
|
|
190
|
+
def crawl_step(scope_handle: str, job_id: str) -> dict:
|
|
191
|
+
"""Read crawl progress instantly; resumes a dead worker automatically."""
|
|
192
|
+
s = _scope(scope_handle)
|
|
193
|
+
idx = _index_for(s["key"])
|
|
194
|
+
job = idx.job(job_id)
|
|
195
|
+
if job is None:
|
|
196
|
+
raise AccessError(f"unknown job {job_id}")
|
|
197
|
+
def progress() -> dict:
|
|
198
|
+
p = idx.job_progress(job_id)
|
|
199
|
+
import json
|
|
200
|
+
warns = idx.meta_get(f"warns:{job_id}")
|
|
201
|
+
if warns:
|
|
202
|
+
p["warnings"] = json.loads(warns)
|
|
203
|
+
return p
|
|
204
|
+
|
|
205
|
+
if job["state"] == "running":
|
|
206
|
+
if not _models_ready.is_set():
|
|
207
|
+
_start_warmup()
|
|
208
|
+
note = LOADING_NOTE
|
|
209
|
+
else:
|
|
210
|
+
note = None
|
|
211
|
+
_ensure_worker(s, job_id)
|
|
212
|
+
p = progress()
|
|
213
|
+
if note:
|
|
214
|
+
p["note"] = note
|
|
215
|
+
return p
|
|
216
|
+
p = progress()
|
|
217
|
+
if p["state"] == "done":
|
|
218
|
+
p["note"] = "no further calls needed — the index is published"
|
|
219
|
+
return p
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _coverage(idx: Index) -> str:
|
|
223
|
+
job = idx.latest_job()
|
|
224
|
+
if job is None or job["state"] == "abandoned":
|
|
225
|
+
return "unknown"
|
|
226
|
+
if job["state"] == "running":
|
|
227
|
+
return "refresh in progress — answering from the previous snapshot"
|
|
228
|
+
return "partial" if job["errors"] else "complete"
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
@mcp.tool()
|
|
232
|
+
def search(scope_handle: str, question: str) -> dict:
|
|
233
|
+
"""Hybrid retrieve + local rerank; returns bounded, cited evidence."""
|
|
234
|
+
s = _scope(scope_handle)
|
|
235
|
+
idx = _index_for(s["key"])
|
|
236
|
+
st = idx.stats()
|
|
237
|
+
if st["pages"] == 0:
|
|
238
|
+
job = idx.latest_job()
|
|
239
|
+
if job and job["state"] == "running":
|
|
240
|
+
return {"evidence": [], "coverage": "crawling",
|
|
241
|
+
"note": f"crawl {job['job_id']} still running — keep calling crawl_step "
|
|
242
|
+
"until state is done, then search"}
|
|
243
|
+
return {"evidence": [], "coverage": "empty",
|
|
244
|
+
"note": "no files indexed — run a crawl first"}
|
|
245
|
+
if not _models_loaded(10.0):
|
|
246
|
+
return {"evidence": [], "coverage": "loading", "note": LOADING_NOTE}
|
|
247
|
+
ev = retrieve(idx, question, _embedder(), _reranker())
|
|
248
|
+
set_id = f"ev:{len(_evidence_sets)}:{s['key'][:8]}"
|
|
249
|
+
_evidence_sets[set_id] = ev
|
|
250
|
+
return {
|
|
251
|
+
"evidence_set": set_id,
|
|
252
|
+
"coverage": _coverage(idx),
|
|
253
|
+
"as_of": st["as_of"],
|
|
254
|
+
"evidence": [
|
|
255
|
+
{"chunk_id": e["chunk_id"], "page_title": e["page_title"],
|
|
256
|
+
"page_url": e["page_url"], "heading": e["heading"],
|
|
257
|
+
"score": e.get("rerank_score"), "text": e["text"][:2000]}
|
|
258
|
+
for e in ev
|
|
259
|
+
],
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
@mcp.tool()
|
|
264
|
+
def get_evidence(scope_handle: str, chunk_ids: list[str]) -> dict:
|
|
265
|
+
"""Return full text of previously returned chunks — same scope only."""
|
|
266
|
+
s = _scope(scope_handle)
|
|
267
|
+
idx = _index_for(s["key"])
|
|
268
|
+
all_chunks = {c["chunk_id"]: c for c in idx.active_chunks()}
|
|
269
|
+
out = []
|
|
270
|
+
for cid in chunk_ids[:16]:
|
|
271
|
+
c = all_chunks.get(cid)
|
|
272
|
+
if c is None:
|
|
273
|
+
raise SecurityException(f"chunk {cid} not in this scope's index")
|
|
274
|
+
p = idx.active_pages().get(c["page_id"], {})
|
|
275
|
+
out.append({"chunk_id": cid, "text": c["text"],
|
|
276
|
+
"page_title": p.get("title", ""), "page_url": p.get("url", "")})
|
|
277
|
+
return {"chunks": out}
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
@mcp.tool()
|
|
281
|
+
def verify_citations(scope_handle: str, evidence_set: str,
|
|
282
|
+
mappings: list[dict]) -> dict:
|
|
283
|
+
_scope(scope_handle)
|
|
284
|
+
ev = _evidence_sets.get(evidence_set)
|
|
285
|
+
if ev is None or not evidence_set.endswith(_scopes[scope_handle]["key"][:8]):
|
|
286
|
+
raise SecurityException(
|
|
287
|
+
"unknown or cross-scope evidence set — run search again and cite its evidence_set")
|
|
288
|
+
return check_citations(ev, mappings)
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def main() -> None:
|
|
292
|
+
logging.getLogger("httpx").setLevel(logging.WARNING)
|
|
293
|
+
_start_warmup()
|
|
294
|
+
mcp.run()
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
if __name__ == "__main__":
|
|
298
|
+
main()
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Filesystem layout for local-knowledge — ~/.local-knowledge, LOCAL_KNOWLEDGE_*.
|
|
2
|
+
|
|
3
|
+
The engine is confluence_knowledge's settings machinery parametrized for the
|
|
4
|
+
local connector; no credentials exist (connect reads only paths).
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import hashlib
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
from confluence_knowledge.settings import (
|
|
16
|
+
SetupError,
|
|
17
|
+
default_home,
|
|
18
|
+
resolve_workdir,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
APP_NAME = "local-knowledge"
|
|
22
|
+
INDEX_DIR_NAME = ".local_knowledge_index"
|
|
23
|
+
PROFILE_NAME = "profile.json"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def home() -> Path:
|
|
27
|
+
return default_home("LOCAL_KNOWLEDGE_HOME", ".local-knowledge")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def workdir() -> Path:
|
|
31
|
+
return resolve_workdir("LOCAL_KNOWLEDGE_WORKDIR",
|
|
32
|
+
"LOCAL_KNOWLEDGE_HOME", ".local-knowledge")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def index_root() -> Path:
|
|
36
|
+
return workdir() / INDEX_DIR_NAME
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@dataclass(frozen=True)
|
|
40
|
+
class LocalProfile:
|
|
41
|
+
"""A non-secret connect profile — just the root path that was indexed."""
|
|
42
|
+
|
|
43
|
+
root_path: str # canonical realpath of the file/folder
|
|
44
|
+
device: str # machine the path lives on (informational)
|
|
45
|
+
|
|
46
|
+
def scope_key(self) -> str:
|
|
47
|
+
raw = f"{self.root_path}|{self.device}"
|
|
48
|
+
return hashlib.sha256(raw.encode()).hexdigest()[:24]
|
|
49
|
+
|
|
50
|
+
def save(self) -> Path:
|
|
51
|
+
p = home() / PROFILE_NAME
|
|
52
|
+
p.parent.mkdir(parents=True, exist_ok=True)
|
|
53
|
+
tmp = p.with_suffix(".tmp")
|
|
54
|
+
tmp.write_text(json.dumps({
|
|
55
|
+
"root_path": self.root_path, "device": self.device,
|
|
56
|
+
}, indent=2))
|
|
57
|
+
os.replace(tmp, p)
|
|
58
|
+
return p
|
|
59
|
+
|
|
60
|
+
@staticmethod
|
|
61
|
+
def load() -> LocalProfile:
|
|
62
|
+
p = home() / PROFILE_NAME
|
|
63
|
+
if not p.exists():
|
|
64
|
+
raise SetupError("no local profile — run `local-knowledge connect` "
|
|
65
|
+
"with a path via the agent, or `local-knowledge doctor`")
|
|
66
|
+
d = json.loads(p.read_text())
|
|
67
|
+
return LocalProfile(d["root_path"], d.get("device", "local"))
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: local-knowledge
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Local file/folder indexer + grounded-Q&A MCP service — same engine as confluence-knowledge, no credentials needed
|
|
5
|
+
License: MIT
|
|
6
|
+
Project-URL: Repository, https://github.com/nkunal1992/agents
|
|
7
|
+
Project-URL: Documentation, https://github.com/nkunal1992/agents/blob/main/docs/local-knowledge.md
|
|
8
|
+
Project-URL: Issues, https://github.com/nkunal1992/agents/issues
|
|
9
|
+
Classifier: Operating System :: OS Independent
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
13
|
+
Requires-Python: >=3.11
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
Requires-Dist: confluence-knowledge>=0.4.0
|
|
16
|
+
Requires-Dist: python-docx>=1.1
|
|
17
|
+
Requires-Dist: pypdf>=4.0
|
|
18
|
+
Requires-Dist: python-pptx>=1.0
|
|
19
|
+
Requires-Dist: openpyxl>=3.1
|
|
20
|
+
Requires-Dist: mcp<2,>=1.6
|
|
21
|
+
Requires-Dist: pip-system-certs>=5.0
|
|
22
|
+
Provides-Extra: models
|
|
23
|
+
Requires-Dist: confluence-knowledge[models]>=0.4.0; extra == "models"
|
|
24
|
+
Provides-Extra: dev
|
|
25
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
26
|
+
Requires-Dist: pytest-cov>=5.0; extra == "dev"
|
|
27
|
+
Requires-Dist: ruff>=0.6; extra == "dev"
|
|
28
|
+
|
|
29
|
+
# local-knowledge
|
|
30
|
+
|
|
31
|
+
Local file/folder indexer + grounded-Q&A MCP service — the same engine as
|
|
32
|
+
[`confluence-knowledge`](../confluence-knowledge), pointed at the filesystem
|
|
33
|
+
instead of Confluence. No credentials, no HTTP.
|
|
34
|
+
|
|
35
|
+
- `local_knowledge/crawl_local.py` — durable filesystem BFS (secret-file
|
|
36
|
+
exclusions, symlink-escape guards, checkpointed resume)
|
|
37
|
+
- `local_knowledge/server_local.py` — the `local-knowledge/*` MCP tools
|
|
38
|
+
- `local_knowledge/cli_local.py` — `local-knowledge` CLI (no `setup`; there's
|
|
39
|
+
nothing to authenticate)
|
|
40
|
+
|
|
41
|
+
Shares `confluence_knowledge`'s index, extraction (`extract` +
|
|
42
|
+
`extract_files`), retrieval, registration and auth plumbing.
|
|
43
|
+
|
|
44
|
+
See `docs/local-knowledge.md` in the repo root for the full guide and
|
|
45
|
+
`bundle/local-README.md` for the store-listing copy.
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
local_knowledge/__init__.py,sha256=rreRBSwDo9eaGZBHs6idZUrXyNwH22MGS--mBeL3N3Y,81
|
|
2
|
+
local_knowledge/cli_local.py,sha256=T5hOoexTxQoTZYyCSGtP35ki0a4PnCVFIpzn7eJKR34,2974
|
|
3
|
+
local_knowledge/crawl_local.py,sha256=fOBadO-VPpJX_bqdpj8LV935CUicJ_odgyFEsI5-QQk,9134
|
|
4
|
+
local_knowledge/server_local.py,sha256=Xh-iI0yJnYyoXUCw7fUB0JYwh0_r-lDYUNIgWODQukQ,9423
|
|
5
|
+
local_knowledge/settings_local.py,sha256=YoHkz236ut6AoiG-Zl2NY_kYGqovEUOjKUxM4H14e2M,1983
|
|
6
|
+
local_knowledge-0.1.0.dist-info/METADATA,sha256=cBrJ51KTCL7GM9XXqNRIL1fae0UEH55H1EnI2pblOVA,1940
|
|
7
|
+
local_knowledge-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
8
|
+
local_knowledge-0.1.0.dist-info/entry_points.txt,sha256=f0GOJGe4zd6e4DiNrBp7Z6akVnaR9zur87-9bEJD9fc,67
|
|
9
|
+
local_knowledge-0.1.0.dist-info/top_level.txt,sha256=-5WJHYmSUJKOgmdKScXvwxS7ND8OzSilB78sfAnwMlQ,16
|
|
10
|
+
local_knowledge-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
local_knowledge
|