confluence-knowledge 0.3.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- confluence_knowledge/__init__.py +3 -0
- confluence_knowledge/auth.py +83 -0
- confluence_knowledge/cli.py +180 -0
- confluence_knowledge/client.py +152 -0
- confluence_knowledge/crawl.py +236 -0
- confluence_knowledge/extract.py +118 -0
- confluence_knowledge/index.py +193 -0
- confluence_knowledge/models.lock.json +15 -0
- confluence_knowledge/registration.py +134 -0
- confluence_knowledge/retrieval.py +207 -0
- confluence_knowledge/server.py +341 -0
- confluence_knowledge/settings.py +156 -0
- confluence_knowledge-0.3.1.dist-info/METADATA +77 -0
- confluence_knowledge-0.3.1.dist-info/RECORD +17 -0
- confluence_knowledge-0.3.1.dist-info/WHEEL +5 -0
- confluence_knowledge-0.3.1.dist-info/entry_points.txt +2 -0
- confluence_knowledge-0.3.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,341 @@
|
|
|
1
|
+
"""stdio MCP server — the only surface the Copilot agent can reach.
|
|
2
|
+
|
|
3
|
+
Exposes narrow operations; no arbitrary HTTP/shell/file tools. Scope handles
|
|
4
|
+
are opaque and bound to the site+account+root partition.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import logging
|
|
10
|
+
import os
|
|
11
|
+
import threading
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
from mcp.server.fastmcp import FastMCP
|
|
15
|
+
|
|
16
|
+
from .auth import get_credential
|
|
17
|
+
from .client import AccessError, ConfluenceClient, SecurityException, make_client
|
|
18
|
+
from .crawl import Crawler
|
|
19
|
+
from .index import Index
|
|
20
|
+
from .retrieval import LocalEmbedder, LocalReranker, check_citations, retrieve
|
|
21
|
+
from .settings import (
|
|
22
|
+
SetupError,
|
|
23
|
+
SiteProfile,
|
|
24
|
+
default_home,
|
|
25
|
+
extract_page_id,
|
|
26
|
+
resolve_workdir,
|
|
27
|
+
workspace_index_root,
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
mcp = FastMCP("confluence-knowledge")
|
|
31
|
+
|
|
32
|
+
# in-process caches (per server lifetime)
|
|
33
|
+
_scopes: dict[str, dict] = {}
|
|
34
|
+
_evidence_sets: dict[str, list[dict]] = {}
|
|
35
|
+
_indexes: dict[str, Index] = {} # one connection per DB for the server lifetime
|
|
36
|
+
_clients: dict[str, ConfluenceClient] = {} # one HTTP pool per credential
|
|
37
|
+
_EMBEDDER = None
|
|
38
|
+
_RERANKER = None
|
|
39
|
+
_model_lock = threading.Lock()
|
|
40
|
+
_models_ready = threading.Event()
|
|
41
|
+
_warm_lock = threading.Lock()
|
|
42
|
+
_warm_started = False
|
|
43
|
+
|
|
44
|
+
LOADING_NOTE = "loading local models — call this tool again"
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _home() -> Path:
|
|
48
|
+
return default_home()
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _workdir() -> Path:
|
|
52
|
+
return resolve_workdir()
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _step_seconds() -> float:
|
|
56
|
+
"""Upper bound for one tool call, kept well under IDE request timeouts."""
|
|
57
|
+
return float(os.environ.get("CONF_KNOWLEDGE_STEP_SECONDS", "10"))
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _client_for(profile: SiteProfile):
|
|
61
|
+
c = _clients.get(profile.credential_ref)
|
|
62
|
+
if c is None:
|
|
63
|
+
token = get_credential(_workdir(), profile.credential_ref)
|
|
64
|
+
if not token:
|
|
65
|
+
raise SetupError(
|
|
66
|
+
"no stored token — run `confluence-knowledge setup` to configure credentials")
|
|
67
|
+
c = _clients[profile.credential_ref] = make_client(profile, token)
|
|
68
|
+
return c
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _index_for(scope_key: str) -> Index:
|
|
72
|
+
idx = _indexes.get(scope_key)
|
|
73
|
+
if idx is None:
|
|
74
|
+
idx = _indexes[scope_key] = Index(workspace_index_root(_workdir()) / f"{scope_key}.db")
|
|
75
|
+
return idx
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _embedder():
|
|
79
|
+
global _EMBEDDER
|
|
80
|
+
with _model_lock:
|
|
81
|
+
if _EMBEDDER is None:
|
|
82
|
+
_EMBEDDER = LocalEmbedder(_home())
|
|
83
|
+
return _EMBEDDER
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _reranker():
|
|
87
|
+
global _RERANKER
|
|
88
|
+
with _model_lock:
|
|
89
|
+
if _RERANKER is None:
|
|
90
|
+
_RERANKER = LocalReranker(_home())
|
|
91
|
+
return _RERANKER
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _warm_models() -> None:
|
|
95
|
+
try:
|
|
96
|
+
_embedder()
|
|
97
|
+
_reranker()
|
|
98
|
+
except SetupError:
|
|
99
|
+
pass # reported by the tool that needs the model
|
|
100
|
+
finally:
|
|
101
|
+
_models_ready.set()
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _start_warmup() -> None:
|
|
105
|
+
"""Load torch + both models off the request path; the first load can take
|
|
106
|
+
longer than an IDE's request timeout."""
|
|
107
|
+
global _warm_started
|
|
108
|
+
with _warm_lock:
|
|
109
|
+
if _warm_started:
|
|
110
|
+
return
|
|
111
|
+
_warm_started = True
|
|
112
|
+
threading.Thread(target=_warm_models, name="model-warmup", daemon=True).start()
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _models_loaded(wait: float) -> bool:
|
|
116
|
+
_start_warmup()
|
|
117
|
+
return _models_ready.wait(wait)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _scope(scope_handle: str) -> dict:
|
|
121
|
+
s = _scopes.get(scope_handle)
|
|
122
|
+
if s is None:
|
|
123
|
+
# the IDE may restart the server (e.g. after a request timeout) and
|
|
124
|
+
# drop in-memory scopes; handles carry the root so they re-bind after
|
|
125
|
+
# the same identity check connect() performs
|
|
126
|
+
parts = scope_handle.split(":")
|
|
127
|
+
if len(parts) == 3 and parts[0] == "scope" and connect(parts[2])["scope_handle"] == scope_handle:
|
|
128
|
+
return _scopes[scope_handle]
|
|
129
|
+
raise SetupError("unknown scope — call connect first")
|
|
130
|
+
return s
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
@mcp.tool()
|
|
134
|
+
def status() -> dict:
|
|
135
|
+
"""Report setup readiness — never returns secrets."""
|
|
136
|
+
home = _home()
|
|
137
|
+
try:
|
|
138
|
+
profile = SiteProfile.load(home)
|
|
139
|
+
cfg = {"env": profile.env, "site": profile.base_url}
|
|
140
|
+
except SetupError as e:
|
|
141
|
+
cfg = {"error": str(e)}
|
|
142
|
+
models = home / "models"
|
|
143
|
+
return {
|
|
144
|
+
"configured": "error" not in cfg,
|
|
145
|
+
"profile": cfg,
|
|
146
|
+
"embedding_model_ready": (models / "all-MiniLM-L6-v2").exists(),
|
|
147
|
+
"reranker_model_ready": (models / "ms-marco-MiniLM-L6-v2").exists(),
|
|
148
|
+
"models_loaded": _models_ready.is_set() and _EMBEDDER is not None,
|
|
149
|
+
"index_root": str(workspace_index_root(_workdir())),
|
|
150
|
+
"live_scopes": sorted(_scopes),
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
@mcp.tool()
|
|
155
|
+
def connect(root: str) -> dict:
|
|
156
|
+
"""Validate credentials + root readability; bind a scope handle."""
|
|
157
|
+
profile = SiteProfile.load(_home())
|
|
158
|
+
root_id = extract_page_id(root)
|
|
159
|
+
_clients.pop(profile.credential_ref, None) # re-read the token: setup may have rotated it
|
|
160
|
+
client = _client_for(profile)
|
|
161
|
+
who = client.get("wiki/rest/api/user/current" if profile.env == "cloud"
|
|
162
|
+
else "rest/api/user/current").json()
|
|
163
|
+
principal = who.get("accountId") or who.get("userKey") or who.get("username") or "?"
|
|
164
|
+
scope_key = profile.scope_key(principal, root_id)
|
|
165
|
+
idx = _index_for(scope_key)
|
|
166
|
+
st = idx.stats()
|
|
167
|
+
handle = f"scope:{scope_key}:{root_id}"
|
|
168
|
+
_scopes[handle] = {"key": scope_key, "profile": profile, "root": root_id}
|
|
169
|
+
_start_warmup()
|
|
170
|
+
return {
|
|
171
|
+
"scope_handle": handle,
|
|
172
|
+
"root_page_id": root_id,
|
|
173
|
+
"principal": principal,
|
|
174
|
+
"cached": st["pages"] > 0,
|
|
175
|
+
"cached_pages": st["pages"],
|
|
176
|
+
"cached_as_of": st["as_of"],
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _crawler(s: dict, embedder=None) -> Crawler:
|
|
181
|
+
return Crawler(_client_for(s["profile"]), _index_for(s["key"]), s["profile"].env,
|
|
182
|
+
embedder=embedder, site_url=s["profile"].base_url)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
# background crawl workers — a crawl runs to completion server-side so a big
|
|
186
|
+
# subtree doesn't need hundreds of tool calls (each one an IDE LLM turn and a
|
|
187
|
+
# timeout surface, #81). crawl_step is then only a progress read.
|
|
188
|
+
_workers: dict[str, threading.Thread] = {}
|
|
189
|
+
_worker_errors: dict[str, str] = {}
|
|
190
|
+
_workers_lock = threading.Lock()
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def _worker_loop(s: dict, job_id: str) -> None:
|
|
194
|
+
idx = _index_for(s["key"])
|
|
195
|
+
try:
|
|
196
|
+
# models may still be warming; the worker waits, not the tool call
|
|
197
|
+
_models_ready.wait()
|
|
198
|
+
c = _crawler(s, embedder=_embedder())
|
|
199
|
+
while True:
|
|
200
|
+
r = c.step(job_id, batch=None, budget_s=None)
|
|
201
|
+
if r["state"] != "running":
|
|
202
|
+
break
|
|
203
|
+
except Exception as e: # noqa: BLE001 — surfaced via job state
|
|
204
|
+
err = f"{type(e).__name__}: {e}"
|
|
205
|
+
_worker_errors[f"{s['key']}:{job_id}"] = err
|
|
206
|
+
job = idx.job(job_id)
|
|
207
|
+
if job and job["state"] == "running":
|
|
208
|
+
idx.save_job(job_id, "error", job["queue"], job["errors"] + [err])
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _ensure_worker(s: dict, job_id: str) -> None:
|
|
212
|
+
"""Start the crawl worker if none is alive — also the resume path after
|
|
213
|
+
the server restarts mid-crawl (the queue is durable, the thread isn't)."""
|
|
214
|
+
key = f"{s['key']}:{job_id}"
|
|
215
|
+
with _workers_lock:
|
|
216
|
+
t = _workers.get(key)
|
|
217
|
+
if t is not None and t.is_alive():
|
|
218
|
+
return
|
|
219
|
+
_workers[key] = threading.Thread(
|
|
220
|
+
target=_worker_loop, args=(s, job_id),
|
|
221
|
+
name=f"crawl-{job_id}", daemon=True)
|
|
222
|
+
_workers[key].start()
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
@mcp.tool()
|
|
226
|
+
def start_crawl(scope_handle: str, refresh: bool = False) -> dict:
|
|
227
|
+
"""Begin crawling the root page and all descendants in a background
|
|
228
|
+
thread. The crawl runs to completion without further input — call
|
|
229
|
+
crawl_step only to read progress."""
|
|
230
|
+
s = _scope(scope_handle)
|
|
231
|
+
job_id = _crawler(s).start(s["root"])
|
|
232
|
+
_ensure_worker(s, job_id)
|
|
233
|
+
return {"job_id": job_id, "state": "running",
|
|
234
|
+
"note": "crawl runs in the background — call crawl_step to poll"}
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
@mcp.tool()
|
|
238
|
+
def crawl_step(scope_handle: str, job_id: str) -> dict:
|
|
239
|
+
"""Read crawl progress instantly. A running job with no live worker (e.g.
|
|
240
|
+
the server restarted mid-crawl) is resumed automatically; the queue and
|
|
241
|
+
per-page checkpoints are durable."""
|
|
242
|
+
s = _scope(scope_handle)
|
|
243
|
+
idx = _index_for(s["key"])
|
|
244
|
+
job = idx.job(job_id)
|
|
245
|
+
if job is None:
|
|
246
|
+
raise AccessError(f"unknown job {job_id}")
|
|
247
|
+
if job["state"] == "running":
|
|
248
|
+
if not _models_ready.is_set():
|
|
249
|
+
_start_warmup()
|
|
250
|
+
note = LOADING_NOTE
|
|
251
|
+
else:
|
|
252
|
+
note = None
|
|
253
|
+
_ensure_worker(s, job_id)
|
|
254
|
+
p = idx.job_progress(job_id)
|
|
255
|
+
if note:
|
|
256
|
+
p["note"] = note
|
|
257
|
+
return p
|
|
258
|
+
p = idx.job_progress(job_id)
|
|
259
|
+
if p["state"] == "done":
|
|
260
|
+
p["note"] = "no further calls needed — the index is published"
|
|
261
|
+
return p
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _coverage(idx: Index) -> str:
|
|
265
|
+
job = idx.latest_job()
|
|
266
|
+
if job is None or job["state"] == "abandoned":
|
|
267
|
+
return "unknown"
|
|
268
|
+
if job["state"] == "running":
|
|
269
|
+
return "refresh in progress — answering from the previous snapshot"
|
|
270
|
+
return "partial" if job["errors"] else "complete"
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
@mcp.tool()
|
|
274
|
+
def search(scope_handle: str, question: str) -> dict:
|
|
275
|
+
"""Hybrid retrieve + local rerank; returns bounded, cited evidence."""
|
|
276
|
+
s = _scope(scope_handle)
|
|
277
|
+
idx = _index_for(s["key"])
|
|
278
|
+
st = idx.stats()
|
|
279
|
+
if st["pages"] == 0:
|
|
280
|
+
job = idx.latest_job()
|
|
281
|
+
if job and job["state"] == "running":
|
|
282
|
+
return {"evidence": [], "coverage": "crawling",
|
|
283
|
+
"note": f"crawl {job['job_id']} still running — keep calling crawl_step "
|
|
284
|
+
"until state is done, then search"}
|
|
285
|
+
return {"evidence": [], "coverage": "empty",
|
|
286
|
+
"note": "no pages indexed — run a crawl first"}
|
|
287
|
+
if not _models_loaded(_step_seconds()):
|
|
288
|
+
return {"evidence": [], "coverage": "loading", "note": LOADING_NOTE}
|
|
289
|
+
ev = retrieve(idx, question, _embedder(), _reranker())
|
|
290
|
+
set_id = f"ev:{len(_evidence_sets)}:{s['key'][:8]}"
|
|
291
|
+
_evidence_sets[set_id] = ev
|
|
292
|
+
return {
|
|
293
|
+
"evidence_set": set_id,
|
|
294
|
+
"coverage": _coverage(idx),
|
|
295
|
+
"as_of": st["as_of"],
|
|
296
|
+
"evidence": [
|
|
297
|
+
{"chunk_id": e["chunk_id"], "page_title": e["page_title"],
|
|
298
|
+
"page_url": e["page_url"], "heading": e["heading"],
|
|
299
|
+
"score": e.get("rerank_score"), "text": e["text"][:2000]}
|
|
300
|
+
for e in ev
|
|
301
|
+
],
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
@mcp.tool()
|
|
306
|
+
def get_evidence(scope_handle: str, chunk_ids: list[str]) -> dict:
|
|
307
|
+
"""Return full text of previously returned chunks — same scope only."""
|
|
308
|
+
s = _scope(scope_handle)
|
|
309
|
+
idx = _index_for(s["key"])
|
|
310
|
+
all_chunks = {c["chunk_id"]: c for c in idx.active_chunks()}
|
|
311
|
+
out = []
|
|
312
|
+
for cid in chunk_ids[:16]:
|
|
313
|
+
c = all_chunks.get(cid)
|
|
314
|
+
if c is None:
|
|
315
|
+
raise SecurityException(f"chunk {cid} not in this scope's index")
|
|
316
|
+
p = idx.active_pages().get(c["page_id"], {})
|
|
317
|
+
out.append({"chunk_id": cid, "text": c["text"],
|
|
318
|
+
"page_title": p.get("title", ""), "page_url": p.get("url", "")})
|
|
319
|
+
return {"chunks": out}
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
@mcp.tool()
|
|
323
|
+
def verify_citations(scope_handle: str, evidence_set: str,
|
|
324
|
+
mappings: list[dict]) -> dict:
|
|
325
|
+
_scope(scope_handle)
|
|
326
|
+
ev = _evidence_sets.get(evidence_set)
|
|
327
|
+
if ev is None or not evidence_set.endswith(_scopes[scope_handle]["key"][:8]):
|
|
328
|
+
raise SecurityException(
|
|
329
|
+
"unknown or cross-scope evidence set — run search again and cite its evidence_set")
|
|
330
|
+
return check_citations(ev, mappings)
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def main() -> None:
|
|
334
|
+
# httpx logs every request URL at INFO; keep the IDE's MCP log readable
|
|
335
|
+
logging.getLogger("httpx").setLevel(logging.WARNING)
|
|
336
|
+
_start_warmup()
|
|
337
|
+
mcp.run()
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
if __name__ == "__main__":
|
|
341
|
+
main()
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
"""Filesystem layout and validated settings for confluence-knowledge.
|
|
2
|
+
|
|
3
|
+
All mutable state lives under the workspace's .conf_knowledge_index/ directory
|
|
4
|
+
(or CONF_KNOWLEDGE_HOME). Nothing is written outside declared paths.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import hashlib
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
import re
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from urllib.parse import urlparse
|
|
16
|
+
|
|
17
|
+
INDEX_DIR_NAME = ".conf_knowledge_index"
|
|
18
|
+
AUTH_FILE_NAME = ".conf_auth.json"
|
|
19
|
+
APP_NAME = "confluence-knowledge"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class SetupError(RuntimeError):
|
|
23
|
+
"""A problem the user can fix via setup instructions."""
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def default_home() -> Path:
|
|
27
|
+
env = os.environ.get("CONF_KNOWLEDGE_HOME")
|
|
28
|
+
if env:
|
|
29
|
+
return Path(env).expanduser().resolve()
|
|
30
|
+
return (Path.home() / ".confluence-knowledge").resolve()
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def resolve_workdir() -> Path:
|
|
34
|
+
"""Folder for the index and the file-fallback credential, shared by the CLI
|
|
35
|
+
and the server. CONF_KNOWLEDGE_WORKDIR wins; otherwise a per-user folder,
|
|
36
|
+
never the process cwd (IDEs start servers from read-only install dirs)."""
|
|
37
|
+
env = os.environ.get("CONF_KNOWLEDGE_WORKDIR")
|
|
38
|
+
if env:
|
|
39
|
+
return Path(env).expanduser().resolve()
|
|
40
|
+
return default_home() / "work"
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def workspace_index_root(workdir: Path) -> Path:
|
|
44
|
+
return workdir / INDEX_DIR_NAME
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _is_windows() -> bool:
|
|
48
|
+
return os.name == "nt"
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def reject_unsafe_path(path: Path) -> Path:
|
|
52
|
+
"""Refuse symlinked/reparse destinations so secrets/indices can't leak."""
|
|
53
|
+
resolved = path.expanduser()
|
|
54
|
+
for part in [resolved, *resolved.parents]:
|
|
55
|
+
if part.exists() and part.is_symlink():
|
|
56
|
+
raise SetupError(f"refusing to write through symlink: {part}")
|
|
57
|
+
if _is_windows() and part.exists():
|
|
58
|
+
try:
|
|
59
|
+
if part.stat().st_file_attributes & 0x400: # FILE_ATTRIBUTE_REPARSE_POINT
|
|
60
|
+
raise SetupError(f"refusing to write through reparse point: {part}")
|
|
61
|
+
except (OSError, AttributeError):
|
|
62
|
+
pass
|
|
63
|
+
return resolved
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def ensure_gitignore(path: Path, entries: list[str]) -> None:
|
|
67
|
+
"""Append ignore entries before any secret/cache write. Idempotent.
|
|
68
|
+
|
|
69
|
+
Raises SetupError if a protected path is already tracked by git.
|
|
70
|
+
"""
|
|
71
|
+
gi = path / ".gitignore"
|
|
72
|
+
existing = set(gi.read_text().splitlines()) if gi.exists() else set()
|
|
73
|
+
missing = [e for e in entries if e not in existing]
|
|
74
|
+
if missing:
|
|
75
|
+
with gi.open("a") as f:
|
|
76
|
+
for e in missing:
|
|
77
|
+
f.write(("\n" if existing else "") + e + "\n")
|
|
78
|
+
existing.update(missing)
|
|
79
|
+
_assert_not_tracked(path)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _assert_not_tracked(workdir: Path) -> None:
|
|
83
|
+
import subprocess
|
|
84
|
+
|
|
85
|
+
for rel in (AUTH_FILE_NAME, INDEX_DIR_NAME):
|
|
86
|
+
try:
|
|
87
|
+
r = subprocess.run(
|
|
88
|
+
["git", "-C", str(workdir), "ls-files", "--error-unmatch", rel],
|
|
89
|
+
check=False, capture_output=True, timeout=10,
|
|
90
|
+
)
|
|
91
|
+
except (OSError, subprocess.SubprocessError):
|
|
92
|
+
return # not a git repo / git absent — nothing tracked
|
|
93
|
+
if r.returncode == 0:
|
|
94
|
+
raise SetupError(
|
|
95
|
+
f"{rel} is already tracked by git — refusing to write secrets/cache there. "
|
|
96
|
+
f"Remove it from the index first: git rm --cached -r {rel}"
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@dataclass(frozen=True)
|
|
101
|
+
class SiteProfile:
|
|
102
|
+
"""A non-secret connection profile. Tokens live in a credential store."""
|
|
103
|
+
|
|
104
|
+
env: str # "cloud" | "datacenter"
|
|
105
|
+
base_url: str # canonical https base incl. context path (DC)
|
|
106
|
+
email: str | None # cloud only
|
|
107
|
+
credential_ref: str # opaque key — not the token itself
|
|
108
|
+
|
|
109
|
+
@staticmethod
|
|
110
|
+
def canonical_base(env: str, raw_url: str) -> str:
|
|
111
|
+
u = urlparse(raw_url.strip())
|
|
112
|
+
if u.scheme not in ("https",) and u.hostname not in ("localhost", "127.0.0.1"):
|
|
113
|
+
raise SetupError("Confluence URL must be https (localhost allowed for tests only)")
|
|
114
|
+
if u.username or u.password:
|
|
115
|
+
raise SetupError("credentials must not be embedded in the URL")
|
|
116
|
+
base = f"{u.scheme}://{u.netloc}{u.path.rstrip('/')}"
|
|
117
|
+
if env == "cloud" and not re.search(r"\.atlassian\.net$", u.hostname or ""):
|
|
118
|
+
raise SetupError("cloud environment requires an *.atlassian.net site URL")
|
|
119
|
+
return base
|
|
120
|
+
|
|
121
|
+
def scope_key(self, principal: str, root_page_id: str) -> str:
|
|
122
|
+
"""Partition identity: site + authenticated account + root page."""
|
|
123
|
+
raw = f"{self.base_url}|{principal}|{root_page_id}"
|
|
124
|
+
return hashlib.sha256(raw.encode()).hexdigest()[:24]
|
|
125
|
+
|
|
126
|
+
def save(self, home: Path) -> Path:
|
|
127
|
+
p = home / "profile.json"
|
|
128
|
+
p.parent.mkdir(parents=True, exist_ok=True)
|
|
129
|
+
tmp = p.with_suffix(".tmp")
|
|
130
|
+
tmp.write_text(json.dumps({
|
|
131
|
+
"env": self.env, "base_url": self.base_url,
|
|
132
|
+
"email": self.email, "credential_ref": self.credential_ref,
|
|
133
|
+
}, indent=2))
|
|
134
|
+
os.replace(tmp, p) # atomic
|
|
135
|
+
return p
|
|
136
|
+
|
|
137
|
+
@staticmethod
|
|
138
|
+
def load(home: Path) -> SiteProfile:
|
|
139
|
+
p = home / "profile.json"
|
|
140
|
+
if not p.exists():
|
|
141
|
+
raise SetupError("no Confluence profile configured — run `confluence-knowledge setup`")
|
|
142
|
+
d = json.loads(p.read_text())
|
|
143
|
+
return SiteProfile(d["env"], d["base_url"], d.get("email"), d["credential_ref"])
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def extract_page_id(root: str) -> str:
|
|
147
|
+
"""Accept a numeric ID or a Confluence page URL."""
|
|
148
|
+
if re.fullmatch(r"\d+", root.strip()):
|
|
149
|
+
return root.strip()
|
|
150
|
+
m = re.search(r"pageId=(\d+)", root) or re.search(r"/pages/(\d+)", root)
|
|
151
|
+
if m:
|
|
152
|
+
return m.group(1)
|
|
153
|
+
raise SetupError(
|
|
154
|
+
"can't resolve the root page ID from that URL — provide the numeric pageId "
|
|
155
|
+
"(open the page → Copy Link → look for pageId=…)"
|
|
156
|
+
)
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: confluence-knowledge
|
|
3
|
+
Version: 0.3.1
|
|
4
|
+
Summary: Local, GET-only Confluence subtree indexer + grounded-Q&A MCP service
|
|
5
|
+
License: MIT
|
|
6
|
+
Project-URL: Repository, https://github.com/nkunal1992/agents
|
|
7
|
+
Project-URL: Documentation, https://github.com/nkunal1992/agents/blob/main/docs/confluence-knowledge.md
|
|
8
|
+
Project-URL: Issues, https://github.com/nkunal1992/agents/issues
|
|
9
|
+
Classifier: Operating System :: OS Independent
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
13
|
+
Requires-Python: >=3.11
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
Requires-Dist: httpx<0.29,>=0.27
|
|
16
|
+
Requires-Dist: beautifulsoup4>=4.12
|
|
17
|
+
Requires-Dist: numpy>=1.26
|
|
18
|
+
Requires-Dist: keyring>=25.2
|
|
19
|
+
Requires-Dist: mcp<2,>=1.6
|
|
20
|
+
Requires-Dist: pip-system-certs>=5.0
|
|
21
|
+
Provides-Extra: models
|
|
22
|
+
Requires-Dist: sentence-transformers<4.0,>=3.0; extra == "models"
|
|
23
|
+
Requires-Dist: huggingface-hub>=0.26; extra == "models"
|
|
24
|
+
Provides-Extra: dev
|
|
25
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
26
|
+
Requires-Dist: pytest-cov>=5.0; extra == "dev"
|
|
27
|
+
Requires-Dist: ruff>=0.6; extra == "dev"
|
|
28
|
+
|
|
29
|
+
# confluence-knowledge (local MCP service)
|
|
30
|
+
|
|
31
|
+
Local companion service for the `confluence-knowledge` Copilot agent.
|
|
32
|
+
See `../../docs/confluence-knowledge.md` for the full guide.
|
|
33
|
+
|
|
34
|
+
- GET-only `ConfluenceClient` — non-GET raises `SecurityException` pre-flight.
|
|
35
|
+
- Recursive subtree crawl (Cloud v2 + Data Center), resumable.
|
|
36
|
+
- Local embeddings (`all-MiniLM-L6-v2`) + local reranker
|
|
37
|
+
(`ms-marco-MiniLM-L6-v2`) — Apache-2.0, pinned in `models.lock.json`,
|
|
38
|
+
never a cloud embedding call.
|
|
39
|
+
- SQLite + FTS5 + vector-blob index under `.conf_knowledge_index/`.
|
|
40
|
+
- Credentials via OS keyring / env var / chmod-600 `.conf_auth.json`.
|
|
41
|
+
|
|
42
|
+
## Dev
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
python3.11+ -m venv .venv && .venv/bin/pip install -e '.[dev]'
|
|
46
|
+
.venv/bin/python -m pytest # all offline — no network needed
|
|
47
|
+
.venv/bin/python -m ruff check src tests
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
Before every release, run the end-to-end smoke test. It spawns the real stdio
|
|
51
|
+
server through the MCP client SDK and uses the real models against a fake
|
|
52
|
+
Confluence on localhost, with simulated network latency:
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
.venv/bin/pip install -e '.[models]'
|
|
56
|
+
CONF_KNOWLEDGE_HOME=/tmp/ck/home .venv/bin/confluence-knowledge prepare-models
|
|
57
|
+
CONF_KNOWLEDGE_HOME=/tmp/ck/home .venv/bin/python scripts/e2e_smoke.py # ends with "E2E OK"
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
`e2e_bootstrap.py` goes further — it replays the agent's entire install flow
|
|
61
|
+
in a sandboxed $HOME (python discovery → venv → `pip install` from the
|
|
62
|
+
shipped `dist/` bundle → models → credentials → `register` → the real server
|
|
63
|
+
over stdio → a full Q&A round-trip). Run it before every release:
|
|
64
|
+
|
|
65
|
+
```bash
|
|
66
|
+
CK_MODEL_CACHE=/tmp/ck/home/models .venv/bin/python scripts/e2e_bootstrap.py # "E2E BOOTSTRAP OK"
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
## CLI
|
|
70
|
+
|
|
71
|
+
| Command | Purpose |
|
|
72
|
+
|---|---|
|
|
73
|
+
| `confluence-knowledge setup` | guided env/site/token setup (hidden input, keyring) |
|
|
74
|
+
| `confluence-knowledge prepare-models` | fetch pinned local models once |
|
|
75
|
+
| `confluence-knowledge doctor` | verify config + heartbeat + models |
|
|
76
|
+
| `confluence-knowledge serve` | run the stdio MCP server (what the IDE launches) |
|
|
77
|
+
| `confluence-knowledge clean` | delete the local index (`~/.confluence-knowledge/work` unless `CONF_KNOWLEDGE_WORKDIR` is set) |
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
confluence_knowledge/__init__.py,sha256=9NQCzCFTQ9AtKCldl_dQPCKLpnVZgOC7OWSrvXecSN0,107
|
|
2
|
+
confluence_knowledge/auth.py,sha256=GSyfyxVT-HWIHDneNW2a3WzG31S8k_CZFdU3dvW1v5Y,2612
|
|
3
|
+
confluence_knowledge/cli.py,sha256=kkTXadtO0Q1McXS2pwtwaFsVYp4w3BhHM0WVNMMZUO8,6771
|
|
4
|
+
confluence_knowledge/client.py,sha256=OPDT4x8pNAhfCZ3DawJRDVPY8bDZHvZkhIXvuel90pA,6157
|
|
5
|
+
confluence_knowledge/crawl.py,sha256=Oiqg1g_z2neS6byEpM3RpC0jJDdzqkXgLDFbBwzrI7M,9884
|
|
6
|
+
confluence_knowledge/extract.py,sha256=zjPwx3Wc3h_OPIWEDBtpvlAay1XSRYIdigvBSdvv3qE,4211
|
|
7
|
+
confluence_knowledge/index.py,sha256=r1sHyZvf6j9Dhm6bDl8sViaFjX0hy27NR1s4UFT3WQk,8692
|
|
8
|
+
confluence_knowledge/models.lock.json,sha256=an-IDRAuNBSk4ooyuxugY-UsPYntolf2TvOdrD3pF8g,522
|
|
9
|
+
confluence_knowledge/registration.py,sha256=r-EoLShv4SzQr5_ygmJpU8UifMGdZ_6vjuPW61mMTFY,4737
|
|
10
|
+
confluence_knowledge/retrieval.py,sha256=SYoI4TYKKek6okKtU08nwtmpHKQIpqKbpXR2oVCEHwE,8139
|
|
11
|
+
confluence_knowledge/server.py,sha256=LhSMV4SfI60skIal6WrVXqQftg8oxPEqZf0hLEk6SZI,11581
|
|
12
|
+
confluence_knowledge/settings.py,sha256=sNDqBp-OIvp4aw6e7m6sxUYGZ4GZ17wnh-21rEI1nAU,5778
|
|
13
|
+
confluence_knowledge-0.3.1.dist-info/METADATA,sha256=1eWgeNGnXOsmyUJOEDHhgWofMuT0GGwNExDSolkKQDU,3292
|
|
14
|
+
confluence_knowledge-0.3.1.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
15
|
+
confluence_knowledge-0.3.1.dist-info/entry_points.txt,sha256=C7QwouYG11ah2TdR2bKpOFf2F9c91799_DFNT0ocrNE,71
|
|
16
|
+
confluence_knowledge-0.3.1.dist-info/top_level.txt,sha256=oMELxCmqPUkKpODS3xuggz-ZMJbsow5L9EPF_4DhydU,21
|
|
17
|
+
confluence_knowledge-0.3.1.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
confluence_knowledge
|