confluence-knowledge 0.3.1__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/PKG-INFO +6 -1
- {confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/pyproject.toml +7 -1
- {confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/src/confluence_knowledge/auth.py +17 -13
- confluence_knowledge-0.4.0/src/confluence_knowledge/extract_files.py +227 -0
- {confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/src/confluence_knowledge/registration.py +16 -12
- {confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/src/confluence_knowledge/retrieval.py +9 -10
- {confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/src/confluence_knowledge/settings.py +19 -14
- {confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/src/confluence_knowledge.egg-info/PKG-INFO +6 -1
- {confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/src/confluence_knowledge.egg-info/SOURCES.txt +1 -0
- {confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/src/confluence_knowledge.egg-info/requires.txt +6 -0
- {confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/tests/test_secrets.py +2 -2
- {confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/README.md +0 -0
- {confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/setup.cfg +0 -0
- {confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/src/confluence_knowledge/__init__.py +0 -0
- {confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/src/confluence_knowledge/cli.py +0 -0
- {confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/src/confluence_knowledge/client.py +0 -0
- {confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/src/confluence_knowledge/crawl.py +0 -0
- {confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/src/confluence_knowledge/extract.py +0 -0
- {confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/src/confluence_knowledge/index.py +0 -0
- {confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/src/confluence_knowledge/models.lock.json +0 -0
- {confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/src/confluence_knowledge/server.py +0 -0
- {confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/src/confluence_knowledge.egg-info/dependency_links.txt +0 -0
- {confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/src/confluence_knowledge.egg-info/entry_points.txt +0 -0
- {confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/src/confluence_knowledge.egg-info/top_level.txt +0 -0
- {confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/tests/test_agent_doc_consistency.py +0 -0
- {confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/tests/test_client_get_only.py +0 -0
- {confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/tests/test_crawl.py +0 -0
- {confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/tests/test_extract.py +0 -0
- {confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/tests/test_index.py +0 -0
- {confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/tests/test_prepare_models.py +0 -0
- {confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/tests/test_registration.py +0 -0
- {confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/tests/test_retrieval.py +0 -0
- {confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/tests/test_server_flow.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: confluence-knowledge
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Local, GET-only Confluence subtree indexer + grounded-Q&A MCP service
|
|
5
5
|
License: MIT
|
|
6
6
|
Project-URL: Repository, https://github.com/nkunal1992/agents
|
|
@@ -21,6 +21,11 @@ Requires-Dist: pip-system-certs>=5.0
|
|
|
21
21
|
Provides-Extra: models
|
|
22
22
|
Requires-Dist: sentence-transformers<4.0,>=3.0; extra == "models"
|
|
23
23
|
Requires-Dist: huggingface-hub>=0.26; extra == "models"
|
|
24
|
+
Provides-Extra: files
|
|
25
|
+
Requires-Dist: python-docx>=1.1; extra == "files"
|
|
26
|
+
Requires-Dist: pypdf>=4.0; extra == "files"
|
|
27
|
+
Requires-Dist: python-pptx>=1.0; extra == "files"
|
|
28
|
+
Requires-Dist: openpyxl>=3.1; extra == "files"
|
|
24
29
|
Provides-Extra: dev
|
|
25
30
|
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
26
31
|
Requires-Dist: pytest-cov>=5.0; extra == "dev"
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "confluence-knowledge"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.4.0"
|
|
8
8
|
description = "Local, GET-only Confluence subtree indexer + grounded-Q&A MCP service"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.11"
|
|
@@ -34,6 +34,12 @@ models = [
|
|
|
34
34
|
"sentence-transformers>=3.0,<4.0",
|
|
35
35
|
"huggingface-hub>=0.26",
|
|
36
36
|
]
|
|
37
|
+
files = [
|
|
38
|
+
"python-docx>=1.1",
|
|
39
|
+
"pypdf>=4.0",
|
|
40
|
+
"python-pptx>=1.0",
|
|
41
|
+
"openpyxl>=3.1",
|
|
42
|
+
]
|
|
37
43
|
dev = [
|
|
38
44
|
"pytest>=8.0",
|
|
39
45
|
"pytest-cov>=5.0",
|
|
@@ -18,36 +18,38 @@ ENV_TOKEN = "CONF_KNOWLEDGE_TOKEN"
|
|
|
18
18
|
ENV_EMAIL = "CONF_KNOWLEDGE_EMAIL"
|
|
19
19
|
|
|
20
20
|
|
|
21
|
-
def _keyring_store(ref: str, secret: str) -> bool:
|
|
21
|
+
def _keyring_store(ref: str, secret: str, service: str = SERVICE) -> bool:
|
|
22
22
|
try:
|
|
23
23
|
import keyring
|
|
24
24
|
from keyring.backends.fail import Keyring as FailBackend
|
|
25
25
|
if isinstance(keyring.get_keyring(), FailBackend):
|
|
26
26
|
return False
|
|
27
|
-
keyring.set_password(
|
|
27
|
+
keyring.set_password(service, ref, secret)
|
|
28
28
|
return True
|
|
29
29
|
except Exception: # noqa: BLE001 — keyring backends fail in many ways; fallback is intentional
|
|
30
30
|
return False
|
|
31
31
|
|
|
32
32
|
|
|
33
|
-
def _keyring_get(ref: str) -> str | None:
|
|
33
|
+
def _keyring_get(ref: str, service: str = SERVICE) -> str | None:
|
|
34
34
|
try:
|
|
35
35
|
import keyring
|
|
36
|
-
return keyring.get_password(
|
|
36
|
+
return keyring.get_password(service, ref)
|
|
37
37
|
except Exception: # noqa: BLE001 — unavailable keyring → try next provider
|
|
38
38
|
return None
|
|
39
39
|
|
|
40
40
|
|
|
41
|
-
def store_credential(workdir: Path, home: Path, ref: str, token: str
|
|
41
|
+
def store_credential(workdir: Path, home: Path, ref: str, token: str,
|
|
42
|
+
service: str = SERVICE, auth_file: str = AUTH_FILE_NAME,
|
|
43
|
+
index_dir: str = ".conf_knowledge_index") -> str:
|
|
42
44
|
"""Persist token: keyring preferred, protected file as explicit fallback.
|
|
43
45
|
|
|
44
46
|
Returns a human description of where it went (never the token).
|
|
45
47
|
"""
|
|
46
|
-
if _keyring_store(ref, token):
|
|
48
|
+
if _keyring_store(ref, token, service):
|
|
47
49
|
return "OS credential store"
|
|
48
50
|
# file fallback — gitignore BEFORE writing
|
|
49
|
-
ensure_gitignore(workdir, [
|
|
50
|
-
p = reject_unsafe_path(workdir /
|
|
51
|
+
ensure_gitignore(workdir, [auth_file, index_dir + "/"])
|
|
52
|
+
p = reject_unsafe_path(workdir / auth_file)
|
|
51
53
|
data = json.loads(p.read_text()) if p.exists() else {}
|
|
52
54
|
data[ref] = token
|
|
53
55
|
tmp = p.with_suffix(".tmp")
|
|
@@ -58,19 +60,21 @@ def store_credential(workdir: Path, home: Path, ref: str, token: str) -> str:
|
|
|
58
60
|
return f"protected file {p} (chmod 600)"
|
|
59
61
|
|
|
60
62
|
|
|
61
|
-
def get_credential(workdir: Path, ref: str
|
|
63
|
+
def get_credential(workdir: Path, ref: str, env: str = ENV_TOKEN,
|
|
64
|
+
service: str = SERVICE, auth_file: str = AUTH_FILE_NAME,
|
|
65
|
+
home_dir: str = ".confluence-knowledge") -> str | None:
|
|
62
66
|
"""Lookup order: env var (CI/scripts) → keyring → protected file."""
|
|
63
|
-
t = os.environ.get(
|
|
67
|
+
t = os.environ.get(env)
|
|
64
68
|
if t:
|
|
65
69
|
return t
|
|
66
|
-
t = _keyring_get(ref)
|
|
70
|
+
t = _keyring_get(ref, service)
|
|
67
71
|
if t:
|
|
68
72
|
return t
|
|
69
|
-
p = workdir /
|
|
73
|
+
p = workdir / auth_file
|
|
70
74
|
if p.exists():
|
|
71
75
|
data = json.loads(p.read_text())
|
|
72
76
|
return data.get(ref)
|
|
73
|
-
p2 = Path.home() /
|
|
77
|
+
p2 = Path.home() / home_dir / auth_file
|
|
74
78
|
if p2.exists():
|
|
75
79
|
return json.loads(p2.read_text()).get(ref)
|
|
76
80
|
return None
|
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
"""Bytes → chunks for office/text file formats.
|
|
2
|
+
|
|
3
|
+
Shared by the local-files and SharePoint connectors — both end up with the
|
|
4
|
+
same Chunk objects the index and retriever already understand. Office/PDF
|
|
5
|
+
parsers are imported lazily so this module costs nothing when a file type
|
|
6
|
+
isn't used. Unsupported/binary files produce a warning, never a crash.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import csv
|
|
12
|
+
import io
|
|
13
|
+
import zipfile
|
|
14
|
+
|
|
15
|
+
from .extract import Chunk, extract_chunks
|
|
16
|
+
from .settings import SetupError
|
|
17
|
+
|
|
18
|
+
MAX_FILE_WORDS = 200_000 # ~1MB of text — beyond that, index metadata only
|
|
19
|
+
|
|
20
|
+
SUPPORTED = {".txt", ".md", ".markdown", ".csv", ".html", ".htm",
|
|
21
|
+
".docx", ".pptx", ".xlsx", ".pdf"}
|
|
22
|
+
|
|
23
|
+
# Names that look like credentials/secrets — excluded by default and
|
|
24
|
+
# disclosed in coverage warnings.
|
|
25
|
+
SECRET_NAMES = {".env", ".envrc", "id_rsa", "id_dsa", "id_ed25519", "id_ecdsa",
|
|
26
|
+
"credentials", "credentials.json", ".netrc", ".pgpass"}
|
|
27
|
+
SECRET_SUFFIXES = {".pem", ".key", ".p12", ".pfx", ".keystore", ".jks"}
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def is_secret_name(filename: str) -> bool:
|
|
31
|
+
"""True when the filename strongly suggests a credential/secret file."""
|
|
32
|
+
from pathlib import Path as P
|
|
33
|
+
name = P(filename).name.lower()
|
|
34
|
+
return (name in SECRET_NAMES
|
|
35
|
+
or name.startswith(".env.") or name.endswith(".env")
|
|
36
|
+
or P(name).suffix in SECRET_SUFFIXES)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _need(pkg: str, fmt: str):
|
|
40
|
+
raise SetupError(
|
|
41
|
+
f"'{pkg}' isn't installed — install the file extra: "
|
|
42
|
+
f"`pip install confluence-knowledge[files]` (needed for {fmt})")
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _decode_text(data: bytes) -> str:
|
|
46
|
+
for enc in ("utf-8", "utf-16", "latin-1"):
|
|
47
|
+
try:
|
|
48
|
+
return data.decode(enc)
|
|
49
|
+
except UnicodeDecodeError:
|
|
50
|
+
continue
|
|
51
|
+
return data.decode("utf-8", errors="replace")
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _text_chunks(text: str, heading: str, kind: str = "text") -> list[Chunk]:
|
|
55
|
+
"""Split plain text into word-bounded chunks (same shape as extract.py)."""
|
|
56
|
+
words = text.split()
|
|
57
|
+
out: list[Chunk] = []
|
|
58
|
+
step = 280 - 40
|
|
59
|
+
for i in range(0, len(words), step):
|
|
60
|
+
seg = " ".join(words[i:i + 280])
|
|
61
|
+
if seg.strip():
|
|
62
|
+
out.append(Chunk(seg, heading, kind, len(out)))
|
|
63
|
+
return out
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _docx_chunks(data: bytes) -> list[Chunk]:
|
|
67
|
+
try:
|
|
68
|
+
import docx # python-docx
|
|
69
|
+
except ImportError:
|
|
70
|
+
_need("python-docx", ".docx")
|
|
71
|
+
doc = docx.Document(io.BytesIO(data))
|
|
72
|
+
chunks: list[Chunk] = []
|
|
73
|
+
heading: list[str] = []
|
|
74
|
+
buf: list[str] = []
|
|
75
|
+
|
|
76
|
+
def flush():
|
|
77
|
+
text = "\n".join(buf).strip()
|
|
78
|
+
buf.clear()
|
|
79
|
+
if text:
|
|
80
|
+
chunks.extend(_text_chunks(text, " > ".join(heading)))
|
|
81
|
+
|
|
82
|
+
for el in doc.element.body.iter():
|
|
83
|
+
tag = el.tag.split("}")[-1]
|
|
84
|
+
if tag == "p":
|
|
85
|
+
# find the paragraph object text via docx API per element
|
|
86
|
+
texts = [n.text for n in el.iter() if n.tag.split("}")[-1] == "t" and n.text]
|
|
87
|
+
line = "".join(texts).strip()
|
|
88
|
+
if not line:
|
|
89
|
+
continue
|
|
90
|
+
style = el.find("{*}pPr/{*}pStyle")
|
|
91
|
+
sty = style.get("{http://schemas.openxmlformats.org/wordprocessingml/2006/main}val", "") if style is not None else ""
|
|
92
|
+
if sty.lower().startswith("heading"):
|
|
93
|
+
flush()
|
|
94
|
+
level = int(sty[7] or 1) if len(sty) > 7 else 1
|
|
95
|
+
heading = heading[: level - 1]
|
|
96
|
+
while len(heading) < level - 1:
|
|
97
|
+
heading.append("")
|
|
98
|
+
heading.append(line)
|
|
99
|
+
buf.append(line)
|
|
100
|
+
elif tag == "tbl":
|
|
101
|
+
flush()
|
|
102
|
+
rows = []
|
|
103
|
+
for tr in el.iter("{http://schemas.openxmlformats.org/wordprocessingml/2006/main}tr"):
|
|
104
|
+
cells = []
|
|
105
|
+
for tc in tr.iter("{http://schemas.openxmlformats.org/wordprocessingml/2006/main}tc"):
|
|
106
|
+
cells.append("".join(t.text or "" for t in tc.iter(
|
|
107
|
+
"{http://schemas.openxmlformats.org/wordprocessingml/2006/main}t")).strip())
|
|
108
|
+
if any(cells):
|
|
109
|
+
rows.append(" | ".join(cells))
|
|
110
|
+
if rows:
|
|
111
|
+
header, *body = rows
|
|
112
|
+
t = "\n".join([f"Table — columns: {header}"]
|
|
113
|
+
+ [f"row {i}: {r}" for i, r in enumerate(body, 1)])
|
|
114
|
+
chunks.extend(_text_chunks(t, " > ".join(heading), "table"))
|
|
115
|
+
flush()
|
|
116
|
+
return chunks
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _pptx_chunks(data: bytes) -> list[Chunk]:
|
|
120
|
+
try:
|
|
121
|
+
from pptx import Presentation
|
|
122
|
+
except ImportError:
|
|
123
|
+
_need("python-pptx", ".pptx")
|
|
124
|
+
prs = Presentation(io.BytesIO(data))
|
|
125
|
+
chunks: list[Chunk] = []
|
|
126
|
+
for i, slide in enumerate(prs.slides, 1):
|
|
127
|
+
texts = []
|
|
128
|
+
for shape in slide.shapes:
|
|
129
|
+
if shape.has_text_frame:
|
|
130
|
+
texts.extend(p.text for p in shape.text_frame.paragraphs)
|
|
131
|
+
if shape.has_table:
|
|
132
|
+
for row in shape.table.rows:
|
|
133
|
+
texts.append(" | ".join(c.text for c in row.cells))
|
|
134
|
+
if slide.has_notes_slide and slide.notes_slide.notes_text_frame.text.strip():
|
|
135
|
+
texts.append("Notes: " + slide.notes_slide.notes_text_frame.text)
|
|
136
|
+
text = "\n".join(t for t in texts if t and t.strip())
|
|
137
|
+
if text.strip():
|
|
138
|
+
chunks.extend(_text_chunks(text, f"Slide {i}"))
|
|
139
|
+
return chunks
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _xlsx_chunks(data: bytes) -> list[Chunk]:
|
|
143
|
+
try:
|
|
144
|
+
import openpyxl
|
|
145
|
+
except ImportError:
|
|
146
|
+
_need("openpyxl", ".xlsx")
|
|
147
|
+
wb = openpyxl.load_workbook(io.BytesIO(data), read_only=True, data_only=True)
|
|
148
|
+
chunks: list[Chunk] = []
|
|
149
|
+
for ws in wb.worksheets:
|
|
150
|
+
rows = []
|
|
151
|
+
for row in ws.iter_rows(values_only=True):
|
|
152
|
+
vals = [str(c) if c is not None else "" for c in row]
|
|
153
|
+
if any(v.strip() for v in vals):
|
|
154
|
+
rows.append(" | ".join(vals))
|
|
155
|
+
if not rows:
|
|
156
|
+
continue
|
|
157
|
+
header, *body = rows
|
|
158
|
+
text = "\n".join([f"Table — columns: {header}"]
|
|
159
|
+
+ [f"row {i}: {r}" for i, r in enumerate(body, 1)])
|
|
160
|
+
chunks.extend(_text_chunks(text, f"Sheet: {ws.title}", "table"))
|
|
161
|
+
wb.close()
|
|
162
|
+
return chunks
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _pdf_chunks(data: bytes) -> tuple[list[Chunk], list[str]]:
|
|
166
|
+
try:
|
|
167
|
+
from pypdf import PdfReader
|
|
168
|
+
except ImportError:
|
|
169
|
+
_need("pypdf", ".pdf")
|
|
170
|
+
chunks: list[Chunk] = []
|
|
171
|
+
warns: list[str] = []
|
|
172
|
+
reader = PdfReader(io.BytesIO(data))
|
|
173
|
+
for i, page in enumerate(reader.pages, 1):
|
|
174
|
+
try:
|
|
175
|
+
text = page.extract_text() or ""
|
|
176
|
+
except Exception: # noqa: BLE001 — a bad page shouldn't kill the file
|
|
177
|
+
warns.append(f"page {i}: no extractable text (scanned/image?)")
|
|
178
|
+
continue
|
|
179
|
+
if text.strip():
|
|
180
|
+
chunks.extend(_text_chunks(text, f"Page {i}"))
|
|
181
|
+
if not chunks and not warns:
|
|
182
|
+
warns.append("no extractable text — likely a scanned document (OCR not supported)")
|
|
183
|
+
return chunks, warns
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _csv_chunks(data: bytes) -> list[Chunk]:
|
|
187
|
+
text = _decode_text(data)
|
|
188
|
+
rows = [r for r in csv.reader(io.StringIO(text)) if any(c.strip() for c in r)]
|
|
189
|
+
if not rows:
|
|
190
|
+
return []
|
|
191
|
+
header, *body = rows
|
|
192
|
+
out = "\n".join([f"Table — columns: {' | '.join(header)}"]
|
|
193
|
+
+ [f"row {i}: {' | '.join(r)}" for i, r in enumerate(body, 1)])
|
|
194
|
+
return _text_chunks(out, "", "table")
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def extract_file_chunks(filename: str, data: bytes) -> tuple[list[Chunk], list[str]]:
|
|
198
|
+
"""Dispatch on extension → (chunks, warnings). Never guesses at binaries."""
|
|
199
|
+
from pathlib import Path as P
|
|
200
|
+
ext = P(filename).suffix.lower()
|
|
201
|
+
if ext not in SUPPORTED:
|
|
202
|
+
return [], [f"unsupported type '{ext or filename}' — skipped"]
|
|
203
|
+
|
|
204
|
+
words = len(data) // 6 # rough guard against absurd inputs
|
|
205
|
+
if words > MAX_FILE_WORDS * 6: # bytes→words estimate too big
|
|
206
|
+
return [], [f"file too large ({len(data)} bytes) — skipped"]
|
|
207
|
+
|
|
208
|
+
try:
|
|
209
|
+
if ext in (".txt", ".md", ".markdown"):
|
|
210
|
+
return _text_chunks(_decode_text(data), ""), []
|
|
211
|
+
if ext in (".html", ".htm"):
|
|
212
|
+
return extract_chunks(_decode_text(data))
|
|
213
|
+
if ext == ".csv":
|
|
214
|
+
return _csv_chunks(data), []
|
|
215
|
+
if ext == ".docx":
|
|
216
|
+
return _docx_chunks(data), []
|
|
217
|
+
if ext == ".pptx":
|
|
218
|
+
return _pptx_chunks(data), []
|
|
219
|
+
if ext == ".xlsx":
|
|
220
|
+
return _xlsx_chunks(data), []
|
|
221
|
+
if ext == ".pdf":
|
|
222
|
+
return _pdf_chunks(data)
|
|
223
|
+
except zipfile.BadZipFile:
|
|
224
|
+
return [], ["corrupt office file (not a valid zip) — skipped"]
|
|
225
|
+
except Exception as e: # noqa: BLE001 — one bad file must not kill a crawl
|
|
226
|
+
return [], [f"parse error: {type(e).__name__}: {e}"]
|
|
227
|
+
return [], [f"unsupported type '{ext}' — skipped"]
|
{confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/src/confluence_knowledge/registration.py
RENAMED
|
@@ -16,15 +16,17 @@ from pathlib import Path
|
|
|
16
16
|
from .settings import SetupError
|
|
17
17
|
|
|
18
18
|
SERVER_ID = "confluence-knowledge"
|
|
19
|
+
SERVER_MODULE = "confluence_knowledge.server"
|
|
20
|
+
WORKDIR_ENV = "CONF_KNOWLEDGE_WORKDIR"
|
|
19
21
|
|
|
20
22
|
|
|
21
|
-
def _server_entry(workdir: Path | None) -> dict:
|
|
23
|
+
def _server_entry(workdir: Path | None, module: str, env_var: str) -> dict:
|
|
22
24
|
entry = {
|
|
23
25
|
"command": sys.executable, # the venv python — no PATH guessing
|
|
24
|
-
"args": ["-m",
|
|
26
|
+
"args": ["-m", module],
|
|
25
27
|
}
|
|
26
28
|
if workdir is not None:
|
|
27
|
-
entry["env"] = {
|
|
29
|
+
entry["env"] = {env_var: str(workdir)}
|
|
28
30
|
return entry
|
|
29
31
|
|
|
30
32
|
|
|
@@ -78,8 +80,8 @@ def targets(ide: str = "auto", workspace: Path | None = None) -> list[Path]:
|
|
|
78
80
|
return dedup
|
|
79
81
|
|
|
80
82
|
|
|
81
|
-
def _merge(path: Path, entry: dict | None) -> str:
|
|
82
|
-
"""Write `entry` under servers[
|
|
83
|
+
def _merge(path: Path, entry: dict | None, server_id: str = SERVER_ID) -> str:
|
|
84
|
+
"""Write `entry` under servers[server_id], or remove it when None.
|
|
83
85
|
|
|
84
86
|
Merges with existing JSON; backs up before overwriting. A file that isn't
|
|
85
87
|
parseable (JSONC comments etc.) is backed up and replaced — the agent
|
|
@@ -97,10 +99,10 @@ def _merge(path: Path, entry: dict | None) -> str:
|
|
|
97
99
|
shutil.copy2(path, path.with_suffix(path.suffix + ".bak"))
|
|
98
100
|
servers = data.setdefault("servers", {})
|
|
99
101
|
if entry is None:
|
|
100
|
-
servers.pop(
|
|
102
|
+
servers.pop(server_id, None)
|
|
101
103
|
action = "removed"
|
|
102
104
|
else:
|
|
103
|
-
servers[
|
|
105
|
+
servers[server_id] = entry
|
|
104
106
|
action = "backup-replaced" if replaced else "written"
|
|
105
107
|
tmp = path.with_suffix(".tmp")
|
|
106
108
|
tmp.write_text(json.dumps(data, indent=2) + "\n")
|
|
@@ -109,7 +111,9 @@ def _merge(path: Path, entry: dict | None) -> str:
|
|
|
109
111
|
|
|
110
112
|
|
|
111
113
|
def register(ide: str = "auto", workspace: Path | None = None,
|
|
112
|
-
workdir: Path | None = None, remove: bool = False
|
|
114
|
+
workdir: Path | None = None, remove: bool = False,
|
|
115
|
+
server_id: str = SERVER_ID, module: str = SERVER_MODULE,
|
|
116
|
+
env_var: str = WORKDIR_ENV) -> list[tuple[Path, str]]:
|
|
113
117
|
"""Write/remove our server entry in each target. Returns (path, action)."""
|
|
114
118
|
if ide not in ("auto", "all", "vscode", "jetbrains"):
|
|
115
119
|
raise SetupError(f"unknown --ide '{ide}' (auto|all|vscode|jetbrains)")
|
|
@@ -117,11 +121,11 @@ def register(ide: str = "auto", workspace: Path | None = None,
|
|
|
117
121
|
if not files:
|
|
118
122
|
raise SetupError("no IDE config locations found — no Copilot/JetBrains "
|
|
119
123
|
"or VS Code profile detected")
|
|
120
|
-
entry = None if remove else _server_entry(workdir)
|
|
121
|
-
return [(f, _merge(f, entry)) for f in files]
|
|
124
|
+
entry = None if remove else _server_entry(workdir, module, env_var)
|
|
125
|
+
return [(f, _merge(f, entry, server_id)) for f in files]
|
|
122
126
|
|
|
123
127
|
|
|
124
|
-
def registered_paths() -> list[Path]:
|
|
128
|
+
def registered_paths(server_id: str = SERVER_ID) -> list[Path]:
|
|
125
129
|
"""Existing config files that already contain our server — for doctor."""
|
|
126
130
|
out = []
|
|
127
131
|
for p in targets("all"):
|
|
@@ -129,6 +133,6 @@ def registered_paths() -> list[Path]:
|
|
|
129
133
|
servers = json.loads(p.read_text()).get("servers", {})
|
|
130
134
|
except (OSError, json.JSONDecodeError, UnicodeDecodeError):
|
|
131
135
|
continue
|
|
132
|
-
if p.exists() and
|
|
136
|
+
if p.exists() and server_id in servers:
|
|
133
137
|
out.append(p)
|
|
134
138
|
return out
|
{confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/src/confluence_knowledge/retrieval.py
RENAMED
|
@@ -27,20 +27,20 @@ def _models_dir(home: Path) -> Path:
|
|
|
27
27
|
return home / "models"
|
|
28
28
|
|
|
29
29
|
|
|
30
|
-
def _load_sentence_transformer(model_key: str, home: Path
|
|
30
|
+
def _load_sentence_transformer(model_key: str, home: Path,
|
|
31
|
+
cli: str = "confluence-knowledge"):
|
|
31
32
|
"""Local-only model load — fails closed if artifacts aren't prepared."""
|
|
32
33
|
try:
|
|
33
34
|
from sentence_transformers import SentenceTransformer
|
|
34
35
|
except ImportError as e:
|
|
35
36
|
raise SetupError(
|
|
36
|
-
"sentence-transformers isn't installed — run "
|
|
37
|
-
"`confluence-knowledge prepare-models` (see docs/confluence-knowledge.md)"
|
|
37
|
+
f"sentence-transformers isn't installed — run `{cli} prepare-models`"
|
|
38
38
|
) from e
|
|
39
39
|
local = _models_dir(home) / model_key.split("/")[-1]
|
|
40
40
|
if not local.exists():
|
|
41
41
|
raise SetupError(
|
|
42
42
|
f"local model '{model_key}' not prepared — run "
|
|
43
|
-
"`
|
|
43
|
+
f"`{cli} prepare-models`. Cloud embeddings are never used."
|
|
44
44
|
)
|
|
45
45
|
os.environ.setdefault("TRANSFORMERS_OFFLINE", "1")
|
|
46
46
|
os.environ.setdefault("HF_HUB_OFFLINE", "1")
|
|
@@ -51,8 +51,8 @@ def _load_sentence_transformer(model_key: str, home: Path):
|
|
|
51
51
|
|
|
52
52
|
|
|
53
53
|
class LocalEmbedder:
|
|
54
|
-
def __init__(self, home: Path):
|
|
55
|
-
self._m = _load_sentence_transformer(EMBED_MODEL, home)
|
|
54
|
+
def __init__(self, home: Path, cli: str = "confluence-knowledge"):
|
|
55
|
+
self._m = _load_sentence_transformer(EMBED_MODEL, home, cli)
|
|
56
56
|
|
|
57
57
|
def embed(self, texts: list[str]) -> np.ndarray:
|
|
58
58
|
v = self._m.encode(texts, convert_to_numpy=True, normalize_embeddings=True,
|
|
@@ -61,19 +61,18 @@ class LocalEmbedder:
|
|
|
61
61
|
|
|
62
62
|
|
|
63
63
|
class LocalReranker:
|
|
64
|
-
def __init__(self, home: Path):
|
|
64
|
+
def __init__(self, home: Path, cli: str = "confluence-knowledge"):
|
|
65
65
|
try:
|
|
66
66
|
from sentence_transformers import CrossEncoder
|
|
67
67
|
except ImportError as e:
|
|
68
68
|
raise SetupError(
|
|
69
|
-
"sentence-transformers isn't installed — run "
|
|
70
|
-
"`confluence-knowledge prepare-models`"
|
|
69
|
+
f"sentence-transformers isn't installed — run `{cli} prepare-models`"
|
|
71
70
|
) from e
|
|
72
71
|
local = _models_dir(home) / RERANK_MODEL.split("/")[-1]
|
|
73
72
|
if not local.exists():
|
|
74
73
|
raise SetupError(
|
|
75
74
|
f"local reranker '{RERANK_MODEL}' not prepared — run "
|
|
76
|
-
"`
|
|
75
|
+
f"`{cli} prepare-models`.")
|
|
77
76
|
self._m = CrossEncoder(str(local), device="cpu",
|
|
78
77
|
trust_remote_code=False)
|
|
79
78
|
|
{confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/src/confluence_knowledge/settings.py
RENAMED
|
@@ -23,25 +23,28 @@ class SetupError(RuntimeError):
|
|
|
23
23
|
"""A problem the user can fix via setup instructions."""
|
|
24
24
|
|
|
25
25
|
|
|
26
|
-
def default_home(
|
|
27
|
-
|
|
26
|
+
def default_home(env_var: str = "CONF_KNOWLEDGE_HOME",
|
|
27
|
+
dir_name: str = ".confluence-knowledge") -> Path:
|
|
28
|
+
env = os.environ.get(env_var)
|
|
28
29
|
if env:
|
|
29
30
|
return Path(env).expanduser().resolve()
|
|
30
|
-
return (Path.home() /
|
|
31
|
+
return (Path.home() / dir_name).resolve()
|
|
31
32
|
|
|
32
33
|
|
|
33
|
-
def resolve_workdir(
|
|
34
|
+
def resolve_workdir(env_var: str = "CONF_KNOWLEDGE_WORKDIR",
|
|
35
|
+
home_env: str = "CONF_KNOWLEDGE_HOME",
|
|
36
|
+
home_dir: str = ".confluence-knowledge") -> Path:
|
|
34
37
|
"""Folder for the index and the file-fallback credential, shared by the CLI
|
|
35
|
-
and the server.
|
|
36
|
-
|
|
37
|
-
env = os.environ.get(
|
|
38
|
+
and the server. The env var wins; otherwise a per-user folder, never the
|
|
39
|
+
process cwd (IDEs start servers from read-only install dirs)."""
|
|
40
|
+
env = os.environ.get(env_var)
|
|
38
41
|
if env:
|
|
39
42
|
return Path(env).expanduser().resolve()
|
|
40
|
-
return default_home() / "work"
|
|
43
|
+
return default_home(home_env, home_dir) / "work"
|
|
41
44
|
|
|
42
45
|
|
|
43
|
-
def workspace_index_root(workdir: Path) -> Path:
|
|
44
|
-
return workdir /
|
|
46
|
+
def workspace_index_root(workdir: Path, index_dir: str = INDEX_DIR_NAME) -> Path:
|
|
47
|
+
return workdir / index_dir
|
|
45
48
|
|
|
46
49
|
|
|
47
50
|
def _is_windows() -> bool:
|
|
@@ -63,7 +66,9 @@ def reject_unsafe_path(path: Path) -> Path:
|
|
|
63
66
|
return resolved
|
|
64
67
|
|
|
65
68
|
|
|
66
|
-
def ensure_gitignore(path: Path, entries: list[str]
|
|
69
|
+
def ensure_gitignore(path: Path, entries: list[str],
|
|
70
|
+
protected: tuple[str, ...] = (AUTH_FILE_NAME,
|
|
71
|
+
INDEX_DIR_NAME)) -> None:
|
|
67
72
|
"""Append ignore entries before any secret/cache write. Idempotent.
|
|
68
73
|
|
|
69
74
|
Raises SetupError if a protected path is already tracked by git.
|
|
@@ -76,13 +81,13 @@ def ensure_gitignore(path: Path, entries: list[str]) -> None:
|
|
|
76
81
|
for e in missing:
|
|
77
82
|
f.write(("\n" if existing else "") + e + "\n")
|
|
78
83
|
existing.update(missing)
|
|
79
|
-
_assert_not_tracked(path)
|
|
84
|
+
_assert_not_tracked(path, protected)
|
|
80
85
|
|
|
81
86
|
|
|
82
|
-
def _assert_not_tracked(workdir: Path) -> None:
|
|
87
|
+
def _assert_not_tracked(workdir: Path, protected: tuple[str, ...]) -> None:
|
|
83
88
|
import subprocess
|
|
84
89
|
|
|
85
|
-
for rel in
|
|
90
|
+
for rel in protected:
|
|
86
91
|
try:
|
|
87
92
|
r = subprocess.run(
|
|
88
93
|
["git", "-C", str(workdir), "ls-files", "--error-unmatch", rel],
|
{confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/src/confluence_knowledge.egg-info/PKG-INFO
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: confluence-knowledge
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Local, GET-only Confluence subtree indexer + grounded-Q&A MCP service
|
|
5
5
|
License: MIT
|
|
6
6
|
Project-URL: Repository, https://github.com/nkunal1992/agents
|
|
@@ -21,6 +21,11 @@ Requires-Dist: pip-system-certs>=5.0
|
|
|
21
21
|
Provides-Extra: models
|
|
22
22
|
Requires-Dist: sentence-transformers<4.0,>=3.0; extra == "models"
|
|
23
23
|
Requires-Dist: huggingface-hub>=0.26; extra == "models"
|
|
24
|
+
Provides-Extra: files
|
|
25
|
+
Requires-Dist: python-docx>=1.1; extra == "files"
|
|
26
|
+
Requires-Dist: pypdf>=4.0; extra == "files"
|
|
27
|
+
Requires-Dist: python-pptx>=1.0; extra == "files"
|
|
28
|
+
Requires-Dist: openpyxl>=3.1; extra == "files"
|
|
24
29
|
Provides-Extra: dev
|
|
25
30
|
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
26
31
|
Requires-Dist: pytest-cov>=5.0; extra == "dev"
|
|
@@ -6,6 +6,7 @@ src/confluence_knowledge/cli.py
|
|
|
6
6
|
src/confluence_knowledge/client.py
|
|
7
7
|
src/confluence_knowledge/crawl.py
|
|
8
8
|
src/confluence_knowledge/extract.py
|
|
9
|
+
src/confluence_knowledge/extract_files.py
|
|
9
10
|
src/confluence_knowledge/index.py
|
|
10
11
|
src/confluence_knowledge/models.lock.json
|
|
11
12
|
src/confluence_knowledge/registration.py
|
|
@@ -9,7 +9,7 @@ from confluence_knowledge.settings import AUTH_FILE_NAME
|
|
|
9
9
|
|
|
10
10
|
def test_gitignore_written_before_secret(tmp_path, monkeypatch):
|
|
11
11
|
from confluence_knowledge import auth
|
|
12
|
-
monkeypatch.setattr(auth, "_keyring_store", lambda r, s: False)
|
|
12
|
+
monkeypatch.setattr(auth, "_keyring_store", lambda r, s, service=None: False)
|
|
13
13
|
where = store_credential(tmp_path, tmp_path / "home", "ref1", "tok_ABC123")
|
|
14
14
|
gi = (tmp_path / ".gitignore").read_text()
|
|
15
15
|
assert AUTH_FILE_NAME in gi and ".conf_knowledge_index/" in gi
|
|
@@ -19,7 +19,7 @@ def test_gitignore_written_before_secret(tmp_path, monkeypatch):
|
|
|
19
19
|
def test_secret_file_permissions(tmp_path, monkeypatch):
|
|
20
20
|
# force file fallback: make keyring unusable
|
|
21
21
|
from confluence_knowledge import auth
|
|
22
|
-
monkeypatch.setattr(auth, "_keyring_store", lambda r, s: False)
|
|
22
|
+
monkeypatch.setattr(auth, "_keyring_store", lambda r, s, service=None: False)
|
|
23
23
|
store_credential(tmp_path, tmp_path / "home", "ref1", "tok_ABC123")
|
|
24
24
|
if os.name != "nt":
|
|
25
25
|
mode = (tmp_path / AUTH_FILE_NAME).stat().st_mode
|
|
File without changes
|
|
File without changes
|
{confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/src/confluence_knowledge/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
{confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/src/confluence_knowledge/client.py
RENAMED
|
File without changes
|
|
File without changes
|
{confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/src/confluence_knowledge/extract.py
RENAMED
|
File without changes
|
|
File without changes
|
{confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/src/confluence_knowledge/models.lock.json
RENAMED
|
File without changes
|
{confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/src/confluence_knowledge/server.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{confluence_knowledge-0.3.1 → confluence_knowledge-0.4.0}/tests/test_agent_doc_consistency.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|