taskflow-doc-search 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- doc_search/__init__.py +1 -0
- doc_search/embeddings.py +192 -0
- doc_search/server.py +336 -0
- taskflow_doc_search-0.1.0.dist-info/METADATA +137 -0
- taskflow_doc_search-0.1.0.dist-info/RECORD +9 -0
- taskflow_doc_search-0.1.0.dist-info/WHEEL +5 -0
- taskflow_doc_search-0.1.0.dist-info/entry_points.txt +2 -0
- taskflow_doc_search-0.1.0.dist-info/licenses/LICENSE +21 -0
- taskflow_doc_search-0.1.0.dist-info/top_level.txt +1 -0
doc_search/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Documentation search tools served over MCP stdio."""
|
doc_search/embeddings.py
ADDED
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
"""Section embeddings with a content-validated, local derived index.
|
|
2
|
+
|
|
3
|
+
Importing this module does not import sentence-transformers or torch.
|
|
4
|
+
The fake embedder is a deterministic test double, not a semantic model.
|
|
5
|
+
"""
|
|
6
|
+
import hashlib
|
|
7
|
+
import json
|
|
8
|
+
import math
|
|
9
|
+
import os
|
|
10
|
+
import re
|
|
11
|
+
from functools import lru_cache
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
from .server import ROOT
|
|
15
|
+
|
|
16
|
+
# Keep the checkout cache beside the compatibility launcher, even when the
|
|
17
|
+
# workspace is overridden. Installed packages cache under the workspace,
|
|
18
|
+
# avoiding writes to site-packages.
|
|
19
|
+
_SERVER_DIR = Path(__file__).resolve().parents[1]
|
|
20
|
+
INDEX_DIR = (
|
|
21
|
+
_SERVER_DIR if (_SERVER_DIR / "server.py").is_file()
|
|
22
|
+
else ROOT / "mcp" / "doc-search"
|
|
23
|
+
) / ".index"
|
|
24
|
+
DEFAULT_MODEL = "sentence-transformers/all-MiniLM-L6-v2"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class _FakeEmbedder:
|
|
28
|
+
def encode(self, texts):
|
|
29
|
+
from .server import _tokenize
|
|
30
|
+
|
|
31
|
+
vectors = []
|
|
32
|
+
for text in texts:
|
|
33
|
+
vector = [0.0] * 64
|
|
34
|
+
for token in _tokenize(text):
|
|
35
|
+
slot = int(hashlib.sha256(token.encode("utf-8")).hexdigest(), 16) % 64
|
|
36
|
+
vector[slot] += 1.0
|
|
37
|
+
norm = math.sqrt(sum(v * v for v in vector))
|
|
38
|
+
vectors.append([v / norm for v in vector] if norm else vector)
|
|
39
|
+
return vectors
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class _RealEmbedder:
|
|
43
|
+
def __init__(self, model):
|
|
44
|
+
try:
|
|
45
|
+
from sentence_transformers import SentenceTransformer
|
|
46
|
+
except ImportError as exc:
|
|
47
|
+
raise RuntimeError(
|
|
48
|
+
"sentence-transformers is not installed; "
|
|
49
|
+
"install the optional embeddings dependencies with: "
|
|
50
|
+
"pip install taskflow-doc-search[embeddings]"
|
|
51
|
+
) from exc
|
|
52
|
+
self.model = SentenceTransformer(model)
|
|
53
|
+
|
|
54
|
+
def encode(self, texts):
|
|
55
|
+
return self.model.encode(texts, normalize_embeddings=True).tolist()
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@lru_cache(maxsize=2)
|
|
59
|
+
def _embedder(name, model):
|
|
60
|
+
if name == "fake":
|
|
61
|
+
return _FakeEmbedder()
|
|
62
|
+
if name != "sentence-transformers":
|
|
63
|
+
raise ValueError(f"unknown DOC_SEARCH_EMBEDDER: {name}")
|
|
64
|
+
return _RealEmbedder(model)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def get_embedder():
|
|
68
|
+
"""Return the configured embedder, loading the real model only on demand."""
|
|
69
|
+
return _embedder(
|
|
70
|
+
os.environ.get("DOC_SEARCH_EMBEDDER", "sentence-transformers"),
|
|
71
|
+
os.environ.get("DOC_SEARCH_EMBEDDING_MODEL", DEFAULT_MODEL),
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def chunk_document(path, text):
|
|
76
|
+
"""Split ATX sections into body chunks carrying their full heading trail.
|
|
77
|
+
|
|
78
|
+
The preamble uses the document title. Empty section bodies are retained
|
|
79
|
+
so a heading alone remains searchable. Fenced code is treated as body.
|
|
80
|
+
"""
|
|
81
|
+
from .server import ROOT, _doc_title
|
|
82
|
+
|
|
83
|
+
path = Path(path)
|
|
84
|
+
title = _doc_title(text, path)
|
|
85
|
+
relative = path.relative_to(ROOT).as_posix()
|
|
86
|
+
chunks, trail, body = [], [], []
|
|
87
|
+
heading = title
|
|
88
|
+
fence = None
|
|
89
|
+
|
|
90
|
+
def flush():
|
|
91
|
+
chunks.append({"path": relative, "title": title,
|
|
92
|
+
"heading": heading, "text": "\n".join(body).strip()})
|
|
93
|
+
|
|
94
|
+
for line in text.splitlines():
|
|
95
|
+
marker = re.match(r"^ {0,3}(`{3,}|~{3,})(.*)$", line)
|
|
96
|
+
if marker:
|
|
97
|
+
run, tail = marker.groups()
|
|
98
|
+
if fence is None:
|
|
99
|
+
fence = run
|
|
100
|
+
elif run[0] == fence[0] and len(run) >= len(fence) and not tail.strip():
|
|
101
|
+
fence = None
|
|
102
|
+
body.append(line)
|
|
103
|
+
continue
|
|
104
|
+
match = None if fence else re.match(r"^ {0,3}(#{1,6})(?:[ \t]+(.*)|$)", line)
|
|
105
|
+
if match:
|
|
106
|
+
if trail or any(part.strip() for part in body):
|
|
107
|
+
flush()
|
|
108
|
+
level = len(match[1])
|
|
109
|
+
label = re.sub(r"[ \t]+#+[ \t]*$", "", match[2] or "").strip()
|
|
110
|
+
trail = [(n, label_) for n, label_ in trail if n < level]
|
|
111
|
+
trail.append((level, label))
|
|
112
|
+
heading = " > ".join(label_ for _, label_ in trail)
|
|
113
|
+
body = []
|
|
114
|
+
else:
|
|
115
|
+
body.append(line)
|
|
116
|
+
if trail or any(part.strip() for part in body):
|
|
117
|
+
flush()
|
|
118
|
+
return chunks
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def build_index(documents, manifest):
|
|
122
|
+
"""Embed a byte snapshot of the corpus and persist its derived cache."""
|
|
123
|
+
chunks = []
|
|
124
|
+
for path, content in documents:
|
|
125
|
+
chunks.extend(chunk_document(path, content.decode("utf-8", errors="replace")))
|
|
126
|
+
if chunks:
|
|
127
|
+
vectors = get_embedder().encode([
|
|
128
|
+
f"{c['title']} > {c['heading']}\n{c['text']}" for c in chunks
|
|
129
|
+
])
|
|
130
|
+
for chunk, vector in zip(chunks, vectors):
|
|
131
|
+
chunk["embedding"] = vector
|
|
132
|
+
INDEX_DIR.mkdir(parents=True, exist_ok=True)
|
|
133
|
+
(INDEX_DIR / "chunks.jsonl").write_text(
|
|
134
|
+
"".join(json.dumps(c, ensure_ascii=False) + "\n" for c in chunks),
|
|
135
|
+
encoding="utf-8",
|
|
136
|
+
)
|
|
137
|
+
(INDEX_DIR / "manifest.json").write_text(
|
|
138
|
+
json.dumps({**manifest, "chunk_count": len(chunks)}, indent=2), encoding="utf-8"
|
|
139
|
+
)
|
|
140
|
+
return chunks
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def load_index():
|
|
144
|
+
"""Reuse the cache only when corpus bytes, model and embedder match."""
|
|
145
|
+
from .server import ROOT, _iter_docs
|
|
146
|
+
|
|
147
|
+
documents = [(p, p.read_bytes()) for p in _iter_docs()]
|
|
148
|
+
hashes = sorted((p.relative_to(ROOT).as_posix(), hashlib.sha256(b).hexdigest())
|
|
149
|
+
for p, b in documents)
|
|
150
|
+
manifest = {
|
|
151
|
+
"model": os.environ.get("DOC_SEARCH_EMBEDDING_MODEL", DEFAULT_MODEL),
|
|
152
|
+
"embedder": os.environ.get("DOC_SEARCH_EMBEDDER", "sentence-transformers"),
|
|
153
|
+
"fingerprint": hashlib.sha256(json.dumps(hashes).encode("utf-8")).hexdigest(),
|
|
154
|
+
}
|
|
155
|
+
try:
|
|
156
|
+
saved = json.loads((INDEX_DIR / "manifest.json").read_text(encoding="utf-8"))
|
|
157
|
+
if all(saved.get(k) == v for k, v in manifest.items()):
|
|
158
|
+
chunks = [json.loads(line) for line in
|
|
159
|
+
(INDEX_DIR / "chunks.jsonl").read_text(encoding="utf-8").splitlines()]
|
|
160
|
+
if len(chunks) == saved.get("chunk_count"):
|
|
161
|
+
return chunks
|
|
162
|
+
except (OSError, ValueError):
|
|
163
|
+
pass
|
|
164
|
+
return build_index(documents, manifest)
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def search(query, limit=8):
|
|
168
|
+
"""Rank section chunks by pure-Python cosine; cache writes are local."""
|
|
169
|
+
from .server import _tokenize
|
|
170
|
+
|
|
171
|
+
if limit <= 0 or not _tokenize(query):
|
|
172
|
+
return []
|
|
173
|
+
chunks = load_index()
|
|
174
|
+
if not chunks:
|
|
175
|
+
return []
|
|
176
|
+
vector = get_embedder().encode([query])[0]
|
|
177
|
+
norm = math.sqrt(sum(v * v for v in vector))
|
|
178
|
+
results = []
|
|
179
|
+
for chunk in chunks:
|
|
180
|
+
other = chunk["embedding"]
|
|
181
|
+
denominator = norm * math.sqrt(sum(v * v for v in other))
|
|
182
|
+
score = round(sum(a * b for a, b in zip(vector, other)) / denominator, 4) if denominator else 0
|
|
183
|
+
if not score:
|
|
184
|
+
continue
|
|
185
|
+
results.append({
|
|
186
|
+
"path": chunk["path"], "title": chunk["title"], "score": score,
|
|
187
|
+
"snippet": next((line.strip()[:200] for line in chunk["text"].splitlines()
|
|
188
|
+
if line.strip()), ""),
|
|
189
|
+
"heading": chunk["heading"],
|
|
190
|
+
})
|
|
191
|
+
results.sort(key=lambda r: (-r["score"], r["path"], r["heading"]))
|
|
192
|
+
return results[:limit]
|
doc_search/server.py
ADDED
|
@@ -0,0 +1,336 @@
|
|
|
1
|
+
"""doc-search — an MCP server for the ai-workspace-kit workspace.
|
|
2
|
+
|
|
3
|
+
Tools (local documents, no credentials; embeddings maintains a derived cache):
|
|
4
|
+
|
|
5
|
+
- search_docs(query) keyword search across the documentation repos that
|
|
6
|
+
sync checked out under projects/
|
|
7
|
+
- search_docs_semantic(query)
|
|
8
|
+
TF-IDF vector-space search over the same corpus
|
|
9
|
+
- search_docs_embeddings(query)
|
|
10
|
+
section embeddings over the same corpus; lazily loaded,
|
|
11
|
+
with a derived cache in the server's .index/ directory
|
|
12
|
+
- get_doc(path) fetch one document by workspace-relative path
|
|
13
|
+
- list_tickets(status) list the pre-TF-115 Markdown archive in tickets/
|
|
14
|
+
- propose_ticket(title, body)
|
|
15
|
+
write a ticket DRAFT under tickets/drafts/ — the
|
|
16
|
+
server's only content write, draft-first by construction:
|
|
17
|
+
it can never create or modify a real ticket; a human
|
|
18
|
+
files an Issue via scripts/promote-draft
|
|
19
|
+
|
|
20
|
+
Workspace root resolution: the AI_WORKSPACE_ROOT environment variable wins;
|
|
21
|
+
otherwise the server assumes it lives at <root>/mcp/doc-search/server.py.
|
|
22
|
+
|
|
23
|
+
Run (stdio): python server.py
|
|
24
|
+
"""
|
|
25
|
+
import math
|
|
26
|
+
import os
|
|
27
|
+
import re
|
|
28
|
+
import sys
|
|
29
|
+
from collections import Counter
|
|
30
|
+
from pathlib import Path
|
|
31
|
+
|
|
32
|
+
from fastmcp import FastMCP
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _force_utf8_stdio() -> None:
|
|
36
|
+
"""Force UTF-8 on stdin/stdout/stderr.
|
|
37
|
+
|
|
38
|
+
On Windows machines whose system locale is not UTF-8 (e.g. Chinese
|
|
39
|
+
Windows defaults Python's stdio to GBK), a stdio MCP server otherwise
|
|
40
|
+
emits non-UTF-8 bytes. Strict hosts (Codex's rmcp launcher) read the
|
|
41
|
+
streams as UTF-8, fail to decode them, and tear the connection down
|
|
42
|
+
during the initialize handshake. Pinning UTF-8 here makes the server
|
|
43
|
+
behave identically on any locale. Clients can also set PYTHONUTF8=1
|
|
44
|
+
in the server env; doing both is harmless.
|
|
45
|
+
"""
|
|
46
|
+
for stream in (sys.stdin, sys.stdout, sys.stderr):
|
|
47
|
+
reconfigure = getattr(stream, "reconfigure", None)
|
|
48
|
+
if reconfigure is not None:
|
|
49
|
+
try:
|
|
50
|
+
reconfigure(encoding="utf-8", errors="replace")
|
|
51
|
+
except (ValueError, OSError):
|
|
52
|
+
pass
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
_force_utf8_stdio()
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def workspace_root() -> Path:
|
|
59
|
+
env = os.environ.get("AI_WORKSPACE_ROOT")
|
|
60
|
+
if env:
|
|
61
|
+
return Path(env).resolve()
|
|
62
|
+
return Path(__file__).resolve().parents[3]
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
ROOT = workspace_root()
|
|
66
|
+
PROJECTS_DIR = ROOT / "projects"
|
|
67
|
+
TICKETS_DIR = ROOT / "tickets"
|
|
68
|
+
|
|
69
|
+
# These are dependency installs and tool caches, never content directories;
|
|
70
|
+
# adding a name is a deliberate corpus change.
|
|
71
|
+
EXCLUDED_DIR_NAMES = {".git", ".venv", "venv", "node_modules", "__pycache__", ".pytest_cache"}
|
|
72
|
+
|
|
73
|
+
mcp = FastMCP("doc-search")
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _iter_docs():
|
|
77
|
+
if not PROJECTS_DIR.exists():
|
|
78
|
+
return
|
|
79
|
+
for p in sorted(PROJECTS_DIR.rglob("*.md")):
|
|
80
|
+
if EXCLUDED_DIR_NAMES.intersection(p.parts):
|
|
81
|
+
continue
|
|
82
|
+
yield p
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _doc_title(text: str, path: Path) -> str:
|
|
86
|
+
for line in text.splitlines():
|
|
87
|
+
if line.startswith("# "):
|
|
88
|
+
return line[2:].strip()
|
|
89
|
+
return path.stem
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
# The search/read tools read local documents; embeddings also writes a
|
|
93
|
+
# derived local cache (and may download its model on first use). Declaring that in the MCP
|
|
94
|
+
# annotations matters: clients such as Codex auto-approve read-only
|
|
95
|
+
# tools but stop un-annotated ones for manual approval (and refuse them
|
|
96
|
+
# outright in non-interactive runs).
|
|
97
|
+
READ_ONLY = {"readOnlyHint": True, "destructiveHint": False, "openWorldHint": False}
|
|
98
|
+
|
|
99
|
+
# The one write tool (propose_ticket) is annotated honestly as a write:
|
|
100
|
+
# clients will ask the user before running it, which is exactly the
|
|
101
|
+
# human-in-the-loop point of draft-first.
|
|
102
|
+
DRAFT_WRITE = {"readOnlyHint": False, "destructiveHint": False, "openWorldHint": False}
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
@mcp.tool(annotations=READ_ONLY)
|
|
106
|
+
def search_docs(query: str, limit: int = 8) -> list[dict]:
|
|
107
|
+
"""Keyword search over the workspace documentation repos.
|
|
108
|
+
|
|
109
|
+
Ranking is deliberately simple and inspectable: a hit in a document
|
|
110
|
+
title counts 3, in a heading 2, in body text 1. Returns workspace-
|
|
111
|
+
relative path, title, score, and the first matching line as a snippet.
|
|
112
|
+
"""
|
|
113
|
+
terms = [t for t in re.split(r"\W+", query.lower()) if t]
|
|
114
|
+
if not terms:
|
|
115
|
+
return []
|
|
116
|
+
results = []
|
|
117
|
+
for p in _iter_docs():
|
|
118
|
+
text = p.read_text(encoding="utf-8", errors="replace")
|
|
119
|
+
title = _doc_title(text, p)
|
|
120
|
+
score = sum(3 * title.lower().count(t) for t in terms)
|
|
121
|
+
snippet = ""
|
|
122
|
+
for line in text.splitlines():
|
|
123
|
+
low = line.lower()
|
|
124
|
+
hits = sum(low.count(t) for t in terms)
|
|
125
|
+
if not hits:
|
|
126
|
+
continue
|
|
127
|
+
score += 2 * hits if line.lstrip().startswith("#") else hits
|
|
128
|
+
if not snippet:
|
|
129
|
+
snippet = line.strip()[:200]
|
|
130
|
+
if score:
|
|
131
|
+
results.append(
|
|
132
|
+
{
|
|
133
|
+
"path": p.relative_to(ROOT).as_posix(),
|
|
134
|
+
"title": title,
|
|
135
|
+
"score": score,
|
|
136
|
+
"snippet": snippet,
|
|
137
|
+
}
|
|
138
|
+
)
|
|
139
|
+
results.sort(key=lambda r: (-r["score"], r["path"]))
|
|
140
|
+
return results[:limit]
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _tokenize(text: str) -> list[str]:
|
|
144
|
+
return [t for t in re.split(r"\W+", text.lower()) if t]
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
@mcp.tool(annotations=READ_ONLY)
|
|
148
|
+
def search_docs_semantic(query: str, limit: int = 8) -> list[dict]:
|
|
149
|
+
"""Vector-space (TF-IDF, cosine similarity) search over the docs.
|
|
150
|
+
|
|
151
|
+
The contrast with search_docs is deliberate. Keyword search counts
|
|
152
|
+
exact hits with hand-set weights (title x3, heading x2, body x1), so
|
|
153
|
+
a document that repeats a common term many times can outrank the
|
|
154
|
+
topically closest one. Here each document is a TF-IDF vector: terms
|
|
155
|
+
that appear in many documents (low information) are down-weighted by
|
|
156
|
+
the IDF factor, and ranking is by cosine similarity to the query
|
|
157
|
+
vector, i.e. overall topical closeness across all query terms.
|
|
158
|
+
|
|
159
|
+
It is still a bag-of-words model - no synonym or embedding magic.
|
|
160
|
+
Pure stdlib; the corpus is re-indexed on every call, which is fine
|
|
161
|
+
at documentation-repo scale and keeps the server stateless.
|
|
162
|
+
"""
|
|
163
|
+
query_tokens = _tokenize(query)
|
|
164
|
+
if not query_tokens:
|
|
165
|
+
return []
|
|
166
|
+
docs = []
|
|
167
|
+
for p in _iter_docs():
|
|
168
|
+
text = p.read_text(encoding="utf-8", errors="replace")
|
|
169
|
+
docs.append((p, text, _tokenize(text)))
|
|
170
|
+
if not docs:
|
|
171
|
+
return []
|
|
172
|
+
|
|
173
|
+
n = len(docs)
|
|
174
|
+
doc_freq: Counter = Counter()
|
|
175
|
+
for _, _, tokens in docs:
|
|
176
|
+
doc_freq.update(set(tokens))
|
|
177
|
+
idf = {t: math.log((n + 1) / (c + 1)) + 1.0 for t, c in doc_freq.items()}
|
|
178
|
+
|
|
179
|
+
def vectorize(tokens: list[str]) -> dict[str, float]:
|
|
180
|
+
counts = Counter(tokens)
|
|
181
|
+
total = max(len(tokens), 1)
|
|
182
|
+
return {t: (c / total) * idf[t] for t, c in counts.items() if t in idf}
|
|
183
|
+
|
|
184
|
+
query_vec = vectorize(query_tokens)
|
|
185
|
+
query_norm = math.sqrt(sum(v * v for v in query_vec.values()))
|
|
186
|
+
if not query_norm:
|
|
187
|
+
return []
|
|
188
|
+
|
|
189
|
+
results = []
|
|
190
|
+
for p, text, tokens in docs:
|
|
191
|
+
doc_vec = vectorize(tokens)
|
|
192
|
+
dot = sum(query_vec[t] * doc_vec.get(t, 0.0) for t in query_vec)
|
|
193
|
+
doc_norm = math.sqrt(sum(v * v for v in doc_vec.values()))
|
|
194
|
+
if not doc_norm or dot <= 0:
|
|
195
|
+
continue
|
|
196
|
+
snippet = ""
|
|
197
|
+
terms = set(query_tokens)
|
|
198
|
+
for line in text.splitlines():
|
|
199
|
+
if terms & set(_tokenize(line)):
|
|
200
|
+
snippet = line.strip()[:200]
|
|
201
|
+
break
|
|
202
|
+
results.append(
|
|
203
|
+
{
|
|
204
|
+
"path": p.relative_to(ROOT).as_posix(),
|
|
205
|
+
"title": _doc_title(text, p),
|
|
206
|
+
"score": round(dot / (query_norm * doc_norm), 4),
|
|
207
|
+
"snippet": snippet,
|
|
208
|
+
}
|
|
209
|
+
)
|
|
210
|
+
results.sort(key=lambda r: (-r["score"], r["path"]))
|
|
211
|
+
return results[:limit]
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
@mcp.tool(annotations=READ_ONLY)
|
|
215
|
+
def search_docs_embeddings(query: str, limit: int = 8) -> list[dict]:
|
|
216
|
+
"""Search document sections using locally computed sentence embeddings.
|
|
217
|
+
|
|
218
|
+
Returns path, title, cosine score rounded to four places, the first
|
|
219
|
+
non-empty body line (up to 200 characters) as snippet, and heading trail.
|
|
220
|
+
Writes only a derived cache in this server's .index/ directory. The
|
|
221
|
+
optional model downloads on first use, then runs locally. Model imports
|
|
222
|
+
are deferred until this tool is called, keeping startup torch-free.
|
|
223
|
+
"""
|
|
224
|
+
from . import embeddings
|
|
225
|
+
|
|
226
|
+
return embeddings.search(query, limit)
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
@mcp.tool(annotations=READ_ONLY)
|
|
230
|
+
def get_doc(path: str) -> str:
|
|
231
|
+
"""Fetch one Markdown document by workspace-relative path, e.g.
|
|
232
|
+
"projects/taskflow-docs/system-overview.md" or "tickets/TF-101.md".
|
|
233
|
+
Paths outside projects/ and tickets/ are refused.
|
|
234
|
+
"""
|
|
235
|
+
target = (ROOT / path).resolve()
|
|
236
|
+
allowed = [PROJECTS_DIR.resolve(), TICKETS_DIR.resolve()]
|
|
237
|
+
if (
|
|
238
|
+
target.suffix != ".md"
|
|
239
|
+
or not target.exists()
|
|
240
|
+
or not any(str(target).startswith(str(a)) for a in allowed)
|
|
241
|
+
):
|
|
242
|
+
raise ValueError(f"document not found or not allowed: {path}")
|
|
243
|
+
return target.read_text(encoding="utf-8", errors="replace")
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def _parse_ticket(p: Path) -> dict:
|
|
247
|
+
text = p.read_text(encoding="utf-8", errors="replace")
|
|
248
|
+
meta: dict[str, str] = {}
|
|
249
|
+
if text.startswith("---"):
|
|
250
|
+
parts = text.split("---", 2)
|
|
251
|
+
if len(parts) >= 3:
|
|
252
|
+
for line in parts[1].splitlines():
|
|
253
|
+
if ":" in line:
|
|
254
|
+
k, v = line.split(":", 1)
|
|
255
|
+
meta[k.strip()] = v.strip()
|
|
256
|
+
return {
|
|
257
|
+
"id": meta.get("id", p.stem),
|
|
258
|
+
"title": meta.get("title", ""),
|
|
259
|
+
"status": meta.get("status", ""),
|
|
260
|
+
"priority": meta.get("priority", ""),
|
|
261
|
+
"path": p.relative_to(ROOT).as_posix(),
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
@mcp.tool(annotations=READ_ONLY)
|
|
266
|
+
def list_tickets(status: str | None = None) -> list[dict]:
|
|
267
|
+
"""List the pre-TF-115 archive (Markdown files in tickets/).
|
|
268
|
+
Optionally filter by status: open, in-progress, done. New work lives
|
|
269
|
+
in GitHub Issues; humans promote drafts via scripts/promote-draft.
|
|
270
|
+
"""
|
|
271
|
+
out = []
|
|
272
|
+
if TICKETS_DIR.exists():
|
|
273
|
+
for p in sorted(TICKETS_DIR.glob("*.md")):
|
|
274
|
+
t = _parse_ticket(p)
|
|
275
|
+
if status and t["status"] != status:
|
|
276
|
+
continue
|
|
277
|
+
out.append(t)
|
|
278
|
+
return out
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
@mcp.tool(annotations=DRAFT_WRITE)
|
|
282
|
+
def propose_ticket(title: str, body: str, priority: str = "medium") -> dict:
|
|
283
|
+
"""Propose a new ticket as a DRAFT under tickets/drafts/.
|
|
284
|
+
|
|
285
|
+
Draft-first is enforced here, not left to convention: this is the
|
|
286
|
+
server's only write tool, and it can only create a new file in
|
|
287
|
+
tickets/drafts/ with status "draft". It cannot create, modify, or
|
|
288
|
+
delete the pre-TF-115 archive in tickets/. A human reviews the draft
|
|
289
|
+
and files an Issue via scripts/promote-draft if it should become
|
|
290
|
+
a ticket. The server stays credential-free. Returns the workspace-relative
|
|
291
|
+
path.
|
|
292
|
+
"""
|
|
293
|
+
# One line, and no "---": the ticket parser splits front matter on
|
|
294
|
+
# that marker, so a title containing it would corrupt the draft.
|
|
295
|
+
clean_title = " ".join(title.split()).replace("---", "-")
|
|
296
|
+
slug = re.sub(r"[^a-z0-9]+", "-", clean_title.lower()).strip("-")[:60]
|
|
297
|
+
slug = slug.strip("-") or "untitled"
|
|
298
|
+
if priority not in ("low", "medium", "high"):
|
|
299
|
+
raise ValueError("priority must be low, medium or high")
|
|
300
|
+
drafts_dir = TICKETS_DIR / "drafts"
|
|
301
|
+
drafts_dir.mkdir(parents=True, exist_ok=True)
|
|
302
|
+
path = drafts_dir / f"draft-{slug}.md"
|
|
303
|
+
if path.exists():
|
|
304
|
+
raise ValueError(
|
|
305
|
+
f"a draft already exists at tickets/drafts/{path.name};"
|
|
306
|
+
" choose a different title"
|
|
307
|
+
)
|
|
308
|
+
text = (
|
|
309
|
+
"---\n"
|
|
310
|
+
f"id: draft-{slug}\n"
|
|
311
|
+
f"title: {clean_title}\n"
|
|
312
|
+
"status: draft\n"
|
|
313
|
+
f"priority: {priority}\n"
|
|
314
|
+
"---\n\n"
|
|
315
|
+
f"{body.strip()}\n"
|
|
316
|
+
)
|
|
317
|
+
path.write_text(text, encoding="utf-8")
|
|
318
|
+
return {
|
|
319
|
+
"path": path.relative_to(ROOT).as_posix(),
|
|
320
|
+
"status": "draft",
|
|
321
|
+
"note": (
|
|
322
|
+
"Draft only: a human must review it and file an Issue via"
|
|
323
|
+
" scripts/promote-draft; tickets/ is the pre-TF-115 archive."
|
|
324
|
+
),
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
def main() -> None:
|
|
329
|
+
# No startup banner: a stdio server should keep stderr quiet, and the
|
|
330
|
+
# banner's box-drawing characters are exactly the kind of non-ASCII
|
|
331
|
+
# output that breaks under non-UTF-8 locales (see _force_utf8_stdio).
|
|
332
|
+
mcp.run(show_banner=False) # stdio transport by default
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
if __name__ == "__main__":
|
|
336
|
+
main()
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: taskflow-doc-search
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: MCP server for keyword search over the ai-workspace-kit documentation repos and demo tickets
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Project-URL: Repository, https://github.com/cynthiawang109-sudo/ai-workspace-kit
|
|
7
|
+
Project-URL: Documentation, https://github.com/cynthiawang109-sudo/ai-workspace-kit/tree/main/mcp/doc-search
|
|
8
|
+
Requires-Python: >=3.10
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Requires-Dist: fastmcp>=4.1.7
|
|
12
|
+
Provides-Extra: dev
|
|
13
|
+
Requires-Dist: pytest>=9.2.0; extra == "dev"
|
|
14
|
+
Provides-Extra: embeddings
|
|
15
|
+
Requires-Dist: sentence-transformers>=3; extra == "embeddings"
|
|
16
|
+
Dynamic: license-file
|
|
17
|
+
|
|
18
|
+
# doc-search MCP server
|
|
19
|
+
|
|
20
|
+
A small [MCP](https://modelcontextprotocol.io) server written with FastMCP.
|
|
21
|
+
It gives an AI coding agent structured access to this workspace's
|
|
22
|
+
documentation and demo tickets — the local, public stand-in for the
|
|
23
|
+
Jira/Confluence side of the original production workspace. Five tools are
|
|
24
|
+
annotated read-only (embeddings maintains a derived local cache); the sixth can only write ticket *drafts* (see below).
|
|
25
|
+
|
|
26
|
+
## Tools
|
|
27
|
+
|
|
28
|
+
| Tool | What it does |
|
|
29
|
+
|---|---|
|
|
30
|
+
| `search_docs(query, limit=8)` | Keyword search across every Markdown doc under `projects/`; title hits ×3, heading hits ×2, body hits ×1; returns path, title, score, snippet |
|
|
31
|
+
| `search_docs_semantic(query, limit=8)` | TF-IDF vector-space search over the same corpus (cosine similarity); returns path, title, score (0–1), snippet |
|
|
32
|
+
| `search_docs_embeddings(query, limit=8)` | Sentence embeddings over heading chunks; returns path, title, cosine score, snippet, heading; caches in `.index/` |
|
|
33
|
+
| `get_doc(path)` | Fetch one document by workspace-relative path (only `projects/` and `tickets/` are readable) |
|
|
34
|
+
| `list_tickets(status?)` | List the pre-TF-115 archive from `tickets/*.md` front matter, optionally filtered by status |
|
|
35
|
+
| `propose_ticket(title, body, priority="medium")` | Write a ticket **draft** to `tickets/drafts/draft-<slug>.md` — the server's only source-content write; it cannot create or modify real tickets |
|
|
36
|
+
|
|
37
|
+
## Draft-first writes
|
|
38
|
+
|
|
39
|
+
`propose_ticket` mirrors the production workspace's rule for outward
|
|
40
|
+
writes (there: Jira changes start as drafts a human approves). The
|
|
41
|
+
boundary is enforced by the tool, not by politeness: it writes only
|
|
42
|
+
new files under `tickets/drafts/`, always with `status: draft`, it
|
|
43
|
+
refuses to overwrite an existing draft, and it is annotated as a
|
|
44
|
+
write, so clients ask the user before calling it. `tickets/` is the pre-TF-115
|
|
45
|
+
archive; `list_tickets` never shows drafts as tickets. A human reviews and
|
|
46
|
+
promotes a draft with `scripts/promote-draft.ps1` (or `.sh`), which files
|
|
47
|
+
an Issue instead of moving the file into `tickets/`. Set `AI_WORKSPACE_PROJECT`
|
|
48
|
+
to the workspace Project number, or pass `-Project N` / `--project N`, to
|
|
49
|
+
also add the Issue to the board. Without a Project, the script prints the
|
|
50
|
+
Issue URL and notes that it was not added to a board. The draft is preserved.
|
|
51
|
+
The doc-search server stays credential-free and never calls GitHub.
|
|
52
|
+
|
|
53
|
+
## Three rungs of document search
|
|
54
|
+
|
|
55
|
+
1. `search_docs` counts exact term hits with title/heading/body weights.
|
|
56
|
+
It is predictable for known terms, but repetition can dominate ranking.
|
|
57
|
+
2. `search_docs_semantic` uses TF-IDF cosine similarity over whole documents.
|
|
58
|
+
It down-weights common terms and balances multi-term queries. Both of
|
|
59
|
+
these first two rungs are bag-of-words models: neither understands synonyms.
|
|
60
|
+
They use only the standard library and rebuild their vectors per call.
|
|
61
|
+
3. `search_docs_embeddings` uses sentence-transformers over ATX heading
|
|
62
|
+
sections, embedding the document title, heading trail and body. It can
|
|
63
|
+
retrieve paraphrases and synonyms, but relevance is model-dependent.
|
|
64
|
+
Cosine ranking uses pure Python. A content-hash fingerprint invalidates
|
|
65
|
+
the derived `.index/` cache when documents or the model change.
|
|
66
|
+
|
|
67
|
+
- **Local documents**: searches never modify the source documents. Embeddings
|
|
68
|
+
writes a derived cache; `propose_ticket` writes drafts under the rules above.
|
|
69
|
+
- **No credentials**: keyword and TF-IDF need no network. Embeddings downloads
|
|
70
|
+
its model once, then runs fully locally.
|
|
71
|
+
- **Client-agnostic**: plain MCP over stdio.
|
|
72
|
+
|
|
73
|
+
## Setup
|
|
74
|
+
|
|
75
|
+
```powershell
|
|
76
|
+
cd mcp\doc-search
|
|
77
|
+
python -m venv .venv
|
|
78
|
+
.venv\Scripts\python -m pip install -r requirements.txt pytest
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
Optional embeddings dependencies (includes torch):
|
|
82
|
+
|
|
83
|
+
```powershell
|
|
84
|
+
.venv\Scripts\python -m pip install -r requirements-embeddings.txt
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
`DOC_SEARCH_EMBEDDING_MODEL` overrides the default
|
|
88
|
+
`sentence-transformers/all-MiniLM-L6-v2`. `DOC_SEARCH_EMBEDDER=fake` selects
|
|
89
|
+
only a deterministic hashed-token test double, without semantic understanding.
|
|
90
|
+
|
|
91
|
+
Evaluate the synced corpus with `python scripts/run_eval.py`; add `--write`
|
|
92
|
+
to record the dated results in [EVAL.md](EVAL.md). The eval is not run in CI.
|
|
93
|
+
|
|
94
|
+
## Run the tests
|
|
95
|
+
|
|
96
|
+
```powershell
|
|
97
|
+
.venv\Scripts\python -m pytest
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
The tests build a throwaway fixture workspace, so they pass whether or not
|
|
101
|
+
`sync` has populated `projects/`.
|
|
102
|
+
|
|
103
|
+
## Register it
|
|
104
|
+
|
|
105
|
+
**Codex** — add to `~/.codex/config.toml`:
|
|
106
|
+
|
|
107
|
+
```toml
|
|
108
|
+
[mcp_servers.doc-search]
|
|
109
|
+
command = 'C:\path\to\ai-workspace-kit\mcp\doc-search\.venv\Scripts\python.exe'
|
|
110
|
+
args = ['C:\path\to\ai-workspace-kit\mcp\doc-search\server.py']
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
**Claude Code**:
|
|
114
|
+
|
|
115
|
+
```powershell
|
|
116
|
+
claude mcp add doc-search -- C:\path\to\ai-workspace-kit\mcp\doc-search\.venv\Scripts\python.exe C:\path\to\ai-workspace-kit\mcp\doc-search\server.py
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
The server finds the workspace root from its own location
|
|
120
|
+
(`<root>/mcp/doc-search/server.py`); set `AI_WORKSPACE_ROOT` to override.
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
## Install the distribution
|
|
124
|
+
|
|
125
|
+
Once published, install with `pip install taskflow-doc-search` (or
|
|
126
|
+
`pip install "taskflow-doc-search[embeddings]"` for sentence embeddings).
|
|
127
|
+
For a local checkout, `pip install .` from this directory works today.
|
|
128
|
+
Run `taskflow-doc-search` with `AI_WORKSPACE_ROOT` set to the absolute path
|
|
129
|
+
of the workspace to search. It serves MCP over stdio; no command arguments
|
|
130
|
+
are needed. The existing `python server.py` launcher and its automatic
|
|
131
|
+
workspace-root detection remain supported.
|
|
132
|
+
|
|
133
|
+
Release instructions: [PUBLISHING.md](PUBLISHING.md).
|
|
134
|
+
The included [server.json](server.json) is prepared for an optional MCP
|
|
135
|
+
registry listing and has not been published.
|
|
136
|
+
|
|
137
|
+
<!-- mcp-name: io.github.cynthiawang109-sudo/taskflow-doc-search -->
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
doc_search/__init__.py,sha256=IMwKvHk4stxVESkd9exVdyNtX35CV5DZpUvMgA83Zkc,56
|
|
2
|
+
doc_search/embeddings.py,sha256=4fEtlhSSTegJS93aElY_0sgucCd_a1X3ITVvW1ySzhw,6963
|
|
3
|
+
doc_search/server.py,sha256=7SQcamb8FkTcCq3T5KX5ZFnGMwF_5EOyi8-FYFhPOD8,12529
|
|
4
|
+
taskflow_doc_search-0.1.0.dist-info/licenses/LICENSE,sha256=oZK015jWgItvrrP5jytEfeTcAdL0q1oWjSQel7-40VA,1069
|
|
5
|
+
taskflow_doc_search-0.1.0.dist-info/METADATA,sha256=QzBWvQZ0y2TewzzgI9KWws0J2UAEVc_nVg4rV-29UlI,6347
|
|
6
|
+
taskflow_doc_search-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
7
|
+
taskflow_doc_search-0.1.0.dist-info/entry_points.txt,sha256=X2XRafobqIZr_Kmc0P9ZH0OdmprSdPMOcOzGJhgHOKk,63
|
|
8
|
+
taskflow_doc_search-0.1.0.dist-info/top_level.txt,sha256=ZnrU1qKAxzuXmSW7FrEmIm-w6mqu_ZjTcgIzWLFvN4E,11
|
|
9
|
+
taskflow_doc_search-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Cynthia Wang
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
doc_search
|