semcode-mcp 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- semantic_code_mcp/__init__.py +3 -0
- semantic_code_mcp/chunker.py +399 -0
- semantic_code_mcp/embedder.py +192 -0
- semantic_code_mcp/expander.py +114 -0
- semantic_code_mcp/indexer.py +416 -0
- semantic_code_mcp/retriever.py +705 -0
- semantic_code_mcp/server.py +284 -0
- semantic_code_mcp/store.py +488 -0
- semantic_code_mcp/watcher.py +156 -0
- semantic_code_mcp/workspace.py +218 -0
- semcode_mcp-0.1.0.dist-info/METADATA +272 -0
- semcode_mcp-0.1.0.dist-info/RECORD +15 -0
- semcode_mcp-0.1.0.dist-info/WHEEL +4 -0
- semcode_mcp-0.1.0.dist-info/entry_points.txt +2 -0
- semcode_mcp-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,488 @@
|
|
|
1
|
+
"""存储层:sqlite-vec 向量存储 + FTS5 全文索引 + 文件指纹增量表。
|
|
2
|
+
|
|
3
|
+
三张核心表:
|
|
4
|
+
chunks 代码块元数据 + 源码(blob_hash 唯一,实现内容寻址去重)
|
|
5
|
+
vec_chunks vec0 虚拟表,rowid = chunks.id,存 embedding
|
|
6
|
+
fts_chunks fts5 虚拟表,rowid = chunks.id,用于 BM25 词法检索
|
|
7
|
+
files file_path -> file_hash,用于增量同步对比
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import re
|
|
12
|
+
import sqlite3
|
|
13
|
+
|
|
14
|
+
import sqlite_vec
|
|
15
|
+
|
|
16
|
+
from .chunker import CodeChunk
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
# FTS5 query 至少保留长度 >= 此值的 token
|
|
20
|
+
_MIN_TOKEN_LEN = 2
|
|
21
|
+
# trigram 分词器要求 token 长度 >= 3 才能命中
|
|
22
|
+
_TRIGRAM_MIN_TOKEN_LEN = 3
|
|
23
|
+
# Call Graph 同名保护:symbol 对应的定义超过此数视为无区分度,跳过图扩展
|
|
24
|
+
_MAX_SYMBOL_FANOUT = 5
|
|
25
|
+
# Call Graph 热点保护:被调用方(distinct caller)超过此数的符号视为全局工具函数(日期/字符串工具等),跳过图扩展
|
|
26
|
+
_MAX_CALLEE_CALLERS = 20
|
|
27
|
+
# Java bean 访问器模式(getX/setX/isX):作为图扩展目标零信息量,纯噪音
|
|
28
|
+
_ACCESSOR_RE = re.compile(r"^(?:get|set|is)[A-Z]\w*$")
|
|
29
|
+
# 语言级样板方法名:同理跳过
|
|
30
|
+
_BOILERPLATE_NAMES = {
|
|
31
|
+
"toString", "equals", "hashCode", "clone", "compareTo",
|
|
32
|
+
"builder", "build", "valueOf", "readObject", "writeObject",
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _is_boilerplate_symbol(name: str) -> bool:
|
|
37
|
+
"""accessor / 样板方法:不值得作为 call graph 扩展的目标或反查起点。"""
|
|
38
|
+
return bool(_ACCESSOR_RE.match(name)) or name in _BOILERPLATE_NAMES
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class CodeStore:
|
|
42
|
+
"""单个工作区的索引存储。"""
|
|
43
|
+
|
|
44
|
+
def __init__(self, db_path: str, dim: int, dtype: str = "float") -> None:
|
|
45
|
+
self.db_path = db_path
|
|
46
|
+
self.dim = dim
|
|
47
|
+
self.dtype = dtype if dtype in ("float", "int8") else "float"
|
|
48
|
+
self.db = sqlite3.connect(db_path, check_same_thread=False)
|
|
49
|
+
self.db.enable_load_extension(True)
|
|
50
|
+
sqlite_vec.load(self.db)
|
|
51
|
+
self.db.enable_load_extension(False)
|
|
52
|
+
self._init_schema()
|
|
53
|
+
|
|
54
|
+
@staticmethod
|
|
55
|
+
def _trigram_supported(cur) -> bool:
|
|
56
|
+
"""探测当前 SQLite 是否支持 FTS5 trigram 分词器(>= 3.34)。"""
|
|
57
|
+
try:
|
|
58
|
+
cur.execute("CREATE VIRTUAL TABLE temp.__scm_trig_probe USING fts5(x, tokenize='trigram')")
|
|
59
|
+
cur.execute("DROP TABLE temp.__scm_trig_probe")
|
|
60
|
+
return True
|
|
61
|
+
except sqlite3.OperationalError:
|
|
62
|
+
return False
|
|
63
|
+
|
|
64
|
+
@staticmethod
|
|
65
|
+
def _migrate_fts(cur) -> None:
|
|
66
|
+
"""旧 FTS5 schema 迁移:列缺 file_path,或被旧版建成了 trigram(主表必须 unicode61
|
|
67
|
+
保英文精确分词,trigram 子串匹配对英文标识符查询是噪音源),则重建。"""
|
|
68
|
+
try:
|
|
69
|
+
# 检查 fts_chunks 是否已存在且列数/分词器是否正确
|
|
70
|
+
row = cur.execute(
|
|
71
|
+
"SELECT sql FROM sqlite_master WHERE type='table' AND name='fts_chunks'"
|
|
72
|
+
).fetchone()
|
|
73
|
+
if row and ("file_path" not in row[0] or "trigram" in row[0]):
|
|
74
|
+
cur.execute("DROP TABLE fts_chunks")
|
|
75
|
+
except Exception:
|
|
76
|
+
pass
|
|
77
|
+
|
|
78
|
+
@staticmethod
|
|
79
|
+
def _rebuild_fts_if_empty(cur, table: str = "fts_chunks") -> None:
|
|
80
|
+
"""FTS 表为空但 chunks 有数据时(如迁移刚 DROP 重建),从 chunks 回填。"""
|
|
81
|
+
try:
|
|
82
|
+
n_chunks = cur.execute("SELECT COUNT(*) FROM chunks").fetchone()[0]
|
|
83
|
+
n_fts = cur.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0]
|
|
84
|
+
if n_chunks and not n_fts:
|
|
85
|
+
cur.execute(
|
|
86
|
+
f"INSERT INTO {table} (rowid, code, file_path, symbol) "
|
|
87
|
+
"SELECT id, code, file_path, symbol FROM chunks"
|
|
88
|
+
)
|
|
89
|
+
except Exception:
|
|
90
|
+
pass
|
|
91
|
+
|
|
92
|
+
def _init_schema(self) -> None:
|
|
93
|
+
cur = self.db.cursor()
|
|
94
|
+
cur.execute(
|
|
95
|
+
"""
|
|
96
|
+
CREATE TABLE IF NOT EXISTS chunks (
|
|
97
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
98
|
+
blob_hash TEXT UNIQUE,
|
|
99
|
+
file_path TEXT NOT NULL,
|
|
100
|
+
language TEXT,
|
|
101
|
+
symbol TEXT,
|
|
102
|
+
start_line INTEGER,
|
|
103
|
+
end_line INTEGER,
|
|
104
|
+
code TEXT
|
|
105
|
+
)
|
|
106
|
+
"""
|
|
107
|
+
)
|
|
108
|
+
cur.execute("CREATE INDEX IF NOT EXISTS idx_chunks_file ON chunks(file_path)")
|
|
109
|
+
vec_type = "int8" if self.dtype == "int8" else "float"
|
|
110
|
+
cur.execute(
|
|
111
|
+
f"""
|
|
112
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS vec_chunks USING vec0(
|
|
113
|
+
embedding {vec_type}[{self.dim}]
|
|
114
|
+
)
|
|
115
|
+
"""
|
|
116
|
+
)
|
|
117
|
+
# FTS5 双表:
|
|
118
|
+
# fts_chunks unicode61 精确分词 —— 英文标识符查询主力(基线行为,trigram 子串匹配会引入噪音)
|
|
119
|
+
# fts_chunks_tri trigram 子串匹配 —— 仅 CJK 查询时作为额外召回路
|
|
120
|
+
# (unicode61 把连续汉字并成单 token,中文查询在主表上全程失效)
|
|
121
|
+
self._migrate_fts(cur)
|
|
122
|
+
cur.execute(
|
|
123
|
+
"""
|
|
124
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS fts_chunks USING fts5(
|
|
125
|
+
code, file_path, symbol, tokenize='unicode61'
|
|
126
|
+
)
|
|
127
|
+
"""
|
|
128
|
+
)
|
|
129
|
+
self._rebuild_fts_if_empty(cur, "fts_chunks")
|
|
130
|
+
self.fts_trigram = self._trigram_supported(cur)
|
|
131
|
+
if self.fts_trigram:
|
|
132
|
+
cur.execute(
|
|
133
|
+
"""
|
|
134
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS fts_chunks_tri USING fts5(
|
|
135
|
+
code, file_path, symbol, tokenize='trigram'
|
|
136
|
+
)
|
|
137
|
+
"""
|
|
138
|
+
)
|
|
139
|
+
self._rebuild_fts_if_empty(cur, "fts_chunks_tri")
|
|
140
|
+
cur.execute(
|
|
141
|
+
"""
|
|
142
|
+
CREATE TABLE IF NOT EXISTS files (
|
|
143
|
+
file_path TEXT PRIMARY KEY,
|
|
144
|
+
file_hash TEXT NOT NULL,
|
|
145
|
+
mtime REAL DEFAULT 0,
|
|
146
|
+
size INTEGER DEFAULT 0
|
|
147
|
+
)
|
|
148
|
+
"""
|
|
149
|
+
)
|
|
150
|
+
# 迁移:旧 DB 无 mtime/size 列时自动补充
|
|
151
|
+
try:
|
|
152
|
+
cur.execute("ALTER TABLE files ADD COLUMN mtime REAL DEFAULT 0")
|
|
153
|
+
except Exception:
|
|
154
|
+
pass
|
|
155
|
+
try:
|
|
156
|
+
cur.execute("ALTER TABLE files ADD COLUMN size INTEGER DEFAULT 0")
|
|
157
|
+
except Exception:
|
|
158
|
+
pass
|
|
159
|
+
# call graph 边:caller_id(chunks.id)-> callee_name(被调用函数名)
|
|
160
|
+
cur.execute(
|
|
161
|
+
"""
|
|
162
|
+
CREATE TABLE IF NOT EXISTS edges (
|
|
163
|
+
caller_id INTEGER NOT NULL,
|
|
164
|
+
callee_name TEXT NOT NULL
|
|
165
|
+
)
|
|
166
|
+
"""
|
|
167
|
+
)
|
|
168
|
+
cur.execute("CREATE INDEX IF NOT EXISTS idx_edges_caller ON edges(caller_id)")
|
|
169
|
+
cur.execute("CREATE INDEX IF NOT EXISTS idx_edges_callee ON edges(callee_name)")
|
|
170
|
+
self.db.commit()
|
|
171
|
+
|
|
172
|
+
# ---------- 写入 ----------
|
|
173
|
+
|
|
174
|
+
def _serialize_vec(self, emb: list) -> bytes:
|
|
175
|
+
"""按 dtype 序列化向量:float32 或 int8。"""
|
|
176
|
+
if self.dtype == "int8":
|
|
177
|
+
return sqlite_vec.serialize_int8(emb)
|
|
178
|
+
return sqlite_vec.serialize_float32(emb)
|
|
179
|
+
|
|
180
|
+
def add_chunks(self, chunks: list[CodeChunk], embeddings: list[list[float]]) -> int:
|
|
181
|
+
"""批量写入代码块及其向量。blob_hash 重复的自动跳过。返回新增条数。"""
|
|
182
|
+
cur = self.db.cursor()
|
|
183
|
+
added = 0
|
|
184
|
+
for chunk, emb in zip(chunks, embeddings):
|
|
185
|
+
cur.execute(
|
|
186
|
+
"INSERT OR IGNORE INTO chunks "
|
|
187
|
+
"(blob_hash, file_path, language, symbol, start_line, end_line, code) "
|
|
188
|
+
"VALUES (?, ?, ?, ?, ?, ?, ?)",
|
|
189
|
+
(
|
|
190
|
+
chunk.blob_hash,
|
|
191
|
+
chunk.file_path,
|
|
192
|
+
chunk.language,
|
|
193
|
+
chunk.symbol,
|
|
194
|
+
chunk.start_line,
|
|
195
|
+
chunk.end_line,
|
|
196
|
+
chunk.code,
|
|
197
|
+
),
|
|
198
|
+
)
|
|
199
|
+
if cur.rowcount == 0:
|
|
200
|
+
continue # 内容已存在,去重
|
|
201
|
+
chunk_id = cur.lastrowid
|
|
202
|
+
vec_wrap = "vec_int8(?)" if self.dtype == "int8" else "?"
|
|
203
|
+
cur.execute(
|
|
204
|
+
f"INSERT INTO vec_chunks (rowid, embedding) VALUES (?, {vec_wrap})",
|
|
205
|
+
(chunk_id, self._serialize_vec(emb)),
|
|
206
|
+
)
|
|
207
|
+
cur.execute(
|
|
208
|
+
"INSERT INTO fts_chunks (rowid, code, file_path, symbol) VALUES (?, ?, ?, ?)",
|
|
209
|
+
(chunk_id, chunk.code, chunk.file_path, chunk.symbol),
|
|
210
|
+
)
|
|
211
|
+
if self.fts_trigram:
|
|
212
|
+
cur.execute(
|
|
213
|
+
"INSERT INTO fts_chunks_tri (rowid, code, file_path, symbol) VALUES (?, ?, ?, ?)",
|
|
214
|
+
(chunk_id, chunk.code, chunk.file_path, chunk.symbol),
|
|
215
|
+
)
|
|
216
|
+
for callee in getattr(chunk, "calls", None) or []:
|
|
217
|
+
cur.execute(
|
|
218
|
+
"INSERT INTO edges (caller_id, callee_name) VALUES (?, ?)",
|
|
219
|
+
(chunk_id, callee),
|
|
220
|
+
)
|
|
221
|
+
added += 1
|
|
222
|
+
self.db.commit()
|
|
223
|
+
return added
|
|
224
|
+
|
|
225
|
+
def delete_file(self, file_path: str) -> None:
|
|
226
|
+
"""删除某文件的所有代码块(向量 + 全文 + 元数据)。"""
|
|
227
|
+
cur = self.db.cursor()
|
|
228
|
+
rows = cur.execute("SELECT id FROM chunks WHERE file_path = ?", (file_path,)).fetchall()
|
|
229
|
+
ids = [r[0] for r in rows]
|
|
230
|
+
if ids:
|
|
231
|
+
marks = ",".join("?" * len(ids))
|
|
232
|
+
cur.execute(f"DELETE FROM vec_chunks WHERE rowid IN ({marks})", ids)
|
|
233
|
+
cur.execute(f"DELETE FROM fts_chunks WHERE rowid IN ({marks})", ids)
|
|
234
|
+
if self.fts_trigram:
|
|
235
|
+
cur.execute(f"DELETE FROM fts_chunks_tri WHERE rowid IN ({marks})", ids)
|
|
236
|
+
cur.execute(f"DELETE FROM edges WHERE caller_id IN ({marks})", ids)
|
|
237
|
+
cur.execute("DELETE FROM chunks WHERE file_path = ?", (file_path,))
|
|
238
|
+
self.db.commit()
|
|
239
|
+
|
|
240
|
+
# ---------- 文件指纹(增量) ----------
|
|
241
|
+
|
|
242
|
+
def get_file_hash(self, file_path: str) -> str | None:
|
|
243
|
+
row = self.db.execute(
|
|
244
|
+
"SELECT file_hash FROM files WHERE file_path = ?", (file_path,)
|
|
245
|
+
).fetchone()
|
|
246
|
+
return row[0] if row else None
|
|
247
|
+
|
|
248
|
+
def get_file_stat(self, file_path: str) -> dict | None:
|
|
249
|
+
"""返回文件的完整元数据 {hash, mtime, size},不存在返回 None。"""
|
|
250
|
+
row = self.db.execute(
|
|
251
|
+
"SELECT file_hash, mtime, size FROM files WHERE file_path = ?", (file_path,)
|
|
252
|
+
).fetchone()
|
|
253
|
+
if not row:
|
|
254
|
+
return None
|
|
255
|
+
return {"hash": row[0], "mtime": row[1] or 0.0, "size": row[2] or 0}
|
|
256
|
+
|
|
257
|
+
def set_file_hash(self, file_path: str, file_hash: str, mtime: float = 0.0, size: int = 0) -> None:
|
|
258
|
+
self.db.execute(
|
|
259
|
+
"INSERT INTO files (file_path, file_hash, mtime, size) VALUES (?, ?, ?, ?) "
|
|
260
|
+
"ON CONFLICT(file_path) DO UPDATE SET file_hash = excluded.file_hash, "
|
|
261
|
+
"mtime = excluded.mtime, size = excluded.size",
|
|
262
|
+
(file_path, file_hash, mtime, size),
|
|
263
|
+
)
|
|
264
|
+
self.db.commit()
|
|
265
|
+
|
|
266
|
+
def remove_file_record(self, file_path: str) -> None:
|
|
267
|
+
self.db.execute("DELETE FROM files WHERE file_path = ?", (file_path,))
|
|
268
|
+
self.db.commit()
|
|
269
|
+
|
|
270
|
+
def all_indexed_files(self) -> set[str]:
|
|
271
|
+
return {r[0] for r in self.db.execute("SELECT file_path FROM files").fetchall()}
|
|
272
|
+
|
|
273
|
+
# ---------- 检索 ----------
|
|
274
|
+
|
|
275
|
+
@staticmethod
|
|
276
|
+
def _fts_query(text: str) -> str | None:
|
|
277
|
+
"""把自然语言 query 转为安全的 FTS5 MATCH 表达式(token OR 连接,unicode61 语义)。"""
|
|
278
|
+
tokens = re.findall(r"\w+", text)
|
|
279
|
+
tokens = [t for t in tokens if len(t) >= _MIN_TOKEN_LEN]
|
|
280
|
+
if not tokens:
|
|
281
|
+
return None
|
|
282
|
+
return " OR ".join(f'"{t}"' for t in tokens)
|
|
283
|
+
|
|
284
|
+
@staticmethod
|
|
285
|
+
def _fts_query_trigram(text: str) -> str | None:
|
|
286
|
+
"""trigram 表的 MATCH 表达式:token 需 >= 3 字符;CJK 连续串整体作为 phrase(子串匹配)。"""
|
|
287
|
+
tokens = re.findall(r"\w+", text)
|
|
288
|
+
tokens = [t for t in tokens if len(t) >= _TRIGRAM_MIN_TOKEN_LEN]
|
|
289
|
+
if not tokens:
|
|
290
|
+
return None
|
|
291
|
+
return " OR ".join(f'"{t}"' for t in tokens)
|
|
292
|
+
|
|
293
|
+
def search_vector(self, query_embedding: list[float], k: int) -> list[tuple[int, float]]:
|
|
294
|
+
"""向量 KNN 检索,返回 [(chunk_id, distance)],distance 越小越相似。"""
|
|
295
|
+
match_expr = "vec_int8(?)" if self.dtype == "int8" else "?"
|
|
296
|
+
rows = self.db.execute(
|
|
297
|
+
f"SELECT rowid, distance FROM vec_chunks "
|
|
298
|
+
f"WHERE embedding MATCH {match_expr} AND k = ? ORDER BY distance",
|
|
299
|
+
(self._serialize_vec(query_embedding), k),
|
|
300
|
+
).fetchall()
|
|
301
|
+
return [(int(r[0]), float(r[1])) for r in rows]
|
|
302
|
+
|
|
303
|
+
def search_fts(self, query_text: str, k: int) -> list[tuple[int, float]]:
|
|
304
|
+
"""BM25 词法检索(unicode61 精确分词),返回 [(chunk_id, rank)],rank 越小越相关。"""
|
|
305
|
+
q = self._fts_query(query_text)
|
|
306
|
+
if not q:
|
|
307
|
+
return []
|
|
308
|
+
try:
|
|
309
|
+
rows = self.db.execute(
|
|
310
|
+
"SELECT rowid, rank FROM fts_chunks WHERE fts_chunks MATCH ? "
|
|
311
|
+
"ORDER BY rank LIMIT ?",
|
|
312
|
+
(q, k),
|
|
313
|
+
).fetchall()
|
|
314
|
+
except sqlite3.OperationalError:
|
|
315
|
+
return []
|
|
316
|
+
return [(int(r[0]), float(r[1])) for r in rows]
|
|
317
|
+
|
|
318
|
+
def search_fts_trigram(self, query_text: str, k: int) -> list[tuple[int, float]]:
|
|
319
|
+
"""trigram BM25 检索(CJK 子串可匹配),仅供含 CJK 的查询作额外召回路。"""
|
|
320
|
+
if not getattr(self, "fts_trigram", False):
|
|
321
|
+
return []
|
|
322
|
+
q = self._fts_query_trigram(query_text)
|
|
323
|
+
if not q:
|
|
324
|
+
return []
|
|
325
|
+
try:
|
|
326
|
+
rows = self.db.execute(
|
|
327
|
+
"SELECT rowid, rank FROM fts_chunks_tri WHERE fts_chunks_tri MATCH ? "
|
|
328
|
+
"ORDER BY rank LIMIT ?",
|
|
329
|
+
(q, k),
|
|
330
|
+
).fetchall()
|
|
331
|
+
except sqlite3.OperationalError:
|
|
332
|
+
return []
|
|
333
|
+
return [(int(r[0]), float(r[1])) for r in rows]
|
|
334
|
+
|
|
335
|
+
def get_chunks(self, ids: list[int]) -> dict[int, dict]:
|
|
336
|
+
"""按 id 批量取代码块元数据。"""
|
|
337
|
+
if not ids:
|
|
338
|
+
return {}
|
|
339
|
+
marks = ",".join("?" * len(ids))
|
|
340
|
+
rows = self.db.execute(
|
|
341
|
+
f"SELECT id, file_path, language, symbol, start_line, end_line, code "
|
|
342
|
+
f"FROM chunks WHERE id IN ({marks})",
|
|
343
|
+
ids,
|
|
344
|
+
).fetchall()
|
|
345
|
+
result: dict[int, dict] = {}
|
|
346
|
+
for r in rows:
|
|
347
|
+
result[int(r[0])] = {
|
|
348
|
+
"id": int(r[0]),
|
|
349
|
+
"file_path": r[1],
|
|
350
|
+
"language": r[2],
|
|
351
|
+
"symbol": r[3],
|
|
352
|
+
"start_line": r[4],
|
|
353
|
+
"end_line": r[5],
|
|
354
|
+
"code": r[6],
|
|
355
|
+
}
|
|
356
|
+
return result
|
|
357
|
+
|
|
358
|
+
# ---------- call graph ----------
|
|
359
|
+
|
|
360
|
+
def hot_callees(self, names: list[str]) -> set[str]:
|
|
361
|
+
"""返回 names 中的"全局热点"符号(被调 caller 数超过 _MAX_CALLEE_CALLERS)。
|
|
362
|
+
|
|
363
|
+
热点符号(日期/字符串工具等被全库调用的函数)作为图扩展目标毫无信息量,
|
|
364
|
+
反而把关联位灌满噪音,需要过滤。
|
|
365
|
+
"""
|
|
366
|
+
names = [n for n in names if n]
|
|
367
|
+
if not names:
|
|
368
|
+
return set()
|
|
369
|
+
marks = ",".join("?" * len(names))
|
|
370
|
+
rows = self.db.execute(
|
|
371
|
+
f"SELECT callee_name, COUNT(DISTINCT caller_id) FROM edges "
|
|
372
|
+
f"WHERE callee_name IN ({marks}) GROUP BY callee_name",
|
|
373
|
+
names,
|
|
374
|
+
).fetchall()
|
|
375
|
+
return {r[0] for r in rows if int(r[1]) > _MAX_CALLEE_CALLERS}
|
|
376
|
+
|
|
377
|
+
def callers_of(self, callee_name: str, limit: int = 20) -> list[dict]:
|
|
378
|
+
"""结构化查询:返回调用了 callee_name 的所有 chunk(按 id 去重)。
|
|
379
|
+
|
|
380
|
+
用于"谁调用了 X"类意图查询,直接走 edges 表,不依赖语义召回。
|
|
381
|
+
"""
|
|
382
|
+
if not callee_name:
|
|
383
|
+
return []
|
|
384
|
+
rows = self.db.execute(
|
|
385
|
+
"SELECT DISTINCT caller_id FROM edges WHERE callee_name = ? LIMIT ?",
|
|
386
|
+
(callee_name, limit),
|
|
387
|
+
).fetchall()
|
|
388
|
+
ids = [int(r[0]) for r in rows]
|
|
389
|
+
chunk_map = self.get_chunks(ids)
|
|
390
|
+
return [chunk_map[i] for i in ids if i in chunk_map]
|
|
391
|
+
|
|
392
|
+
def expand_graph(
|
|
393
|
+
self,
|
|
394
|
+
chunk_ids: list[int],
|
|
395
|
+
symbols: list[str],
|
|
396
|
+
limit: int = 10,
|
|
397
|
+
extra_callee_names: list[str] | None = None,
|
|
398
|
+
) -> list[dict]:
|
|
399
|
+
"""图扩展:返回与给定 chunk 有调用关系的相关 chunk。
|
|
400
|
+
|
|
401
|
+
callees:给定 chunk 调用的函数(edges.caller_id 命中 -> 匹配 symbol,热点过滤)
|
|
402
|
+
callers:调用了给定 symbol / extra_callee_names 的函数(edges.callee_name 命中 -> caller_id)
|
|
403
|
+
extra_callee_names 用于类/接口级 chunk:其内部声明的方法名不是 chunk symbol,
|
|
404
|
+
由调用方提取后传入,否则 caller 方向永远查不到方法调用边。
|
|
405
|
+
两个方向都过滤 accessor / 样板方法(setRemark 这类 bean setter 是纯噪音)。
|
|
406
|
+
每条结果带 relation 字段("callee"/"caller");跨模块同名同内容副本去重。
|
|
407
|
+
"""
|
|
408
|
+
origin = set(chunk_ids)
|
|
409
|
+
related: dict[int, str] = {} # chunk_id -> relation
|
|
410
|
+
|
|
411
|
+
# callees:这些 chunk 调用了哪些函数名 -> 找对应定义(先滤掉全局热点工具函数)
|
|
412
|
+
if chunk_ids:
|
|
413
|
+
marks = ",".join("?" * len(chunk_ids))
|
|
414
|
+
callee_names = [
|
|
415
|
+
r[0] for r in self.db.execute(
|
|
416
|
+
f"SELECT DISTINCT callee_name FROM edges WHERE caller_id IN ({marks})",
|
|
417
|
+
chunk_ids,
|
|
418
|
+
).fetchall()
|
|
419
|
+
]
|
|
420
|
+
callee_names = [n for n in callee_names if n and not _is_boilerplate_symbol(n)]
|
|
421
|
+
if callee_names:
|
|
422
|
+
hot = self.hot_callees(callee_names)
|
|
423
|
+
callee_names = [n for n in callee_names if n not in hot]
|
|
424
|
+
if callee_names:
|
|
425
|
+
nmarks = ",".join("?" * len(callee_names))
|
|
426
|
+
rows = self.db.execute(
|
|
427
|
+
f"SELECT id, symbol FROM chunks WHERE symbol IN ({nmarks})", callee_names
|
|
428
|
+
).fetchall()
|
|
429
|
+
by_sym: dict[str, list[int]] = {}
|
|
430
|
+
for r in rows:
|
|
431
|
+
by_sym.setdefault(r[1], []).append(int(r[0]))
|
|
432
|
+
for sym, cids in by_sym.items():
|
|
433
|
+
if len(cids) > _MAX_SYMBOL_FANOUT:
|
|
434
|
+
continue
|
|
435
|
+
for cid in cids:
|
|
436
|
+
if cid not in origin:
|
|
437
|
+
related.setdefault(cid, "callee")
|
|
438
|
+
|
|
439
|
+
# callers:谁调用了这些 symbol / 块内声明的方法名(热点符号跳过,防工具函数反查爆炸)
|
|
440
|
+
caller_targets = [s for s in symbols if s] + [n for n in (extra_callee_names or []) if n]
|
|
441
|
+
caller_targets = [n for n in dict.fromkeys(caller_targets) if not _is_boilerplate_symbol(n)]
|
|
442
|
+
if caller_targets:
|
|
443
|
+
hot = self.hot_callees(caller_targets)
|
|
444
|
+
caller_targets = [n for n in caller_targets if n not in hot]
|
|
445
|
+
if caller_targets:
|
|
446
|
+
smarks = ",".join("?" * len(caller_targets))
|
|
447
|
+
for r in self.db.execute(
|
|
448
|
+
f"SELECT DISTINCT caller_id FROM edges WHERE callee_name IN ({smarks})",
|
|
449
|
+
caller_targets,
|
|
450
|
+
).fetchall():
|
|
451
|
+
cid = int(r[0])
|
|
452
|
+
if cid not in origin and cid not in related:
|
|
453
|
+
related[cid] = "caller"
|
|
454
|
+
|
|
455
|
+
if not related:
|
|
456
|
+
return []
|
|
457
|
+
ids = list(related.keys())[: max(limit * 3, limit)]
|
|
458
|
+
chunk_map = self.get_chunks(ids)
|
|
459
|
+
out: list[dict] = []
|
|
460
|
+
seen_content: set[tuple] = set()
|
|
461
|
+
for cid in ids:
|
|
462
|
+
ch = chunk_map.get(cid)
|
|
463
|
+
if not ch:
|
|
464
|
+
continue
|
|
465
|
+
# 跨模块复制的同名同内容块(api/domin 双份 DTO)只保留一份
|
|
466
|
+
key = (ch.get("symbol"), (ch.get("code") or "")[:200])
|
|
467
|
+
if key in seen_content:
|
|
468
|
+
continue
|
|
469
|
+
seen_content.add(key)
|
|
470
|
+
ch = dict(ch)
|
|
471
|
+
ch["relation"] = related[cid]
|
|
472
|
+
out.append(ch)
|
|
473
|
+
if len(out) >= limit:
|
|
474
|
+
break
|
|
475
|
+
return out
|
|
476
|
+
|
|
477
|
+
# ---------- 杂项 ----------
|
|
478
|
+
|
|
479
|
+
def stats(self) -> dict:
|
|
480
|
+
n_chunks = self.db.execute("SELECT COUNT(*) FROM chunks").fetchone()[0]
|
|
481
|
+
n_files = self.db.execute("SELECT COUNT(*) FROM files").fetchone()[0]
|
|
482
|
+
return {"chunks": n_chunks, "files": n_files}
|
|
483
|
+
|
|
484
|
+
def close(self) -> None:
|
|
485
|
+
try:
|
|
486
|
+
self.db.close()
|
|
487
|
+
except Exception:
|
|
488
|
+
pass
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
"""文件监控:watchdog + debounce,实时追踪工作区变更。
|
|
2
|
+
|
|
3
|
+
对齐 augment-context-mcp 的 fs.watch 能力:
|
|
4
|
+
- 文件创建/修改/删除 → 标记 dirty
|
|
5
|
+
- 2s debounce 合并高频变更
|
|
6
|
+
- 主动通知 workspace 触发增量同步
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import threading
|
|
11
|
+
import time
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Callable
|
|
14
|
+
|
|
15
|
+
from watchdog.events import FileSystemEvent, FileSystemEventHandler
|
|
16
|
+
from watchdog.observers import Observer
|
|
17
|
+
|
|
18
|
+
from .indexer import DEFAULT_IGNORE_DIRS, FORCE_INCLUDE_FILENAMES
|
|
19
|
+
|
|
20
|
+
# debounce 间隔(秒)
|
|
21
|
+
DEBOUNCE_SECONDS = 2.0
|
|
22
|
+
# 累积变更超过此数时立即触发(不等 debounce)
|
|
23
|
+
FLUSH_THRESHOLD = 50
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class _Handler(FileSystemEventHandler):
|
|
27
|
+
"""watchdog 事件 → dirty set。"""
|
|
28
|
+
|
|
29
|
+
def __init__(self, watcher: FileWatcher) -> None:
|
|
30
|
+
super().__init__()
|
|
31
|
+
self._watcher = watcher
|
|
32
|
+
|
|
33
|
+
def on_any_event(self, event: FileSystemEvent) -> None:
|
|
34
|
+
if event.is_directory:
|
|
35
|
+
return
|
|
36
|
+
src = event.src_path
|
|
37
|
+
if src:
|
|
38
|
+
self._watcher._on_file_event(src)
|
|
39
|
+
# 移动事件有 dest_path
|
|
40
|
+
dest = getattr(event, "dest_path", None)
|
|
41
|
+
if dest:
|
|
42
|
+
self._watcher._on_file_event(dest)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class FileWatcher:
|
|
46
|
+
"""单个工作区的文件监控器。
|
|
47
|
+
|
|
48
|
+
追踪 dirty 文件集合,debounce 后回调通知上层做增量同步。
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
def __init__(
|
|
52
|
+
self,
|
|
53
|
+
root: str,
|
|
54
|
+
on_dirty: Callable[[], None] | None = None,
|
|
55
|
+
extensions: set[str] | None = None,
|
|
56
|
+
) -> None:
|
|
57
|
+
self.root = Path(root).resolve()
|
|
58
|
+
self._on_dirty = on_dirty
|
|
59
|
+
self._extensions = extensions # 为 None 时不过滤扩展名
|
|
60
|
+
self._dirty: set[str] = set()
|
|
61
|
+
self._deleted: set[str] = set()
|
|
62
|
+
self._lock = threading.Lock()
|
|
63
|
+
self._timer: threading.Timer | None = None
|
|
64
|
+
self._observer: Observer | None = None
|
|
65
|
+
self._stopped = False
|
|
66
|
+
|
|
67
|
+
@property
|
|
68
|
+
def dirty_count(self) -> int:
|
|
69
|
+
with self._lock:
|
|
70
|
+
return len(self._dirty) + len(self._deleted)
|
|
71
|
+
|
|
72
|
+
def take_dirty(self) -> tuple[set[str], set[str]]:
|
|
73
|
+
"""取出并清空 dirty/deleted 集合(原子操作)。"""
|
|
74
|
+
with self._lock:
|
|
75
|
+
d, r = self._dirty, self._deleted
|
|
76
|
+
self._dirty = set()
|
|
77
|
+
self._deleted = set()
|
|
78
|
+
return d, r
|
|
79
|
+
|
|
80
|
+
def start(self) -> None:
|
|
81
|
+
"""启动文件监控。"""
|
|
82
|
+
if self._observer is not None:
|
|
83
|
+
return
|
|
84
|
+
handler = _Handler(self)
|
|
85
|
+
self._observer = Observer()
|
|
86
|
+
self._observer.schedule(handler, str(self.root), recursive=True)
|
|
87
|
+
self._observer.daemon = True
|
|
88
|
+
self._observer.start()
|
|
89
|
+
|
|
90
|
+
def stop(self) -> None:
|
|
91
|
+
"""停止文件监控并清理。"""
|
|
92
|
+
self._stopped = True
|
|
93
|
+
if self._timer:
|
|
94
|
+
self._timer.cancel()
|
|
95
|
+
self._timer = None
|
|
96
|
+
if self._observer:
|
|
97
|
+
self._observer.stop()
|
|
98
|
+
try:
|
|
99
|
+
self._observer.join(timeout=2)
|
|
100
|
+
except Exception:
|
|
101
|
+
pass
|
|
102
|
+
self._observer = None
|
|
103
|
+
|
|
104
|
+
def _on_file_event(self, filepath: str) -> None:
|
|
105
|
+
"""处理单个文件事件。"""
|
|
106
|
+
if self._stopped:
|
|
107
|
+
return
|
|
108
|
+
p = Path(filepath)
|
|
109
|
+
# 过滤:忽略目录中的文件
|
|
110
|
+
try:
|
|
111
|
+
rel = p.relative_to(self.root)
|
|
112
|
+
except ValueError:
|
|
113
|
+
return
|
|
114
|
+
# 跳过忽略目录
|
|
115
|
+
parts = rel.parts
|
|
116
|
+
if any(part in DEFAULT_IGNORE_DIRS or part.startswith(".") for part in parts[:-1]):
|
|
117
|
+
return
|
|
118
|
+
# 过滤扩展名(豁免文件如 .windsurfrules 无扩展名,不受此限)
|
|
119
|
+
if (
|
|
120
|
+
self._extensions
|
|
121
|
+
and p.suffix.lower() not in self._extensions
|
|
122
|
+
and p.name not in FORCE_INCLUDE_FILENAMES
|
|
123
|
+
):
|
|
124
|
+
return
|
|
125
|
+
with self._lock:
|
|
126
|
+
if p.exists():
|
|
127
|
+
self._dirty.add(str(p))
|
|
128
|
+
self._deleted.discard(str(p))
|
|
129
|
+
else:
|
|
130
|
+
self._deleted.add(str(p))
|
|
131
|
+
self._dirty.discard(str(p))
|
|
132
|
+
count = len(self._dirty) + len(self._deleted)
|
|
133
|
+
# debounce 或立即 flush
|
|
134
|
+
if count >= FLUSH_THRESHOLD:
|
|
135
|
+
self._fire_callback()
|
|
136
|
+
else:
|
|
137
|
+
self._schedule_debounce()
|
|
138
|
+
|
|
139
|
+
def _schedule_debounce(self) -> None:
|
|
140
|
+
"""重置 debounce 计时器。"""
|
|
141
|
+
if self._timer:
|
|
142
|
+
self._timer.cancel()
|
|
143
|
+
self._timer = threading.Timer(DEBOUNCE_SECONDS, self._fire_callback)
|
|
144
|
+
self._timer.daemon = True
|
|
145
|
+
self._timer.start()
|
|
146
|
+
|
|
147
|
+
def _fire_callback(self) -> None:
|
|
148
|
+
"""触发 dirty 回调。"""
|
|
149
|
+
if self._timer:
|
|
150
|
+
self._timer.cancel()
|
|
151
|
+
self._timer = None
|
|
152
|
+
if self._on_dirty and not self._stopped:
|
|
153
|
+
try:
|
|
154
|
+
self._on_dirty()
|
|
155
|
+
except Exception:
|
|
156
|
+
pass
|