semcode-mcp 0.1.1__tar.gz → 0.1.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {semcode_mcp-0.1.1 → semcode_mcp-0.1.2}/PKG-INFO +13 -4
- {semcode_mcp-0.1.1 → semcode_mcp-0.1.2}/README.md +12 -3
- {semcode_mcp-0.1.1 → semcode_mcp-0.1.2}/pyproject.toml +1 -1
- {semcode_mcp-0.1.1 → semcode_mcp-0.1.2}/src/semantic_code_mcp/chunker.py +11 -5
- {semcode_mcp-0.1.1 → semcode_mcp-0.1.2}/src/semantic_code_mcp/embedder.py +43 -19
- {semcode_mcp-0.1.1 → semcode_mcp-0.1.2}/src/semantic_code_mcp/indexer.py +14 -9
- {semcode_mcp-0.1.1 → semcode_mcp-0.1.2}/src/semantic_code_mcp/server.py +1 -1
- {semcode_mcp-0.1.1 → semcode_mcp-0.1.2}/src/semantic_code_mcp/store.py +101 -3
- {semcode_mcp-0.1.1 → semcode_mcp-0.1.2}/src/semantic_code_mcp/workspace.py +2 -1
- {semcode_mcp-0.1.1 → semcode_mcp-0.1.2}/.env.example +0 -0
- {semcode_mcp-0.1.1 → semcode_mcp-0.1.2}/.gitignore +0 -0
- {semcode_mcp-0.1.1 → semcode_mcp-0.1.2}/LICENSE +0 -0
- {semcode_mcp-0.1.1 → semcode_mcp-0.1.2}/npm/README.md +0 -0
- {semcode_mcp-0.1.1 → semcode_mcp-0.1.2}/src/semantic_code_mcp/__init__.py +0 -0
- {semcode_mcp-0.1.1 → semcode_mcp-0.1.2}/src/semantic_code_mcp/expander.py +0 -0
- {semcode_mcp-0.1.1 → semcode_mcp-0.1.2}/src/semantic_code_mcp/retriever.py +0 -0
- {semcode_mcp-0.1.1 → semcode_mcp-0.1.2}/src/semantic_code_mcp/watcher.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: semcode-mcp
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.2
|
|
4
4
|
Summary: Semantic code search MCP server: Tree-sitter AST chunking + voyage-code-3 embeddings + hybrid retrieval (sqlite-vec + FTS5 + RRF) + Voyage rerank-2.5 + call graph expansion
|
|
5
5
|
Project-URL: Homepage, https://github.com/huawang1258/semantic-code-mcp
|
|
6
6
|
Project-URL: Repository, https://github.com/huawang1258/semantic-code-mcp
|
|
@@ -51,7 +51,7 @@ Description-Content-Type: text/markdown
|
|
|
51
51
|
|
|
52
52
|
「效果优先」技术栈:**Tree-sitter AST 切分 + voyage-code-3 embedding(int8 量化)+ sqlite-vec 向量库 + BM25 全文 + RRF 融合 + Voyage rerank-2.5 重排 + Call Graph 图扩展**。一个 `VOYAGE_API_KEY` 跑通 embedding + rerank 全链路。
|
|
53
53
|
|
|
54
|
-
工程特性:**watchdog 文件实时监控 + 多工作区 LRU 管理 + MCP progress notification +
|
|
54
|
+
工程特性:**watchdog 文件实时监控 + 多工作区 LRU 管理 + MCP progress notification + 协程级取消**(取消后立即返回;已在飞的索引批次在后台跑完并持久化,不丢数据也不重复计费重建)。
|
|
55
55
|
|
|
56
56
|
## 架构
|
|
57
57
|
|
|
@@ -69,7 +69,7 @@ codebase_search(query, dir)
|
|
|
69
69
|
Retriever ── 向量召回 + BM25召回 ── RRF融合 ── rerank(Voyage/Cohere)── Call Graph扩展 ── top_n 代码片段
|
|
70
70
|
│
|
|
71
71
|
▼
|
|
72
|
-
MCP Server ── async tools + progress notification +
|
|
72
|
+
MCP Server ── async tools + progress notification + 协程级取消
|
|
73
73
|
```
|
|
74
74
|
|
|
75
75
|
| 模块 | 职责 |
|
|
@@ -186,7 +186,16 @@ npx 用法把 `command/args` 换成 `"npx", ["-y", "semcode-mcp"]`;
|
|
|
186
186
|
| Cohere(可选后端) | 查询文本 + top-50 候选代码块全文 | 仅 `SCM_RERANK_BACKEND=cohere` 时;默认零发送 |
|
|
187
187
|
| 自配 LLM 端点(可选,默认关) | 仅查询文本(不含代码) | `SCM_QUERY_EXPANSION=on` 时 |
|
|
188
188
|
|
|
189
|
-
|
|
189
|
+
### ⚠️ Voyage 数据条款(重要,索引闭源代码前必读)
|
|
190
|
+
|
|
191
|
+
按 Voyage [TOS](https://www.voyageai.com/tos) Section 3(iii)(2026-07 复核):
|
|
192
|
+
|
|
193
|
+
- **默认(opt-out 前)提交内容可被 Voyage 用于训练和改进模型**
|
|
194
|
+
- Opt-out 需要:账户绑定支付方式 + 组织 Admin 身份,dashboard → Terms of Service → 底部 toggle 切到 Opted Out
|
|
195
|
+
- Opt-out 后为 zero-day retention(处理完即删),但**不追溯**之前提交的数据,且免费额度可能被作废
|
|
196
|
+
- Batch API 上传的文件会保留 30 天(本工具不使用 Batch API)
|
|
197
|
+
|
|
198
|
+
**用于公司/闭源代码前,先完成 opt-out 或使用全本地模式。** Cohere(可选 rerank 后端)条款请自行评估。
|
|
190
199
|
|
|
191
200
|
### 始终留在本地的
|
|
192
201
|
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
「效果优先」技术栈:**Tree-sitter AST 切分 + voyage-code-3 embedding(int8 量化)+ sqlite-vec 向量库 + BM25 全文 + RRF 融合 + Voyage rerank-2.5 重排 + Call Graph 图扩展**。一个 `VOYAGE_API_KEY` 跑通 embedding + rerank 全链路。
|
|
6
6
|
|
|
7
|
-
工程特性:**watchdog 文件实时监控 + 多工作区 LRU 管理 + MCP progress notification +
|
|
7
|
+
工程特性:**watchdog 文件实时监控 + 多工作区 LRU 管理 + MCP progress notification + 协程级取消**(取消后立即返回;已在飞的索引批次在后台跑完并持久化,不丢数据也不重复计费重建)。
|
|
8
8
|
|
|
9
9
|
## 架构
|
|
10
10
|
|
|
@@ -22,7 +22,7 @@ codebase_search(query, dir)
|
|
|
22
22
|
Retriever ── 向量召回 + BM25召回 ── RRF融合 ── rerank(Voyage/Cohere)── Call Graph扩展 ── top_n 代码片段
|
|
23
23
|
│
|
|
24
24
|
▼
|
|
25
|
-
MCP Server ── async tools + progress notification +
|
|
25
|
+
MCP Server ── async tools + progress notification + 协程级取消
|
|
26
26
|
```
|
|
27
27
|
|
|
28
28
|
| 模块 | 职责 |
|
|
@@ -139,7 +139,16 @@ npx 用法把 `command/args` 换成 `"npx", ["-y", "semcode-mcp"]`;
|
|
|
139
139
|
| Cohere(可选后端) | 查询文本 + top-50 候选代码块全文 | 仅 `SCM_RERANK_BACKEND=cohere` 时;默认零发送 |
|
|
140
140
|
| 自配 LLM 端点(可选,默认关) | 仅查询文本(不含代码) | `SCM_QUERY_EXPANSION=on` 时 |
|
|
141
141
|
|
|
142
|
-
|
|
142
|
+
### ⚠️ Voyage 数据条款(重要,索引闭源代码前必读)
|
|
143
|
+
|
|
144
|
+
按 Voyage [TOS](https://www.voyageai.com/tos) Section 3(iii)(2026-07 复核):
|
|
145
|
+
|
|
146
|
+
- **默认(opt-out 前)提交内容可被 Voyage 用于训练和改进模型**
|
|
147
|
+
- Opt-out 需要:账户绑定支付方式 + 组织 Admin 身份,dashboard → Terms of Service → 底部 toggle 切到 Opted Out
|
|
148
|
+
- Opt-out 后为 zero-day retention(处理完即删),但**不追溯**之前提交的数据,且免费额度可能被作废
|
|
149
|
+
- Batch API 上传的文件会保留 30 天(本工具不使用 Batch API)
|
|
150
|
+
|
|
151
|
+
**用于公司/闭源代码前,先完成 opt-out 或使用全本地模式。** Cohere(可选 rerank 后端)条款请自行评估。
|
|
143
152
|
|
|
144
153
|
### 始终留在本地的
|
|
145
154
|
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "semcode-mcp"
|
|
7
|
-
version = "0.1.
|
|
7
|
+
version = "0.1.2"
|
|
8
8
|
description = "Semantic code search MCP server: Tree-sitter AST chunking + voyage-code-3 embeddings + hybrid retrieval (sqlite-vec + FTS5 + RRF) + Voyage rerank-2.5 + call graph expansion"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "MIT"
|
|
@@ -7,6 +7,7 @@ from __future__ import annotations
|
|
|
7
7
|
|
|
8
8
|
import hashlib
|
|
9
9
|
import importlib
|
|
10
|
+
import threading
|
|
10
11
|
from dataclasses import dataclass, field
|
|
11
12
|
from pathlib import Path
|
|
12
13
|
|
|
@@ -125,7 +126,9 @@ LINE_OVERLAP = 10
|
|
|
125
126
|
MIN_CHUNK_LINES = 3
|
|
126
127
|
MIN_CHUNK_CHARS = 40
|
|
127
128
|
|
|
128
|
-
|
|
129
|
+
# per-thread parser 缓存:tree-sitter Parser 对象非线程安全,多 workspace
|
|
130
|
+
# 并发 sync 时共享同一 parser 会产生数据竞争(错误解析/崩溃)
|
|
131
|
+
_PARSER_TLS = threading.local()
|
|
129
132
|
|
|
130
133
|
|
|
131
134
|
@dataclass
|
|
@@ -170,9 +173,12 @@ _LANG_MODULES = {
|
|
|
170
173
|
|
|
171
174
|
|
|
172
175
|
def _get_parser(lang: str):
|
|
173
|
-
"""获取并缓存 parser
|
|
174
|
-
|
|
175
|
-
|
|
176
|
+
"""获取并缓存 parser(每线程独立),失败返回 None(该语言回退按行切)。"""
|
|
177
|
+
cache = getattr(_PARSER_TLS, "cache", None)
|
|
178
|
+
if cache is None:
|
|
179
|
+
cache = _PARSER_TLS.cache = {}
|
|
180
|
+
if lang in cache:
|
|
181
|
+
return cache[lang]
|
|
176
182
|
parser = None
|
|
177
183
|
spec = _LANG_MODULES.get(lang)
|
|
178
184
|
if spec is not None:
|
|
@@ -183,7 +189,7 @@ def _get_parser(lang: str):
|
|
|
183
189
|
parser = Parser(language)
|
|
184
190
|
except Exception:
|
|
185
191
|
parser = None
|
|
186
|
-
|
|
192
|
+
cache[lang] = parser
|
|
187
193
|
return parser
|
|
188
194
|
|
|
189
195
|
|
|
@@ -65,6 +65,9 @@ class Embedder:
|
|
|
65
65
|
self._dim: int | None = None
|
|
66
66
|
self._query_cache: OrderedDict[str, list[float]] = OrderedDict()
|
|
67
67
|
self._query_cache_max = 32
|
|
68
|
+
# 缓存锁:多 workspace 并发检索共享同一 Embedder,OrderedDict 读改
|
|
69
|
+
# 无锁会在淘汰竞态下 KeyError(重复 miss 多发一次请求可接受,无害)
|
|
70
|
+
self._cache_lock = threading.Lock()
|
|
68
71
|
|
|
69
72
|
@property
|
|
70
73
|
def dim(self) -> int:
|
|
@@ -89,17 +92,24 @@ class Embedder:
|
|
|
89
92
|
复合查询主查询 + 子查询合并成一次请求(省 N-1 次网络往返);
|
|
90
93
|
命中缓存的不重发,只有未命中子集上行。
|
|
91
94
|
"""
|
|
92
|
-
|
|
95
|
+
with self._cache_lock:
|
|
96
|
+
cached = {t: self._query_cache[t] for t in texts if t in self._query_cache}
|
|
97
|
+
for t in cached:
|
|
98
|
+
self._query_cache.move_to_end(t)
|
|
99
|
+
missing = [t for t in dict.fromkeys(texts) if t not in cached]
|
|
100
|
+
fresh: dict[str, list[float]] = {}
|
|
93
101
|
if missing:
|
|
102
|
+
# 网络调用在锁外:持锁发请求会把并发查询串行化
|
|
94
103
|
for t, vec in zip(missing, self._embed(missing, "query")):
|
|
95
|
-
|
|
96
|
-
|
|
104
|
+
fresh[t] = vec
|
|
105
|
+
with self._cache_lock:
|
|
106
|
+
for t, vec in fresh.items():
|
|
107
|
+
self._query_cache[t] = vec
|
|
108
|
+
self._query_cache.move_to_end(t)
|
|
109
|
+
while len(self._query_cache) > self._query_cache_max:
|
|
97
110
|
self._query_cache.popitem(last=False)
|
|
98
|
-
|
|
99
|
-
for t in texts
|
|
100
|
-
self._query_cache.move_to_end(t)
|
|
101
|
-
out.append(self._query_cache[t])
|
|
102
|
-
return out
|
|
111
|
+
# 输出只从本地快照组装,不依赖缓存在锁外的存活性(并发淘汰不影响正确性)
|
|
112
|
+
return [cached[t] if t in cached else fresh[t] for t in texts]
|
|
103
113
|
|
|
104
114
|
def _embed(self, texts: list[str], input_type: str) -> list[list[float]]:
|
|
105
115
|
results: list[list[float]] = []
|
|
@@ -166,6 +176,10 @@ class LocalEmbedder:
|
|
|
166
176
|
self._dim: int | None = None
|
|
167
177
|
self._query_cache: OrderedDict[str, list[float]] = OrderedDict()
|
|
168
178
|
self._query_cache_max = 32
|
|
179
|
+
# 与 Embedder 同款缓存锁;另加 encode 串行锁——SentenceTransformer 推理
|
|
180
|
+
# 非线程安全,且 CPU 推理并发也无吞吐收益
|
|
181
|
+
self._cache_lock = threading.Lock()
|
|
182
|
+
self._encode_lock = threading.Lock()
|
|
169
183
|
|
|
170
184
|
@property
|
|
171
185
|
def dim(self) -> int:
|
|
@@ -175,23 +189,33 @@ class LocalEmbedder:
|
|
|
175
189
|
|
|
176
190
|
def embed_documents(self, texts: list[str]) -> list[list[float]]:
|
|
177
191
|
clipped = [(t or " ")[:_MAX_CHARS_PER_TEXT] for t in texts]
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
192
|
+
with self._encode_lock:
|
|
193
|
+
embs = self.st.encode(
|
|
194
|
+
clipped, batch_size=self.batch_size,
|
|
195
|
+
normalize_embeddings=True, show_progress_bar=False,
|
|
196
|
+
)
|
|
182
197
|
return [e.tolist() for e in embs]
|
|
183
198
|
|
|
184
199
|
def embed_query(self, text: str) -> list[float]:
|
|
185
|
-
|
|
186
|
-
self._query_cache
|
|
187
|
-
|
|
200
|
+
with self._cache_lock:
|
|
201
|
+
if text in self._query_cache:
|
|
202
|
+
self._query_cache.move_to_end(text)
|
|
203
|
+
return self._query_cache[text]
|
|
188
204
|
kwargs = {"normalize_embeddings": True, "show_progress_bar": False}
|
|
189
205
|
if self._query_prompt:
|
|
190
206
|
kwargs["prompt_name"] = self._query_prompt
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
self.
|
|
207
|
+
with self._encode_lock:
|
|
208
|
+
# double-check:排队等锁期间同 key 可能已被先行线程算完,
|
|
209
|
+
# 不查会让 N 个并发同 key 请求重复推理 N 次(本地模型延迟放大)
|
|
210
|
+
with self._cache_lock:
|
|
211
|
+
if text in self._query_cache:
|
|
212
|
+
self._query_cache.move_to_end(text)
|
|
213
|
+
return self._query_cache[text]
|
|
214
|
+
result = self.st.encode([text[:_MAX_CHARS_PER_TEXT]], **kwargs)[0].tolist()
|
|
215
|
+
with self._cache_lock:
|
|
216
|
+
self._query_cache[text] = result
|
|
217
|
+
while len(self._query_cache) > self._query_cache_max:
|
|
218
|
+
self._query_cache.popitem(last=False)
|
|
195
219
|
return result
|
|
196
220
|
|
|
197
221
|
def embed_queries(self, texts: list[str]) -> list[list[float]]:
|
|
@@ -43,6 +43,9 @@ SKIP_FILENAMES = {
|
|
|
43
43
|
}
|
|
44
44
|
# agent 指导文件:架构/规范信息密度最高,很多仓库把它们 gitignore 了,索引时豁免
|
|
45
45
|
FORCE_INCLUDE_FILENAMES = {"AGENTS.md", "CLAUDE.md", "GEMINI.md", "AGENT.md", ".windsurfrules", ".cursorrules"}
|
|
46
|
+
# 点目录白名单:隐藏目录默认全排除,但 CI workflow / agent 配置的检索价值高
|
|
47
|
+
# ("发布流程在哪定义"类查询必需);仍受 gitignore 约束
|
|
48
|
+
DOT_DIR_ALLOWLIST = {".github", ".gitlab", ".windsurf", ".devin", ".claude", ".cursor", ".codex", ".agent"}
|
|
46
49
|
# 单批 embedding 的最大累计块数(跨文件合并,减少 API 往返)
|
|
47
50
|
BATCH_CHUNKS = 256
|
|
48
51
|
|
|
@@ -116,10 +119,10 @@ class _EmbedPipeline:
|
|
|
116
119
|
offset = 0
|
|
117
120
|
for fp, fh, mt, sz, cs in batch:
|
|
118
121
|
n = len(cs)
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
122
|
+
# 单事务原子替换:删旧 + 插新 + 指纹,中途崩溃整体回滚,
|
|
123
|
+
# 不会出现旧数据已删但新数据未写的中间态
|
|
124
|
+
store.replace_file(fp, cs, embeddings[offset:offset + n], fh, mtime=mt, size=sz)
|
|
125
|
+
offset += n
|
|
123
126
|
if self.on_write:
|
|
124
127
|
self.on_write(len(batch), len(all_chunks))
|
|
125
128
|
|
|
@@ -189,7 +192,8 @@ class Indexer:
|
|
|
189
192
|
for dirpath, dirnames, filenames in os.walk(self.root):
|
|
190
193
|
dirnames[:] = [
|
|
191
194
|
d for d in dirnames
|
|
192
|
-
if d not in DEFAULT_IGNORE_DIRS
|
|
195
|
+
if d not in DEFAULT_IGNORE_DIRS
|
|
196
|
+
and (not d.startswith(".") or d in DOT_DIR_ALLOWLIST)
|
|
193
197
|
]
|
|
194
198
|
if ".gitignore" in filenames:
|
|
195
199
|
gi = Path(dirpath) / ".gitignore"
|
|
@@ -242,10 +246,11 @@ class Indexer:
|
|
|
242
246
|
|
|
243
247
|
def _iter_source_files(self):
|
|
244
248
|
for dirpath, dirnames, filenames in os.walk(self.root):
|
|
245
|
-
#
|
|
249
|
+
# 原地过滤忽略目录与隐藏目录(白名单内的点目录放行)
|
|
246
250
|
dirnames[:] = [
|
|
247
251
|
d for d in dirnames
|
|
248
|
-
if d not in DEFAULT_IGNORE_DIRS
|
|
252
|
+
if d not in DEFAULT_IGNORE_DIRS
|
|
253
|
+
and (not d.startswith(".") or d in DOT_DIR_ALLOWLIST)
|
|
249
254
|
]
|
|
250
255
|
for fn in filenames:
|
|
251
256
|
p = Path(dirpath) / fn
|
|
@@ -308,7 +313,7 @@ class Indexer:
|
|
|
308
313
|
buf_chunks = 0
|
|
309
314
|
try:
|
|
310
315
|
for done, (fp, fh, mt, sz) in enumerate(to_index, 1):
|
|
311
|
-
|
|
316
|
+
# 不提前删旧数据:由 replace_file 在写回时单事务完成删+插
|
|
312
317
|
chunks = chunk_file(fp)
|
|
313
318
|
chunks_total += len(chunks)
|
|
314
319
|
buf.append((fp, fh, mt, sz, chunks))
|
|
@@ -397,7 +402,7 @@ class Indexer:
|
|
|
397
402
|
buf_chunks = 0
|
|
398
403
|
try:
|
|
399
404
|
for done, (fp, fh, mt, sz) in enumerate(to_index, 1):
|
|
400
|
-
|
|
405
|
+
# 不提前删旧数据:由 replace_file 在写回时单事务完成删+插
|
|
401
406
|
chunks = chunk_file(fp)
|
|
402
407
|
buf.append((fp, fh, mt, sz, chunks))
|
|
403
408
|
buf_chunks += len(chunks)
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
- 多工作区 LRU 管理(SCM_MAX_WORKSPACES,默认 8)
|
|
9
9
|
- watchdog 文件监控 + 2s debounce(SCM_WATCH=0 禁用)
|
|
10
10
|
- MCP progress notification(索引进度实时回报)
|
|
11
|
-
-
|
|
11
|
+
- 协程级取消(取消后立即返回;已在飞的索引批次后台跑完并持久化,不丢数据)
|
|
12
12
|
- 首次全量索引 / 后续 watcher 增量(O(变更数) 极速)
|
|
13
13
|
"""
|
|
14
14
|
from __future__ import annotations
|
|
@@ -38,10 +38,16 @@ def _is_boilerplate_symbol(name: str) -> bool:
|
|
|
38
38
|
return bool(_ACCESSOR_RE.match(name)) or name in _BOILERPLATE_NAMES
|
|
39
39
|
|
|
40
40
|
|
|
41
|
+
# 索引格式版本:切块算法 / embed 上下文文本格式(_embed_text)变更时必须 bump。
|
|
42
|
+
# 文件 hash 没变不代表块/向量还有效:切块边界或上下文头变了,旧块与新块
|
|
43
|
+
# 不可比,必须拒绝静默混用(由 _check_profile 拦截并提示重建)。
|
|
44
|
+
INDEX_FORMAT = "1"
|
|
45
|
+
|
|
46
|
+
|
|
41
47
|
class CodeStore:
|
|
42
48
|
"""单个工作区的索引存储。"""
|
|
43
49
|
|
|
44
|
-
def __init__(self, db_path: str, dim: int, dtype: str = "float") -> None:
|
|
50
|
+
def __init__(self, db_path: str, dim: int, dtype: str = "float", embed_model: str = "") -> None:
|
|
45
51
|
self.db_path = db_path
|
|
46
52
|
self.dim = dim
|
|
47
53
|
self.dtype = dtype if dtype in ("float", "int8") else "float"
|
|
@@ -49,7 +55,52 @@ class CodeStore:
|
|
|
49
55
|
self.db.enable_load_extension(True)
|
|
50
56
|
sqlite_vec.load(self.db)
|
|
51
57
|
self.db.enable_load_extension(False)
|
|
58
|
+
# 跨进程治理:WAL 允许多进程并发读 + 单写不互斥读;busy_timeout 让并发写
|
|
59
|
+
# 排队等待而不是立即抛 SQLITE_BUSY(本机源码版 + uvx 发布版可能同时运行)
|
|
60
|
+
try:
|
|
61
|
+
self.db.execute("PRAGMA journal_mode=WAL")
|
|
62
|
+
self.db.execute("PRAGMA busy_timeout=5000")
|
|
63
|
+
except Exception:
|
|
64
|
+
pass
|
|
52
65
|
self._init_schema()
|
|
66
|
+
self._check_profile(embed_model)
|
|
67
|
+
|
|
68
|
+
def _check_profile(self, embed_model: str) -> None:
|
|
69
|
+
"""索引 profile 校验:换 embedding 模型/维度后旧向量不可比,必须拒绝混用。
|
|
70
|
+
|
|
71
|
+
meta 表记录 (embed_model, dim, dtype)。不匹配时抛错并提示删库重建,
|
|
72
|
+
而不是静默把不同模型的向量混进同一个向量空间。embed_model 为空
|
|
73
|
+
(单元测试 fake embedder)时跳过模型名校验,但仍校验 dim/dtype。
|
|
74
|
+
"""
|
|
75
|
+
cur = self.db.cursor()
|
|
76
|
+
cur.execute("CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)")
|
|
77
|
+
rows = dict(cur.execute("SELECT key, value FROM meta").fetchall())
|
|
78
|
+
want = {
|
|
79
|
+
"embed_model": embed_model,
|
|
80
|
+
"dim": str(self.dim),
|
|
81
|
+
"dtype": self.dtype,
|
|
82
|
+
"index_format": INDEX_FORMAT,
|
|
83
|
+
}
|
|
84
|
+
has_chunks = bool(cur.execute("SELECT 1 FROM chunks LIMIT 1").fetchone())
|
|
85
|
+
for key, val in want.items():
|
|
86
|
+
if key == "embed_model" and not val:
|
|
87
|
+
continue
|
|
88
|
+
old = rows.get(key)
|
|
89
|
+
if old is None:
|
|
90
|
+
# 缺失时认领当前值:仅对「库确实由当前算法/模型构建」成立。
|
|
91
|
+
# 历史上 chunker/默认模型从未变过,认领事实正确;今后变更
|
|
92
|
+
# 必须 bump INDEX_FORMAT/换模型名,让不匹配分支拦住旧库。
|
|
93
|
+
cur.execute("INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)", (key, val))
|
|
94
|
+
elif old != val and has_chunks:
|
|
95
|
+
self.db.commit()
|
|
96
|
+
raise RuntimeError(
|
|
97
|
+
f"索引 profile 不匹配:DB 的 {key}={old!r},当前配置 {key}={val!r}。"
|
|
98
|
+
f"不同 embedding 配置的向量不可混用,请删除索引文件后重建:{self.db_path}"
|
|
99
|
+
)
|
|
100
|
+
elif old != val:
|
|
101
|
+
# 库还是空的:直接更新 profile(如 fake 测试库先建后配)
|
|
102
|
+
cur.execute("INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)", (key, val))
|
|
103
|
+
self.db.commit()
|
|
53
104
|
|
|
54
105
|
@staticmethod
|
|
55
106
|
def _trigram_supported(cur) -> bool:
|
|
@@ -180,6 +231,19 @@ class CodeStore:
|
|
|
180
231
|
def add_chunks(self, chunks: list[CodeChunk], embeddings: list[list[float]]) -> int:
|
|
181
232
|
"""批量写入代码块及其向量。blob_hash 重复的自动跳过。返回新增条数。"""
|
|
182
233
|
cur = self.db.cursor()
|
|
234
|
+
added = self._insert_chunk_rows(cur, chunks, embeddings)
|
|
235
|
+
self.db.commit()
|
|
236
|
+
return added
|
|
237
|
+
|
|
238
|
+
def _insert_chunk_rows(self, cur, chunks: list[CodeChunk], embeddings: list[list[float]]) -> int:
|
|
239
|
+
"""插入 chunk 相关全部行(chunks/vec/fts/edges),不 commit(由调用方控制事务)。"""
|
|
240
|
+
if len(chunks) != len(embeddings):
|
|
241
|
+
# zip 静默截断是数据完整性事故:供应商少返回向量时部分写入
|
|
242
|
+
# + 指纹照更新 = 永久静默缺索引。抱错让 replace_file 整体回滚,
|
|
243
|
+
# 下次 sync 重试该文件。
|
|
244
|
+
raise ValueError(
|
|
245
|
+
f"chunks({len(chunks)}) 与 embeddings({len(embeddings)}) 数量不一致,拒绝部分写入"
|
|
246
|
+
)
|
|
183
247
|
added = 0
|
|
184
248
|
for chunk, emb in zip(chunks, embeddings):
|
|
185
249
|
cur.execute(
|
|
@@ -219,12 +283,16 @@ class CodeStore:
|
|
|
219
283
|
(chunk_id, callee),
|
|
220
284
|
)
|
|
221
285
|
added += 1
|
|
222
|
-
self.db.commit()
|
|
223
286
|
return added
|
|
224
287
|
|
|
225
288
|
def delete_file(self, file_path: str) -> None:
|
|
226
289
|
"""删除某文件的所有代码块(向量 + 全文 + 元数据)。"""
|
|
227
290
|
cur = self.db.cursor()
|
|
291
|
+
self._delete_file_rows(cur, file_path)
|
|
292
|
+
self.db.commit()
|
|
293
|
+
|
|
294
|
+
def _delete_file_rows(self, cur, file_path: str) -> None:
|
|
295
|
+
"""删除某文件全部行,不 commit(由调用方控制事务)。"""
|
|
228
296
|
rows = cur.execute("SELECT id FROM chunks WHERE file_path = ?", (file_path,)).fetchall()
|
|
229
297
|
ids = [r[0] for r in rows]
|
|
230
298
|
if ids:
|
|
@@ -235,7 +303,37 @@ class CodeStore:
|
|
|
235
303
|
cur.execute(f"DELETE FROM fts_chunks_tri WHERE rowid IN ({marks})", ids)
|
|
236
304
|
cur.execute(f"DELETE FROM edges WHERE caller_id IN ({marks})", ids)
|
|
237
305
|
cur.execute("DELETE FROM chunks WHERE file_path = ?", (file_path,))
|
|
238
|
-
|
|
306
|
+
|
|
307
|
+
def replace_file(
|
|
308
|
+
self,
|
|
309
|
+
file_path: str,
|
|
310
|
+
chunks: list[CodeChunk],
|
|
311
|
+
embeddings: list[list[float]],
|
|
312
|
+
file_hash: str,
|
|
313
|
+
mtime: float = 0.0,
|
|
314
|
+
size: int = 0,
|
|
315
|
+
) -> int:
|
|
316
|
+
"""单事务原子替换一个文件的索引:删旧行 + 插新行 + 更新指纹。
|
|
317
|
+
|
|
318
|
+
任何一步失败整体回滚,不会出现「旧数据已删、新数据未写、指纹还在」
|
|
319
|
+
的静默丢失状态(旧实现 delete/add/set_hash 三次独立 commit,进程在
|
|
320
|
+
中间崩溃会让该文件永远不被重新索引)。
|
|
321
|
+
"""
|
|
322
|
+
cur = self.db.cursor()
|
|
323
|
+
try:
|
|
324
|
+
self._delete_file_rows(cur, file_path)
|
|
325
|
+
added = self._insert_chunk_rows(cur, chunks, embeddings)
|
|
326
|
+
cur.execute(
|
|
327
|
+
"INSERT INTO files (file_path, file_hash, mtime, size) VALUES (?, ?, ?, ?) "
|
|
328
|
+
"ON CONFLICT(file_path) DO UPDATE SET file_hash = excluded.file_hash, "
|
|
329
|
+
"mtime = excluded.mtime, size = excluded.size",
|
|
330
|
+
(file_path, file_hash, mtime, size),
|
|
331
|
+
)
|
|
332
|
+
self.db.commit()
|
|
333
|
+
except Exception:
|
|
334
|
+
self.db.rollback()
|
|
335
|
+
raise
|
|
336
|
+
return added
|
|
239
337
|
|
|
240
338
|
# ---------- 文件指纹(增量) ----------
|
|
241
339
|
|
|
@@ -166,7 +166,8 @@ class WorkspaceManager:
|
|
|
166
166
|
def _create_workspace(self, resolved: str) -> Workspace:
|
|
167
167
|
"""创建新的 workspace 实例。"""
|
|
168
168
|
db_path = self._db_path_for(resolved)
|
|
169
|
-
store = CodeStore(db_path, self.embedder.dim, dtype=self.embedder.output_dtype
|
|
169
|
+
store = CodeStore(db_path, self.embedder.dim, dtype=self.embedder.output_dtype,
|
|
170
|
+
embed_model=getattr(self.embedder, "model", ""))
|
|
170
171
|
# rerank 后端/模型由 Retriever 按 SCM_RERANK_BACKEND / SCM_RERANK_MODEL 自行选择
|
|
171
172
|
retriever = Retriever(store, self.embedder)
|
|
172
173
|
indexer = Indexer(resolved, store, self.embedder)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|