thincoder 0.7.0 → 0.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -1
- package/bin/thincoder.mjs +28 -1
- package/package.json +1 -1
- package/src/SYSTEM_PROMPT.md +1 -0
- package/src/agent.mjs +96 -64
- package/src/checkpoint.mjs +6 -3
- package/src/config.mjs +10 -4
- package/src/context.mjs +37 -8
- package/src/distill.mjs +6 -2
- package/src/embedding.mjs +11 -2
- package/src/gitmem.mjs +6 -2
- package/src/markdown.mjs +11 -4
- package/src/mcp.mjs +118 -36
- package/src/memory.mjs +214 -74
- package/src/provider.mjs +160 -21
- package/src/repomap.mjs +117 -16
- package/src/session.mjs +23 -9
- package/src/skills.mjs +6 -2
- package/src/tools/apply_patch.md +11 -0
- package/src/tools/checkpoint.md +11 -0
- package/src/tools.mjs +311 -25
- package/src/tui.mjs +506 -426
package/src/memory.mjs
CHANGED
|
@@ -19,7 +19,7 @@ import { embed, cosine, toBlob, fromBlob } from "./embedding.mjs"
|
|
|
19
19
|
import { commitAndPush } from "./gitmem.mjs"
|
|
20
20
|
|
|
21
21
|
const VALID_TYPES = new Set(["rule", "knowledge", "decision", "pattern"])
|
|
22
|
-
const SCHEMA_VERSION =
|
|
22
|
+
const SCHEMA_VERSION = 8
|
|
23
23
|
|
|
24
24
|
// 代码索引:源码文件扩展名
|
|
25
25
|
const CODE_EXTS = new Set([".mjs", ".js", ".ts", ".tsx", ".jsx", ".py", ".rs", ".go", ".java", ".c", ".h", ".cpp", ".hpp", ".rb", ".swift", ".kt", ".sh", ".bash", ".sql", ".yaml", ".yml", ".toml", ".json", ".css", ".html", ".vue", ".svelte"])
|
|
@@ -37,6 +37,9 @@ const BIG_FILE_LINES = 2000
|
|
|
37
37
|
export function createMemory({ dbPath }) {
|
|
38
38
|
mkdirSync(dirname(dbPath), { recursive: true })
|
|
39
39
|
const db = new DatabaseSync(dbPath)
|
|
40
|
+
// WAL 读写不互锁(TUI 检索和后台索引可并发);busy_timeout 防多进程同库直接 SQLITE_BUSY
|
|
41
|
+
db.exec(`PRAGMA journal_mode = WAL`)
|
|
42
|
+
db.exec(`PRAGMA busy_timeout = 3000`)
|
|
40
43
|
|
|
41
44
|
db.exec(`
|
|
42
45
|
CREATE TABLE IF NOT EXISTS entries (
|
|
@@ -57,11 +60,13 @@ export function createMemory({ dbPath }) {
|
|
|
57
60
|
return { db }
|
|
58
61
|
}
|
|
59
62
|
|
|
60
|
-
/** 按 user_version
|
|
63
|
+
/** 按 user_version 逐步迁移。整体单事务——任何一步失败都回滚,不留半成品 schema */
|
|
61
64
|
function migrate(db) {
|
|
62
65
|
const { user_version: version } = db.prepare(`PRAGMA user_version`).get()
|
|
63
66
|
if (version >= SCHEMA_VERSION) return
|
|
64
67
|
|
|
68
|
+
db.exec("BEGIN IMMEDIATE")
|
|
69
|
+
try {
|
|
65
70
|
if (version < 2) {
|
|
66
71
|
// v1(trigram) 或空库 → v2(unicode61 + CJK 逐字):重建 FTS 和触发器
|
|
67
72
|
db.exec(`
|
|
@@ -260,6 +265,107 @@ function migrate(db) {
|
|
|
260
265
|
`)
|
|
261
266
|
db.exec(`PRAGMA user_version = 7`)
|
|
262
267
|
}
|
|
268
|
+
|
|
269
|
+
if (version < 8) {
|
|
270
|
+
// v8:code_chunks/doc_chunks 加 origin 列(项目根目录绝对路径),主键改为 (origin, path, line_start)。
|
|
271
|
+
// 旧主键不含 origin,多项目共用一个记忆库时同相对路径互相覆盖、codeSync(B) 会把 A 的块当 stale 清掉。
|
|
272
|
+
// SQLite 不能 ALTER 主键,且索引是易失品 → 直接删表重建(下次 codeSync/docSync 自动重索引)。
|
|
273
|
+
db.exec(`
|
|
274
|
+
DROP TRIGGER IF EXISTS code_chunks_ai;
|
|
275
|
+
DROP TRIGGER IF EXISTS code_chunks_ad;
|
|
276
|
+
DROP TRIGGER IF EXISTS code_chunks_au;
|
|
277
|
+
DROP TABLE IF EXISTS code_chunks_fts;
|
|
278
|
+
DROP TABLE IF EXISTS code_chunks;
|
|
279
|
+
DROP TRIGGER IF EXISTS doc_chunks_ai;
|
|
280
|
+
DROP TRIGGER IF EXISTS doc_chunks_ad;
|
|
281
|
+
DROP TRIGGER IF EXISTS doc_chunks_au;
|
|
282
|
+
DROP TABLE IF EXISTS doc_chunks_fts;
|
|
283
|
+
DROP TABLE IF EXISTS doc_chunks;
|
|
284
|
+
`)
|
|
285
|
+
db.exec(`
|
|
286
|
+
CREATE TABLE code_chunks (
|
|
287
|
+
origin TEXT NOT NULL DEFAULT '',
|
|
288
|
+
path TEXT NOT NULL,
|
|
289
|
+
language TEXT NOT NULL,
|
|
290
|
+
chunk_type TEXT NOT NULL CHECK(chunk_type IN ('file','symbol')),
|
|
291
|
+
symbol_name TEXT NOT NULL DEFAULT '',
|
|
292
|
+
content TEXT NOT NULL,
|
|
293
|
+
line_start INTEGER NOT NULL DEFAULT 0,
|
|
294
|
+
line_end INTEGER NOT NULL DEFAULT 0,
|
|
295
|
+
mtime_ms INTEGER NOT NULL DEFAULT 0,
|
|
296
|
+
embedding BLOB,
|
|
297
|
+
seg_content TEXT NOT NULL DEFAULT '',
|
|
298
|
+
PRIMARY KEY (origin, path, line_start)
|
|
299
|
+
)
|
|
300
|
+
`)
|
|
301
|
+
db.exec(`
|
|
302
|
+
CREATE VIRTUAL TABLE code_chunks_fts USING fts5(
|
|
303
|
+
path, symbol_name, seg_content,
|
|
304
|
+
content='code_chunks', content_rowid='rowid',
|
|
305
|
+
tokenize='unicode61'
|
|
306
|
+
)
|
|
307
|
+
`)
|
|
308
|
+
db.exec(`
|
|
309
|
+
CREATE TRIGGER code_chunks_ai AFTER INSERT ON code_chunks BEGIN
|
|
310
|
+
INSERT INTO code_chunks_fts(rowid, path, symbol_name, seg_content)
|
|
311
|
+
VALUES (new.rowid, new.path, new.symbol_name, new.seg_content);
|
|
312
|
+
END;
|
|
313
|
+
CREATE TRIGGER code_chunks_ad AFTER DELETE ON code_chunks BEGIN
|
|
314
|
+
INSERT INTO code_chunks_fts(code_chunks_fts, rowid, path, symbol_name, seg_content)
|
|
315
|
+
VALUES ('delete', old.rowid, old.path, old.symbol_name, old.seg_content);
|
|
316
|
+
END;
|
|
317
|
+
CREATE TRIGGER code_chunks_au AFTER UPDATE ON code_chunks BEGIN
|
|
318
|
+
INSERT INTO code_chunks_fts(code_chunks_fts, rowid, path, symbol_name, seg_content)
|
|
319
|
+
VALUES ('delete', old.rowid, old.path, old.symbol_name, old.seg_content);
|
|
320
|
+
INSERT INTO code_chunks_fts(rowid, path, symbol_name, seg_content)
|
|
321
|
+
VALUES (new.rowid, new.path, new.symbol_name, new.seg_content);
|
|
322
|
+
END;
|
|
323
|
+
`)
|
|
324
|
+
db.exec(`
|
|
325
|
+
CREATE TABLE doc_chunks (
|
|
326
|
+
origin TEXT NOT NULL DEFAULT '',
|
|
327
|
+
path TEXT NOT NULL,
|
|
328
|
+
language TEXT NOT NULL DEFAULT 'markdown',
|
|
329
|
+
heading TEXT NOT NULL DEFAULT '',
|
|
330
|
+
content TEXT NOT NULL,
|
|
331
|
+
line_start INTEGER NOT NULL DEFAULT 0,
|
|
332
|
+
line_end INTEGER NOT NULL DEFAULT 0,
|
|
333
|
+
mtime_ms INTEGER NOT NULL DEFAULT 0,
|
|
334
|
+
embedding BLOB,
|
|
335
|
+
seg_content TEXT NOT NULL DEFAULT '',
|
|
336
|
+
PRIMARY KEY (origin, path, line_start)
|
|
337
|
+
)
|
|
338
|
+
`)
|
|
339
|
+
db.exec(`
|
|
340
|
+
CREATE VIRTUAL TABLE doc_chunks_fts USING fts5(
|
|
341
|
+
path, heading, seg_content,
|
|
342
|
+
content='doc_chunks', content_rowid='rowid',
|
|
343
|
+
tokenize='unicode61'
|
|
344
|
+
)
|
|
345
|
+
`)
|
|
346
|
+
db.exec(`
|
|
347
|
+
CREATE TRIGGER doc_chunks_ai AFTER INSERT ON doc_chunks BEGIN
|
|
348
|
+
INSERT INTO doc_chunks_fts(rowid, path, heading, seg_content)
|
|
349
|
+
VALUES (new.rowid, new.path, new.heading, new.seg_content);
|
|
350
|
+
END;
|
|
351
|
+
CREATE TRIGGER doc_chunks_ad AFTER DELETE ON doc_chunks BEGIN
|
|
352
|
+
INSERT INTO doc_chunks_fts(doc_chunks_fts, rowid, path, heading, seg_content)
|
|
353
|
+
VALUES ('delete', old.rowid, old.path, old.heading, old.seg_content);
|
|
354
|
+
END;
|
|
355
|
+
CREATE TRIGGER doc_chunks_au AFTER UPDATE ON doc_chunks BEGIN
|
|
356
|
+
INSERT INTO doc_chunks_fts(doc_chunks_fts, rowid, path, heading, seg_content)
|
|
357
|
+
VALUES ('delete', old.rowid, old.path, old.heading, old.seg_content);
|
|
358
|
+
INSERT INTO doc_chunks_fts(rowid, path, heading, seg_content)
|
|
359
|
+
VALUES (new.rowid, new.path, new.heading, new.seg_content);
|
|
360
|
+
END;
|
|
361
|
+
`)
|
|
362
|
+
db.exec(`PRAGMA user_version = 8`)
|
|
363
|
+
}
|
|
364
|
+
db.exec("COMMIT")
|
|
365
|
+
} catch (err) {
|
|
366
|
+
db.exec("ROLLBACK")
|
|
367
|
+
throw err
|
|
368
|
+
}
|
|
263
369
|
}
|
|
264
370
|
|
|
265
371
|
/**
|
|
@@ -326,7 +432,10 @@ export async function search(memory, query, { limit = 5 } = {}) {
|
|
|
326
432
|
return [...scores.entries()]
|
|
327
433
|
.sort((a, b) => b[1] - a[1])
|
|
328
434
|
.slice(0, limit)
|
|
329
|
-
.map(([id, score]) =>
|
|
435
|
+
.map(([id, score]) => {
|
|
436
|
+
const entry = fetchEntry(memory, id)
|
|
437
|
+
return entry ? { ...entry, rrf: score } : null // fetchEntry 为 null 时不能展开(会漏出 { rrf } 空壳)
|
|
438
|
+
})
|
|
330
439
|
.filter(Boolean)
|
|
331
440
|
}
|
|
332
441
|
|
|
@@ -670,18 +779,18 @@ function yieldTick() {
|
|
|
670
779
|
return new Promise((r) => setTimeout(r, 0))
|
|
671
780
|
}
|
|
672
781
|
|
|
673
|
-
function _upsertCodeFile(memory, rel, lines, lang, mtimeMs) {
|
|
782
|
+
function _upsertCodeFile(memory, origin, rel, lines, lang, mtimeMs) {
|
|
674
783
|
const chunks = chunkCode(lines, rel)
|
|
675
784
|
memory.db.exec("BEGIN")
|
|
676
785
|
try {
|
|
677
|
-
memory.db.prepare(`DELETE FROM code_chunks WHERE path = ?`).run(rel)
|
|
786
|
+
memory.db.prepare(`DELETE FROM code_chunks WHERE origin = ? AND path = ?`).run(origin, rel)
|
|
678
787
|
const insert = memory.db.prepare(`
|
|
679
|
-
INSERT INTO code_chunks (path, language, chunk_type, symbol_name, content, line_start, line_end, mtime_ms, seg_content)
|
|
680
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
788
|
+
INSERT INTO code_chunks (origin, path, language, chunk_type, symbol_name, content, line_start, line_end, mtime_ms, seg_content)
|
|
789
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
681
790
|
`)
|
|
682
791
|
for (const c of chunks) {
|
|
683
792
|
const isFile = c.name === rel
|
|
684
|
-
insert.run(rel, lang, isFile ? "file" : "symbol", isFile ? "" : c.name.slice(rel.length + 1), c.content, c.line_start, c.line_end, mtimeMs, segmentCJK(c.content))
|
|
793
|
+
insert.run(origin, rel, lang, isFile ? "file" : "symbol", isFile ? "" : c.name.slice(rel.length + 1), c.content, c.line_start, c.line_end, mtimeMs, segmentCJK(c.content))
|
|
685
794
|
}
|
|
686
795
|
memory.db.exec("COMMIT")
|
|
687
796
|
} catch (e) {
|
|
@@ -690,18 +799,18 @@ function _upsertCodeFile(memory, rel, lines, lang, mtimeMs) {
|
|
|
690
799
|
}
|
|
691
800
|
}
|
|
692
801
|
|
|
693
|
-
function _upsertDocFile(memory, rel, lines, mtimeMs) {
|
|
802
|
+
function _upsertDocFile(memory, origin, rel, lines, mtimeMs) {
|
|
694
803
|
const chunks = chunkMarkdown(lines, rel)
|
|
695
804
|
const lang = rel.endsWith(".rst") ? "rst" : rel.endsWith(".adoc") ? "asciidoc" : rel.endsWith(".txt") ? "text" : "markdown"
|
|
696
805
|
memory.db.exec("BEGIN")
|
|
697
806
|
try {
|
|
698
|
-
memory.db.prepare(`DELETE FROM doc_chunks WHERE path = ?`).run(rel)
|
|
807
|
+
memory.db.prepare(`DELETE FROM doc_chunks WHERE origin = ? AND path = ?`).run(origin, rel)
|
|
699
808
|
const insert = memory.db.prepare(`
|
|
700
|
-
INSERT INTO doc_chunks (path, language, heading, content, line_start, line_end, mtime_ms, seg_content)
|
|
701
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
809
|
+
INSERT INTO doc_chunks (origin, path, language, heading, content, line_start, line_end, mtime_ms, seg_content)
|
|
810
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
702
811
|
`)
|
|
703
812
|
for (const c of chunks) {
|
|
704
|
-
insert.run(rel, lang, c.heading, c.content, c.line_start, c.line_end, mtimeMs, segmentCJK(c.content))
|
|
813
|
+
insert.run(origin, rel, lang, c.heading, c.content, c.line_start, c.line_end, mtimeMs, segmentCJK(c.content))
|
|
705
814
|
}
|
|
706
815
|
memory.db.exec("COMMIT")
|
|
707
816
|
} catch (e) {
|
|
@@ -733,15 +842,16 @@ export async function codeSync(memory, dir, { onProgress } = {}) {
|
|
|
733
842
|
}
|
|
734
843
|
await walk(dir)
|
|
735
844
|
|
|
736
|
-
// 取已索引文件的 mtime
|
|
845
|
+
// 取已索引文件的 mtime 快照(只看本 origin,别的项目的块不归这里管)
|
|
737
846
|
const indexed = new Map(
|
|
738
|
-
memory.db.prepare(`SELECT path, mtime_ms FROM code_chunks
|
|
847
|
+
memory.db.prepare(`SELECT path, mtime_ms FROM code_chunks WHERE origin = ?`).all(dir).map((r) => [r.path, r.mtime_ms])
|
|
739
848
|
)
|
|
740
849
|
const seen = new Set()
|
|
741
850
|
|
|
742
851
|
onProgress?.({ phase: "scan", total: files.length })
|
|
743
852
|
|
|
744
|
-
let updated = 0, removed = 0, skipped = 0
|
|
853
|
+
let updated = 0, removed = 0, skipped = 0, failed = 0
|
|
854
|
+
const errors = []
|
|
745
855
|
for (let i = 0; i < files.length; i++) {
|
|
746
856
|
const abs = files[i]
|
|
747
857
|
const rel = abs.slice(dir.length + 1).replaceAll("\\", "/")
|
|
@@ -754,72 +864,81 @@ export async function codeSync(memory, dir, { onProgress } = {}) {
|
|
|
754
864
|
continue
|
|
755
865
|
}
|
|
756
866
|
|
|
757
|
-
//
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
867
|
+
// 单文件失败不拖垮整轮同步:记下错误继续,调用方看 failed/errors
|
|
868
|
+
try {
|
|
869
|
+
const text = await readFile(abs, "utf8")
|
|
870
|
+
const lines = text.split("\n")
|
|
871
|
+
const lang = detectLanguage(abs)
|
|
872
|
+
_upsertCodeFile(memory, dir, rel, lines, lang, mtimeMs)
|
|
873
|
+
updated++
|
|
874
|
+
} catch (e) {
|
|
875
|
+
failed++
|
|
876
|
+
if (errors.length < 5) errors.push(`${rel}: ${e.message}`)
|
|
877
|
+
}
|
|
764
878
|
await yieldTick()
|
|
765
879
|
|
|
766
880
|
if (onProgress && i % 10 === 0) {
|
|
767
|
-
onProgress({ phase: "index", current: i + 1, total: files.length, updated, removed, skipped })
|
|
881
|
+
onProgress({ phase: "index", current: i + 1, total: files.length, updated, removed, skipped, failed })
|
|
768
882
|
}
|
|
769
883
|
}
|
|
770
884
|
|
|
771
|
-
//
|
|
885
|
+
// 清理磁盘上已消失的文件块(仅本 origin)
|
|
772
886
|
for (const stale of indexed.keys()) {
|
|
773
887
|
if (!seen.has(stale)) {
|
|
774
|
-
memory.db.prepare(`DELETE FROM code_chunks WHERE path = ?`).run(stale)
|
|
888
|
+
memory.db.prepare(`DELETE FROM code_chunks WHERE origin = ? AND path = ?`).run(dir, stale)
|
|
775
889
|
removed++
|
|
776
890
|
}
|
|
777
891
|
}
|
|
778
892
|
|
|
779
|
-
onProgress?.({ phase: "done", total: files.length, updated, removed, skipped })
|
|
780
|
-
return { updated, removed, skipped, total: files.length }
|
|
893
|
+
onProgress?.({ phase: "done", total: files.length, updated, removed, skipped, failed })
|
|
894
|
+
return { updated, removed, skipped, failed, errors, total: files.length }
|
|
781
895
|
}
|
|
782
896
|
|
|
783
897
|
/**
|
|
784
898
|
* 代码检索:FTS5(BM25) + 可选向量余弦,RRF 合并。
|
|
785
|
-
* 无 embedder 时退化为纯 FTS
|
|
786
|
-
* 返回 [{ path, language, symbol_name, content, line_start, line_end
|
|
899
|
+
* 无 embedder 时退化为纯 FTS;ftsQuery 为空(纯标点查询)且有 embedder 时退化为纯向量。
|
|
900
|
+
* 返回 [{ path, language, symbol_name, content, line_start, line_end }]
|
|
787
901
|
*/
|
|
788
902
|
export async function codeSearch(memory, query, { limit = 5 } = {}) {
|
|
789
903
|
const ftsQuery = buildFtsQuery(query)
|
|
790
|
-
if (!ftsQuery) return []
|
|
904
|
+
if (!ftsQuery && !memory.embedder) return []
|
|
905
|
+
|
|
906
|
+
// codeOrigin 设置时只检索本项目(与 files 表的 projectOrigin 过滤同模式);未设置时不过滤
|
|
907
|
+
const ftsOriginFilter = memory.codeOrigin ? `AND c.origin = ?` : ""
|
|
908
|
+
const vecOriginFilter = memory.codeOrigin ? `AND origin = ?` : ""
|
|
909
|
+
const originParams = memory.codeOrigin ? [memory.codeOrigin] : []
|
|
791
910
|
|
|
792
|
-
const ftsList = memory.db.prepare(`
|
|
793
|
-
SELECT c.path, c.language, c.symbol_name, c.content, c.line_start, c.line_end, bm25(code_chunks_fts) AS rank
|
|
911
|
+
const ftsList = ftsQuery ? memory.db.prepare(`
|
|
912
|
+
SELECT c.rowid, c.path, c.language, c.symbol_name, c.content, c.line_start, c.line_end, bm25(code_chunks_fts) AS rank
|
|
794
913
|
FROM code_chunks_fts JOIN code_chunks c ON c.rowid = code_chunks_fts.rowid
|
|
795
|
-
WHERE code_chunks_fts MATCH ?
|
|
914
|
+
WHERE code_chunks_fts MATCH ? ${ftsOriginFilter}
|
|
796
915
|
ORDER BY rank LIMIT ?
|
|
797
|
-
`).all(ftsQuery, Math.max(limit * 4, 20))
|
|
916
|
+
`).all(ftsQuery, ...originParams, Math.max(limit * 4, 20)) : []
|
|
798
917
|
|
|
799
918
|
if (!memory.embedder) return ftsList.slice(0, limit)
|
|
800
919
|
|
|
801
920
|
// 向量通道
|
|
802
921
|
await ensureEmbeddings(memory)
|
|
803
922
|
const [qvec] = await embed(memory.embedder, [query])
|
|
804
|
-
const rows = memory.db.prepare(`SELECT
|
|
923
|
+
const rows = memory.db.prepare(`SELECT rowid, embedding FROM code_chunks WHERE embedding IS NOT NULL ${vecOriginFilter}`).all(...originParams)
|
|
805
924
|
const vecList = rows
|
|
806
|
-
.map((r) => ({
|
|
925
|
+
.map((r) => ({ rowid: r.rowid, score: cosine(qvec, fromBlob(r.embedding)) }))
|
|
807
926
|
.sort((a, b) => b.score - a.score)
|
|
808
927
|
.slice(0, Math.max(limit * 4, 20))
|
|
809
928
|
|
|
810
|
-
// RRF
|
|
929
|
+
// RRF 合并(按 rowid 对齐两个通道;解析结果回表取,纯向量命中的块也能浮现)
|
|
811
930
|
const K = 60
|
|
812
931
|
const scores = new Map()
|
|
813
|
-
ftsList.forEach((r, i) => scores.set(
|
|
814
|
-
vecList.forEach((r, i) => scores.set(r.
|
|
932
|
+
ftsList.forEach((r, i) => scores.set(r.rowid, (scores.get(r.rowid) ?? 0) + 1 / (K + i + 1)))
|
|
933
|
+
vecList.forEach((r, i) => scores.set(r.rowid, (scores.get(r.rowid) ?? 0) + 1 / (K + i + 1)))
|
|
815
934
|
|
|
935
|
+
const fetchChunk = memory.db.prepare(`
|
|
936
|
+
SELECT path, language, symbol_name, content, line_start, line_end FROM code_chunks WHERE rowid = ?
|
|
937
|
+
`)
|
|
816
938
|
return [...scores.entries()]
|
|
817
939
|
.sort((a, b) => b[1] - a[1])
|
|
818
940
|
.slice(0, limit)
|
|
819
|
-
.map(([
|
|
820
|
-
const [path, lineStr] = key.split(/:(?=\d+$)/)
|
|
821
|
-
return ftsList.find((r) => r.path === path && String(r.line_start) === lineStr)
|
|
822
|
-
})
|
|
941
|
+
.map(([rowid]) => fetchChunk.get(rowid))
|
|
823
942
|
.filter(Boolean)
|
|
824
943
|
}
|
|
825
944
|
|
|
@@ -878,13 +997,18 @@ export function codeSearchTool(memory) {
|
|
|
878
997
|
export async function reindexFile(memory, cwd, absPath) {
|
|
879
998
|
const ext = absPath.slice(absPath.lastIndexOf(".")).toLowerCase()
|
|
880
999
|
const rel = relative(cwd, absPath).replaceAll("\\", "/")
|
|
881
|
-
|
|
1000
|
+
// 越界路径拒索引:必须是 ".." 或 "../..." 才算越界——文件名以 .. 开头(如 ..foo.js)不算
|
|
1001
|
+
if (rel === ".." || rel.startsWith("../")) return
|
|
1002
|
+
// 与 walk 策略一致:跳过隐藏目录和 SKIP_DIRS 里的文件(node_modules/.git/dist…),
|
|
1003
|
+
// 否则这些文件虽然全量扫描时被跳过,单文件增量更新却会漏进来
|
|
1004
|
+
const dirs = rel.split("/").slice(0, -1)
|
|
1005
|
+
if (dirs.some((d) => SKIP_DIRS.has(d) || d.startsWith("."))) return
|
|
882
1006
|
|
|
883
1007
|
let text
|
|
884
1008
|
try { text = await readFile(absPath, "utf8") } catch {
|
|
885
1009
|
// 文件已删:清理索引
|
|
886
|
-
if (CODE_EXTS.has(ext)) memory.db.prepare(`DELETE FROM code_chunks WHERE path = ?`).run(rel)
|
|
887
|
-
else if (DOC_EXTS.has(ext)) memory.db.prepare(`DELETE FROM doc_chunks WHERE path = ?`).run(rel)
|
|
1010
|
+
if (CODE_EXTS.has(ext)) memory.db.prepare(`DELETE FROM code_chunks WHERE origin = ? AND path = ?`).run(cwd, rel)
|
|
1011
|
+
else if (DOC_EXTS.has(ext)) memory.db.prepare(`DELETE FROM doc_chunks WHERE origin = ? AND path = ?`).run(cwd, rel)
|
|
888
1012
|
return
|
|
889
1013
|
}
|
|
890
1014
|
const lines = text.split("\n")
|
|
@@ -893,11 +1017,11 @@ export async function reindexFile(memory, cwd, absPath) {
|
|
|
893
1017
|
const lang = detectLanguage(absPath)
|
|
894
1018
|
let mtimeMs = 0
|
|
895
1019
|
try { mtimeMs = Math.floor((await stat(absPath)).mtimeMs) } catch { /* 新文件 */ }
|
|
896
|
-
_upsertCodeFile(memory, rel, lines, lang, mtimeMs)
|
|
1020
|
+
_upsertCodeFile(memory, cwd, rel, lines, lang, mtimeMs)
|
|
897
1021
|
} else if (DOC_EXTS.has(ext)) {
|
|
898
1022
|
let mtimeMs = 0
|
|
899
1023
|
try { mtimeMs = Math.floor((await stat(absPath)).mtimeMs) } catch { /* 新文件 */ }
|
|
900
|
-
_upsertDocFile(memory, rel, lines, mtimeMs)
|
|
1024
|
+
_upsertDocFile(memory, cwd, rel, lines, mtimeMs)
|
|
901
1025
|
}
|
|
902
1026
|
}
|
|
903
1027
|
|
|
@@ -951,13 +1075,14 @@ export async function docSync(memory, dir, { onProgress } = {}) {
|
|
|
951
1075
|
await walk(dir)
|
|
952
1076
|
|
|
953
1077
|
const indexed = new Map(
|
|
954
|
-
memory.db.prepare(`SELECT path, mtime_ms FROM doc_chunks
|
|
1078
|
+
memory.db.prepare(`SELECT path, mtime_ms FROM doc_chunks WHERE origin = ?`).all(dir).map((r) => [r.path, r.mtime_ms])
|
|
955
1079
|
)
|
|
956
1080
|
const seen = new Set()
|
|
957
1081
|
|
|
958
1082
|
onProgress?.({ phase: "scan", total: files.length })
|
|
959
1083
|
|
|
960
|
-
let updated = 0, removed = 0, skipped = 0
|
|
1084
|
+
let updated = 0, removed = 0, skipped = 0, failed = 0
|
|
1085
|
+
const errors = []
|
|
961
1086
|
for (let i = 0; i < files.length; i++) {
|
|
962
1087
|
const abs = files[i]
|
|
963
1088
|
const rel = abs.slice(dir.length + 1).replaceAll("\\", "/")
|
|
@@ -970,65 +1095,80 @@ export async function docSync(memory, dir, { onProgress } = {}) {
|
|
|
970
1095
|
continue
|
|
971
1096
|
}
|
|
972
1097
|
|
|
973
|
-
|
|
974
|
-
try {
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
1098
|
+
// 单文件失败不拖垮整轮同步(与 codeSync 同策略)
|
|
1099
|
+
try {
|
|
1100
|
+
const text = await readFile(abs, "utf8")
|
|
1101
|
+
const lines = text.split("\n")
|
|
1102
|
+
_upsertDocFile(memory, dir, rel, lines, mtimeMs)
|
|
1103
|
+
updated++
|
|
1104
|
+
} catch (e) {
|
|
1105
|
+
failed++
|
|
1106
|
+
if (errors.length < 5) errors.push(`${rel}: ${e.message}`)
|
|
1107
|
+
}
|
|
978
1108
|
await yieldTick()
|
|
979
1109
|
|
|
980
1110
|
if (onProgress && i % 10 === 0) {
|
|
981
|
-
onProgress({ phase: "index", current: i + 1, total: files.length, updated, removed, skipped })
|
|
1111
|
+
onProgress({ phase: "index", current: i + 1, total: files.length, updated, removed, skipped, failed })
|
|
982
1112
|
}
|
|
983
1113
|
}
|
|
984
1114
|
|
|
985
|
-
//
|
|
1115
|
+
// 清理磁盘上消失的文件(仅本 origin)
|
|
986
1116
|
for (const stale of indexed.keys()) {
|
|
987
1117
|
if (!seen.has(stale)) {
|
|
988
|
-
memory.db.prepare(`DELETE FROM doc_chunks WHERE path = ?`).run(stale)
|
|
1118
|
+
memory.db.prepare(`DELETE FROM doc_chunks WHERE origin = ? AND path = ?`).run(dir, stale)
|
|
989
1119
|
removed++
|
|
990
1120
|
}
|
|
991
1121
|
}
|
|
992
1122
|
|
|
993
|
-
onProgress?.({ phase: "done", total: files.length, updated, removed, skipped })
|
|
994
|
-
return { updated, removed, skipped, total: files.length }
|
|
1123
|
+
onProgress?.({ phase: "done", total: files.length, updated, removed, skipped, failed })
|
|
1124
|
+
return { updated, removed, skipped, failed, errors, total: files.length }
|
|
995
1125
|
}
|
|
996
1126
|
|
|
997
1127
|
/**
|
|
998
1128
|
* 文档检索:FTS5(BM25) + 可选向量余弦,RRF 合并。
|
|
1129
|
+
* 无 embedder 时退化为纯 FTS;ftsQuery 为空(纯标点查询)且有 embedder 时退化为纯向量。
|
|
1130
|
+
* 返回 [{ path, language, heading, content, line_start, line_end }]
|
|
999
1131
|
*/
|
|
1000
1132
|
export async function docSearch(memory, query, { limit = 5 } = {}) {
|
|
1001
1133
|
const ftsQuery = buildFtsQuery(query)
|
|
1002
|
-
if (!ftsQuery) return []
|
|
1134
|
+
if (!ftsQuery && !memory.embedder) return []
|
|
1003
1135
|
|
|
1004
|
-
|
|
1005
|
-
|
|
1136
|
+
// codeOrigin 设置时只检索本项目(与 codeSearch 同模式);未设置时不过滤
|
|
1137
|
+
const ftsOriginFilter = memory.codeOrigin ? `AND d.origin = ?` : ""
|
|
1138
|
+
const vecOriginFilter = memory.codeOrigin ? `AND origin = ?` : ""
|
|
1139
|
+
const originParams = memory.codeOrigin ? [memory.codeOrigin] : []
|
|
1140
|
+
|
|
1141
|
+
const ftsList = ftsQuery ? memory.db.prepare(`
|
|
1142
|
+
SELECT d.rowid, d.path, d.language, d.heading, d.content, d.line_start, d.line_end, bm25(doc_chunks_fts) AS rank
|
|
1006
1143
|
FROM doc_chunks_fts JOIN doc_chunks d ON d.rowid = doc_chunks_fts.rowid
|
|
1007
|
-
WHERE doc_chunks_fts MATCH ?
|
|
1144
|
+
WHERE doc_chunks_fts MATCH ? ${ftsOriginFilter}
|
|
1008
1145
|
ORDER BY rank LIMIT ?
|
|
1009
|
-
`).all(ftsQuery, Math.max(limit * 4, 20))
|
|
1146
|
+
`).all(ftsQuery, ...originParams, Math.max(limit * 4, 20)) : []
|
|
1010
1147
|
|
|
1011
1148
|
if (!memory.embedder) return ftsList.slice(0, limit)
|
|
1012
1149
|
|
|
1150
|
+
// 向量通道
|
|
1151
|
+
await ensureDocEmbeddings(memory)
|
|
1013
1152
|
const [qvec] = await embed(memory.embedder, [query])
|
|
1014
|
-
const rows = memory.db.prepare(`SELECT
|
|
1153
|
+
const rows = memory.db.prepare(`SELECT rowid, embedding FROM doc_chunks WHERE embedding IS NOT NULL ${vecOriginFilter}`).all(...originParams)
|
|
1015
1154
|
const vecList = rows
|
|
1016
|
-
.map((r) => ({
|
|
1155
|
+
.map((r) => ({ rowid: r.rowid, score: cosine(qvec, fromBlob(r.embedding)) }))
|
|
1017
1156
|
.sort((a, b) => b.score - a.score)
|
|
1018
1157
|
.slice(0, Math.max(limit * 4, 20))
|
|
1019
1158
|
|
|
1159
|
+
// RRF 合并(按 rowid 对齐两个通道;解析结果回表取,纯向量命中的块也能浮现)
|
|
1020
1160
|
const K = 60
|
|
1021
1161
|
const scores = new Map()
|
|
1022
|
-
ftsList.forEach((r, i) => scores.set(
|
|
1023
|
-
vecList.forEach((r, i) => scores.set(r.
|
|
1162
|
+
ftsList.forEach((r, i) => scores.set(r.rowid, (scores.get(r.rowid) ?? 0) + 1 / (K + i + 1)))
|
|
1163
|
+
vecList.forEach((r, i) => scores.set(r.rowid, (scores.get(r.rowid) ?? 0) + 1 / (K + i + 1)))
|
|
1024
1164
|
|
|
1165
|
+
const fetchChunk = memory.db.prepare(`
|
|
1166
|
+
SELECT path, language, heading, content, line_start, line_end FROM doc_chunks WHERE rowid = ?
|
|
1167
|
+
`)
|
|
1025
1168
|
return [...scores.entries()]
|
|
1026
1169
|
.sort((a, b) => b[1] - a[1])
|
|
1027
1170
|
.slice(0, limit)
|
|
1028
|
-
.map(([
|
|
1029
|
-
const [path, lineStr] = key.split(/:(?=\d+$)/)
|
|
1030
|
-
return ftsList.find((r) => r.path === path && String(r.line_start) === lineStr)
|
|
1031
|
-
})
|
|
1171
|
+
.map(([rowid]) => fetchChunk.get(rowid))
|
|
1032
1172
|
.filter(Boolean)
|
|
1033
1173
|
}
|
|
1034
1174
|
|