thincoder 0.6.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -1
- package/bin/thincoder.mjs +25 -1
- package/package.json +1 -1
- package/src/SYSTEM_PROMPT.md +6 -3
- package/src/agent.mjs +147 -64
- package/src/checkpoint.mjs +6 -3
- package/src/coder-overlay.md +1 -1
- package/src/config.mjs +37 -27
- package/src/context.mjs +37 -8
- package/src/distill.mjs +6 -2
- package/src/embedding.mjs +11 -2
- package/src/gitmem.mjs +6 -2
- package/src/markdown.mjs +11 -4
- package/src/mcp.mjs +197 -24
- package/src/memory.mjs +242 -81
- package/src/provider.mjs +35 -14
- package/src/repomap.mjs +17 -6
- package/src/session.mjs +8 -5
- package/src/skills.mjs +6 -2
- package/src/tools/apply_patch.md +11 -0
- package/src/tools/bash.md +13 -1
- package/src/tools/checkpoint.md +11 -0
- package/src/tools/grep.md +3 -0
- package/src/tools/insert_after.md +13 -0
- package/src/tools/question.md +1 -0
- package/src/tools/syntax_check.md +10 -0
- package/src/tools/websearch.md +1 -0
- package/src/tools.mjs +483 -84
- package/src/tui.mjs +156 -55
package/src/memory.mjs
CHANGED
|
@@ -19,7 +19,7 @@ import { embed, cosine, toBlob, fromBlob } from "./embedding.mjs"
|
|
|
19
19
|
import { commitAndPush } from "./gitmem.mjs"
|
|
20
20
|
|
|
21
21
|
const VALID_TYPES = new Set(["rule", "knowledge", "decision", "pattern"])
|
|
22
|
-
const SCHEMA_VERSION =
|
|
22
|
+
const SCHEMA_VERSION = 8
|
|
23
23
|
|
|
24
24
|
// 代码索引:源码文件扩展名
|
|
25
25
|
const CODE_EXTS = new Set([".mjs", ".js", ".ts", ".tsx", ".jsx", ".py", ".rs", ".go", ".java", ".c", ".h", ".cpp", ".hpp", ".rb", ".swift", ".kt", ".sh", ".bash", ".sql", ".yaml", ".yml", ".toml", ".json", ".css", ".html", ".vue", ".svelte"])
|
|
@@ -37,6 +37,9 @@ const BIG_FILE_LINES = 2000
|
|
|
37
37
|
export function createMemory({ dbPath }) {
|
|
38
38
|
mkdirSync(dirname(dbPath), { recursive: true })
|
|
39
39
|
const db = new DatabaseSync(dbPath)
|
|
40
|
+
// WAL 读写不互锁(TUI 检索和后台索引可并发);busy_timeout 防多进程同库直接 SQLITE_BUSY
|
|
41
|
+
db.exec(`PRAGMA journal_mode = WAL`)
|
|
42
|
+
db.exec(`PRAGMA busy_timeout = 3000`)
|
|
40
43
|
|
|
41
44
|
db.exec(`
|
|
42
45
|
CREATE TABLE IF NOT EXISTS entries (
|
|
@@ -57,11 +60,13 @@ export function createMemory({ dbPath }) {
|
|
|
57
60
|
return { db }
|
|
58
61
|
}
|
|
59
62
|
|
|
60
|
-
/** 按 user_version
|
|
63
|
+
/** 按 user_version 逐步迁移。整体单事务——任何一步失败都回滚,不留半成品 schema */
|
|
61
64
|
function migrate(db) {
|
|
62
65
|
const { user_version: version } = db.prepare(`PRAGMA user_version`).get()
|
|
63
66
|
if (version >= SCHEMA_VERSION) return
|
|
64
67
|
|
|
68
|
+
db.exec("BEGIN IMMEDIATE")
|
|
69
|
+
try {
|
|
65
70
|
if (version < 2) {
|
|
66
71
|
// v1(trigram) 或空库 → v2(unicode61 + CJK 逐字):重建 FTS 和触发器
|
|
67
72
|
db.exec(`
|
|
@@ -260,6 +265,107 @@ function migrate(db) {
|
|
|
260
265
|
`)
|
|
261
266
|
db.exec(`PRAGMA user_version = 7`)
|
|
262
267
|
}
|
|
268
|
+
|
|
269
|
+
if (version < 8) {
|
|
270
|
+
// v8:code_chunks/doc_chunks 加 origin 列(项目根目录绝对路径),主键改为 (origin, path, line_start)。
|
|
271
|
+
// 旧主键不含 origin,多项目共用一个记忆库时同相对路径互相覆盖、codeSync(B) 会把 A 的块当 stale 清掉。
|
|
272
|
+
// SQLite 不能 ALTER 主键,且索引是易失品 → 直接删表重建(下次 codeSync/docSync 自动重索引)。
|
|
273
|
+
db.exec(`
|
|
274
|
+
DROP TRIGGER IF EXISTS code_chunks_ai;
|
|
275
|
+
DROP TRIGGER IF EXISTS code_chunks_ad;
|
|
276
|
+
DROP TRIGGER IF EXISTS code_chunks_au;
|
|
277
|
+
DROP TABLE IF EXISTS code_chunks_fts;
|
|
278
|
+
DROP TABLE IF EXISTS code_chunks;
|
|
279
|
+
DROP TRIGGER IF EXISTS doc_chunks_ai;
|
|
280
|
+
DROP TRIGGER IF EXISTS doc_chunks_ad;
|
|
281
|
+
DROP TRIGGER IF EXISTS doc_chunks_au;
|
|
282
|
+
DROP TABLE IF EXISTS doc_chunks_fts;
|
|
283
|
+
DROP TABLE IF EXISTS doc_chunks;
|
|
284
|
+
`)
|
|
285
|
+
db.exec(`
|
|
286
|
+
CREATE TABLE code_chunks (
|
|
287
|
+
origin TEXT NOT NULL DEFAULT '',
|
|
288
|
+
path TEXT NOT NULL,
|
|
289
|
+
language TEXT NOT NULL,
|
|
290
|
+
chunk_type TEXT NOT NULL CHECK(chunk_type IN ('file','symbol')),
|
|
291
|
+
symbol_name TEXT NOT NULL DEFAULT '',
|
|
292
|
+
content TEXT NOT NULL,
|
|
293
|
+
line_start INTEGER NOT NULL DEFAULT 0,
|
|
294
|
+
line_end INTEGER NOT NULL DEFAULT 0,
|
|
295
|
+
mtime_ms INTEGER NOT NULL DEFAULT 0,
|
|
296
|
+
embedding BLOB,
|
|
297
|
+
seg_content TEXT NOT NULL DEFAULT '',
|
|
298
|
+
PRIMARY KEY (origin, path, line_start)
|
|
299
|
+
)
|
|
300
|
+
`)
|
|
301
|
+
db.exec(`
|
|
302
|
+
CREATE VIRTUAL TABLE code_chunks_fts USING fts5(
|
|
303
|
+
path, symbol_name, seg_content,
|
|
304
|
+
content='code_chunks', content_rowid='rowid',
|
|
305
|
+
tokenize='unicode61'
|
|
306
|
+
)
|
|
307
|
+
`)
|
|
308
|
+
db.exec(`
|
|
309
|
+
CREATE TRIGGER code_chunks_ai AFTER INSERT ON code_chunks BEGIN
|
|
310
|
+
INSERT INTO code_chunks_fts(rowid, path, symbol_name, seg_content)
|
|
311
|
+
VALUES (new.rowid, new.path, new.symbol_name, new.seg_content);
|
|
312
|
+
END;
|
|
313
|
+
CREATE TRIGGER code_chunks_ad AFTER DELETE ON code_chunks BEGIN
|
|
314
|
+
INSERT INTO code_chunks_fts(code_chunks_fts, rowid, path, symbol_name, seg_content)
|
|
315
|
+
VALUES ('delete', old.rowid, old.path, old.symbol_name, old.seg_content);
|
|
316
|
+
END;
|
|
317
|
+
CREATE TRIGGER code_chunks_au AFTER UPDATE ON code_chunks BEGIN
|
|
318
|
+
INSERT INTO code_chunks_fts(code_chunks_fts, rowid, path, symbol_name, seg_content)
|
|
319
|
+
VALUES ('delete', old.rowid, old.path, old.symbol_name, old.seg_content);
|
|
320
|
+
INSERT INTO code_chunks_fts(rowid, path, symbol_name, seg_content)
|
|
321
|
+
VALUES (new.rowid, new.path, new.symbol_name, new.seg_content);
|
|
322
|
+
END;
|
|
323
|
+
`)
|
|
324
|
+
db.exec(`
|
|
325
|
+
CREATE TABLE doc_chunks (
|
|
326
|
+
origin TEXT NOT NULL DEFAULT '',
|
|
327
|
+
path TEXT NOT NULL,
|
|
328
|
+
language TEXT NOT NULL DEFAULT 'markdown',
|
|
329
|
+
heading TEXT NOT NULL DEFAULT '',
|
|
330
|
+
content TEXT NOT NULL,
|
|
331
|
+
line_start INTEGER NOT NULL DEFAULT 0,
|
|
332
|
+
line_end INTEGER NOT NULL DEFAULT 0,
|
|
333
|
+
mtime_ms INTEGER NOT NULL DEFAULT 0,
|
|
334
|
+
embedding BLOB,
|
|
335
|
+
seg_content TEXT NOT NULL DEFAULT '',
|
|
336
|
+
PRIMARY KEY (origin, path, line_start)
|
|
337
|
+
)
|
|
338
|
+
`)
|
|
339
|
+
db.exec(`
|
|
340
|
+
CREATE VIRTUAL TABLE doc_chunks_fts USING fts5(
|
|
341
|
+
path, heading, seg_content,
|
|
342
|
+
content='doc_chunks', content_rowid='rowid',
|
|
343
|
+
tokenize='unicode61'
|
|
344
|
+
)
|
|
345
|
+
`)
|
|
346
|
+
db.exec(`
|
|
347
|
+
CREATE TRIGGER doc_chunks_ai AFTER INSERT ON doc_chunks BEGIN
|
|
348
|
+
INSERT INTO doc_chunks_fts(rowid, path, heading, seg_content)
|
|
349
|
+
VALUES (new.rowid, new.path, new.heading, new.seg_content);
|
|
350
|
+
END;
|
|
351
|
+
CREATE TRIGGER doc_chunks_ad AFTER DELETE ON doc_chunks BEGIN
|
|
352
|
+
INSERT INTO doc_chunks_fts(doc_chunks_fts, rowid, path, heading, seg_content)
|
|
353
|
+
VALUES ('delete', old.rowid, old.path, old.heading, old.seg_content);
|
|
354
|
+
END;
|
|
355
|
+
CREATE TRIGGER doc_chunks_au AFTER UPDATE ON doc_chunks BEGIN
|
|
356
|
+
INSERT INTO doc_chunks_fts(doc_chunks_fts, rowid, path, heading, seg_content)
|
|
357
|
+
VALUES ('delete', old.rowid, old.path, old.heading, old.seg_content);
|
|
358
|
+
INSERT INTO doc_chunks_fts(rowid, path, heading, seg_content)
|
|
359
|
+
VALUES (new.rowid, new.path, new.heading, new.seg_content);
|
|
360
|
+
END;
|
|
361
|
+
`)
|
|
362
|
+
db.exec(`PRAGMA user_version = 8`)
|
|
363
|
+
}
|
|
364
|
+
db.exec("COMMIT")
|
|
365
|
+
} catch (err) {
|
|
366
|
+
db.exec("ROLLBACK")
|
|
367
|
+
throw err
|
|
368
|
+
}
|
|
263
369
|
}
|
|
264
370
|
|
|
265
371
|
/**
|
|
@@ -326,7 +432,10 @@ export async function search(memory, query, { limit = 5 } = {}) {
|
|
|
326
432
|
return [...scores.entries()]
|
|
327
433
|
.sort((a, b) => b[1] - a[1])
|
|
328
434
|
.slice(0, limit)
|
|
329
|
-
.map(([id, score]) =>
|
|
435
|
+
.map(([id, score]) => {
|
|
436
|
+
const entry = fetchEntry(memory, id)
|
|
437
|
+
return entry ? { ...entry, rrf: score } : null // fetchEntry 为 null 时不能展开(会漏出 { rrf } 空壳)
|
|
438
|
+
})
|
|
330
439
|
.filter(Boolean)
|
|
331
440
|
}
|
|
332
441
|
|
|
@@ -665,29 +774,48 @@ function extractLeadingDoc(lines, lineNum, ext) {
|
|
|
665
774
|
}
|
|
666
775
|
|
|
667
776
|
/** 单文件入索引:删除旧块 → 分块 → 插入新块(codeSync 和 reindexFile 共用) */
|
|
668
|
-
|
|
777
|
+
/** 将控制权交还给事件循环一个 tick(让键盘输入有机会被处理) */
|
|
778
|
+
function yieldTick() {
|
|
779
|
+
return new Promise((r) => setTimeout(r, 0))
|
|
780
|
+
}
|
|
781
|
+
|
|
782
|
+
function _upsertCodeFile(memory, origin, rel, lines, lang, mtimeMs) {
|
|
669
783
|
const chunks = chunkCode(lines, rel)
|
|
670
|
-
memory.db.
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
784
|
+
memory.db.exec("BEGIN")
|
|
785
|
+
try {
|
|
786
|
+
memory.db.prepare(`DELETE FROM code_chunks WHERE origin = ? AND path = ?`).run(origin, rel)
|
|
787
|
+
const insert = memory.db.prepare(`
|
|
788
|
+
INSERT INTO code_chunks (origin, path, language, chunk_type, symbol_name, content, line_start, line_end, mtime_ms, seg_content)
|
|
789
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
790
|
+
`)
|
|
791
|
+
for (const c of chunks) {
|
|
792
|
+
const isFile = c.name === rel
|
|
793
|
+
insert.run(origin, rel, lang, isFile ? "file" : "symbol", isFile ? "" : c.name.slice(rel.length + 1), c.content, c.line_start, c.line_end, mtimeMs, segmentCJK(c.content))
|
|
794
|
+
}
|
|
795
|
+
memory.db.exec("COMMIT")
|
|
796
|
+
} catch (e) {
|
|
797
|
+
memory.db.exec("ROLLBACK")
|
|
798
|
+
throw e
|
|
678
799
|
}
|
|
679
800
|
}
|
|
680
801
|
|
|
681
|
-
function _upsertDocFile(memory, rel, lines, mtimeMs) {
|
|
802
|
+
function _upsertDocFile(memory, origin, rel, lines, mtimeMs) {
|
|
682
803
|
const chunks = chunkMarkdown(lines, rel)
|
|
683
804
|
const lang = rel.endsWith(".rst") ? "rst" : rel.endsWith(".adoc") ? "asciidoc" : rel.endsWith(".txt") ? "text" : "markdown"
|
|
684
|
-
memory.db.
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
805
|
+
memory.db.exec("BEGIN")
|
|
806
|
+
try {
|
|
807
|
+
memory.db.prepare(`DELETE FROM doc_chunks WHERE origin = ? AND path = ?`).run(origin, rel)
|
|
808
|
+
const insert = memory.db.prepare(`
|
|
809
|
+
INSERT INTO doc_chunks (origin, path, language, heading, content, line_start, line_end, mtime_ms, seg_content)
|
|
810
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
811
|
+
`)
|
|
812
|
+
for (const c of chunks) {
|
|
813
|
+
insert.run(origin, rel, lang, c.heading, c.content, c.line_start, c.line_end, mtimeMs, segmentCJK(c.content))
|
|
814
|
+
}
|
|
815
|
+
memory.db.exec("COMMIT")
|
|
816
|
+
} catch (e) {
|
|
817
|
+
memory.db.exec("ROLLBACK")
|
|
818
|
+
throw e
|
|
691
819
|
}
|
|
692
820
|
}
|
|
693
821
|
|
|
@@ -714,15 +842,16 @@ export async function codeSync(memory, dir, { onProgress } = {}) {
|
|
|
714
842
|
}
|
|
715
843
|
await walk(dir)
|
|
716
844
|
|
|
717
|
-
// 取已索引文件的 mtime
|
|
845
|
+
// 取已索引文件的 mtime 快照(只看本 origin,别的项目的块不归这里管)
|
|
718
846
|
const indexed = new Map(
|
|
719
|
-
memory.db.prepare(`SELECT path, mtime_ms FROM code_chunks
|
|
847
|
+
memory.db.prepare(`SELECT path, mtime_ms FROM code_chunks WHERE origin = ?`).all(dir).map((r) => [r.path, r.mtime_ms])
|
|
720
848
|
)
|
|
721
849
|
const seen = new Set()
|
|
722
850
|
|
|
723
851
|
onProgress?.({ phase: "scan", total: files.length })
|
|
724
852
|
|
|
725
|
-
let updated = 0, removed = 0, skipped = 0
|
|
853
|
+
let updated = 0, removed = 0, skipped = 0, failed = 0
|
|
854
|
+
const errors = []
|
|
726
855
|
for (let i = 0; i < files.length; i++) {
|
|
727
856
|
const abs = files[i]
|
|
728
857
|
const rel = abs.slice(dir.length + 1).replaceAll("\\", "/")
|
|
@@ -735,71 +864,81 @@ export async function codeSync(memory, dir, { onProgress } = {}) {
|
|
|
735
864
|
continue
|
|
736
865
|
}
|
|
737
866
|
|
|
738
|
-
//
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
867
|
+
// 单文件失败不拖垮整轮同步:记下错误继续,调用方看 failed/errors
|
|
868
|
+
try {
|
|
869
|
+
const text = await readFile(abs, "utf8")
|
|
870
|
+
const lines = text.split("\n")
|
|
871
|
+
const lang = detectLanguage(abs)
|
|
872
|
+
_upsertCodeFile(memory, dir, rel, lines, lang, mtimeMs)
|
|
873
|
+
updated++
|
|
874
|
+
} catch (e) {
|
|
875
|
+
failed++
|
|
876
|
+
if (errors.length < 5) errors.push(`${rel}: ${e.message}`)
|
|
877
|
+
}
|
|
878
|
+
await yieldTick()
|
|
745
879
|
|
|
746
880
|
if (onProgress && i % 10 === 0) {
|
|
747
|
-
onProgress({ phase: "index", current: i + 1, total: files.length, updated, removed, skipped })
|
|
881
|
+
onProgress({ phase: "index", current: i + 1, total: files.length, updated, removed, skipped, failed })
|
|
748
882
|
}
|
|
749
883
|
}
|
|
750
884
|
|
|
751
|
-
//
|
|
885
|
+
// 清理磁盘上已消失的文件块(仅本 origin)
|
|
752
886
|
for (const stale of indexed.keys()) {
|
|
753
887
|
if (!seen.has(stale)) {
|
|
754
|
-
memory.db.prepare(`DELETE FROM code_chunks WHERE path = ?`).run(stale)
|
|
888
|
+
memory.db.prepare(`DELETE FROM code_chunks WHERE origin = ? AND path = ?`).run(dir, stale)
|
|
755
889
|
removed++
|
|
756
890
|
}
|
|
757
891
|
}
|
|
758
892
|
|
|
759
|
-
onProgress?.({ phase: "done", total: files.length, updated, removed, skipped })
|
|
760
|
-
return { updated, removed, skipped, total: files.length }
|
|
893
|
+
onProgress?.({ phase: "done", total: files.length, updated, removed, skipped, failed })
|
|
894
|
+
return { updated, removed, skipped, failed, errors, total: files.length }
|
|
761
895
|
}
|
|
762
896
|
|
|
763
897
|
/**
|
|
764
898
|
* 代码检索:FTS5(BM25) + 可选向量余弦,RRF 合并。
|
|
765
|
-
* 无 embedder 时退化为纯 FTS
|
|
766
|
-
* 返回 [{ path, language, symbol_name, content, line_start, line_end
|
|
899
|
+
* 无 embedder 时退化为纯 FTS;ftsQuery 为空(纯标点查询)且有 embedder 时退化为纯向量。
|
|
900
|
+
* 返回 [{ path, language, symbol_name, content, line_start, line_end }]
|
|
767
901
|
*/
|
|
768
902
|
export async function codeSearch(memory, query, { limit = 5 } = {}) {
|
|
769
903
|
const ftsQuery = buildFtsQuery(query)
|
|
770
|
-
if (!ftsQuery) return []
|
|
904
|
+
if (!ftsQuery && !memory.embedder) return []
|
|
905
|
+
|
|
906
|
+
// codeOrigin 设置时只检索本项目(与 files 表的 projectOrigin 过滤同模式);未设置时不过滤
|
|
907
|
+
const ftsOriginFilter = memory.codeOrigin ? `AND c.origin = ?` : ""
|
|
908
|
+
const vecOriginFilter = memory.codeOrigin ? `AND origin = ?` : ""
|
|
909
|
+
const originParams = memory.codeOrigin ? [memory.codeOrigin] : []
|
|
771
910
|
|
|
772
|
-
const ftsList = memory.db.prepare(`
|
|
773
|
-
SELECT c.path, c.language, c.symbol_name, c.content, c.line_start, c.line_end, bm25(code_chunks_fts) AS rank
|
|
911
|
+
const ftsList = ftsQuery ? memory.db.prepare(`
|
|
912
|
+
SELECT c.rowid, c.path, c.language, c.symbol_name, c.content, c.line_start, c.line_end, bm25(code_chunks_fts) AS rank
|
|
774
913
|
FROM code_chunks_fts JOIN code_chunks c ON c.rowid = code_chunks_fts.rowid
|
|
775
|
-
WHERE code_chunks_fts MATCH ?
|
|
914
|
+
WHERE code_chunks_fts MATCH ? ${ftsOriginFilter}
|
|
776
915
|
ORDER BY rank LIMIT ?
|
|
777
|
-
`).all(ftsQuery, Math.max(limit * 4, 20))
|
|
916
|
+
`).all(ftsQuery, ...originParams, Math.max(limit * 4, 20)) : []
|
|
778
917
|
|
|
779
918
|
if (!memory.embedder) return ftsList.slice(0, limit)
|
|
780
919
|
|
|
781
920
|
// 向量通道
|
|
782
921
|
await ensureEmbeddings(memory)
|
|
783
922
|
const [qvec] = await embed(memory.embedder, [query])
|
|
784
|
-
const rows = memory.db.prepare(`SELECT
|
|
923
|
+
const rows = memory.db.prepare(`SELECT rowid, embedding FROM code_chunks WHERE embedding IS NOT NULL ${vecOriginFilter}`).all(...originParams)
|
|
785
924
|
const vecList = rows
|
|
786
|
-
.map((r) => ({
|
|
925
|
+
.map((r) => ({ rowid: r.rowid, score: cosine(qvec, fromBlob(r.embedding)) }))
|
|
787
926
|
.sort((a, b) => b.score - a.score)
|
|
788
927
|
.slice(0, Math.max(limit * 4, 20))
|
|
789
928
|
|
|
790
|
-
// RRF
|
|
929
|
+
// RRF 合并(按 rowid 对齐两个通道;解析结果回表取,纯向量命中的块也能浮现)
|
|
791
930
|
const K = 60
|
|
792
931
|
const scores = new Map()
|
|
793
|
-
ftsList.forEach((r, i) => scores.set(
|
|
794
|
-
vecList.forEach((r, i) => scores.set(r.
|
|
932
|
+
ftsList.forEach((r, i) => scores.set(r.rowid, (scores.get(r.rowid) ?? 0) + 1 / (K + i + 1)))
|
|
933
|
+
vecList.forEach((r, i) => scores.set(r.rowid, (scores.get(r.rowid) ?? 0) + 1 / (K + i + 1)))
|
|
795
934
|
|
|
935
|
+
const fetchChunk = memory.db.prepare(`
|
|
936
|
+
SELECT path, language, symbol_name, content, line_start, line_end FROM code_chunks WHERE rowid = ?
|
|
937
|
+
`)
|
|
796
938
|
return [...scores.entries()]
|
|
797
939
|
.sort((a, b) => b[1] - a[1])
|
|
798
940
|
.slice(0, limit)
|
|
799
|
-
.map(([
|
|
800
|
-
const [path, lineStr] = key.split(/:(?=\d+$)/)
|
|
801
|
-
return ftsList.find((r) => r.path === path && String(r.line_start) === lineStr)
|
|
802
|
-
})
|
|
941
|
+
.map(([rowid]) => fetchChunk.get(rowid))
|
|
803
942
|
.filter(Boolean)
|
|
804
943
|
}
|
|
805
944
|
|
|
@@ -858,13 +997,18 @@ export function codeSearchTool(memory) {
|
|
|
858
997
|
export async function reindexFile(memory, cwd, absPath) {
|
|
859
998
|
const ext = absPath.slice(absPath.lastIndexOf(".")).toLowerCase()
|
|
860
999
|
const rel = relative(cwd, absPath).replaceAll("\\", "/")
|
|
861
|
-
|
|
1000
|
+
// 越界路径拒索引:必须是 ".." 或 "../..." 才算越界——文件名以 .. 开头(如 ..foo.js)不算
|
|
1001
|
+
if (rel === ".." || rel.startsWith("../")) return
|
|
1002
|
+
// 与 walk 策略一致:跳过隐藏目录和 SKIP_DIRS 里的文件(node_modules/.git/dist…),
|
|
1003
|
+
// 否则这些文件虽然全量扫描时被跳过,单文件增量更新却会漏进来
|
|
1004
|
+
const dirs = rel.split("/").slice(0, -1)
|
|
1005
|
+
if (dirs.some((d) => SKIP_DIRS.has(d) || d.startsWith("."))) return
|
|
862
1006
|
|
|
863
1007
|
let text
|
|
864
1008
|
try { text = await readFile(absPath, "utf8") } catch {
|
|
865
1009
|
// 文件已删:清理索引
|
|
866
|
-
if (CODE_EXTS.has(ext)) memory.db.prepare(`DELETE FROM code_chunks WHERE path = ?`).run(rel)
|
|
867
|
-
else if (DOC_EXTS.has(ext)) memory.db.prepare(`DELETE FROM doc_chunks WHERE path = ?`).run(rel)
|
|
1010
|
+
if (CODE_EXTS.has(ext)) memory.db.prepare(`DELETE FROM code_chunks WHERE origin = ? AND path = ?`).run(cwd, rel)
|
|
1011
|
+
else if (DOC_EXTS.has(ext)) memory.db.prepare(`DELETE FROM doc_chunks WHERE origin = ? AND path = ?`).run(cwd, rel)
|
|
868
1012
|
return
|
|
869
1013
|
}
|
|
870
1014
|
const lines = text.split("\n")
|
|
@@ -873,11 +1017,11 @@ export async function reindexFile(memory, cwd, absPath) {
|
|
|
873
1017
|
const lang = detectLanguage(absPath)
|
|
874
1018
|
let mtimeMs = 0
|
|
875
1019
|
try { mtimeMs = Math.floor((await stat(absPath)).mtimeMs) } catch { /* 新文件 */ }
|
|
876
|
-
_upsertCodeFile(memory, rel, lines, lang, mtimeMs)
|
|
1020
|
+
_upsertCodeFile(memory, cwd, rel, lines, lang, mtimeMs)
|
|
877
1021
|
} else if (DOC_EXTS.has(ext)) {
|
|
878
1022
|
let mtimeMs = 0
|
|
879
1023
|
try { mtimeMs = Math.floor((await stat(absPath)).mtimeMs) } catch { /* 新文件 */ }
|
|
880
|
-
_upsertDocFile(memory, rel, lines, mtimeMs)
|
|
1024
|
+
_upsertDocFile(memory, cwd, rel, lines, mtimeMs)
|
|
881
1025
|
}
|
|
882
1026
|
}
|
|
883
1027
|
|
|
@@ -931,13 +1075,14 @@ export async function docSync(memory, dir, { onProgress } = {}) {
|
|
|
931
1075
|
await walk(dir)
|
|
932
1076
|
|
|
933
1077
|
const indexed = new Map(
|
|
934
|
-
memory.db.prepare(`SELECT path, mtime_ms FROM doc_chunks
|
|
1078
|
+
memory.db.prepare(`SELECT path, mtime_ms FROM doc_chunks WHERE origin = ?`).all(dir).map((r) => [r.path, r.mtime_ms])
|
|
935
1079
|
)
|
|
936
1080
|
const seen = new Set()
|
|
937
1081
|
|
|
938
1082
|
onProgress?.({ phase: "scan", total: files.length })
|
|
939
1083
|
|
|
940
|
-
let updated = 0, removed = 0, skipped = 0
|
|
1084
|
+
let updated = 0, removed = 0, skipped = 0, failed = 0
|
|
1085
|
+
const errors = []
|
|
941
1086
|
for (let i = 0; i < files.length; i++) {
|
|
942
1087
|
const abs = files[i]
|
|
943
1088
|
const rel = abs.slice(dir.length + 1).replaceAll("\\", "/")
|
|
@@ -950,64 +1095,80 @@ export async function docSync(memory, dir, { onProgress } = {}) {
|
|
|
950
1095
|
continue
|
|
951
1096
|
}
|
|
952
1097
|
|
|
953
|
-
|
|
954
|
-
try {
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
1098
|
+
// 单文件失败不拖垮整轮同步(与 codeSync 同策略)
|
|
1099
|
+
try {
|
|
1100
|
+
const text = await readFile(abs, "utf8")
|
|
1101
|
+
const lines = text.split("\n")
|
|
1102
|
+
_upsertDocFile(memory, dir, rel, lines, mtimeMs)
|
|
1103
|
+
updated++
|
|
1104
|
+
} catch (e) {
|
|
1105
|
+
failed++
|
|
1106
|
+
if (errors.length < 5) errors.push(`${rel}: ${e.message}`)
|
|
1107
|
+
}
|
|
1108
|
+
await yieldTick()
|
|
958
1109
|
|
|
959
1110
|
if (onProgress && i % 10 === 0) {
|
|
960
|
-
onProgress({ phase: "index", current: i + 1, total: files.length, updated, removed, skipped })
|
|
1111
|
+
onProgress({ phase: "index", current: i + 1, total: files.length, updated, removed, skipped, failed })
|
|
961
1112
|
}
|
|
962
1113
|
}
|
|
963
1114
|
|
|
964
|
-
//
|
|
1115
|
+
// 清理磁盘上消失的文件(仅本 origin)
|
|
965
1116
|
for (const stale of indexed.keys()) {
|
|
966
1117
|
if (!seen.has(stale)) {
|
|
967
|
-
memory.db.prepare(`DELETE FROM doc_chunks WHERE path = ?`).run(stale)
|
|
1118
|
+
memory.db.prepare(`DELETE FROM doc_chunks WHERE origin = ? AND path = ?`).run(dir, stale)
|
|
968
1119
|
removed++
|
|
969
1120
|
}
|
|
970
1121
|
}
|
|
971
1122
|
|
|
972
|
-
onProgress?.({ phase: "done", total: files.length, updated, removed, skipped })
|
|
973
|
-
return { updated, removed, skipped, total: files.length }
|
|
1123
|
+
onProgress?.({ phase: "done", total: files.length, updated, removed, skipped, failed })
|
|
1124
|
+
return { updated, removed, skipped, failed, errors, total: files.length }
|
|
974
1125
|
}
|
|
975
1126
|
|
|
976
1127
|
/**
|
|
977
1128
|
* 文档检索:FTS5(BM25) + 可选向量余弦,RRF 合并。
|
|
1129
|
+
* 无 embedder 时退化为纯 FTS;ftsQuery 为空(纯标点查询)且有 embedder 时退化为纯向量。
|
|
1130
|
+
* 返回 [{ path, language, heading, content, line_start, line_end }]
|
|
978
1131
|
*/
|
|
979
1132
|
export async function docSearch(memory, query, { limit = 5 } = {}) {
|
|
980
1133
|
const ftsQuery = buildFtsQuery(query)
|
|
981
|
-
if (!ftsQuery) return []
|
|
1134
|
+
if (!ftsQuery && !memory.embedder) return []
|
|
1135
|
+
|
|
1136
|
+
// codeOrigin 设置时只检索本项目(与 codeSearch 同模式);未设置时不过滤
|
|
1137
|
+
const ftsOriginFilter = memory.codeOrigin ? `AND d.origin = ?` : ""
|
|
1138
|
+
const vecOriginFilter = memory.codeOrigin ? `AND origin = ?` : ""
|
|
1139
|
+
const originParams = memory.codeOrigin ? [memory.codeOrigin] : []
|
|
982
1140
|
|
|
983
|
-
const ftsList = memory.db.prepare(`
|
|
984
|
-
SELECT d.path, d.language, d.heading, d.content, d.line_start, d.line_end, bm25(doc_chunks_fts) AS rank
|
|
1141
|
+
const ftsList = ftsQuery ? memory.db.prepare(`
|
|
1142
|
+
SELECT d.rowid, d.path, d.language, d.heading, d.content, d.line_start, d.line_end, bm25(doc_chunks_fts) AS rank
|
|
985
1143
|
FROM doc_chunks_fts JOIN doc_chunks d ON d.rowid = doc_chunks_fts.rowid
|
|
986
|
-
WHERE doc_chunks_fts MATCH ?
|
|
1144
|
+
WHERE doc_chunks_fts MATCH ? ${ftsOriginFilter}
|
|
987
1145
|
ORDER BY rank LIMIT ?
|
|
988
|
-
`).all(ftsQuery, Math.max(limit * 4, 20))
|
|
1146
|
+
`).all(ftsQuery, ...originParams, Math.max(limit * 4, 20)) : []
|
|
989
1147
|
|
|
990
1148
|
if (!memory.embedder) return ftsList.slice(0, limit)
|
|
991
1149
|
|
|
1150
|
+
// 向量通道
|
|
1151
|
+
await ensureDocEmbeddings(memory)
|
|
992
1152
|
const [qvec] = await embed(memory.embedder, [query])
|
|
993
|
-
const rows = memory.db.prepare(`SELECT
|
|
1153
|
+
const rows = memory.db.prepare(`SELECT rowid, embedding FROM doc_chunks WHERE embedding IS NOT NULL ${vecOriginFilter}`).all(...originParams)
|
|
994
1154
|
const vecList = rows
|
|
995
|
-
.map((r) => ({
|
|
1155
|
+
.map((r) => ({ rowid: r.rowid, score: cosine(qvec, fromBlob(r.embedding)) }))
|
|
996
1156
|
.sort((a, b) => b.score - a.score)
|
|
997
1157
|
.slice(0, Math.max(limit * 4, 20))
|
|
998
1158
|
|
|
1159
|
+
// RRF 合并(按 rowid 对齐两个通道;解析结果回表取,纯向量命中的块也能浮现)
|
|
999
1160
|
const K = 60
|
|
1000
1161
|
const scores = new Map()
|
|
1001
|
-
ftsList.forEach((r, i) => scores.set(
|
|
1002
|
-
vecList.forEach((r, i) => scores.set(r.
|
|
1162
|
+
ftsList.forEach((r, i) => scores.set(r.rowid, (scores.get(r.rowid) ?? 0) + 1 / (K + i + 1)))
|
|
1163
|
+
vecList.forEach((r, i) => scores.set(r.rowid, (scores.get(r.rowid) ?? 0) + 1 / (K + i + 1)))
|
|
1003
1164
|
|
|
1165
|
+
const fetchChunk = memory.db.prepare(`
|
|
1166
|
+
SELECT path, language, heading, content, line_start, line_end FROM doc_chunks WHERE rowid = ?
|
|
1167
|
+
`)
|
|
1004
1168
|
return [...scores.entries()]
|
|
1005
1169
|
.sort((a, b) => b[1] - a[1])
|
|
1006
1170
|
.slice(0, limit)
|
|
1007
|
-
.map(([
|
|
1008
|
-
const [path, lineStr] = key.split(/:(?=\d+$)/)
|
|
1009
|
-
return ftsList.find((r) => r.path === path && String(r.line_start) === lineStr)
|
|
1010
|
-
})
|
|
1171
|
+
.map(([rowid]) => fetchChunk.get(rowid))
|
|
1011
1172
|
.filter(Boolean)
|
|
1012
1173
|
}
|
|
1013
1174
|
|
package/src/provider.mjs
CHANGED
|
@@ -47,6 +47,7 @@ export function createProvider(config) {
|
|
|
47
47
|
* 思考模式不支持前缀续写,已产出 reasoning 时放弃续写
|
|
48
48
|
*/
|
|
49
49
|
export async function chat(provider, { messages, tools, onToken, onReasoning, signal }) {
|
|
50
|
+
const spec = specForModel(provider.model)
|
|
50
51
|
const body = {
|
|
51
52
|
model: provider.model,
|
|
52
53
|
messages,
|
|
@@ -54,16 +55,32 @@ export async function chat(provider, { messages, tools, onToken, onReasoning, si
|
|
|
54
55
|
stream_options: { include_usage: true },
|
|
55
56
|
}
|
|
56
57
|
if (provider.maxTokens) body.max_tokens = provider.maxTokens
|
|
57
|
-
if (provider.temperature != null)
|
|
58
|
+
if (provider.temperature != null) {
|
|
59
|
+
// 按规格表裁剪 temperature:GLM [0,1] 限两位小数,DeepSeek ≤2,未声明则不裁剪
|
|
60
|
+
let t = provider.temperature
|
|
61
|
+
if (spec.tempRange) {
|
|
62
|
+
t = Math.min(spec.tempRange[1], Math.max(spec.tempRange[0], t))
|
|
63
|
+
t = Math.round(t * 100) / 100
|
|
64
|
+
}
|
|
65
|
+
body.temperature = t
|
|
66
|
+
}
|
|
58
67
|
if (provider.thinking) body.thinking = provider.thinking
|
|
59
|
-
if (provider.reasoningEffort)
|
|
68
|
+
if (provider.reasoningEffort) {
|
|
69
|
+
// 按规格表校验 reasoning_effort 枚举:不在枚举内则报错,不映射、不猜测
|
|
70
|
+
if (spec.reasoningEffortEnum && !spec.reasoningEffortEnum.includes(provider.reasoningEffort)) {
|
|
71
|
+
throw new Error(
|
|
72
|
+
`reasoning_effort "${provider.reasoningEffort}" not supported by model "${provider.model}"; ` +
|
|
73
|
+
`valid values: ${spec.reasoningEffortEnum.join(", ")}`
|
|
74
|
+
)
|
|
75
|
+
}
|
|
76
|
+
body.reasoning_effort = provider.reasoningEffort
|
|
77
|
+
}
|
|
60
78
|
if (tools?.length) body.tools = tools
|
|
61
79
|
|
|
62
80
|
const response = await requestWithRetry(provider, body, signal)
|
|
63
81
|
const result = await readSSE(response, { onToken, onReasoning })
|
|
64
82
|
|
|
65
83
|
// 截断续写:仅规格表声明续写协议的模型(其他端点不认识 partial/prefix 字段,可能 400)
|
|
66
|
-
const spec = specForModel(provider.model)
|
|
67
84
|
if (!spec.partialMode && !spec.prefixMode) return result
|
|
68
85
|
// DeepSeek prefix 续写不支持思考模式,已产出 reasoning 时无前缀协议可用
|
|
69
86
|
if (spec.prefixMode && !spec.partialMode && result.reasoning) return result
|
|
@@ -89,21 +106,25 @@ export async function chat(provider, { messages, tools, onToken, onReasoning, si
|
|
|
89
106
|
result.content += continued.content
|
|
90
107
|
result.reasoning += continued.reasoning ?? ""
|
|
91
108
|
for (const tc of continued.toolCalls ?? []) {
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
109
|
+
if (tc.index == null) { result.toolCalls = continued.toolCalls; break }
|
|
110
|
+
const s = result.toolCalls[tc.index] ??= { id: "", name: "", arguments: "" }
|
|
111
|
+
if (tc.id) s.id = tc.id
|
|
112
|
+
s.name += tc.name ?? ""
|
|
113
|
+
s.arguments += tc.arguments ?? ""
|
|
114
|
+
}
|
|
98
115
|
result.finishReason = continued.finishReason
|
|
99
116
|
if (continued.usage) {
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
117
|
+
const sum = (k) => (result.usage?.[k] ?? 0) + (continued.usage[k] ?? 0)
|
|
118
|
+
result.usage = {
|
|
119
|
+
prompt_tokens: sum("prompt_tokens"),
|
|
120
|
+
completion_tokens: sum("completion_tokens"),
|
|
121
|
+
total_tokens: sum("total_tokens"),
|
|
122
|
+
// 缓存命中/未命中也要累计(DeepSeek 计费与状态栏展示依赖这两个字段)
|
|
123
|
+
prompt_cache_hit_tokens: sum("prompt_cache_hit_tokens"),
|
|
124
|
+
prompt_cache_miss_tokens: sum("prompt_cache_miss_tokens"),
|
|
125
|
+
}
|
|
104
126
|
}
|
|
105
127
|
}
|
|
106
|
-
}
|
|
107
128
|
return result
|
|
108
129
|
}
|
|
109
130
|
|
package/src/repomap.mjs
CHANGED
|
@@ -74,17 +74,15 @@ function parsePyOutline(lines) {
|
|
|
74
74
|
for (const line of lines) {
|
|
75
75
|
const fromRe = line.match(/^from\s+(\S+)\s+import\s+(.+)/)
|
|
76
76
|
if (fromRe) {
|
|
77
|
-
const
|
|
78
|
-
if (
|
|
79
|
-
imports.push(normalizeExt(mod.replace(/^\.+/, "")))
|
|
77
|
+
const rel = pyRelPath(fromRe[1])
|
|
78
|
+
if (rel) imports.push(rel)
|
|
80
79
|
continue
|
|
81
80
|
}
|
|
82
81
|
const impRe = line.match(/^import\s+(.+)/)
|
|
83
82
|
if (impRe) {
|
|
84
83
|
for (const mod of impRe[1].split(",")) {
|
|
85
|
-
const
|
|
86
|
-
if (
|
|
87
|
-
imports.push(normalizeExt(m.replace(/^\.+/, "")))
|
|
84
|
+
const rel = pyRelPath(mod.trim().split(/\s+/)[0])
|
|
85
|
+
if (rel) imports.push(rel)
|
|
88
86
|
}
|
|
89
87
|
continue
|
|
90
88
|
}
|
|
@@ -94,6 +92,19 @@ function parsePyOutline(lines) {
|
|
|
94
92
|
return { imports: [...new Set(imports)], symbols: [...new Set(symbols)] }
|
|
95
93
|
}
|
|
96
94
|
|
|
95
|
+
/**
|
|
96
|
+
* Python 相对导入 → 相对文件路径:
|
|
97
|
+
* 前导 n 个点表示上溯 n-1 层("."=当前包),模块点号转路径分隔符。
|
|
98
|
+
* 非相对导入(不以 . 开头)或纯包导入("from . import x")返回 null。
|
|
99
|
+
*/
|
|
100
|
+
function pyRelPath(mod) {
|
|
101
|
+
if (!mod?.startsWith(".")) return null
|
|
102
|
+
const dots = mod.match(/^\.+/)[0].length
|
|
103
|
+
const rest = mod.slice(dots).replaceAll(".", "/")
|
|
104
|
+
if (!rest) return null
|
|
105
|
+
return normalizeExt("../".repeat(dots - 1) + rest)
|
|
106
|
+
}
|
|
107
|
+
|
|
97
108
|
function normalizeExt(p) {
|
|
98
109
|
return p.replace(/\.(m?js|jsx|tsx?)$/i, "")
|
|
99
110
|
}
|
package/src/session.mjs
CHANGED
|
@@ -45,8 +45,8 @@ function saveManifest(cwd, m) {
|
|
|
45
45
|
writeSessionFile(manifestPath(cwd), m)
|
|
46
46
|
}
|
|
47
47
|
|
|
48
|
-
/**
|
|
49
|
-
export function archiveCurrent(cwd) {
|
|
48
|
+
/** 归档当前会话到空闲槽位——满了踢最老;exclude 指定一个不许被踢的槽位(switchToSlot 的目标槽) */
|
|
49
|
+
export function archiveCurrent(cwd, { exclude } = {}) {
|
|
50
50
|
const src = sessionPath(cwd)
|
|
51
51
|
if (!existsSync(src)) return
|
|
52
52
|
const m = loadManifest(cwd)
|
|
@@ -57,11 +57,13 @@ export function archiveCurrent(cwd) {
|
|
|
57
57
|
slot = 1
|
|
58
58
|
while (m.slots[slot]) slot++
|
|
59
59
|
} else {
|
|
60
|
-
|
|
60
|
+
const candidates = entries.filter(([n]) => Number(n) !== exclude)
|
|
61
|
+
slot = Number((candidates.length ? candidates : entries).sort((a, b) => a[1] - b[1])[0][0])
|
|
61
62
|
}
|
|
62
63
|
|
|
63
64
|
const dst = slotPath(cwd, slot)
|
|
64
|
-
|
|
65
|
+
// 复制(rename 会丢当前);走原子写,防中途崩溃留下截断的 JSON 丢归档
|
|
66
|
+
writeSessionFile(dst, JSON.parse(readFileSync(src, "utf8")))
|
|
65
67
|
m.slots[slot] = Date.now()
|
|
66
68
|
delete m.slots._currentName
|
|
67
69
|
saveManifest(cwd, m)
|
|
@@ -82,7 +84,8 @@ export function switchToSlot(cwd, slot) {
|
|
|
82
84
|
if (!m.slots[slot]) return null
|
|
83
85
|
|
|
84
86
|
// 归档当前(内部写 manifest;之后我们的 m 已过期,需重读)
|
|
85
|
-
|
|
87
|
+
// 满槽时排除目标槽:否则最老槽=目标槽,归档会把目标覆盖掉再复制回来,目标会话永久丢失
|
|
88
|
+
archiveCurrent(cwd, { exclude: slot })
|
|
86
89
|
|
|
87
90
|
// 槽位文件 → 当前(copy+unlink,不用 rename:Windows rename 目标已存在会抛 EPERM)
|
|
88
91
|
const src = slotPath(cwd, slot)
|