thincoder 0.7.0 → 0.7.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/memory.mjs CHANGED
@@ -19,7 +19,7 @@ import { embed, cosine, toBlob, fromBlob } from "./embedding.mjs"
19
19
  import { commitAndPush } from "./gitmem.mjs"
20
20
 
21
21
  const VALID_TYPES = new Set(["rule", "knowledge", "decision", "pattern"])
22
- const SCHEMA_VERSION = 7
22
+ const SCHEMA_VERSION = 8
23
23
 
24
24
  // 代码索引:源码文件扩展名
25
25
  const CODE_EXTS = new Set([".mjs", ".js", ".ts", ".tsx", ".jsx", ".py", ".rs", ".go", ".java", ".c", ".h", ".cpp", ".hpp", ".rb", ".swift", ".kt", ".sh", ".bash", ".sql", ".yaml", ".yml", ".toml", ".json", ".css", ".html", ".vue", ".svelte"])
@@ -37,6 +37,9 @@ const BIG_FILE_LINES = 2000
37
37
  export function createMemory({ dbPath }) {
38
38
  mkdirSync(dirname(dbPath), { recursive: true })
39
39
  const db = new DatabaseSync(dbPath)
40
+ // WAL 读写不互锁(TUI 检索和后台索引可并发);busy_timeout 防多进程同库直接 SQLITE_BUSY
41
+ db.exec(`PRAGMA journal_mode = WAL`)
42
+ db.exec(`PRAGMA busy_timeout = 3000`)
40
43
 
41
44
  db.exec(`
42
45
  CREATE TABLE IF NOT EXISTS entries (
@@ -57,11 +60,13 @@ export function createMemory({ dbPath }) {
57
60
  return { db }
58
61
  }
59
62
 
60
- /** 按 user_version 逐步迁移 */
63
+ /** 按 user_version 逐步迁移。整体单事务——任何一步失败都回滚,不留半成品 schema */
61
64
  function migrate(db) {
62
65
  const { user_version: version } = db.prepare(`PRAGMA user_version`).get()
63
66
  if (version >= SCHEMA_VERSION) return
64
67
 
68
+ db.exec("BEGIN IMMEDIATE")
69
+ try {
65
70
  if (version < 2) {
66
71
  // v1(trigram) 或空库 → v2(unicode61 + CJK 逐字):重建 FTS 和触发器
67
72
  db.exec(`
@@ -260,6 +265,107 @@ function migrate(db) {
260
265
  `)
261
266
  db.exec(`PRAGMA user_version = 7`)
262
267
  }
268
+
269
+ if (version < 8) {
270
+ // v8:code_chunks/doc_chunks 加 origin 列(项目根目录绝对路径),主键改为 (origin, path, line_start)。
271
+ // 旧主键不含 origin,多项目共用一个记忆库时同相对路径互相覆盖、codeSync(B) 会把 A 的块当 stale 清掉。
272
+ // SQLite 不能 ALTER 主键,且索引是易失品 → 直接删表重建(下次 codeSync/docSync 自动重索引)。
273
+ db.exec(`
274
+ DROP TRIGGER IF EXISTS code_chunks_ai;
275
+ DROP TRIGGER IF EXISTS code_chunks_ad;
276
+ DROP TRIGGER IF EXISTS code_chunks_au;
277
+ DROP TABLE IF EXISTS code_chunks_fts;
278
+ DROP TABLE IF EXISTS code_chunks;
279
+ DROP TRIGGER IF EXISTS doc_chunks_ai;
280
+ DROP TRIGGER IF EXISTS doc_chunks_ad;
281
+ DROP TRIGGER IF EXISTS doc_chunks_au;
282
+ DROP TABLE IF EXISTS doc_chunks_fts;
283
+ DROP TABLE IF EXISTS doc_chunks;
284
+ `)
285
+ db.exec(`
286
+ CREATE TABLE code_chunks (
287
+ origin TEXT NOT NULL DEFAULT '',
288
+ path TEXT NOT NULL,
289
+ language TEXT NOT NULL,
290
+ chunk_type TEXT NOT NULL CHECK(chunk_type IN ('file','symbol')),
291
+ symbol_name TEXT NOT NULL DEFAULT '',
292
+ content TEXT NOT NULL,
293
+ line_start INTEGER NOT NULL DEFAULT 0,
294
+ line_end INTEGER NOT NULL DEFAULT 0,
295
+ mtime_ms INTEGER NOT NULL DEFAULT 0,
296
+ embedding BLOB,
297
+ seg_content TEXT NOT NULL DEFAULT '',
298
+ PRIMARY KEY (origin, path, line_start)
299
+ )
300
+ `)
301
+ db.exec(`
302
+ CREATE VIRTUAL TABLE code_chunks_fts USING fts5(
303
+ path, symbol_name, seg_content,
304
+ content='code_chunks', content_rowid='rowid',
305
+ tokenize='unicode61'
306
+ )
307
+ `)
308
+ db.exec(`
309
+ CREATE TRIGGER code_chunks_ai AFTER INSERT ON code_chunks BEGIN
310
+ INSERT INTO code_chunks_fts(rowid, path, symbol_name, seg_content)
311
+ VALUES (new.rowid, new.path, new.symbol_name, new.seg_content);
312
+ END;
313
+ CREATE TRIGGER code_chunks_ad AFTER DELETE ON code_chunks BEGIN
314
+ INSERT INTO code_chunks_fts(code_chunks_fts, rowid, path, symbol_name, seg_content)
315
+ VALUES ('delete', old.rowid, old.path, old.symbol_name, old.seg_content);
316
+ END;
317
+ CREATE TRIGGER code_chunks_au AFTER UPDATE ON code_chunks BEGIN
318
+ INSERT INTO code_chunks_fts(code_chunks_fts, rowid, path, symbol_name, seg_content)
319
+ VALUES ('delete', old.rowid, old.path, old.symbol_name, old.seg_content);
320
+ INSERT INTO code_chunks_fts(rowid, path, symbol_name, seg_content)
321
+ VALUES (new.rowid, new.path, new.symbol_name, new.seg_content);
322
+ END;
323
+ `)
324
+ db.exec(`
325
+ CREATE TABLE doc_chunks (
326
+ origin TEXT NOT NULL DEFAULT '',
327
+ path TEXT NOT NULL,
328
+ language TEXT NOT NULL DEFAULT 'markdown',
329
+ heading TEXT NOT NULL DEFAULT '',
330
+ content TEXT NOT NULL,
331
+ line_start INTEGER NOT NULL DEFAULT 0,
332
+ line_end INTEGER NOT NULL DEFAULT 0,
333
+ mtime_ms INTEGER NOT NULL DEFAULT 0,
334
+ embedding BLOB,
335
+ seg_content TEXT NOT NULL DEFAULT '',
336
+ PRIMARY KEY (origin, path, line_start)
337
+ )
338
+ `)
339
+ db.exec(`
340
+ CREATE VIRTUAL TABLE doc_chunks_fts USING fts5(
341
+ path, heading, seg_content,
342
+ content='doc_chunks', content_rowid='rowid',
343
+ tokenize='unicode61'
344
+ )
345
+ `)
346
+ db.exec(`
347
+ CREATE TRIGGER doc_chunks_ai AFTER INSERT ON doc_chunks BEGIN
348
+ INSERT INTO doc_chunks_fts(rowid, path, heading, seg_content)
349
+ VALUES (new.rowid, new.path, new.heading, new.seg_content);
350
+ END;
351
+ CREATE TRIGGER doc_chunks_ad AFTER DELETE ON doc_chunks BEGIN
352
+ INSERT INTO doc_chunks_fts(doc_chunks_fts, rowid, path, heading, seg_content)
353
+ VALUES ('delete', old.rowid, old.path, old.heading, old.seg_content);
354
+ END;
355
+ CREATE TRIGGER doc_chunks_au AFTER UPDATE ON doc_chunks BEGIN
356
+ INSERT INTO doc_chunks_fts(doc_chunks_fts, rowid, path, heading, seg_content)
357
+ VALUES ('delete', old.rowid, old.path, old.heading, old.seg_content);
358
+ INSERT INTO doc_chunks_fts(rowid, path, heading, seg_content)
359
+ VALUES (new.rowid, new.path, new.heading, new.seg_content);
360
+ END;
361
+ `)
362
+ db.exec(`PRAGMA user_version = 8`)
363
+ }
364
+ db.exec("COMMIT")
365
+ } catch (err) {
366
+ db.exec("ROLLBACK")
367
+ throw err
368
+ }
263
369
  }
264
370
 
265
371
  /**
@@ -326,7 +432,10 @@ export async function search(memory, query, { limit = 5 } = {}) {
326
432
  return [...scores.entries()]
327
433
  .sort((a, b) => b[1] - a[1])
328
434
  .slice(0, limit)
329
- .map(([id, score]) => ({ ...fetchEntry(memory, id), rrf: score }))
435
+ .map(([id, score]) => {
436
+ const entry = fetchEntry(memory, id)
437
+ return entry ? { ...entry, rrf: score } : null // fetchEntry 为 null 时不能展开(会漏出 { rrf } 空壳)
438
+ })
330
439
  .filter(Boolean)
331
440
  }
332
441
 
@@ -670,18 +779,18 @@ function yieldTick() {
670
779
  return new Promise((r) => setTimeout(r, 0))
671
780
  }
672
781
 
673
- function _upsertCodeFile(memory, rel, lines, lang, mtimeMs) {
782
+ function _upsertCodeFile(memory, origin, rel, lines, lang, mtimeMs) {
674
783
  const chunks = chunkCode(lines, rel)
675
784
  memory.db.exec("BEGIN")
676
785
  try {
677
- memory.db.prepare(`DELETE FROM code_chunks WHERE path = ?`).run(rel)
786
+ memory.db.prepare(`DELETE FROM code_chunks WHERE origin = ? AND path = ?`).run(origin, rel)
678
787
  const insert = memory.db.prepare(`
679
- INSERT INTO code_chunks (path, language, chunk_type, symbol_name, content, line_start, line_end, mtime_ms, seg_content)
680
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
788
+ INSERT INTO code_chunks (origin, path, language, chunk_type, symbol_name, content, line_start, line_end, mtime_ms, seg_content)
789
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
681
790
  `)
682
791
  for (const c of chunks) {
683
792
  const isFile = c.name === rel
684
- insert.run(rel, lang, isFile ? "file" : "symbol", isFile ? "" : c.name.slice(rel.length + 1), c.content, c.line_start, c.line_end, mtimeMs, segmentCJK(c.content))
793
+ insert.run(origin, rel, lang, isFile ? "file" : "symbol", isFile ? "" : c.name.slice(rel.length + 1), c.content, c.line_start, c.line_end, mtimeMs, segmentCJK(c.content))
685
794
  }
686
795
  memory.db.exec("COMMIT")
687
796
  } catch (e) {
@@ -690,18 +799,18 @@ function _upsertCodeFile(memory, rel, lines, lang, mtimeMs) {
690
799
  }
691
800
  }
692
801
 
693
- function _upsertDocFile(memory, rel, lines, mtimeMs) {
802
+ function _upsertDocFile(memory, origin, rel, lines, mtimeMs) {
694
803
  const chunks = chunkMarkdown(lines, rel)
695
804
  const lang = rel.endsWith(".rst") ? "rst" : rel.endsWith(".adoc") ? "asciidoc" : rel.endsWith(".txt") ? "text" : "markdown"
696
805
  memory.db.exec("BEGIN")
697
806
  try {
698
- memory.db.prepare(`DELETE FROM doc_chunks WHERE path = ?`).run(rel)
807
+ memory.db.prepare(`DELETE FROM doc_chunks WHERE origin = ? AND path = ?`).run(origin, rel)
699
808
  const insert = memory.db.prepare(`
700
- INSERT INTO doc_chunks (path, language, heading, content, line_start, line_end, mtime_ms, seg_content)
701
- VALUES (?, ?, ?, ?, ?, ?, ?, ?)
809
+ INSERT INTO doc_chunks (origin, path, language, heading, content, line_start, line_end, mtime_ms, seg_content)
810
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
702
811
  `)
703
812
  for (const c of chunks) {
704
- insert.run(rel, lang, c.heading, c.content, c.line_start, c.line_end, mtimeMs, segmentCJK(c.content))
813
+ insert.run(origin, rel, lang, c.heading, c.content, c.line_start, c.line_end, mtimeMs, segmentCJK(c.content))
705
814
  }
706
815
  memory.db.exec("COMMIT")
707
816
  } catch (e) {
@@ -733,15 +842,16 @@ export async function codeSync(memory, dir, { onProgress } = {}) {
733
842
  }
734
843
  await walk(dir)
735
844
 
736
- // 取已索引文件的 mtime 快照
845
+ // 取已索引文件的 mtime 快照(只看本 origin,别的项目的块不归这里管)
737
846
  const indexed = new Map(
738
- memory.db.prepare(`SELECT path, mtime_ms FROM code_chunks`).all().map((r) => [r.path, r.mtime_ms])
847
+ memory.db.prepare(`SELECT path, mtime_ms FROM code_chunks WHERE origin = ?`).all(dir).map((r) => [r.path, r.mtime_ms])
739
848
  )
740
849
  const seen = new Set()
741
850
 
742
851
  onProgress?.({ phase: "scan", total: files.length })
743
852
 
744
- let updated = 0, removed = 0, skipped = 0
853
+ let updated = 0, removed = 0, skipped = 0, failed = 0
854
+ const errors = []
745
855
  for (let i = 0; i < files.length; i++) {
746
856
  const abs = files[i]
747
857
  const rel = abs.slice(dir.length + 1).replaceAll("\\", "/")
@@ -754,72 +864,81 @@ export async function codeSync(memory, dir, { onProgress } = {}) {
754
864
  continue
755
865
  }
756
866
 
757
- // 读文件并分块
758
- let text
759
- try { text = await readFile(abs, "utf8") } catch { continue }
760
- const lines = text.split("\n")
761
- const lang = detectLanguage(abs)
762
- _upsertCodeFile(memory, rel, lines, lang, mtimeMs)
763
- updated++
867
+ // 单文件失败不拖垮整轮同步:记下错误继续,调用方看 failed/errors
868
+ try {
869
+ const text = await readFile(abs, "utf8")
870
+ const lines = text.split("\n")
871
+ const lang = detectLanguage(abs)
872
+ _upsertCodeFile(memory, dir, rel, lines, lang, mtimeMs)
873
+ updated++
874
+ } catch (e) {
875
+ failed++
876
+ if (errors.length < 5) errors.push(`${rel}: ${e.message}`)
877
+ }
764
878
  await yieldTick()
765
879
 
766
880
  if (onProgress && i % 10 === 0) {
767
- onProgress({ phase: "index", current: i + 1, total: files.length, updated, removed, skipped })
881
+ onProgress({ phase: "index", current: i + 1, total: files.length, updated, removed, skipped, failed })
768
882
  }
769
883
  }
770
884
 
771
- // 清理磁盘上已消失的文件块
885
+ // 清理磁盘上已消失的文件块(仅本 origin)
772
886
  for (const stale of indexed.keys()) {
773
887
  if (!seen.has(stale)) {
774
- memory.db.prepare(`DELETE FROM code_chunks WHERE path = ?`).run(stale)
888
+ memory.db.prepare(`DELETE FROM code_chunks WHERE origin = ? AND path = ?`).run(dir, stale)
775
889
  removed++
776
890
  }
777
891
  }
778
892
 
779
- onProgress?.({ phase: "done", total: files.length, updated, removed, skipped })
780
- return { updated, removed, skipped, total: files.length }
893
+ onProgress?.({ phase: "done", total: files.length, updated, removed, skipped, failed })
894
+ return { updated, removed, skipped, failed, errors, total: files.length }
781
895
  }
782
896
 
783
897
  /**
784
898
  * 代码检索:FTS5(BM25) + 可选向量余弦,RRF 合并。
785
- * 无 embedder 时退化为纯 FTS
786
- * 返回 [{ path, language, symbol_name, content, line_start, line_end, rank }]
899
+ * 无 embedder 时退化为纯 FTS;ftsQuery 为空(纯标点查询)且有 embedder 时退化为纯向量。
900
+ * 返回 [{ path, language, symbol_name, content, line_start, line_end }]
787
901
  */
788
902
  export async function codeSearch(memory, query, { limit = 5 } = {}) {
789
903
  const ftsQuery = buildFtsQuery(query)
790
- if (!ftsQuery) return []
904
+ if (!ftsQuery && !memory.embedder) return []
905
+
906
+ // codeOrigin 设置时只检索本项目(与 files 表的 projectOrigin 过滤同模式);未设置时不过滤
907
+ const ftsOriginFilter = memory.codeOrigin ? `AND c.origin = ?` : ""
908
+ const vecOriginFilter = memory.codeOrigin ? `AND origin = ?` : ""
909
+ const originParams = memory.codeOrigin ? [memory.codeOrigin] : []
791
910
 
792
- const ftsList = memory.db.prepare(`
793
- SELECT c.path, c.language, c.symbol_name, c.content, c.line_start, c.line_end, bm25(code_chunks_fts) AS rank
911
+ const ftsList = ftsQuery ? memory.db.prepare(`
912
+ SELECT c.rowid, c.path, c.language, c.symbol_name, c.content, c.line_start, c.line_end, bm25(code_chunks_fts) AS rank
794
913
  FROM code_chunks_fts JOIN code_chunks c ON c.rowid = code_chunks_fts.rowid
795
- WHERE code_chunks_fts MATCH ?
914
+ WHERE code_chunks_fts MATCH ? ${ftsOriginFilter}
796
915
  ORDER BY rank LIMIT ?
797
- `).all(ftsQuery, Math.max(limit * 4, 20))
916
+ `).all(ftsQuery, ...originParams, Math.max(limit * 4, 20)) : []
798
917
 
799
918
  if (!memory.embedder) return ftsList.slice(0, limit)
800
919
 
801
920
  // 向量通道
802
921
  await ensureEmbeddings(memory)
803
922
  const [qvec] = await embed(memory.embedder, [query])
804
- const rows = memory.db.prepare(`SELECT path, line_start, embedding FROM code_chunks WHERE embedding IS NOT NULL`).all()
923
+ const rows = memory.db.prepare(`SELECT rowid, embedding FROM code_chunks WHERE embedding IS NOT NULL ${vecOriginFilter}`).all(...originParams)
805
924
  const vecList = rows
806
- .map((r) => ({ key: `${r.path}:${r.line_start}`, score: cosine(qvec, fromBlob(r.embedding)) }))
925
+ .map((r) => ({ rowid: r.rowid, score: cosine(qvec, fromBlob(r.embedding)) }))
807
926
  .sort((a, b) => b.score - a.score)
808
927
  .slice(0, Math.max(limit * 4, 20))
809
928
 
810
- // RRF 合并
929
+ // RRF 合并(按 rowid 对齐两个通道;解析结果回表取,纯向量命中的块也能浮现)
811
930
  const K = 60
812
931
  const scores = new Map()
813
- ftsList.forEach((r, i) => scores.set(`${r.path}:${r.line_start}`, (scores.get(`${r.path}:${r.line_start}`) ?? 0) + 1 / (K + i + 1)))
814
- vecList.forEach((r, i) => scores.set(r.key, (scores.get(r.key) ?? 0) + 1 / (K + i + 1)))
932
+ ftsList.forEach((r, i) => scores.set(r.rowid, (scores.get(r.rowid) ?? 0) + 1 / (K + i + 1)))
933
+ vecList.forEach((r, i) => scores.set(r.rowid, (scores.get(r.rowid) ?? 0) + 1 / (K + i + 1)))
815
934
 
935
+ const fetchChunk = memory.db.prepare(`
936
+ SELECT path, language, symbol_name, content, line_start, line_end FROM code_chunks WHERE rowid = ?
937
+ `)
816
938
  return [...scores.entries()]
817
939
  .sort((a, b) => b[1] - a[1])
818
940
  .slice(0, limit)
819
- .map(([key]) => {
820
- const [path, lineStr] = key.split(/:(?=\d+$)/)
821
- return ftsList.find((r) => r.path === path && String(r.line_start) === lineStr)
822
- })
941
+ .map(([rowid]) => fetchChunk.get(rowid))
823
942
  .filter(Boolean)
824
943
  }
825
944
 
@@ -878,13 +997,18 @@ export function codeSearchTool(memory) {
878
997
  export async function reindexFile(memory, cwd, absPath) {
879
998
  const ext = absPath.slice(absPath.lastIndexOf(".")).toLowerCase()
880
999
  const rel = relative(cwd, absPath).replaceAll("\\", "/")
881
- if (rel.startsWith("..")) return // 越界路径拒索引
1000
+ // 越界路径拒索引:必须是 ".." "../..." 才算越界——文件名以 .. 开头(如 ..foo.js)不算
1001
+ if (rel === ".." || rel.startsWith("../")) return
1002
+ // 与 walk 策略一致:跳过隐藏目录和 SKIP_DIRS 里的文件(node_modules/.git/dist…),
1003
+ // 否则这些文件虽然全量扫描时被跳过,单文件增量更新却会漏进来
1004
+ const dirs = rel.split("/").slice(0, -1)
1005
+ if (dirs.some((d) => SKIP_DIRS.has(d) || d.startsWith("."))) return
882
1006
 
883
1007
  let text
884
1008
  try { text = await readFile(absPath, "utf8") } catch {
885
1009
  // 文件已删:清理索引
886
- if (CODE_EXTS.has(ext)) memory.db.prepare(`DELETE FROM code_chunks WHERE path = ?`).run(rel)
887
- else if (DOC_EXTS.has(ext)) memory.db.prepare(`DELETE FROM doc_chunks WHERE path = ?`).run(rel)
1010
+ if (CODE_EXTS.has(ext)) memory.db.prepare(`DELETE FROM code_chunks WHERE origin = ? AND path = ?`).run(cwd, rel)
1011
+ else if (DOC_EXTS.has(ext)) memory.db.prepare(`DELETE FROM doc_chunks WHERE origin = ? AND path = ?`).run(cwd, rel)
888
1012
  return
889
1013
  }
890
1014
  const lines = text.split("\n")
@@ -893,11 +1017,11 @@ export async function reindexFile(memory, cwd, absPath) {
893
1017
  const lang = detectLanguage(absPath)
894
1018
  let mtimeMs = 0
895
1019
  try { mtimeMs = Math.floor((await stat(absPath)).mtimeMs) } catch { /* 新文件 */ }
896
- _upsertCodeFile(memory, rel, lines, lang, mtimeMs)
1020
+ _upsertCodeFile(memory, cwd, rel, lines, lang, mtimeMs)
897
1021
  } else if (DOC_EXTS.has(ext)) {
898
1022
  let mtimeMs = 0
899
1023
  try { mtimeMs = Math.floor((await stat(absPath)).mtimeMs) } catch { /* 新文件 */ }
900
- _upsertDocFile(memory, rel, lines, mtimeMs)
1024
+ _upsertDocFile(memory, cwd, rel, lines, mtimeMs)
901
1025
  }
902
1026
  }
903
1027
 
@@ -951,13 +1075,14 @@ export async function docSync(memory, dir, { onProgress } = {}) {
951
1075
  await walk(dir)
952
1076
 
953
1077
  const indexed = new Map(
954
- memory.db.prepare(`SELECT path, mtime_ms FROM doc_chunks`).all().map((r) => [r.path, r.mtime_ms])
1078
+ memory.db.prepare(`SELECT path, mtime_ms FROM doc_chunks WHERE origin = ?`).all(dir).map((r) => [r.path, r.mtime_ms])
955
1079
  )
956
1080
  const seen = new Set()
957
1081
 
958
1082
  onProgress?.({ phase: "scan", total: files.length })
959
1083
 
960
- let updated = 0, removed = 0, skipped = 0
1084
+ let updated = 0, removed = 0, skipped = 0, failed = 0
1085
+ const errors = []
961
1086
  for (let i = 0; i < files.length; i++) {
962
1087
  const abs = files[i]
963
1088
  const rel = abs.slice(dir.length + 1).replaceAll("\\", "/")
@@ -970,65 +1095,80 @@ export async function docSync(memory, dir, { onProgress } = {}) {
970
1095
  continue
971
1096
  }
972
1097
 
973
- let text
974
- try { text = await readFile(abs, "utf8") } catch { continue }
975
- const lines = text.split("\n")
976
- _upsertDocFile(memory, rel, lines, mtimeMs)
977
- updated++
1098
+ // 单文件失败不拖垮整轮同步(与 codeSync 同策略)
1099
+ try {
1100
+ const text = await readFile(abs, "utf8")
1101
+ const lines = text.split("\n")
1102
+ _upsertDocFile(memory, dir, rel, lines, mtimeMs)
1103
+ updated++
1104
+ } catch (e) {
1105
+ failed++
1106
+ if (errors.length < 5) errors.push(`${rel}: ${e.message}`)
1107
+ }
978
1108
  await yieldTick()
979
1109
 
980
1110
  if (onProgress && i % 10 === 0) {
981
- onProgress({ phase: "index", current: i + 1, total: files.length, updated, removed, skipped })
1111
+ onProgress({ phase: "index", current: i + 1, total: files.length, updated, removed, skipped, failed })
982
1112
  }
983
1113
  }
984
1114
 
985
- // 清理磁盘上消失的文件
1115
+ // 清理磁盘上消失的文件(仅本 origin)
986
1116
  for (const stale of indexed.keys()) {
987
1117
  if (!seen.has(stale)) {
988
- memory.db.prepare(`DELETE FROM doc_chunks WHERE path = ?`).run(stale)
1118
+ memory.db.prepare(`DELETE FROM doc_chunks WHERE origin = ? AND path = ?`).run(dir, stale)
989
1119
  removed++
990
1120
  }
991
1121
  }
992
1122
 
993
- onProgress?.({ phase: "done", total: files.length, updated, removed, skipped })
994
- return { updated, removed, skipped, total: files.length }
1123
+ onProgress?.({ phase: "done", total: files.length, updated, removed, skipped, failed })
1124
+ return { updated, removed, skipped, failed, errors, total: files.length }
995
1125
  }
996
1126
 
997
1127
  /**
998
1128
  * 文档检索:FTS5(BM25) + 可选向量余弦,RRF 合并。
1129
+ * 无 embedder 时退化为纯 FTS;ftsQuery 为空(纯标点查询)且有 embedder 时退化为纯向量。
1130
+ * 返回 [{ path, language, heading, content, line_start, line_end }]
999
1131
  */
1000
1132
  export async function docSearch(memory, query, { limit = 5 } = {}) {
1001
1133
  const ftsQuery = buildFtsQuery(query)
1002
- if (!ftsQuery) return []
1134
+ if (!ftsQuery && !memory.embedder) return []
1003
1135
 
1004
- const ftsList = memory.db.prepare(`
1005
- SELECT d.path, d.language, d.heading, d.content, d.line_start, d.line_end, bm25(doc_chunks_fts) AS rank
1136
+ // codeOrigin 设置时只检索本项目(与 codeSearch 同模式);未设置时不过滤
1137
+ const ftsOriginFilter = memory.codeOrigin ? `AND d.origin = ?` : ""
1138
+ const vecOriginFilter = memory.codeOrigin ? `AND origin = ?` : ""
1139
+ const originParams = memory.codeOrigin ? [memory.codeOrigin] : []
1140
+
1141
+ const ftsList = ftsQuery ? memory.db.prepare(`
1142
+ SELECT d.rowid, d.path, d.language, d.heading, d.content, d.line_start, d.line_end, bm25(doc_chunks_fts) AS rank
1006
1143
  FROM doc_chunks_fts JOIN doc_chunks d ON d.rowid = doc_chunks_fts.rowid
1007
- WHERE doc_chunks_fts MATCH ?
1144
+ WHERE doc_chunks_fts MATCH ? ${ftsOriginFilter}
1008
1145
  ORDER BY rank LIMIT ?
1009
- `).all(ftsQuery, Math.max(limit * 4, 20))
1146
+ `).all(ftsQuery, ...originParams, Math.max(limit * 4, 20)) : []
1010
1147
 
1011
1148
  if (!memory.embedder) return ftsList.slice(0, limit)
1012
1149
 
1150
+ // 向量通道
1151
+ await ensureDocEmbeddings(memory)
1013
1152
  const [qvec] = await embed(memory.embedder, [query])
1014
- const rows = memory.db.prepare(`SELECT path, line_start, embedding FROM doc_chunks WHERE embedding IS NOT NULL`).all()
1153
+ const rows = memory.db.prepare(`SELECT rowid, embedding FROM doc_chunks WHERE embedding IS NOT NULL ${vecOriginFilter}`).all(...originParams)
1015
1154
  const vecList = rows
1016
- .map((r) => ({ key: `${r.path}:${r.line_start}`, score: cosine(qvec, fromBlob(r.embedding)) }))
1155
+ .map((r) => ({ rowid: r.rowid, score: cosine(qvec, fromBlob(r.embedding)) }))
1017
1156
  .sort((a, b) => b.score - a.score)
1018
1157
  .slice(0, Math.max(limit * 4, 20))
1019
1158
 
1159
+ // RRF 合并(按 rowid 对齐两个通道;解析结果回表取,纯向量命中的块也能浮现)
1020
1160
  const K = 60
1021
1161
  const scores = new Map()
1022
- ftsList.forEach((r, i) => scores.set(`${r.path}:${r.line_start}`, (scores.get(`${r.path}:${r.line_start}`) ?? 0) + 1 / (K + i + 1)))
1023
- vecList.forEach((r, i) => scores.set(r.key, (scores.get(r.key) ?? 0) + 1 / (K + i + 1)))
1162
+ ftsList.forEach((r, i) => scores.set(r.rowid, (scores.get(r.rowid) ?? 0) + 1 / (K + i + 1)))
1163
+ vecList.forEach((r, i) => scores.set(r.rowid, (scores.get(r.rowid) ?? 0) + 1 / (K + i + 1)))
1024
1164
 
1165
+ const fetchChunk = memory.db.prepare(`
1166
+ SELECT path, language, heading, content, line_start, line_end FROM doc_chunks WHERE rowid = ?
1167
+ `)
1025
1168
  return [...scores.entries()]
1026
1169
  .sort((a, b) => b[1] - a[1])
1027
1170
  .slice(0, limit)
1028
- .map(([key]) => {
1029
- const [path, lineStr] = key.split(/:(?=\d+$)/)
1030
- return ftsList.find((r) => r.path === path && String(r.line_start) === lineStr)
1031
- })
1171
+ .map(([rowid]) => fetchChunk.get(rowid))
1032
1172
  .filter(Boolean)
1033
1173
  }
1034
1174