yongle-dadian 1.3.3 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -27,6 +27,7 @@
27
27
  - [4. 快速上手指南](#4-快速上手指南)
28
28
  - [5. 详细命令手册](#5-详细命令手册)
29
29
  - [6. 与交付框架 GSD (Get Shit Done) 协同工作](#6-与交付框架-gsd-get-shit-done-协同工作)
30
+ - [7. DeepSeek Harness 原生集成(插件)](#7-deepseek-harness-原生集成插件)
30
31
  - [🇬🇧 Yongle Dadian — AI Agent Long-term Memory & Hybrid Search Engine](#-yongle-dadian--ai-agent-long-term-memory--hybrid-search-engine-1)
31
32
  - [🗺️ Table of Contents](#️-table-of-contents)
32
33
  - [1. Core Value & Scenarios](#1-core-value--scenarios)
@@ -341,6 +342,29 @@ npx yongle-dadian --global --antigravity
341
342
 
342
343
  ---
343
344
 
345
+ ### 7. DeepSeek Harness 原生集成(插件)
346
+
347
+ 永乐大典以 **原生插件** 形态接入 [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness):
348
+ 模型可直接调用知识库检索、记录与复盘蒸馏,人类侧有 `/yongle-*` 命令。
349
+
350
+ ```sh
351
+ # 从 tarball 安装(或发布后: dsh plugin --profile web add yongle-dadian-dsh)
352
+ cd dsh && pnpm pack
353
+ dsh plugin --profile web add ./yongle-dadian-dsh-*.tgz
354
+ dsh --profile web --dump-config # 确认出现 "# == yongle-dadian-dsh" 层
355
+ dsh --profile web
356
+ ```
357
+
358
+ **能力**:`yongle_search` / `yongle_recall` / `yongle_remember` / `yongle_status` 工具;
359
+ `/yongle-search` `/yongle-record` `/yongle-postmortem` `/yongle-confirm` `/yongle-repair`
360
+ `/yongle-reindex` `/yongle-status` 命令。三级知识库(项目级/用户级/全局级),检索与
361
+ 记录均可区分级别;复盘蒸馏直接从 DSH 会话日志提取素材(固定系统模板,DeepSeek
362
+ 前缀缓存友好)。
363
+
364
+ 详见 [`dsh/README.md`](dsh/README.md) 与 [`docs/永乐大典-DeepSeek-Harness-集成方案.md`](docs/永乐大典-DeepSeek-Harness-集成方案.md)。
365
+
366
+ ---
367
+
344
368
  ## 🇬🇧 Yongle Dadian — AI Agent Long-term Memory & Hybrid Search Engine
345
369
 
346
370
  > **"Knowledge is power, but only knowledge that can be quickly retrieved is productivity."**
package/YONGLE.md CHANGED
@@ -119,10 +119,16 @@
119
119
  | `default_limit` | `5` | `1` ~ `20` | `/yongle-search` 默认返回的结果条数。 |
120
120
  | `semantic_threshold` | `0.7` | `0.0` ~ `1.0` | (预留) 语义搜索的相关度阈值。目前主要用于过滤极低相关度的模糊匹配。 |
121
121
 
122
- #### `yongle.storage` (存储配置)
122
+ #### `yongle.storage` (存储配置 — 三级知识库)
123
+
123
124
  | 字段 | 默认值 | 可选值 | 说明 |
124
125
  | :--- | :--- | :--- | :--- |
125
- | `global_path` | `null` | 绝对路径 | 自定义全局知识库的存储位置。默认为家目录下的 `.yongle_knowledge/`。 |
126
+ | `user_path` | `~/.yongle_knowledge/` | 绝对路径 | **用户级**知识库位置(个人跨项目经验)。也可用环境变量 `YONGLE_HOME` 覆盖。 |
127
+ | `global_path` | `null` | 绝对路径 | **全局级(团队共享)**知识库位置。未配置时 `global` 级别禁用(检索自动跳过,记录报错)。也可用环境变量 `YONGLE_GLOBAL_HOME` 覆盖。 |
128
+
129
+ **三级体系**:`project`(项目级,`<workspace>/.planning/yongle/`)→ `user`(用户级)→ `global`(团队级)。
130
+ - 检索:`auto` 级别按 project → user → global 合并检索(自动跳过未配置级别)。
131
+ - 记录:必须显式指定级别(如 `--level project|user|global`)。
126
132
 
127
133
  #### `embedding` (向量提取配置)
128
134
  | 字段 | 默认值 | 可选值 | 说明 |
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "yongle-dadian",
3
- "version": "1.3.3",
3
+ "version": "1.6.0",
4
4
  "description": "永乐大典 — AI 开发经验知识库引擎。自动复盘、过程追踪、毫秒级检索。独立于 GSD 运行。",
5
5
  "repository": {
6
6
  "type": "git",
@@ -0,0 +1,61 @@
1
+ /**
2
+ * cache.js — 永乐大典核心引擎:内存缓存工具
3
+ * - LruCache:通用 LRU(容量 + TTL)
4
+ * - 查询向量缓存:内存 LRU(首查后免 API 调用)
5
+ * - 检索结果缓存:同 scope+query 重复查询秒回(同时保证字节级稳定输出,
6
+ * 使 DSH 会话中重复检索的工具结果完全一致 → DeepSeek 前缀缓存友好)
7
+ */
8
+ 'use strict';
9
+
10
+ class LruCache {
11
+ /**
12
+ * @param {number} [capacity]
13
+ * @param {number} [ttlMs] 0 = 不过期
14
+ */
15
+ constructor(capacity = 512, ttlMs = 0) {
16
+ this.capacity = capacity;
17
+ this.ttlMs = ttlMs;
18
+ /** @type {Map<string, {v:any, at:number}>} */
19
+ this.map = new Map();
20
+ }
21
+
22
+ get(key) {
23
+ const hit = this.map.get(key);
24
+ if (!hit) return undefined;
25
+ if (this.ttlMs > 0 && Date.now() - hit.at > this.ttlMs) {
26
+ this.map.delete(key);
27
+ return undefined;
28
+ }
29
+ // LRU 触碰:重插到末尾
30
+ this.map.delete(key);
31
+ this.map.set(key, hit);
32
+ return hit.v;
33
+ }
34
+
35
+ set(key, value) {
36
+ if (this.map.has(key)) this.map.delete(key);
37
+ this.map.set(key, { v: value, at: Date.now() });
38
+ while (this.map.size > this.capacity) {
39
+ const oldest = this.map.keys().next().value;
40
+ this.map.delete(oldest);
41
+ }
42
+ }
43
+
44
+ has(key) {
45
+ return this.get(key) !== undefined;
46
+ }
47
+
48
+ delete(key) {
49
+ this.map.delete(key);
50
+ }
51
+
52
+ clear() {
53
+ this.map.clear();
54
+ }
55
+
56
+ get size() {
57
+ return this.map.size;
58
+ }
59
+ }
60
+
61
+ module.exports = { LruCache };
@@ -0,0 +1,356 @@
1
+ /**
2
+ * db.js — 永乐大典核心引擎:进程内 SQLite 存储
3
+ * 基于 node:sqlite(DatabaseSync),替代逐次 spawn sqlite3 子进程的方案:
4
+ * - WAL 模式 + synchronous=NORMAL:读性能与写吞吐兼顾
5
+ * - FTS5 trigram 虚拟表:中文短语/英文子串毫秒级全文检索(≥3 字符)
6
+ * - <3 字符查询(如 2 字中文)自动回退进程内 LIKE 扫描
7
+ * - 增量索引:scanMemoryDir() 按 mtime 扫描 memory 目录,只 upsert 变更
8
+ * - 兼容旧库:打开旧 entries 表自动补建 FTS 并回填
9
+ */
10
+ 'use strict';
11
+
12
+ const fs = require('fs');
13
+ const path = require('path');
14
+ const crypto = require('crypto');
15
+
16
+ /** @type {Map<string, import('node:sqlite').DatabaseSync>} 按 db 路径缓存连接 */
17
+ const _dbs = new Map();
18
+ /** 当前活动连接(open 时切换;所有查询作用于它) */
19
+ let _db = null;
20
+ let _dbPath = null;
21
+
22
+ /**
23
+ * 打开(或复用)指定数据库并设为活动连接,自动迁移 schema
24
+ * 多级别知识库各自独立 db 文件,连接全部缓存,open 切换当前活动连接
25
+ */
26
+ function open(dbPath) {
27
+ if (_dbs.has(dbPath)) {
28
+ _db = _dbs.get(dbPath);
29
+ _dbPath = dbPath;
30
+ return _db;
31
+ }
32
+ const { DatabaseSync } = require('node:sqlite');
33
+ const dir = path.dirname(dbPath);
34
+ if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
35
+ const db = new DatabaseSync(dbPath);
36
+ // 使用回滚日志而非 WAL:单进程同步访问无需 WAL 的并发收益,
37
+ // 且规避混合 SQLite 构建(如旧 32 位 sqlite3 CLI)遗留 WAL 导致的损坏
38
+ db.exec('PRAGMA journal_mode = DELETE;');
39
+ db.exec('PRAGMA synchronous = NORMAL;');
40
+ db.exec('PRAGMA busy_timeout = 5000;');
41
+ migrate(db);
42
+ _dbs.set(dbPath, db);
43
+ _db = db;
44
+ _dbPath = dbPath;
45
+ return db;
46
+ }
47
+
48
+ /** 获取当前打开的数据库(未打开则抛错) */
49
+ function get() {
50
+ if (!_db) throw new Error('yongle core db not opened. Call open(dbPath) first.');
51
+ return _db;
52
+ }
53
+
54
+ function migrate(db) {
55
+ db.exec(`CREATE TABLE IF NOT EXISTS entries (
56
+ id TEXT PRIMARY KEY,
57
+ date TEXT,
58
+ resolution_type TEXT,
59
+ cause_summary TEXT,
60
+ tags TEXT,
61
+ filepath TEXT,
62
+ source TEXT,
63
+ body_index TEXT DEFAULT '',
64
+ last_indexed DATETIME DEFAULT CURRENT_TIMESTAMP
65
+ );`);
66
+ // schema v2:旧库补 body_index 列(正文索引,提升蒸馏知识的召回)
67
+ const cols = db.prepare("PRAGMA table_info(entries)").all().map((c) => c.name);
68
+ if (!cols.includes('body_index')) {
69
+ db.exec("ALTER TABLE entries ADD COLUMN body_index TEXT DEFAULT ''");
70
+ }
71
+ db.exec('CREATE TABLE IF NOT EXISTS meta (k TEXT PRIMARY KEY, v TEXT)');
72
+ db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS entries_fts USING fts5(
73
+ id, cause_summary, tags, body_index,
74
+ content='entries', content_rowid='rowid',
75
+ tokenize='trigram'
76
+ );`);
77
+ db.exec(`CREATE TRIGGER IF NOT EXISTS entries_ai AFTER INSERT ON entries BEGIN
78
+ INSERT INTO entries_fts(rowid, id, cause_summary, tags, body_index)
79
+ VALUES (new.rowid, new.id, new.cause_summary, new.tags, new.body_index); END;`);
80
+ db.exec(`CREATE TRIGGER IF NOT EXISTS entries_ad AFTER DELETE ON entries BEGIN
81
+ INSERT INTO entries_fts(entries_fts, rowid, id, cause_summary, tags, body_index)
82
+ VALUES ('delete', old.rowid, old.id, old.cause_summary, old.tags, old.body_index); END;`);
83
+ db.exec(`CREATE TRIGGER IF NOT EXISTS entries_au AFTER UPDATE ON entries BEGIN
84
+ INSERT INTO entries_fts(entries_fts, rowid, id, cause_summary, tags, body_index)
85
+ VALUES ('delete', old.rowid, old.id, old.cause_summary, old.tags, old.body_index);
86
+ INSERT INTO entries_fts(rowid, id, cause_summary, tags, body_index)
87
+ VALUES (new.rowid, new.id, new.cause_summary, new.tags, new.body_index); END;`);
88
+ // schema 版本管理:v2 = 4 列 FTS(含 body_index);旧 3 列索引需重建
89
+ const ver = db.prepare("SELECT v FROM meta WHERE k = 'schema_version'").get();
90
+ const version = ver ? parseInt(ver.v, 10) : 1;
91
+ if (version < 2) {
92
+ db.exec('DROP TRIGGER IF EXISTS entries_ai');
93
+ db.exec('DROP TRIGGER IF EXISTS entries_ad');
94
+ db.exec('DROP TRIGGER IF EXISTS entries_au');
95
+ db.exec('DROP TABLE IF EXISTS entries_fts');
96
+ db.exec(`CREATE VIRTUAL TABLE entries_fts USING fts5(
97
+ id, cause_summary, tags, body_index,
98
+ content='entries', content_rowid='rowid',
99
+ tokenize='trigram'
100
+ );`);
101
+ db.exec(`CREATE TRIGGER entries_ai AFTER INSERT ON entries BEGIN
102
+ INSERT INTO entries_fts(rowid, id, cause_summary, tags, body_index)
103
+ VALUES (new.rowid, new.id, new.cause_summary, new.tags, new.body_index); END;`);
104
+ db.exec(`CREATE TRIGGER entries_ad AFTER DELETE ON entries BEGIN
105
+ INSERT INTO entries_fts(entries_fts, rowid, id, cause_summary, tags, body_index)
106
+ VALUES ('delete', old.rowid, old.id, old.cause_summary, old.tags, old.body_index); END;`);
107
+ db.exec(`CREATE TRIGGER entries_au AFTER UPDATE ON entries BEGIN
108
+ INSERT INTO entries_fts(entries_fts, rowid, id, cause_summary, tags, body_index)
109
+ VALUES ('delete', old.rowid, old.id, old.cause_summary, old.tags, old.body_index);
110
+ INSERT INTO entries_fts(rowid, id, cause_summary, tags, body_index)
111
+ VALUES (new.rowid, new.id, new.cause_summary, new.tags, new.body_index); END;`);
112
+ db.prepare("INSERT INTO meta (k, v) VALUES ('schema_version', '2') ON CONFLICT(k) DO UPDATE SET v = excluded.v").run();
113
+ }
114
+ // 回填:索引行数少于 entries 时重建(外部内容表必须用 'delete-all',不能用 DELETE)
115
+ const c1 = db.prepare('SELECT COUNT(*) c FROM entries').get().c;
116
+ let c2 = 0;
117
+ try { c2 = db.prepare('SELECT COUNT(*) c FROM entries_fts_data').get().c; } catch { c2 = 0; }
118
+ if (c1 > 0 && c2 < c1) {
119
+ db.exec("INSERT INTO entries_fts(entries_fts) VALUES('delete-all')");
120
+ db.prepare('INSERT INTO entries_fts(rowid, id, cause_summary, tags, body_index) SELECT rowid, id, cause_summary, tags, body_index FROM entries').run();
121
+ }
122
+ }
123
+
124
+ /**
125
+ * 修复 FTS 索引:删除并重建 entries_fts 与触发器,再从 entries 回填
126
+ * (用于索引被破坏的场景,如旧版本误用 DELETE 导致的校验失败)
127
+ */
128
+ function repairFts() {
129
+ const db = get();
130
+ db.exec('DROP TRIGGER IF EXISTS entries_ai');
131
+ db.exec('DROP TRIGGER IF EXISTS entries_ad');
132
+ db.exec('DROP TRIGGER IF EXISTS entries_au');
133
+ db.exec('DROP TABLE IF EXISTS entries_fts');
134
+ migrate(db);
135
+ return { ok: true };
136
+ }
137
+
138
+ /** 解析 Markdown frontmatter(返回 { meta, body }) */
139
+ function parseFrontmatter(content) {
140
+ // 注意:正则必须有两个捕获组 —— 组1 = frontmatter,组2 = 正文
141
+ // 旧实现只有 1 个组却读 m[2],导致带 frontmatter 的文件正文永远为空
142
+ const m = content.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n?([\s\S]*)$/);
143
+ if (!m) return { meta: {}, body: content };
144
+ const meta = {};
145
+ let key = null;
146
+ for (const line of m[1].split(/\r?\n/)) {
147
+ const kv = line.match(/^([A-Za-z_][\w]*):\s*(.*)$/);
148
+ if (kv) {
149
+ key = kv[1];
150
+ const raw = kv[2].trim().replace(/^["']|["']$/g, '');
151
+ if (raw.startsWith('[') && raw.endsWith(']')) {
152
+ meta[key] = raw.slice(1, -1).split(',').map((s) => s.trim().replace(/^["']|["']$/g, '')).filter(Boolean);
153
+ } else {
154
+ meta[key] = raw;
155
+ }
156
+ } else if (key && /^\s+-/.test(line)) {
157
+ const item = line.replace(/^\s+-\s*/, '').trim().replace(/^["']|["']$/g, '');
158
+ if (!Array.isArray(meta[key])) meta[key] = [];
159
+ meta[key].push(item);
160
+ }
161
+ }
162
+ return { meta, body: m[2] || '' };
163
+ }
164
+
165
+ /**
166
+ * 从 Markdown 文件提取条目字段
167
+ * @param {string} filepath
168
+ * @param {string} [scope]
169
+ * @returns {{id:string, date:string, resolution_type:string, cause_summary:string, tags:string, filepath:string, mtimeMs:number, hash:string}|null}
170
+ */
171
+ function extractEntry(filepath, scope) {
172
+ let content;
173
+ try {
174
+ content = fs.readFileSync(filepath, 'utf8');
175
+ } catch {
176
+ return null;
177
+ }
178
+ const { meta, body } = parseFrontmatter(content);
179
+ if (!meta.id && !body.trim()) return null;
180
+ const id = String(meta.id || path.basename(filepath, '.md'));
181
+ const tags = Array.isArray(meta.tags) ? meta.tags : (meta.tags ? [meta.tags] : []);
182
+ const cause = String(meta.cause_summary || body.replace(/\s+/g, ' ').trim().slice(0, 200));
183
+ const st = fs.statSync(filepath);
184
+ return {
185
+ id,
186
+ date: String(meta.date || ''),
187
+ resolution_type: String(meta.resolution_type || 'other'),
188
+ cause_summary: cause,
189
+ tags: tags.join(','),
190
+ filepath: path.resolve(filepath),
191
+ body_index: body.replace(/\s+/g, ' ').trim().slice(0, 3000), // 正文入索引,提升召回
192
+ mtimeMs: st.mtimeMs,
193
+ hash: crypto.createHash('sha1').update(id + '|' + st.mtimeMs + '|' + st.size).digest('hex').slice(0, 16),
194
+ };
195
+ }
196
+
197
+ /**
198
+ * upsert 一条条目(同时经触发器维护 FTS)
199
+ * @param {{id:string, date?:string, resolution_type?:string, cause_summary?:string, tags?:string, filepath?:string, source?:string, body_index?:string}} e
200
+ */
201
+ function upsertEntry(e) {
202
+ const db = get();
203
+ db.prepare(`INSERT INTO entries (id, date, resolution_type, cause_summary, tags, filepath, source, body_index)
204
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?)
205
+ ON CONFLICT(id) DO UPDATE SET
206
+ date=excluded.date, resolution_type=excluded.resolution_type,
207
+ cause_summary=excluded.cause_summary, tags=excluded.tags,
208
+ filepath=excluded.filepath, source=excluded.source,
209
+ body_index=excluded.body_index,
210
+ last_indexed=CURRENT_TIMESTAMP`)
211
+ .run(e.id, e.date || '', e.resolution_type || 'other', e.cause_summary || '', e.tags || '', e.filepath || '', e.source || '', e.body_index || '');
212
+ }
213
+
214
+ /** 按 id 删除条目(触发器同步删 FTS) */
215
+ function removeEntry(id) {
216
+ get().prepare('DELETE FROM entries WHERE id = ?').run(id);
217
+ }
218
+
219
+ /** 判断某条目是否存在 */
220
+ function hasEntry(id) {
221
+ return get().prepare('SELECT 1 c FROM entries WHERE id = ?').get(id) !== undefined;
222
+ }
223
+
224
+ /**
225
+ * 关键词检索:FTS5 trigram(≥3 字符)→ LIKE 兜底(<3 字符或 FTS 失败)
226
+ * @param {string} query
227
+ * @param {number} [limit]
228
+ * @returns {Array<{id:string, cause_summary:string, tags:string, filepath:string, resolution_type:string, source:string, snippet:string}>}
229
+ */
230
+ function keywordSearch(query, limit = 10) {
231
+ const q = String(query || '').trim();
232
+ if (!q) return [];
233
+ const db = get();
234
+ let rows = [];
235
+ if (q.length >= 3) {
236
+ const match = buildMatchQuery(q);
237
+ try {
238
+ rows = db.prepare(
239
+ `SELECT e.rowid, e.id, e.cause_summary, e.tags, e.filepath, e.resolution_type, e.source,
240
+ snippet(entries_fts, 1, '', '', '...', 24) AS snippet
241
+ FROM entries_fts JOIN entries e ON e.rowid = entries_fts.rowid
242
+ WHERE entries_fts MATCH ?
243
+ ORDER BY bm25(entries_fts) LIMIT ?`
244
+ ).all(match, limit);
245
+ } catch {
246
+ rows = [];
247
+ }
248
+ }
249
+ if (rows.length === 0) {
250
+ // LIKE 兜底:短查询(<3 字符,如 2 字中文)或 FTS 语法失败
251
+ // 多词查询逐词 OR,保证 "乱码 提交 端口" 这类输入能命中任一关键词
252
+ const words = q.split(/\s+/).filter(Boolean);
253
+ const conds = [];
254
+ const params = [];
255
+ for (const w of words) {
256
+ const like = `%${w}%`;
257
+ conds.push('(id LIKE ? OR cause_summary LIKE ? OR tags LIKE ? OR body_index LIKE ?)');
258
+ params.push(like, like, like, like);
259
+ }
260
+ if (conds.length > 0) {
261
+ params.push(limit);
262
+ rows = db.prepare(
263
+ `SELECT id, cause_summary, tags, filepath, resolution_type, source
264
+ FROM entries WHERE ${conds.join(' OR ')}
265
+ LIMIT ?`
266
+ ).all(...params);
267
+ }
268
+ }
269
+ return rows.map((r) => ({
270
+ id: r.id,
271
+ cause_summary: r.cause_summary,
272
+ tags: r.tags,
273
+ filepath: r.filepath,
274
+ resolution_type: r.resolution_type,
275
+ source: r.source,
276
+ snippet: r.snippet || r.cause_summary,
277
+ }));
278
+ }
279
+
280
+ /**
281
+ * 构造 FTS5 MATCH 表达式:
282
+ * - 空格分隔的多词 → OR 组合的短语(命中任一词)
283
+ * - 单个短语 → 整体短语匹配(中文子串/英文子串)
284
+ */
285
+ function buildMatchQuery(q) {
286
+ const words = q.split(/\s+/).filter(Boolean);
287
+ const quote = (w) => '"' + w.replace(/"/g, '""') + '"';
288
+ if (words.length > 1) return words.map(quote).join(' OR ');
289
+ return quote(words[0]);
290
+ }
291
+
292
+ /**
293
+ * 增量扫描知识目录:按 mtime 只 upsert 变更/新增,删除已消失条目
294
+ * 维护 meta 表 scan_state:scope|dirs → 逐条目 hash 快照
295
+ * @param {string[]} dirs 待扫描目录(global 通常 [root, root/memory];local 通常 [root/memory])
296
+ * @param {string} [source] global|local
297
+ * @returns {{upserted:number, removed:number, scanned:number}}
298
+ */
299
+ function scanMemoryDir(dirs, source = 'global') {
300
+ const db = get();
301
+ db.exec('CREATE TABLE IF NOT EXISTS scan_state (scope TEXT PRIMARY KEY, state TEXT, updated_at DATETIME DEFAULT CURRENT_TIMESTAMP)');
302
+ const dirList = (Array.isArray(dirs) ? dirs : [dirs]).filter(Boolean);
303
+ const scopeKey = source + '|' + dirList.join('|');
304
+ const prev = db.prepare('SELECT state FROM scan_state WHERE scope = ?').get(scopeKey);
305
+ const prevMap = prev ? JSON.parse(prev.state) : {};
306
+ const nowMap = {};
307
+ let scanned = 0;
308
+ let upserted = 0;
309
+ let removed = 0;
310
+ const seen = new Set();
311
+ for (const dir of dirList) {
312
+ if (!fs.existsSync(dir)) continue;
313
+ for (const f of fs.readdirSync(dir)) {
314
+ if (!f.endsWith('.md')) continue;
315
+ if (f === 'INDEX.md') continue; // 索引文件不是条目
316
+ const fp = path.join(dir, f);
317
+ const entry = extractEntry(fp, source);
318
+ if (!entry) continue;
319
+ scanned++;
320
+ seen.add(entry.id);
321
+ nowMap[entry.id] = entry.hash;
322
+ if (prevMap[entry.id] !== entry.hash) {
323
+ upsertEntry({ ...entry, source });
324
+ upserted++;
325
+ }
326
+ }
327
+ }
328
+ for (const id of Object.keys(prevMap)) {
329
+ if (!seen.has(id)) {
330
+ removeEntry(id);
331
+ removed++;
332
+ }
333
+ }
334
+ db.prepare('INSERT INTO scan_state (scope, state) VALUES (?, ?) ON CONFLICT(scope) DO UPDATE SET state=excluded.state, updated_at=CURRENT_TIMESTAMP')
335
+ .run(scopeKey, JSON.stringify(nowMap));
336
+ return { upserted, removed, scanned };
337
+ }
338
+
339
+ /** 统计 */
340
+ function stats() {
341
+ const db = get();
342
+ const entries = db.prepare('SELECT COUNT(*) c FROM entries').get().c;
343
+ const fts = db.prepare('SELECT COUNT(*) c FROM entries_fts').get().c;
344
+ return { entries, fts };
345
+ }
346
+
347
+ /** 关闭所有数据库连接 */
348
+ function close() {
349
+ for (const db of _dbs.values()) {
350
+ try { db.close(); } catch { /* ignore */ }
351
+ }
352
+ _dbs.clear();
353
+ _db = null;
354
+ }
355
+
356
+ module.exports = { open, get, migrate, repairFts, parseFrontmatter, extractEntry, upsertEntry, removeEntry, hasEntry, keywordSearch, scanMemoryDir, stats, close };
@@ -0,0 +1,101 @@
1
+ /**
2
+ * embed.js — 永乐大典核心引擎:Embedding 获取与磁盘缓存
3
+ * 缓存顺序:磁盘 JSON 缓存(复用旧 query_cache.json,内存常驻)→ API
4
+ * 磁盘写入防抖(300ms),避免每次查询整文件重写;容量上限淘汰最旧项。
5
+ */
6
+ 'use strict';
7
+
8
+ const fs = require('fs');
9
+ const path = require('path');
10
+ const { yongleHome } = require('./paths');
11
+
12
+ /** 磁盘缓存状态 */
13
+ let _fileCache = null;
14
+ let _fileCachePath = null;
15
+ let _writeTimer = null;
16
+ let _dirty = false;
17
+ const FILE_CACHE_MAX = 2000;
18
+
19
+ /** 懒加载磁盘缓存(兼容旧 query_cache.json 格式:{"provider:model:keyword": vector}) */
20
+ function fileCache() {
21
+ if (_fileCache) return _fileCache;
22
+ _fileCachePath = path.join(yongleHome(), 'query_cache.json');
23
+ try {
24
+ if (fs.existsSync(_fileCachePath)) {
25
+ _fileCache = JSON.parse(fs.readFileSync(_fileCachePath, 'utf8'));
26
+ }
27
+ } catch {
28
+ _fileCache = {};
29
+ }
30
+ if (!_fileCache || typeof _fileCache !== 'object') _fileCache = {};
31
+ return _fileCache;
32
+ }
33
+
34
+ function scheduleWrite() {
35
+ _dirty = true;
36
+ if (_writeTimer) return;
37
+ _writeTimer = setTimeout(() => {
38
+ _writeTimer = null;
39
+ if (!_dirty) return;
40
+ _dirty = false;
41
+ try {
42
+ fs.mkdirSync(path.dirname(_fileCachePath), { recursive: true });
43
+ fs.writeFileSync(_fileCachePath, JSON.stringify(_fileCache));
44
+ } catch {
45
+ /* 写入失败不影响检索 */
46
+ }
47
+ }, 300);
48
+ }
49
+
50
+ /**
51
+ * 获取(带缓存)文本向量
52
+ * @param {string} text
53
+ * @param {{provider:string, model:string, apiKey?:string, baseUrl?:string}} embedConfig
54
+ * @param {{force?:boolean}} [opts]
55
+ * @returns {Promise<number[]|null>} null = embedding 被禁用或获取失败
56
+ */
57
+ async function getEmbeddingCached(text, embedConfig, opts = {}) {
58
+ const provider = embedConfig.provider || 'ollama';
59
+ const model = embedConfig.model || 'nomic-embed-text';
60
+ const key = `${provider}:${model}:${text}`;
61
+
62
+ if (!opts.force) {
63
+ const fc = fileCache();
64
+ if (Object.prototype.hasOwnProperty.call(fc, key)) {
65
+ const v = fc[key];
66
+ return v;
67
+ }
68
+ }
69
+
70
+ if (opts.signal?.aborted) return null;
71
+
72
+ const { getEmbedding } = require('../yongle-embed.js');
73
+ const vector = await getEmbedding(text, provider, model, embedConfig.apiKey, embedConfig.baseUrl, { signal: opts.signal });
74
+ if (vector) {
75
+ const fc = fileCache();
76
+ fc[key] = vector;
77
+ const keys = Object.keys(fc);
78
+ while (keys.length > FILE_CACHE_MAX) {
79
+ delete fc[keys.shift()];
80
+ }
81
+ scheduleWrite();
82
+ }
83
+ return vector;
84
+ }
85
+
86
+ /** 立即落盘(进程退出前调用) */
87
+ function flush() {
88
+ if (_writeTimer) {
89
+ clearTimeout(_writeTimer);
90
+ _writeTimer = null;
91
+ }
92
+ if (_dirty && _fileCache) {
93
+ try {
94
+ fs.mkdirSync(path.dirname(_fileCachePath), { recursive: true });
95
+ fs.writeFileSync(_fileCachePath, JSON.stringify(_fileCache));
96
+ } catch { /* ignore */ }
97
+ _dirty = false;
98
+ }
99
+ }
100
+
101
+ module.exports = { getEmbeddingCached, flush };