scream-code 0.16.6 → 0.16.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -3
- package/dist/{app-D9oJ-Y8W.mjs → app-GeQQHNLx.mjs} +335 -88
- package/dist/{dispatch-fjwbqaYI.mjs → dispatch-CNitkOqX.mjs} +1934 -2022
- package/dist/{dispatch-dDbUBD7J.mjs → dispatch-Du8ddILK.mjs} +1 -1
- package/dist/main.mjs +1 -1
- package/dist/public/assets/index-DMGV8LA-.js +240 -0
- package/dist/public/assets/index-DMGV8LA-.js.map +1 -0
- package/dist/public/assets/index-DdPmkwIL.css +1 -0
- package/dist/public/index.html +2 -2
- package/dist/src-CXmZg1VT.mjs +4115 -0
- package/dist/{src-h6ZExh4A.mjs → src-DIAKpkXO.mjs} +1 -1
- package/dist/{text-input-dialog-DBlx8pMR.mjs → text-input-dialog-6v2667rm.mjs} +140 -6
- package/dist/{text-input-dialog-Cj0egbsU.mjs → text-input-dialog-DDUarM0_.mjs} +1 -1
- package/package.json +3 -2
- package/dist/public/assets/index-DQBhBKK1.css +0 -1
- package/dist/public/assets/index-DakPeNBt.js +0 -240
- package/dist/public/assets/index-DakPeNBt.js.map +0 -1
- package/dist/src-tDEINaMV.mjs +0 -2249
|
@@ -0,0 +1,4115 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { fileURLToPath as __cjsShimFileURLToPath } from 'node:url';
|
|
3
|
+
import { dirname as __cjsShimDirname } from 'node:path';
|
|
4
|
+
const __filename = __cjsShimFileURLToPath(import.meta.url);
|
|
5
|
+
const __dirname = __cjsShimDirname(__filename);
|
|
6
|
+
import { createHash } from "node:crypto";
|
|
7
|
+
import { mkdir, readFile, readdir, rename, rm, rmdir, stat, unlink, writeFile } from "node:fs/promises";
|
|
8
|
+
import { createReadStream, existsSync, mkdirSync, rmSync } from "node:fs";
|
|
9
|
+
import { join } from "node:path";
|
|
10
|
+
import { DatabaseSync } from "node:sqlite";
|
|
11
|
+
//#region ../../node_modules/.pnpm/pathe@2.0.3/node_modules/pathe/dist/shared/pathe.M-eThtNZ.mjs
|
|
12
|
+
const _DRIVE_LETTER_START_RE = /^[A-Za-z]:\//;
|
|
13
|
+
function normalizeWindowsPath(input = "") {
|
|
14
|
+
if (!input) return input;
|
|
15
|
+
return input.replace(/\\/g, "/").replace(_DRIVE_LETTER_START_RE, (r) => r.toUpperCase());
|
|
16
|
+
}
|
|
17
|
+
const _UNC_REGEX = /^[/\\]{2}/;
|
|
18
|
+
const _IS_ABSOLUTE_RE = /^[/\\](?![/\\])|^[/\\]{2}(?!\.)|^[A-Za-z]:[/\\]/;
|
|
19
|
+
const _DRIVE_LETTER_RE = /^[A-Za-z]:$/;
|
|
20
|
+
const _ROOT_FOLDER_RE = /^\/([A-Za-z]:)?$/;
|
|
21
|
+
const _EXTNAME_RE = /.(\.[^./]+|\.)$/;
|
|
22
|
+
const _PATH_ROOT_RE = /^[/\\]|^[a-zA-Z]:[/\\]/;
|
|
23
|
+
const normalize$1 = function(path) {
|
|
24
|
+
if (path.length === 0) return ".";
|
|
25
|
+
path = normalizeWindowsPath(path);
|
|
26
|
+
const isUNCPath = path.match(_UNC_REGEX);
|
|
27
|
+
const isPathAbsolute = isAbsolute$1(path);
|
|
28
|
+
const trailingSeparator = path[path.length - 1] === "/";
|
|
29
|
+
path = normalizeString(path, !isPathAbsolute);
|
|
30
|
+
if (path.length === 0) {
|
|
31
|
+
if (isPathAbsolute) return "/";
|
|
32
|
+
return trailingSeparator ? "./" : ".";
|
|
33
|
+
}
|
|
34
|
+
if (trailingSeparator) path += "/";
|
|
35
|
+
if (_DRIVE_LETTER_RE.test(path)) path += "/";
|
|
36
|
+
if (isUNCPath) {
|
|
37
|
+
if (!isPathAbsolute) return `//./${path}`;
|
|
38
|
+
return `//${path}`;
|
|
39
|
+
}
|
|
40
|
+
return isPathAbsolute && !isAbsolute$1(path) ? `/${path}` : path;
|
|
41
|
+
};
|
|
42
|
+
const join$1 = function(...segments) {
|
|
43
|
+
let path = "";
|
|
44
|
+
for (const seg of segments) {
|
|
45
|
+
if (!seg) continue;
|
|
46
|
+
if (path.length > 0) {
|
|
47
|
+
const pathTrailing = path[path.length - 1] === "/";
|
|
48
|
+
const segLeading = seg[0] === "/";
|
|
49
|
+
if (pathTrailing && segLeading) path += seg.slice(1);
|
|
50
|
+
else path += pathTrailing || segLeading ? seg : `/${seg}`;
|
|
51
|
+
} else path += seg;
|
|
52
|
+
}
|
|
53
|
+
return normalize$1(path);
|
|
54
|
+
};
|
|
55
|
+
function cwd() {
|
|
56
|
+
if (typeof process !== "undefined" && typeof process.cwd === "function") return process.cwd().replace(/\\/g, "/");
|
|
57
|
+
return "/";
|
|
58
|
+
}
|
|
59
|
+
const resolve$1 = function(...arguments_) {
|
|
60
|
+
arguments_ = arguments_.map((argument) => normalizeWindowsPath(argument));
|
|
61
|
+
let resolvedPath = "";
|
|
62
|
+
let resolvedAbsolute = false;
|
|
63
|
+
for (let index = arguments_.length - 1; index >= -1 && !resolvedAbsolute; index--) {
|
|
64
|
+
const path = index >= 0 ? arguments_[index] : cwd();
|
|
65
|
+
if (!path || path.length === 0) continue;
|
|
66
|
+
resolvedPath = `${path}/${resolvedPath}`;
|
|
67
|
+
resolvedAbsolute = isAbsolute$1(path);
|
|
68
|
+
}
|
|
69
|
+
resolvedPath = normalizeString(resolvedPath, !resolvedAbsolute);
|
|
70
|
+
if (resolvedAbsolute && !isAbsolute$1(resolvedPath)) return `/${resolvedPath}`;
|
|
71
|
+
return resolvedPath.length > 0 ? resolvedPath : ".";
|
|
72
|
+
};
|
|
73
|
+
function normalizeString(path, allowAboveRoot) {
|
|
74
|
+
let res = "";
|
|
75
|
+
let lastSegmentLength = 0;
|
|
76
|
+
let lastSlash = -1;
|
|
77
|
+
let dots = 0;
|
|
78
|
+
let char = null;
|
|
79
|
+
for (let index = 0; index <= path.length; ++index) {
|
|
80
|
+
if (index < path.length) char = path[index];
|
|
81
|
+
else if (char === "/") break;
|
|
82
|
+
else char = "/";
|
|
83
|
+
if (char === "/") {
|
|
84
|
+
if (lastSlash === index - 1 || dots === 1);
|
|
85
|
+
else if (dots === 2) {
|
|
86
|
+
if (res.length < 2 || lastSegmentLength !== 2 || res[res.length - 1] !== "." || res[res.length - 2] !== ".") {
|
|
87
|
+
if (res.length > 2) {
|
|
88
|
+
const lastSlashIndex = res.lastIndexOf("/");
|
|
89
|
+
if (lastSlashIndex === -1) {
|
|
90
|
+
res = "";
|
|
91
|
+
lastSegmentLength = 0;
|
|
92
|
+
} else {
|
|
93
|
+
res = res.slice(0, lastSlashIndex);
|
|
94
|
+
lastSegmentLength = res.length - 1 - res.lastIndexOf("/");
|
|
95
|
+
}
|
|
96
|
+
lastSlash = index;
|
|
97
|
+
dots = 0;
|
|
98
|
+
continue;
|
|
99
|
+
} else if (res.length > 0) {
|
|
100
|
+
res = "";
|
|
101
|
+
lastSegmentLength = 0;
|
|
102
|
+
lastSlash = index;
|
|
103
|
+
dots = 0;
|
|
104
|
+
continue;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
if (allowAboveRoot) {
|
|
108
|
+
res += res.length > 0 ? "/.." : "..";
|
|
109
|
+
lastSegmentLength = 2;
|
|
110
|
+
}
|
|
111
|
+
} else {
|
|
112
|
+
if (res.length > 0) res += `/${path.slice(lastSlash + 1, index)}`;
|
|
113
|
+
else res = path.slice(lastSlash + 1, index);
|
|
114
|
+
lastSegmentLength = index - lastSlash - 1;
|
|
115
|
+
}
|
|
116
|
+
lastSlash = index;
|
|
117
|
+
dots = 0;
|
|
118
|
+
} else if (char === "." && dots !== -1) ++dots;
|
|
119
|
+
else dots = -1;
|
|
120
|
+
}
|
|
121
|
+
return res;
|
|
122
|
+
}
|
|
123
|
+
const isAbsolute$1 = function(p) {
|
|
124
|
+
return _IS_ABSOLUTE_RE.test(p);
|
|
125
|
+
};
|
|
126
|
+
const extname$1 = function(p) {
|
|
127
|
+
if (p === "..") return "";
|
|
128
|
+
const match = _EXTNAME_RE.exec(normalizeWindowsPath(p));
|
|
129
|
+
return match && match[1] || "";
|
|
130
|
+
};
|
|
131
|
+
const relative$1 = function(from, to) {
|
|
132
|
+
const _from = resolve$1(from).replace(_ROOT_FOLDER_RE, "$1").split("/");
|
|
133
|
+
const _to = resolve$1(to).replace(_ROOT_FOLDER_RE, "$1").split("/");
|
|
134
|
+
if (_to[0][1] === ":" && _from[0][1] === ":" && _from[0] !== _to[0]) return _to.join("/");
|
|
135
|
+
const _fromCopy = [..._from];
|
|
136
|
+
for (const segment of _fromCopy) {
|
|
137
|
+
if (_to[0] !== segment) break;
|
|
138
|
+
_from.shift();
|
|
139
|
+
_to.shift();
|
|
140
|
+
}
|
|
141
|
+
return [..._from.map(() => ".."), ..._to].join("/");
|
|
142
|
+
};
|
|
143
|
+
const dirname$1 = function(p) {
|
|
144
|
+
const segments = normalizeWindowsPath(p).replace(/\/$/, "").split("/").slice(0, -1);
|
|
145
|
+
if (segments.length === 1 && _DRIVE_LETTER_RE.test(segments[0])) segments[0] += "/";
|
|
146
|
+
return segments.join("/") || (isAbsolute$1(p) ? "/" : ".");
|
|
147
|
+
};
|
|
148
|
+
const basename$1 = function(p, extension) {
|
|
149
|
+
const segments = normalizeWindowsPath(p).split("/");
|
|
150
|
+
let lastSegment = "";
|
|
151
|
+
for (let i = segments.length - 1; i >= 0; i--) {
|
|
152
|
+
const val = segments[i];
|
|
153
|
+
if (val) {
|
|
154
|
+
lastSegment = val;
|
|
155
|
+
break;
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
return extension && lastSegment.endsWith(extension) ? lastSegment.slice(0, -extension.length) : lastSegment;
|
|
159
|
+
};
|
|
160
|
+
const parse = function(p) {
|
|
161
|
+
const root = _PATH_ROOT_RE.exec(p)?.[0]?.replace(/\\/g, "/") || "";
|
|
162
|
+
const base = basename$1(p);
|
|
163
|
+
const extension = extname$1(base);
|
|
164
|
+
return {
|
|
165
|
+
root,
|
|
166
|
+
dir: dirname$1(p),
|
|
167
|
+
base,
|
|
168
|
+
ext: extension,
|
|
169
|
+
name: base.slice(0, base.length - extension.length)
|
|
170
|
+
};
|
|
171
|
+
};
|
|
172
|
+
//#endregion
|
|
173
|
+
//#region ../../packages/memory/src/scoring.ts
|
|
174
|
+
const TEMPORAL_STRONG = /昨天|今天|上周|本周|本月|最近|之前|以前|上次|刚刚|刚刚才|刚才|前天|大前天|前几天|这几天|这几天来|近.*[天周月年]/;
|
|
175
|
+
const TEMPORAL_WEAK = /什么时候|何时|多久|几时|哪天|哪一天|多长.*时间/;
|
|
176
|
+
const TEMPORAL_ACTION = /做了|干了|处理了|解决了|修复了|完成了|改了|写了|加了|删了|移除了|部署了|发布了/;
|
|
177
|
+
const CODE_SIGNAL = /[`'"#]|[{}()[\]]|\.(ts|js|tsx|jsx|py|rs|go|java|cpp|c|h)\b|function |const |import |export |class |async |await |type |interface /;
|
|
178
|
+
/**
|
|
179
|
+
* Lightweight regex-based query intent detection.
|
|
180
|
+
* Zero LLM cost — classifies into temporal vs factual bias to adjust scoring weights.
|
|
181
|
+
*/
|
|
182
|
+
function detectQueryIntent(query) {
|
|
183
|
+
const q = query.trim();
|
|
184
|
+
if (q.length === 0) return {
|
|
185
|
+
temporalBias: .3,
|
|
186
|
+
factualBias: .6
|
|
187
|
+
};
|
|
188
|
+
let temporalBias = .3;
|
|
189
|
+
let factualBias = .6;
|
|
190
|
+
if (TEMPORAL_STRONG.test(q)) temporalBias = 1;
|
|
191
|
+
else if (TEMPORAL_WEAK.test(q)) temporalBias = .8;
|
|
192
|
+
else if (TEMPORAL_ACTION.test(q)) temporalBias = .6;
|
|
193
|
+
if (CODE_SIGNAL.test(q) && !TEMPORAL_STRONG.test(q) && !TEMPORAL_WEAK.test(q)) factualBias = 1;
|
|
194
|
+
return {
|
|
195
|
+
temporalBias,
|
|
196
|
+
factualBias
|
|
197
|
+
};
|
|
198
|
+
}
|
|
199
|
+
/**
|
|
200
|
+
* Multi-factor relevance score for a memory memo against a query.
|
|
201
|
+
* Pure deterministic scoring — no LLM call, no network.
|
|
202
|
+
*
|
|
203
|
+
* When `intent` is provided, keyword weight increases for factual queries
|
|
204
|
+
* and recency weight increases for temporal queries.
|
|
205
|
+
*/
|
|
206
|
+
function computeRelevanceScore(memo, query, usageCount = 0, currentProjectDir, projectTagCloud, intent) {
|
|
207
|
+
const ti = intent?.temporalBias ?? .3;
|
|
208
|
+
const fi = intent?.factualBias ?? .6;
|
|
209
|
+
const factors = {
|
|
210
|
+
keywordOverlap: computeKeywordSimilarity(memo, query),
|
|
211
|
+
recency: computeRecency(memo.recordedAt),
|
|
212
|
+
usageBoost: Math.min(.3, usageCount * .1),
|
|
213
|
+
projectBoost: computeProjectBoost(memo.projectDir, currentProjectDir),
|
|
214
|
+
tagOverlap: computeTagOverlap(memo.tags, projectTagCloud)
|
|
215
|
+
};
|
|
216
|
+
const kwWeight = .45 * (1 + (fi - .6) * .5);
|
|
217
|
+
const recencyWeight = .25 * (1 + (ti - .3) * .571);
|
|
218
|
+
const usageWeight = .15;
|
|
219
|
+
const projectWeight = .1;
|
|
220
|
+
const tagWeight = .05;
|
|
221
|
+
const total = kwWeight + recencyWeight + usageWeight + projectWeight + tagWeight;
|
|
222
|
+
return factors.keywordOverlap * (kwWeight / total) + factors.recency * (recencyWeight / total) + factors.usageBoost * (usageWeight / total) + factors.projectBoost * (projectWeight / total) + factors.tagOverlap * (tagWeight / total);
|
|
223
|
+
}
|
|
224
|
+
/**
|
|
225
|
+
* Score multiple memos against a query, returning sorted results.
|
|
226
|
+
*/
|
|
227
|
+
function rankMemos(memos, query, options = {}) {
|
|
228
|
+
const { minScore = .3, maxResults = 3, currentProjectDir, projectTagCloud, vectorScores } = options;
|
|
229
|
+
const intent = detectQueryIntent(query);
|
|
230
|
+
const hasVectorScores = vectorScores !== void 0 && vectorScores.size > 0;
|
|
231
|
+
return memos.map((memo) => {
|
|
232
|
+
const keywordScore = computeRelevanceScore(memo, query, 0, currentProjectDir, projectTagCloud, intent);
|
|
233
|
+
const vectorScore = vectorScores?.get(memo.id) ?? 0;
|
|
234
|
+
return {
|
|
235
|
+
memo,
|
|
236
|
+
score: hasVectorScores ? keywordScore * .6 + vectorScore * .4 : keywordScore
|
|
237
|
+
};
|
|
238
|
+
}).filter((s) => s.score >= minScore).toSorted((a, b) => b.score - a.score).slice(0, maxResults);
|
|
239
|
+
}
|
|
240
|
+
/**
|
|
241
|
+
* Build a tag cloud from memos that belong to the current project.
|
|
242
|
+
*/
|
|
243
|
+
function buildProjectTagCloud(memos, projectDir) {
|
|
244
|
+
const cloud = /* @__PURE__ */ new Set();
|
|
245
|
+
for (const memo of memos) if (memo.projectDir === projectDir && memo.tags !== void 0) for (const tag of memo.tags) cloud.add(tag);
|
|
246
|
+
return cloud;
|
|
247
|
+
}
|
|
248
|
+
function computeProjectBoost(memoProjectDir, currentProjectDir) {
|
|
249
|
+
if (currentProjectDir === void 0 || currentProjectDir.length === 0) return 0;
|
|
250
|
+
if (memoProjectDir === currentProjectDir) return 1;
|
|
251
|
+
if (memoProjectDir.length === 0) return 0;
|
|
252
|
+
const memoParent = parentDir(memoProjectDir);
|
|
253
|
+
const currentParent = parentDir(currentProjectDir);
|
|
254
|
+
if (memoParent.length > 0 && memoParent === currentParent) return .3;
|
|
255
|
+
return 0;
|
|
256
|
+
}
|
|
257
|
+
function parentDir(path) {
|
|
258
|
+
const trimmed = path.replace(/\/+$/, "");
|
|
259
|
+
const lastSep = Math.max(trimmed.lastIndexOf("/"), trimmed.lastIndexOf("\\"));
|
|
260
|
+
return lastSep > 0 ? trimmed.slice(0, lastSep) : "";
|
|
261
|
+
}
|
|
262
|
+
function computeTagOverlap(memoTags, projectTagCloud) {
|
|
263
|
+
if (projectTagCloud === void 0 || projectTagCloud.size === 0) return 0;
|
|
264
|
+
if (memoTags === void 0 || memoTags.length === 0) return 0;
|
|
265
|
+
const memoSet = new Set(memoTags);
|
|
266
|
+
let intersection = 0;
|
|
267
|
+
for (const tag of memoSet) if (projectTagCloud.has(tag)) intersection++;
|
|
268
|
+
const union = new Set([...memoSet, ...projectTagCloud]).size;
|
|
269
|
+
return union === 0 ? 0 : intersection / union;
|
|
270
|
+
}
|
|
271
|
+
const STOP_WORDS = new Set([
|
|
272
|
+
"的",
|
|
273
|
+
"了",
|
|
274
|
+
"在",
|
|
275
|
+
"是",
|
|
276
|
+
"我",
|
|
277
|
+
"有",
|
|
278
|
+
"和",
|
|
279
|
+
"就",
|
|
280
|
+
"不",
|
|
281
|
+
"人",
|
|
282
|
+
"都",
|
|
283
|
+
"一",
|
|
284
|
+
"一个",
|
|
285
|
+
"上",
|
|
286
|
+
"也",
|
|
287
|
+
"很",
|
|
288
|
+
"到",
|
|
289
|
+
"说",
|
|
290
|
+
"要",
|
|
291
|
+
"去",
|
|
292
|
+
"你",
|
|
293
|
+
"会",
|
|
294
|
+
"着",
|
|
295
|
+
"没有",
|
|
296
|
+
"看",
|
|
297
|
+
"好",
|
|
298
|
+
"自己",
|
|
299
|
+
"这",
|
|
300
|
+
"他",
|
|
301
|
+
"她",
|
|
302
|
+
"它",
|
|
303
|
+
"们",
|
|
304
|
+
"那",
|
|
305
|
+
"些",
|
|
306
|
+
"the",
|
|
307
|
+
"a",
|
|
308
|
+
"an",
|
|
309
|
+
"is",
|
|
310
|
+
"are",
|
|
311
|
+
"was",
|
|
312
|
+
"were",
|
|
313
|
+
"be",
|
|
314
|
+
"been",
|
|
315
|
+
"being",
|
|
316
|
+
"have",
|
|
317
|
+
"has",
|
|
318
|
+
"had",
|
|
319
|
+
"do",
|
|
320
|
+
"does",
|
|
321
|
+
"did",
|
|
322
|
+
"will",
|
|
323
|
+
"would",
|
|
324
|
+
"could",
|
|
325
|
+
"should",
|
|
326
|
+
"may",
|
|
327
|
+
"might",
|
|
328
|
+
"can",
|
|
329
|
+
"shall",
|
|
330
|
+
"to",
|
|
331
|
+
"of",
|
|
332
|
+
"in",
|
|
333
|
+
"for",
|
|
334
|
+
"on",
|
|
335
|
+
"with",
|
|
336
|
+
"at",
|
|
337
|
+
"by",
|
|
338
|
+
"from",
|
|
339
|
+
"as",
|
|
340
|
+
"into",
|
|
341
|
+
"through",
|
|
342
|
+
"during",
|
|
343
|
+
"before",
|
|
344
|
+
"after",
|
|
345
|
+
"above",
|
|
346
|
+
"below",
|
|
347
|
+
"between",
|
|
348
|
+
"and",
|
|
349
|
+
"but",
|
|
350
|
+
"or",
|
|
351
|
+
"nor",
|
|
352
|
+
"not",
|
|
353
|
+
"so",
|
|
354
|
+
"yet",
|
|
355
|
+
"both",
|
|
356
|
+
"either",
|
|
357
|
+
"neither",
|
|
358
|
+
"each",
|
|
359
|
+
"every",
|
|
360
|
+
"all",
|
|
361
|
+
"any",
|
|
362
|
+
"few",
|
|
363
|
+
"more",
|
|
364
|
+
"most",
|
|
365
|
+
"other",
|
|
366
|
+
"some",
|
|
367
|
+
"such",
|
|
368
|
+
"no",
|
|
369
|
+
"only",
|
|
370
|
+
"own",
|
|
371
|
+
"same",
|
|
372
|
+
"than",
|
|
373
|
+
"too",
|
|
374
|
+
"very",
|
|
375
|
+
"just",
|
|
376
|
+
"because",
|
|
377
|
+
"about",
|
|
378
|
+
"what",
|
|
379
|
+
"which",
|
|
380
|
+
"who",
|
|
381
|
+
"whom",
|
|
382
|
+
"this",
|
|
383
|
+
"that",
|
|
384
|
+
"these",
|
|
385
|
+
"those",
|
|
386
|
+
"it",
|
|
387
|
+
"its",
|
|
388
|
+
"he",
|
|
389
|
+
"she",
|
|
390
|
+
"they",
|
|
391
|
+
"we",
|
|
392
|
+
"you",
|
|
393
|
+
"how"
|
|
394
|
+
]);
|
|
395
|
+
function extractKeywords(text) {
|
|
396
|
+
const parts = text.toLowerCase().replaceAll(/([一-鿿㐀-䶿])([a-z0-9])/g, "$1 $2").replaceAll(/([a-z0-9])([一-鿿㐀-䶿])/g, "$1 $2").split(/[^a-z0-9一-鿿㐀-䶿]+/);
|
|
397
|
+
const tokens = [];
|
|
398
|
+
for (const part of parts) {
|
|
399
|
+
if (part.length === 0) continue;
|
|
400
|
+
if (/[一-鿿㐀-䶿]/.test(part)) {
|
|
401
|
+
for (const ch of part) if (ch.length > 0 && !STOP_WORDS.has(ch)) tokens.push(ch);
|
|
402
|
+
}
|
|
403
|
+
if (/^[a-z0-9]+$/.test(part) && part.length >= 2 && !STOP_WORDS.has(part)) tokens.push(part);
|
|
404
|
+
}
|
|
405
|
+
return [...new Set(tokens)];
|
|
406
|
+
}
|
|
407
|
+
function computeKeywordSimilarity(memo, query) {
|
|
408
|
+
const memoWords = extractKeywords(`${memo.userNeed} ${memo.approach} ${memo.whatFailed} ${memo.whatWorked}`);
|
|
409
|
+
const queryWords = extractKeywords(query);
|
|
410
|
+
if (memoWords.length === 0 || queryWords.length === 0) return 0;
|
|
411
|
+
const intersection = memoWords.filter((w) => queryWords.includes(w)).length;
|
|
412
|
+
const union = new Set([...memoWords, ...queryWords]).size;
|
|
413
|
+
return union === 0 ? 0 : intersection / union;
|
|
414
|
+
}
|
|
415
|
+
function computeRecency(recordedAt) {
|
|
416
|
+
const daysSince = (Date.now() - recordedAt) / (1e3 * 60 * 60 * 24);
|
|
417
|
+
return Math.max(0, 1 - daysSince / 90);
|
|
418
|
+
}
|
|
419
|
+
//#endregion
|
|
420
|
+
//#region ../../packages/memory/src/tags.ts
|
|
421
|
+
/**
|
|
422
|
+
* Normalize a tag list: lowercase, trim, deduplicate, drop empties,
|
|
423
|
+
* and cap at `max` entries.
|
|
424
|
+
*/
|
|
425
|
+
function normalizeTags(tags, max = 5) {
|
|
426
|
+
if (!Array.isArray(tags)) return [];
|
|
427
|
+
const seen = /* @__PURE__ */ new Set();
|
|
428
|
+
const result = [];
|
|
429
|
+
for (const raw of tags) {
|
|
430
|
+
if (typeof raw !== "string") continue;
|
|
431
|
+
const tag = raw.trim().toLowerCase();
|
|
432
|
+
if (tag.length === 0 || seen.has(tag)) continue;
|
|
433
|
+
seen.add(tag);
|
|
434
|
+
result.push(tag);
|
|
435
|
+
if (result.length >= max) break;
|
|
436
|
+
}
|
|
437
|
+
return result;
|
|
438
|
+
}
|
|
439
|
+
/**
|
|
440
|
+
* Generate a small set of semantic tags from free-form text.
|
|
441
|
+
* Falls back to keyword extraction when the caller does not supply tags.
|
|
442
|
+
*/
|
|
443
|
+
function generateTags(text, max = 5) {
|
|
444
|
+
return normalizeTags(extractKeywords(text), max);
|
|
445
|
+
}
|
|
446
|
+
//#endregion
|
|
447
|
+
//#region ../../packages/memory/src/models.ts
|
|
448
|
+
function generateId$1() {
|
|
449
|
+
return `memo-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
|
|
450
|
+
}
|
|
451
|
+
function createMemoryMemo(partial) {
|
|
452
|
+
return {
|
|
453
|
+
id: partial.id ?? generateId$1(),
|
|
454
|
+
sourceSessionId: partial.sourceSessionId,
|
|
455
|
+
sourceSessionTitle: partial.sourceSessionTitle,
|
|
456
|
+
userNeed: partial.userNeed,
|
|
457
|
+
approach: partial.approach,
|
|
458
|
+
outcome: partial.outcome,
|
|
459
|
+
whatFailed: partial.whatFailed,
|
|
460
|
+
whatWorked: partial.whatWorked,
|
|
461
|
+
extractionSource: partial.extractionSource,
|
|
462
|
+
recordedAt: partial.recordedAt ?? Date.now(),
|
|
463
|
+
projectDir: partial.projectDir ?? "",
|
|
464
|
+
tags: normalizedTags(partial.tags),
|
|
465
|
+
note: partial.note
|
|
466
|
+
};
|
|
467
|
+
}
|
|
468
|
+
function normalizedTags(value) {
|
|
469
|
+
const tags = normalizeTags(value);
|
|
470
|
+
return tags.length > 0 ? tags : void 0;
|
|
471
|
+
}
|
|
472
|
+
function toSummary(memo) {
|
|
473
|
+
return {
|
|
474
|
+
id: memo.id,
|
|
475
|
+
sourceSessionTitle: memo.sourceSessionTitle,
|
|
476
|
+
sourceSessionId: memo.sourceSessionId,
|
|
477
|
+
userNeed: memo.userNeed,
|
|
478
|
+
approach: memo.approach,
|
|
479
|
+
outcome: memo.outcome,
|
|
480
|
+
whatFailed: memo.whatFailed,
|
|
481
|
+
whatWorked: memo.whatWorked,
|
|
482
|
+
extractionSource: memo.extractionSource,
|
|
483
|
+
recordedAt: memo.recordedAt,
|
|
484
|
+
projectDir: memo.projectDir,
|
|
485
|
+
tags: memo.tags,
|
|
486
|
+
note: memo.note
|
|
487
|
+
};
|
|
488
|
+
}
|
|
489
|
+
//#endregion
|
|
490
|
+
//#region ../../packages/memory/src/embeddings.ts
|
|
491
|
+
/**
|
|
492
|
+
* Text used to generate embeddings for a memo.
|
|
493
|
+
* Combines the most semantically meaningful fields.
|
|
494
|
+
*/
|
|
495
|
+
function buildEmbeddingText(memo) {
|
|
496
|
+
return `${memo.userNeed} ${memo.approach} ${memo.whatWorked}`;
|
|
497
|
+
}
|
|
498
|
+
/**
|
|
499
|
+
* Cache of created engines keyed by cacheDir.
|
|
500
|
+
* Guarantees that all callers sharing the same cacheDir reuse the same engine
|
|
501
|
+
* instance and the same in-flight model download/load — avoiding duplicate
|
|
502
|
+
* downloads and file-corruption races when /memory and /knowledge both start
|
|
503
|
+
* before the model is cached.
|
|
504
|
+
*/
|
|
505
|
+
const engineCache = /* @__PURE__ */ new Map();
|
|
506
|
+
/**
|
|
507
|
+
* Create an embedding engine backed by fastembed.
|
|
508
|
+
* Lazily loads the model on first use so startup is not blocked.
|
|
509
|
+
* Engines are cached by cacheDir so repeated calls with the same cacheDir
|
|
510
|
+
* return the same instance, sharing model state and download progress.
|
|
511
|
+
* @param cacheDir Absolute path for model cache (e.g. ~/.scream-code/cache/fastembed).
|
|
512
|
+
* Defaults to "local_cache" (CWD-relative) if not provided — prefer
|
|
513
|
+
* passing an explicit path so the cache doesn't duplicate across CWDs.
|
|
514
|
+
*/
|
|
515
|
+
function createFastEmbedEngine(cacheDir) {
|
|
516
|
+
const key = cacheDir ?? "";
|
|
517
|
+
const cached = engineCache.get(key);
|
|
518
|
+
if (cached !== void 0) return cached;
|
|
519
|
+
const engine = createFastEmbedEngineImpl(cacheDir);
|
|
520
|
+
engineCache.set(key, engine);
|
|
521
|
+
return engine;
|
|
522
|
+
}
|
|
523
|
+
/** Stable identity of the model used by every engine created here. */
|
|
524
|
+
const EMBEDDING_MODEL_NAME = "bge-small-zh-v1.5";
|
|
525
|
+
function createFastEmbedEngineImpl(cacheDir) {
|
|
526
|
+
let embedder = null;
|
|
527
|
+
let initPromise = null;
|
|
528
|
+
let loadFailed = false;
|
|
529
|
+
let lastError;
|
|
530
|
+
return {
|
|
531
|
+
modelName: EMBEDDING_MODEL_NAME,
|
|
532
|
+
get available() {
|
|
533
|
+
return embedder !== null && !loadFailed;
|
|
534
|
+
},
|
|
535
|
+
get lastError() {
|
|
536
|
+
return lastError;
|
|
537
|
+
},
|
|
538
|
+
async embedBatch(texts) {
|
|
539
|
+
if (!this.available) return null;
|
|
540
|
+
if (texts.length === 0) return [];
|
|
541
|
+
try {
|
|
542
|
+
const generator = embedder.embed(texts);
|
|
543
|
+
const vectors = [];
|
|
544
|
+
for await (const batch of generator) for (const vec of batch) vectors.push(new Float32Array(vec));
|
|
545
|
+
return vectors.length > 0 ? vectors : null;
|
|
546
|
+
} catch (error) {
|
|
547
|
+
lastError = error instanceof Error ? error.message : String(error);
|
|
548
|
+
return null;
|
|
549
|
+
}
|
|
550
|
+
},
|
|
551
|
+
cosineSimilarity(a, b) {
|
|
552
|
+
if (a.length !== b.length || a.length === 0) return 0;
|
|
553
|
+
let dot = 0;
|
|
554
|
+
let normA = 0;
|
|
555
|
+
let normB = 0;
|
|
556
|
+
for (let i = 0; i < a.length; i++) {
|
|
557
|
+
dot += a[i] * b[i];
|
|
558
|
+
normA += a[i] * a[i];
|
|
559
|
+
normB += b[i] * b[i];
|
|
560
|
+
}
|
|
561
|
+
const denom = Math.sqrt(normA) * Math.sqrt(normB);
|
|
562
|
+
return denom === 0 ? 0 : dot / denom;
|
|
563
|
+
},
|
|
564
|
+
async ensureReady() {
|
|
565
|
+
if (embedder !== null) {
|
|
566
|
+
loadFailed = false;
|
|
567
|
+
lastError = void 0;
|
|
568
|
+
return true;
|
|
569
|
+
}
|
|
570
|
+
try {
|
|
571
|
+
if (loadFailed || initPromise === null) {
|
|
572
|
+
loadFailed = false;
|
|
573
|
+
lastError = void 0;
|
|
574
|
+
initPromise = loadEmbedder(cacheDir);
|
|
575
|
+
}
|
|
576
|
+
embedder = await initPromise;
|
|
577
|
+
if (embedder === null) {
|
|
578
|
+
initPromise = null;
|
|
579
|
+
lastError = "fastembed returned an empty model";
|
|
580
|
+
return false;
|
|
581
|
+
}
|
|
582
|
+
loadFailed = false;
|
|
583
|
+
lastError = void 0;
|
|
584
|
+
return true;
|
|
585
|
+
} catch (error) {
|
|
586
|
+
initPromise = null;
|
|
587
|
+
loadFailed = true;
|
|
588
|
+
lastError = error instanceof Error ? error.message : String(error);
|
|
589
|
+
return false;
|
|
590
|
+
}
|
|
591
|
+
}
|
|
592
|
+
};
|
|
593
|
+
}
|
|
594
|
+
async function loadEmbedder(cacheDir) {
|
|
595
|
+
const { FlagEmbedding, EmbeddingModel } = await import("fastembed");
|
|
596
|
+
if (cacheDir !== void 0) mkdirSync(cacheDir, { recursive: true });
|
|
597
|
+
const model = EmbeddingModel.BGESmallZH;
|
|
598
|
+
const initOpts = cacheDir !== void 0 ? {
|
|
599
|
+
model,
|
|
600
|
+
cacheDir
|
|
601
|
+
} : { model };
|
|
602
|
+
try {
|
|
603
|
+
return await FlagEmbedding.init(initOpts);
|
|
604
|
+
} catch (initError) {
|
|
605
|
+
const msg = initError instanceof Error ? initError.message : "";
|
|
606
|
+
if (!/Config file not found|Tokenizer file not found|Tokens map file not found/iu.test(msg)) throw initError;
|
|
607
|
+
await ensureFastembedModelSidecars(model, cacheDir);
|
|
608
|
+
return FlagEmbedding.init(initOpts);
|
|
609
|
+
}
|
|
610
|
+
}
|
|
611
|
+
/**
|
|
612
|
+
* Cache entry names fastembed uses for the fixed BGESmallZH model: it extracts
|
|
613
|
+
* into `<cacheDir>/<name>/` from the staged archive `<cacheDir>/<name>.tar.gz`.
|
|
614
|
+
*/
|
|
615
|
+
const BGESMALLZH_CACHE_NAME = "fast-bge-small-zh-v1.5";
|
|
616
|
+
const PLATFORM_FAILURE = /@anush008\/tokenizers|Failed to load native binding|Unsupported (?:OS|platform|architecture)|ERR_DLOPEN_FAILED|invalid ELF header|Exec format error|is not a valid Win32 application|GLIBC_\d+[^"\n]*not found|Error loading shared library|cannot open shared object file/iu;
|
|
617
|
+
const ARCHIVE_FAILURE = /\bTAR_[A-Z_]+|incorrect header check|unexpected end of file|invalid tar|zlib/iu;
|
|
618
|
+
const NETWORK_FAILURE = /\b(?:ENOTFOUND|EAI_AGAIN|ETIMEDOUT|ECONNRESET|ECONNREFUSED|EHOSTUNREACH|socket hang up|fetch failed)\b|network error|HTTP \d{3}/iu;
|
|
619
|
+
/**
|
|
620
|
+
* Classify a load failure so callers can say something true about it: the four
|
|
621
|
+
* causes need four different answers (unsupported platform, corrupt cache,
|
|
622
|
+
* network, unknown), and they used to be reported as one network problem.
|
|
623
|
+
*/
|
|
624
|
+
function classifyEmbeddingFailure(message) {
|
|
625
|
+
if (message === void 0) return "other";
|
|
626
|
+
if (PLATFORM_FAILURE.test(message)) return "platform";
|
|
627
|
+
if (ARCHIVE_FAILURE.test(message)) return "archive";
|
|
628
|
+
if (NETWORK_FAILURE.test(message)) return "network";
|
|
629
|
+
return "other";
|
|
630
|
+
}
|
|
631
|
+
/**
|
|
632
|
+
* Loads only the module entry point — no model download — so callers can tell
|
|
633
|
+
* "this machine cannot run the local model at all" apart from "the download
|
|
634
|
+
* failed". fastembed imports its native tokenizer binding at module scope, which
|
|
635
|
+
* is exactly the step a platform without a published binary cannot pass.
|
|
636
|
+
*/
|
|
637
|
+
async function probeLocalEmbeddingSupport() {
|
|
638
|
+
try {
|
|
639
|
+
await import("fastembed");
|
|
640
|
+
return { supported: true };
|
|
641
|
+
} catch (error) {
|
|
642
|
+
return {
|
|
643
|
+
supported: false,
|
|
644
|
+
error: error instanceof Error ? error.message : String(error)
|
|
645
|
+
};
|
|
646
|
+
}
|
|
647
|
+
}
|
|
648
|
+
/**
|
|
649
|
+
* Remove every trace of a previous BGESmallZH download: the extracted model
|
|
650
|
+
* directory *and* the staged archive it came from.
|
|
651
|
+
*
|
|
652
|
+
* The archive has to go with it. fastembed skips the download whenever
|
|
653
|
+
* `<cacheDir>/<name>.tar.gz` exists, and it only deletes that file after a
|
|
654
|
+
* *successful* extract — its download error paths (response error, file stream
|
|
655
|
+
* error) reject and leave whatever was written behind. So an archive sitting
|
|
656
|
+
* next to a failed attempt is usually a partial leftover, and keeping it makes
|
|
657
|
+
* every retry extract the same junk locally: no network traffic, a few
|
|
658
|
+
* milliseconds, the same failure forever.
|
|
659
|
+
*
|
|
660
|
+
* Never throws: a file locked by another process (on Windows a mapped ONNX file
|
|
661
|
+
* stays locked) must not replace the download error the caller is reporting.
|
|
662
|
+
*/
|
|
663
|
+
function clearEmbeddingModelCache(cacheDir) {
|
|
664
|
+
for (const name of [BGESMALLZH_CACHE_NAME, `${BGESMALLZH_CACHE_NAME}.tar.gz`]) {
|
|
665
|
+
const target = join(cacheDir, name);
|
|
666
|
+
if (!existsSync(target)) continue;
|
|
667
|
+
try {
|
|
668
|
+
rmSync(target, {
|
|
669
|
+
recursive: true,
|
|
670
|
+
force: true
|
|
671
|
+
});
|
|
672
|
+
} catch {}
|
|
673
|
+
}
|
|
674
|
+
}
|
|
675
|
+
/**
|
|
676
|
+
* True when the extracted model weights are already on disk. Loading then
|
|
677
|
+
* never downloads (fastembed skips the fetch whenever the model dir exists),
|
|
678
|
+
* so callers can safely trigger a local load without risking a surprise
|
|
679
|
+
* mid-query download.
|
|
680
|
+
*/
|
|
681
|
+
function hasEmbeddingModelCache(cacheDir) {
|
|
682
|
+
const modelDir = join(cacheDir, BGESMALLZH_CACHE_NAME);
|
|
683
|
+
return existsSync(modelDir) && existsSync(join(modelDir, "model_optimized.onnx"));
|
|
684
|
+
}
|
|
685
|
+
/**
|
|
686
|
+
* Small config/tokenizer files that fastembed expects alongside model.onnx.
|
|
687
|
+
* If these are missing (e.g. GCS download partially failed), fastembed throws.
|
|
688
|
+
* We download them from HuggingFace so the model can load — this covers the
|
|
689
|
+
* case where the GCS tarball was incomplete but the HF repo is reachable.
|
|
690
|
+
*/
|
|
691
|
+
const FASTEMBED_SIDECARS = [
|
|
692
|
+
"config.json",
|
|
693
|
+
"tokenizer.json",
|
|
694
|
+
"tokenizer_config.json",
|
|
695
|
+
"special_tokens_map.json"
|
|
696
|
+
];
|
|
697
|
+
const FASTEMBED_HF_REPOS = { "fast-bge-small-zh-v1.5": "BAAI/bge-small-zh-v1.5" };
|
|
698
|
+
async function ensureFastembedModelSidecars(model, cacheDir) {
|
|
699
|
+
const repo = FASTEMBED_HF_REPOS[model];
|
|
700
|
+
if (repo === void 0) return;
|
|
701
|
+
const modelDir = join(cacheDir ?? "local_cache", model);
|
|
702
|
+
mkdirSync(modelDir, { recursive: true });
|
|
703
|
+
for (const fileName of FASTEMBED_SIDECARS) {
|
|
704
|
+
const target = join(modelDir, fileName);
|
|
705
|
+
try {
|
|
706
|
+
const { access } = await import("node:fs/promises");
|
|
707
|
+
await access(target);
|
|
708
|
+
continue;
|
|
709
|
+
} catch {}
|
|
710
|
+
const hfUrl = `https://huggingface.co/${repo}/resolve/main/${fileName}`;
|
|
711
|
+
try {
|
|
712
|
+
const response = await fetch(hfUrl);
|
|
713
|
+
if (!response.ok) continue;
|
|
714
|
+
const { writeFile } = await import("node:fs/promises");
|
|
715
|
+
await writeFile(target, Buffer.from(await response.arrayBuffer()));
|
|
716
|
+
} catch {}
|
|
717
|
+
}
|
|
718
|
+
}
|
|
719
|
+
//#endregion
|
|
720
|
+
//#region ../../packages/memory/src/store.ts
|
|
721
|
+
const FILE_NAME = "entries.jsonl";
|
|
722
|
+
const MIGRATION_MARKER = ".migrated";
|
|
723
|
+
const SQLITE_MIGRATION_MARKER = ".migrated-to-sqlite";
|
|
724
|
+
var MemoryMemoStore = class MemoryMemoStore {
|
|
725
|
+
projectDir;
|
|
726
|
+
jsonlPath;
|
|
727
|
+
dbPath;
|
|
728
|
+
db;
|
|
729
|
+
initialized = false;
|
|
730
|
+
writeLock = Promise.resolve();
|
|
731
|
+
embeddingEngine;
|
|
732
|
+
embeddingQueue = /* @__PURE__ */ new Set();
|
|
733
|
+
embeddingTimer;
|
|
734
|
+
embeddingFlushing = false;
|
|
735
|
+
embeddingDegraded = false;
|
|
736
|
+
embeddingConsecutiveFailures = 0;
|
|
737
|
+
lastEmbeddingError;
|
|
738
|
+
log;
|
|
739
|
+
constructor(projectDir, log) {
|
|
740
|
+
this.projectDir = projectDir;
|
|
741
|
+
this.jsonlPath = join$1(projectDir, "memory", FILE_NAME);
|
|
742
|
+
this.dbPath = join$1(projectDir, "memory", "memos.sqlite");
|
|
743
|
+
this.log = log ?? {};
|
|
744
|
+
}
|
|
745
|
+
/**
|
|
746
|
+
* Open the SQLite database and run schema migrations. Call this once after
|
|
747
|
+
* construction before relying on reads/writes.
|
|
748
|
+
*
|
|
749
|
+
* Note on async SQLite: Node.js added the asynchronous `Database` class to
|
|
750
|
+
* `node:sqlite` in v23.4.0 (experimental). This package currently supports
|
|
751
|
+
* Node >=22.0.0 and uses `DatabaseSync` because the v22 type definitions do
|
|
752
|
+
* not yet include `Database`. Once the project baseline moves to Node 23+ and
|
|
753
|
+
* the types catch up, the synchronous calls here should be migrated to the
|
|
754
|
+
* async API to avoid blocking the event loop on large operations.
|
|
755
|
+
*/
|
|
756
|
+
initPromise = null;
|
|
757
|
+
async init() {
|
|
758
|
+
if (this.initPromise) return this.initPromise;
|
|
759
|
+
this.initPromise = this._doInit().catch((error) => {
|
|
760
|
+
this.initPromise = null;
|
|
761
|
+
throw error;
|
|
762
|
+
});
|
|
763
|
+
return this.initPromise;
|
|
764
|
+
}
|
|
765
|
+
async _doInit() {
|
|
766
|
+
if (this.initialized) return;
|
|
767
|
+
await this.ensureDir();
|
|
768
|
+
this.db = new DatabaseSync(this.dbPath);
|
|
769
|
+
this.db.exec("PRAGMA journal_mode = WAL;");
|
|
770
|
+
this.db.exec("PRAGMA foreign_keys = ON;");
|
|
771
|
+
this.db.exec("PRAGMA busy_timeout = 5000;");
|
|
772
|
+
this.createSchema();
|
|
773
|
+
await this.migrateFromJsonl();
|
|
774
|
+
this.initialized = true;
|
|
775
|
+
}
|
|
776
|
+
/** Close the database handle and checkpoint WAL. */
|
|
777
|
+
close() {
|
|
778
|
+
if (this.embeddingTimer !== void 0) {
|
|
779
|
+
clearTimeout(this.embeddingTimer);
|
|
780
|
+
this.embeddingTimer = void 0;
|
|
781
|
+
}
|
|
782
|
+
this.embeddingQueue.clear();
|
|
783
|
+
if (this.db !== void 0) {
|
|
784
|
+
this.db.exec("PRAGMA wal_checkpoint(TRUNCATE)");
|
|
785
|
+
this.db.close();
|
|
786
|
+
this.db = void 0;
|
|
787
|
+
}
|
|
788
|
+
this.initialized = false;
|
|
789
|
+
this.initPromise = null;
|
|
790
|
+
}
|
|
791
|
+
/** Iterate all memos from the database, newest first. Optionally filter by project directory. */
|
|
792
|
+
async *read(options) {
|
|
793
|
+
await this.init();
|
|
794
|
+
if (this.db === void 0) return;
|
|
795
|
+
const projectDir = options?.projectDir;
|
|
796
|
+
const stmt = projectDir === void 0 ? this.db.prepare("SELECT * FROM memos ORDER BY recorded_at DESC") : this.db.prepare("SELECT * FROM memos WHERE project_dir = ? OR project_dir = '' ORDER BY recorded_at DESC");
|
|
797
|
+
const rows = projectDir === void 0 ? stmt.all() : stmt.all(projectDir);
|
|
798
|
+
for (const row of rows) yield rowToMemo(row);
|
|
799
|
+
}
|
|
800
|
+
/** Append a memo. */
|
|
801
|
+
async append(entry) {
|
|
802
|
+
return this.withWriteLock(() => this.appendInternal(entry));
|
|
803
|
+
}
|
|
804
|
+
/**
|
|
805
|
+
* True when a memo with the same source session + user need + approach
|
|
806
|
+
* already exists. Used by compaction-summary recovery to re-store memos
|
|
807
|
+
* idempotently after a crash between compaction apply and extraction.
|
|
808
|
+
* The approach field keeps two distinct memos that happen to share the
|
|
809
|
+
* same user need from being collapsed into one.
|
|
810
|
+
*/
|
|
811
|
+
existsBySourceAndNeed(sourceSessionId, userNeed, approach) {
|
|
812
|
+
if (this.db === void 0) return false;
|
|
813
|
+
return this.db.prepare("SELECT 1 FROM memos WHERE source_session_id = ? AND user_need = ? AND approach = ? LIMIT 1").get(sourceSessionId, userNeed, approach) !== void 0;
|
|
814
|
+
}
|
|
815
|
+
/** Delete a memo by id. */
|
|
816
|
+
async delete(id) {
|
|
817
|
+
return this.withWriteLock(() => this.deleteInternal(id));
|
|
818
|
+
}
|
|
819
|
+
/** Get a single memo by ID. */
|
|
820
|
+
async get(id) {
|
|
821
|
+
await this.init();
|
|
822
|
+
if (this.db === void 0) return void 0;
|
|
823
|
+
const row = this.db.prepare("SELECT * FROM memos WHERE id = ?").get(id);
|
|
824
|
+
return row !== void 0 ? rowToMemo(row) : void 0;
|
|
825
|
+
}
|
|
826
|
+
/**
|
|
827
|
+
* Full-text search over memos using the FTS5 index.
|
|
828
|
+
*
|
|
829
|
+
* Returns raw candidates newest first. Callers that need ranking should pass
|
|
830
|
+
* the results to `rankMemos`. An empty or whitespace-only query returns an
|
|
831
|
+
* empty array.
|
|
832
|
+
*/
|
|
833
|
+
async search(query, options) {
|
|
834
|
+
await this.init();
|
|
835
|
+
if (this.db === void 0) return [];
|
|
836
|
+
const ftsQuery = buildFtsQuery$1(query);
|
|
837
|
+
if (ftsQuery === void 0) return [];
|
|
838
|
+
const limit = options?.candidateLimit ?? 200;
|
|
839
|
+
const projectDir = options?.projectDir;
|
|
840
|
+
const stmt = projectDir === void 0 ? this.db.prepare(`SELECT m.* FROM memos m
|
|
841
|
+
JOIN memos_fts f ON m.rowid = f.rowid
|
|
842
|
+
WHERE f.memos_fts MATCH ?
|
|
843
|
+
ORDER BY m.recorded_at DESC LIMIT ?`) : this.db.prepare(`SELECT m.* FROM memos m
|
|
844
|
+
JOIN memos_fts f ON m.rowid = f.rowid
|
|
845
|
+
WHERE f.memos_fts MATCH ? AND (m.project_dir = ? OR m.project_dir = '')
|
|
846
|
+
ORDER BY m.recorded_at DESC LIMIT ?`);
|
|
847
|
+
return (projectDir === void 0 ? stmt.all(ftsQuery, limit) : stmt.all(ftsQuery, projectDir, limit)).map(rowToMemo);
|
|
848
|
+
}
|
|
849
|
+
/** List memos with optional full-text search and pagination. */
|
|
850
|
+
async list(options) {
|
|
851
|
+
await this.init();
|
|
852
|
+
if (this.db === void 0) return {
|
|
853
|
+
memos: [],
|
|
854
|
+
total: 0
|
|
855
|
+
};
|
|
856
|
+
const search = options?.search?.toLowerCase().trim();
|
|
857
|
+
const limit = options?.limit ?? 50;
|
|
858
|
+
const offset = options?.offset ?? 0;
|
|
859
|
+
const projectDir = options?.projectDir;
|
|
860
|
+
if (search !== void 0 && search.length > 0) {
|
|
861
|
+
let candidates = await this.search(search, { projectDir });
|
|
862
|
+
if (candidates.length === 0) for await (const memo of this.read({ projectDir })) candidates.push(memo);
|
|
863
|
+
const filtered = candidates.filter((memo) => memoMatchesSearch(memo, search));
|
|
864
|
+
const total = filtered.length;
|
|
865
|
+
return {
|
|
866
|
+
memos: filtered.slice(offset, offset + limit).map(toSummary),
|
|
867
|
+
total
|
|
868
|
+
};
|
|
869
|
+
}
|
|
870
|
+
const { rows, total } = this.listAll(limit, offset, projectDir);
|
|
871
|
+
return {
|
|
872
|
+
memos: rows.map(toSummary),
|
|
873
|
+
total
|
|
874
|
+
};
|
|
875
|
+
}
|
|
876
|
+
/**
|
|
877
|
+
* One-time migration from per-workDir memory stores to a global store.
|
|
878
|
+
* Reads `<screamHomeDir>/sessions/<workDirKey>/memory/entries.jsonl`
|
|
879
|
+
* and appends valid entries to the global SQLite store.
|
|
880
|
+
* Deletes the legacy per-session memory files afterwards and writes a marker
|
|
881
|
+
* file so the migration only runs once.
|
|
882
|
+
*/
|
|
883
|
+
static async migrateLegacyStores(screamHomeDir) {
|
|
884
|
+
const target = new MemoryMemoStore(screamHomeDir);
|
|
885
|
+
const markerPath = join$1(screamHomeDir, "memory", MIGRATION_MARKER);
|
|
886
|
+
try {
|
|
887
|
+
await stat(markerPath);
|
|
888
|
+
return;
|
|
889
|
+
} catch {}
|
|
890
|
+
const sessionsDir = join$1(screamHomeDir, "sessions");
|
|
891
|
+
let sessionEntries;
|
|
892
|
+
try {
|
|
893
|
+
sessionEntries = await readdir(sessionsDir, { withFileTypes: true }).then((entries) => entries.filter((e) => e.isDirectory()).map((e) => e.name));
|
|
894
|
+
} catch {
|
|
895
|
+
await writeFile(markerPath, "", "utf8").catch(() => {});
|
|
896
|
+
return;
|
|
897
|
+
}
|
|
898
|
+
const migratedIds = /* @__PURE__ */ new Set();
|
|
899
|
+
for await (const memo of target.read()) migratedIds.add(memo.id);
|
|
900
|
+
let migratedCount = 0;
|
|
901
|
+
const legacyPaths = [];
|
|
902
|
+
for (const sessionKey of sessionEntries) {
|
|
903
|
+
const legacyPath = join$1(sessionsDir, sessionKey, "memory", FILE_NAME);
|
|
904
|
+
let stream;
|
|
905
|
+
try {
|
|
906
|
+
stream = createReadStream(legacyPath, { encoding: "utf8" });
|
|
907
|
+
} catch {}
|
|
908
|
+
stream.on("error", () => {});
|
|
909
|
+
let line = "";
|
|
910
|
+
try {
|
|
911
|
+
for await (const chunk of stream) {
|
|
912
|
+
line += chunk;
|
|
913
|
+
let newlineIndex = line.indexOf("\n");
|
|
914
|
+
while (newlineIndex !== -1) {
|
|
915
|
+
const rawLine = line.slice(0, newlineIndex).replace(/\r$/, "");
|
|
916
|
+
line = line.slice(newlineIndex + 1);
|
|
917
|
+
newlineIndex = line.indexOf("\n");
|
|
918
|
+
const memo = target.parseLine(rawLine, 0);
|
|
919
|
+
if (memo === void 0 || migratedIds.has(memo.id)) continue;
|
|
920
|
+
await target.append(memo);
|
|
921
|
+
migratedIds.add(memo.id);
|
|
922
|
+
migratedCount++;
|
|
923
|
+
}
|
|
924
|
+
}
|
|
925
|
+
} catch {
|
|
926
|
+
continue;
|
|
927
|
+
}
|
|
928
|
+
legacyPaths.push(legacyPath);
|
|
929
|
+
}
|
|
930
|
+
for (const legacyPath of legacyPaths) {
|
|
931
|
+
await unlink(legacyPath).catch(() => {});
|
|
932
|
+
await rmdir(dirname$1(legacyPath)).catch(() => {});
|
|
933
|
+
}
|
|
934
|
+
await writeFile(markerPath, `${migratedCount}\n`, "utf8").catch(() => {});
|
|
935
|
+
}
|
|
936
|
+
/** @internal */
|
|
937
|
+
parseLine(rawLine, _lineNumber) {
|
|
938
|
+
if (rawLine.length === 0) return void 0;
|
|
939
|
+
try {
|
|
940
|
+
const record = JSON.parse(rawLine);
|
|
941
|
+
if (record["type"] !== "memory_memo" || !record["entry"]) return void 0;
|
|
942
|
+
const entry = record["entry"];
|
|
943
|
+
if (record["version"] === 1 || entry["userRequirement"] !== void 0 && entry["userNeed"] === void 0) {
|
|
944
|
+
const str = (v, fallback = "") => typeof v === "string" ? v : fallback;
|
|
945
|
+
return {
|
|
946
|
+
id: str(entry["id"]),
|
|
947
|
+
sourceSessionId: str(entry["sourceSessionId"]),
|
|
948
|
+
sourceSessionTitle: str(entry["sourceSessionTitle"], void 0),
|
|
949
|
+
userNeed: str(entry["userRequirement"]),
|
|
950
|
+
approach: str(entry["solution"]),
|
|
951
|
+
outcome: str(entry["completionStatus"]),
|
|
952
|
+
whatFailed: str(entry["problemsEncountered"], "none"),
|
|
953
|
+
whatWorked: "none",
|
|
954
|
+
extractionSource: entry["extractionSource"] === "exit" ? "exit" : "compaction",
|
|
955
|
+
recordedAt: typeof entry["recordedAt"] === "number" ? entry["recordedAt"] : 0,
|
|
956
|
+
projectDir: str(entry["projectDir"])
|
|
957
|
+
};
|
|
958
|
+
}
|
|
959
|
+
const src = entry["extractionSource"];
|
|
960
|
+
if (typeof src === "string" && ![
|
|
961
|
+
"compaction",
|
|
962
|
+
"exit",
|
|
963
|
+
"manual"
|
|
964
|
+
].includes(src)) return;
|
|
965
|
+
return entry;
|
|
966
|
+
} catch {
|
|
967
|
+
return;
|
|
968
|
+
}
|
|
969
|
+
}
|
|
970
|
+
createSchema() {
|
|
971
|
+
if (this.db === void 0) return;
|
|
972
|
+
this.db.exec(`
|
|
973
|
+
CREATE TABLE IF NOT EXISTS memos (
|
|
974
|
+
id TEXT PRIMARY KEY,
|
|
975
|
+
source_session_id TEXT NOT NULL,
|
|
976
|
+
source_session_title TEXT,
|
|
977
|
+
user_need TEXT NOT NULL,
|
|
978
|
+
approach TEXT NOT NULL,
|
|
979
|
+
outcome TEXT NOT NULL,
|
|
980
|
+
what_failed TEXT NOT NULL DEFAULT 'none',
|
|
981
|
+
what_worked TEXT NOT NULL DEFAULT 'none',
|
|
982
|
+
extraction_source TEXT NOT NULL CHECK(extraction_source IN ('compaction', 'exit', 'manual')),
|
|
983
|
+
recorded_at INTEGER NOT NULL,
|
|
984
|
+
project_dir TEXT NOT NULL DEFAULT '',
|
|
985
|
+
tags TEXT NOT NULL DEFAULT '[]'
|
|
986
|
+
);
|
|
987
|
+
|
|
988
|
+
CREATE INDEX IF NOT EXISTS idx_memos_project_dir ON memos(project_dir);
|
|
989
|
+
|
|
990
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS memos_fts USING fts5(
|
|
991
|
+
user_need,
|
|
992
|
+
approach,
|
|
993
|
+
what_failed,
|
|
994
|
+
what_worked,
|
|
995
|
+
source_session_title,
|
|
996
|
+
content=''
|
|
997
|
+
);
|
|
998
|
+
|
|
999
|
+
CREATE TABLE IF NOT EXISTS memory_embeddings (
|
|
1000
|
+
memory_id TEXT PRIMARY KEY REFERENCES memos(id) ON DELETE CASCADE,
|
|
1001
|
+
embedding_json TEXT NOT NULL,
|
|
1002
|
+
model TEXT NOT NULL DEFAULT '${EMBEDDING_MODEL_NAME}',
|
|
1003
|
+
created_at INTEGER NOT NULL
|
|
1004
|
+
);
|
|
1005
|
+
|
|
1006
|
+
CREATE INDEX IF NOT EXISTS idx_memory_embeddings_created_at ON memory_embeddings(created_at DESC);
|
|
1007
|
+
`);
|
|
1008
|
+
this.migrateSchema();
|
|
1009
|
+
}
|
|
1010
|
+
migrateSchema() {
|
|
1011
|
+
if (this.db === void 0) return;
|
|
1012
|
+
const info = this.db.prepare("PRAGMA table_info(memos)").all();
|
|
1013
|
+
if (!info.some((col) => col.name === "project_dir")) this.db.exec("ALTER TABLE memos ADD COLUMN project_dir TEXT NOT NULL DEFAULT ''");
|
|
1014
|
+
if (!info.some((col) => col.name === "tags")) this.db.exec("ALTER TABLE memos ADD COLUMN tags TEXT NOT NULL DEFAULT '[]'");
|
|
1015
|
+
this.db.exec(`
|
|
1016
|
+
CREATE INDEX IF NOT EXISTS idx_memos_project_dir ON memos(project_dir);
|
|
1017
|
+
CREATE INDEX IF NOT EXISTS idx_memory_embeddings_created_at ON memory_embeddings(created_at DESC);
|
|
1018
|
+
`);
|
|
1019
|
+
}
|
|
1020
|
+
async migrateFromJsonl() {
|
|
1021
|
+
const markerPath = join$1(this.projectDir, "memory", SQLITE_MIGRATION_MARKER);
|
|
1022
|
+
try {
|
|
1023
|
+
await stat(markerPath);
|
|
1024
|
+
return;
|
|
1025
|
+
} catch {}
|
|
1026
|
+
const memos = [];
|
|
1027
|
+
let stream;
|
|
1028
|
+
try {
|
|
1029
|
+
stream = createReadStream(this.jsonlPath, { encoding: "utf8" });
|
|
1030
|
+
} catch {
|
|
1031
|
+
await writeFile(markerPath, "", "utf8").catch(() => {});
|
|
1032
|
+
return;
|
|
1033
|
+
}
|
|
1034
|
+
stream.on("error", () => {});
|
|
1035
|
+
let line = "";
|
|
1036
|
+
try {
|
|
1037
|
+
for await (const chunk of stream) {
|
|
1038
|
+
line += chunk;
|
|
1039
|
+
let newlineIndex = line.indexOf("\n");
|
|
1040
|
+
while (newlineIndex !== -1) {
|
|
1041
|
+
const rawLine = line.slice(0, newlineIndex).replace(/\r$/, "");
|
|
1042
|
+
line = line.slice(newlineIndex + 1);
|
|
1043
|
+
newlineIndex = line.indexOf("\n");
|
|
1044
|
+
const memo = this.parseLine(rawLine, 0);
|
|
1045
|
+
if (memo !== void 0) memos.push(memo);
|
|
1046
|
+
}
|
|
1047
|
+
}
|
|
1048
|
+
} catch {}
|
|
1049
|
+
if (memos.length > 0) this.insertMany(memos);
|
|
1050
|
+
await writeFile(markerPath, "", "utf8").catch(() => {});
|
|
1051
|
+
await rename(this.jsonlPath, `${this.jsonlPath}.bak`).catch(() => {});
|
|
1052
|
+
await unlink(join$1(this.projectDir, "memory", "index.json")).catch(() => {});
|
|
1053
|
+
}
|
|
1054
|
+
insertMany(memos) {
|
|
1055
|
+
if (this.db === void 0 || memos.length === 0) return;
|
|
1056
|
+
const insert = this.db.prepare(`INSERT OR IGNORE INTO memos (
|
|
1057
|
+
id, source_session_id, source_session_title, user_need, approach,
|
|
1058
|
+
outcome, what_failed, what_worked, extraction_source, recorded_at, project_dir, tags
|
|
1059
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1060
|
+
RETURNING rowid`);
|
|
1061
|
+
const insertFts = this.db.prepare(`INSERT INTO memos_fts(rowid, user_need, approach, what_failed, what_worked, source_session_title)
|
|
1062
|
+
VALUES (?, ?, ?, ?, ?, ?)`);
|
|
1063
|
+
this.db.exec("BEGIN TRANSACTION");
|
|
1064
|
+
try {
|
|
1065
|
+
for (const memo of memos) {
|
|
1066
|
+
const row = insert.get(memo.id, memo.sourceSessionId, memo.sourceSessionTitle ?? null, memo.userNeed, memo.approach, memo.outcome, memo.whatFailed, memo.whatWorked, memo.extractionSource, memo.recordedAt, memo.projectDir ?? "", JSON.stringify(memo.tags ?? []));
|
|
1067
|
+
if (row === void 0) continue;
|
|
1068
|
+
insertFts.run(row.rowid, toFtsText$1(memo.userNeed), toFtsText$1(memo.approach), toFtsText$1(memo.whatFailed), toFtsText$1(memo.whatWorked), toFtsText$1(memo.sourceSessionTitle ?? ""));
|
|
1069
|
+
}
|
|
1070
|
+
this.db.exec("COMMIT");
|
|
1071
|
+
} catch (error) {
|
|
1072
|
+
this.db.exec("ROLLBACK");
|
|
1073
|
+
throw new Error(`Failed to migrate memos to SQLite: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
|
|
1074
|
+
}
|
|
1075
|
+
}
|
|
1076
|
+
async appendInternal(entry) {
|
|
1077
|
+
await this.init();
|
|
1078
|
+
if (this.db === void 0) return;
|
|
1079
|
+
const insert = this.db.prepare(`INSERT INTO memos (
|
|
1080
|
+
id, source_session_id, source_session_title, user_need, approach,
|
|
1081
|
+
outcome, what_failed, what_worked, extraction_source, recorded_at, project_dir, tags
|
|
1082
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1083
|
+
RETURNING rowid`);
|
|
1084
|
+
const insertFts = this.db.prepare(`INSERT INTO memos_fts(rowid, user_need, approach, what_failed, what_worked, source_session_title)
|
|
1085
|
+
VALUES (?, ?, ?, ?, ?, ?)`);
|
|
1086
|
+
this.db.exec("BEGIN TRANSACTION");
|
|
1087
|
+
try {
|
|
1088
|
+
const row = insert.get(entry.id, entry.sourceSessionId, entry.sourceSessionTitle ?? null, entry.userNeed, entry.approach, entry.outcome, entry.whatFailed, entry.whatWorked, entry.extractionSource, entry.recordedAt, entry.projectDir ?? "", JSON.stringify(entry.tags ?? []));
|
|
1089
|
+
insertFts.run(row.rowid, toFtsText$1(entry.userNeed), toFtsText$1(entry.approach), toFtsText$1(entry.whatFailed), toFtsText$1(entry.whatWorked), toFtsText$1(entry.sourceSessionTitle ?? ""));
|
|
1090
|
+
this.db.exec("COMMIT");
|
|
1091
|
+
this.scheduleEmbedding(entry);
|
|
1092
|
+
} catch {
|
|
1093
|
+
this.db.exec("ROLLBACK");
|
|
1094
|
+
throw new Error("Failed to append memo");
|
|
1095
|
+
}
|
|
1096
|
+
}
|
|
1097
|
+
/** Update a memo by id. Returns true if the memo existed and was updated. */
|
|
1098
|
+
async update(id, patch) {
|
|
1099
|
+
return this.withWriteLock(() => this.updateInternal(id, patch));
|
|
1100
|
+
}
|
|
1101
|
+
/** @internal */
|
|
1102
|
+
async updateInternal(id, patch) {
|
|
1103
|
+
await this.init();
|
|
1104
|
+
if (this.db === void 0) return false;
|
|
1105
|
+
const existing = await this.get(id);
|
|
1106
|
+
if (existing === void 0) return false;
|
|
1107
|
+
const updated = {
|
|
1108
|
+
...existing,
|
|
1109
|
+
...patch
|
|
1110
|
+
};
|
|
1111
|
+
const selectRow = this.db.prepare("SELECT rowid FROM memos WHERE id = ?");
|
|
1112
|
+
const update = this.db.prepare(`UPDATE memos SET
|
|
1113
|
+
rowid = (SELECT COALESCE(MAX(rowid), 0) + 1 FROM memos),
|
|
1114
|
+
source_session_id = ?,
|
|
1115
|
+
source_session_title = ?,
|
|
1116
|
+
user_need = ?,
|
|
1117
|
+
approach = ?,
|
|
1118
|
+
outcome = ?,
|
|
1119
|
+
what_failed = ?,
|
|
1120
|
+
what_worked = ?,
|
|
1121
|
+
extraction_source = ?,
|
|
1122
|
+
recorded_at = ?,
|
|
1123
|
+
project_dir = ?,
|
|
1124
|
+
tags = ?
|
|
1125
|
+
WHERE id = ?
|
|
1126
|
+
RETURNING rowid`);
|
|
1127
|
+
const updateFts = this.db.prepare(`INSERT INTO memos_fts(rowid, user_need, approach, what_failed, what_worked, source_session_title)
|
|
1128
|
+
VALUES (?, ?, ?, ?, ?, ?)`);
|
|
1129
|
+
const deleteFts = this.db.prepare("INSERT INTO memos_fts(memos_fts, rowid) VALUES ('delete', ?)");
|
|
1130
|
+
const deleteEmbedding = this.db.prepare("DELETE FROM memory_embeddings WHERE memory_id = ?");
|
|
1131
|
+
this.db.exec("BEGIN TRANSACTION");
|
|
1132
|
+
try {
|
|
1133
|
+
const oldRow = selectRow.get(id);
|
|
1134
|
+
if (oldRow === void 0) {
|
|
1135
|
+
this.db.exec("ROLLBACK");
|
|
1136
|
+
return false;
|
|
1137
|
+
}
|
|
1138
|
+
const row = update.get(updated.sourceSessionId, updated.sourceSessionTitle ?? null, updated.userNeed, updated.approach, updated.outcome, updated.whatFailed, updated.whatWorked, updated.extractionSource, updated.recordedAt, updated.projectDir ?? "", JSON.stringify(updated.tags ?? []), id);
|
|
1139
|
+
if (row === void 0) {
|
|
1140
|
+
this.db.exec("ROLLBACK");
|
|
1141
|
+
return false;
|
|
1142
|
+
}
|
|
1143
|
+
deleteFts.run(oldRow.rowid);
|
|
1144
|
+
updateFts.run(row.rowid, toFtsText$1(updated.userNeed), toFtsText$1(updated.approach), toFtsText$1(updated.whatFailed), toFtsText$1(updated.whatWorked), toFtsText$1(updated.sourceSessionTitle ?? ""));
|
|
1145
|
+
deleteEmbedding.run(id);
|
|
1146
|
+
this.db.exec("COMMIT");
|
|
1147
|
+
this.scheduleEmbedding(updated);
|
|
1148
|
+
return true;
|
|
1149
|
+
} catch {
|
|
1150
|
+
this.db.exec("ROLLBACK");
|
|
1151
|
+
throw new Error("Failed to update memo");
|
|
1152
|
+
}
|
|
1153
|
+
}
|
|
1154
|
+
async deleteInternal(id) {
|
|
1155
|
+
await this.init();
|
|
1156
|
+
if (this.db === void 0) return false;
|
|
1157
|
+
const row = this.db.prepare("SELECT rowid FROM memos WHERE id = ?").get(id);
|
|
1158
|
+
if (row === void 0) return true;
|
|
1159
|
+
const deleteFts = this.db.prepare("INSERT INTO memos_fts(memos_fts, rowid) VALUES ('delete', ?)");
|
|
1160
|
+
const deleteMemo = this.db.prepare("DELETE FROM memos WHERE id = ?");
|
|
1161
|
+
this.db.exec("BEGIN TRANSACTION");
|
|
1162
|
+
try {
|
|
1163
|
+
deleteFts.run(row.rowid);
|
|
1164
|
+
deleteMemo.run(id);
|
|
1165
|
+
this.db.exec("COMMIT");
|
|
1166
|
+
return true;
|
|
1167
|
+
} catch {
|
|
1168
|
+
this.db.exec("ROLLBACK");
|
|
1169
|
+
throw new Error("Failed to delete memo");
|
|
1170
|
+
}
|
|
1171
|
+
}
|
|
1172
|
+
/** Set the embedding engine. Call once after construction, before any writes. */
|
|
1173
|
+
setEmbeddingEngine(engine) {
|
|
1174
|
+
this.embeddingEngine = engine;
|
|
1175
|
+
}
|
|
1176
|
+
/** Check whether the store has any vector embeddings. */
|
|
1177
|
+
hasEmbeddings() {
|
|
1178
|
+
if (this.db === void 0) return false;
|
|
1179
|
+
return (this.db.prepare("SELECT COUNT(*) as count FROM memory_embeddings").get()?.count ?? 0) > 0;
|
|
1180
|
+
}
|
|
1181
|
+
/** Access the embedding engine (may be undefined if not configured). */
|
|
1182
|
+
getEmbeddingEngine() {
|
|
1183
|
+
return this.embeddingEngine;
|
|
1184
|
+
}
|
|
1185
|
+
/**
|
|
1186
|
+
* Search memos by vector similarity. Returns memos sorted by cosine
|
|
1187
|
+
* similarity (highest first). Falls back to empty if no embeddings exist.
|
|
1188
|
+
*
|
|
1189
|
+
* NOTE: candidates are ordered by created_at DESC, so only the newest
|
|
1190
|
+
* N embeddings are considered. Older but highly relevant memos may be
|
|
1191
|
+
* omitted when the store has more than candidateLimit embeddings.
|
|
1192
|
+
* This is a known limitation — see M-M5 in project review notes.
|
|
1193
|
+
* Performance notes:
|
|
1194
|
+
* - candidateLimit bounds the SQL query so we never load every embedding.
|
|
1195
|
+
* - recencyCutoffDays lets callers ignore very old memos.
|
|
1196
|
+
* - projectDir is pushed into the SQL JOIN so unrelated projects are not
|
|
1197
|
+
* considered at all.
|
|
1198
|
+
*/
|
|
1199
|
+
async searchByVector(queryEmbedding, options) {
|
|
1200
|
+
await this.init();
|
|
1201
|
+
if (this.db === void 0) return [];
|
|
1202
|
+
const limit = options?.candidateLimit ?? 200;
|
|
1203
|
+
const projectDir = options?.projectDir;
|
|
1204
|
+
const recencyCutoffDays = options?.recencyCutoffDays;
|
|
1205
|
+
const cutoffMs = recencyCutoffDays !== void 0 && recencyCutoffDays > 0 ? Date.now() - recencyCutoffDays * 24 * 60 * 60 * 1e3 : void 0;
|
|
1206
|
+
let rows;
|
|
1207
|
+
if (projectDir !== void 0) {
|
|
1208
|
+
const stmt = cutoffMs === void 0 ? this.db.prepare(`
|
|
1209
|
+
SELECT e.memory_id, e.embedding_json
|
|
1210
|
+
FROM memory_embeddings e
|
|
1211
|
+
JOIN memos m ON m.id = e.memory_id
|
|
1212
|
+
WHERE (m.project_dir = ? OR m.project_dir = '')
|
|
1213
|
+
ORDER BY e.created_at DESC
|
|
1214
|
+
LIMIT ?
|
|
1215
|
+
`) : this.db.prepare(`
|
|
1216
|
+
SELECT e.memory_id, e.embedding_json
|
|
1217
|
+
FROM memory_embeddings e
|
|
1218
|
+
JOIN memos m ON m.id = e.memory_id
|
|
1219
|
+
WHERE (m.project_dir = ? OR m.project_dir = '')
|
|
1220
|
+
AND e.created_at > ?
|
|
1221
|
+
ORDER BY e.created_at DESC
|
|
1222
|
+
LIMIT ?
|
|
1223
|
+
`);
|
|
1224
|
+
rows = cutoffMs === void 0 ? stmt.all(projectDir, limit) : stmt.all(projectDir, cutoffMs, limit);
|
|
1225
|
+
} else {
|
|
1226
|
+
const stmt = cutoffMs === void 0 ? this.db.prepare("SELECT memory_id, embedding_json FROM memory_embeddings ORDER BY created_at DESC LIMIT ?") : this.db.prepare("SELECT memory_id, embedding_json FROM memory_embeddings WHERE created_at > ? ORDER BY created_at DESC LIMIT ?");
|
|
1227
|
+
rows = cutoffMs === void 0 ? stmt.all(limit) : stmt.all(cutoffMs, limit);
|
|
1228
|
+
}
|
|
1229
|
+
if (rows.length === 0) return [];
|
|
1230
|
+
const scored = [];
|
|
1231
|
+
for (const row of rows) try {
|
|
1232
|
+
const vec = new Float32Array(JSON.parse(row.embedding_json));
|
|
1233
|
+
const similarity = this.embeddingEngine?.cosineSimilarity(queryEmbedding, vec) ?? 0;
|
|
1234
|
+
if (similarity > 0) scored.push({
|
|
1235
|
+
id: row.memory_id,
|
|
1236
|
+
score: similarity
|
|
1237
|
+
});
|
|
1238
|
+
} catch {}
|
|
1239
|
+
scored.sort((a, b) => b.score - a.score);
|
|
1240
|
+
const topScored = scored.slice(0, limit);
|
|
1241
|
+
const results = [];
|
|
1242
|
+
for (const { id, score } of topScored) {
|
|
1243
|
+
const row = this.db.prepare("SELECT * FROM memos WHERE id = ?").get(id);
|
|
1244
|
+
if (row !== void 0) results.push({
|
|
1245
|
+
memo: rowToMemo(row),
|
|
1246
|
+
score
|
|
1247
|
+
});
|
|
1248
|
+
}
|
|
1249
|
+
return results;
|
|
1250
|
+
}
|
|
1251
|
+
/**
|
|
1252
|
+
* Schedule async embedding generation for a memo. Debounced — the actual
|
|
1253
|
+
* batch flush runs after a short quiet period. Never blocks the caller.
|
|
1254
|
+
*/
|
|
1255
|
+
scheduleEmbedding(memo) {
|
|
1256
|
+
this.embeddingQueue.add(memo.id);
|
|
1257
|
+
if (this.embeddingEngine === void 0) return;
|
|
1258
|
+
if (!this.embeddingEngine.available) {
|
|
1259
|
+
this.embeddingEngine.ensureReady().then((ready) => {
|
|
1260
|
+
if (ready && this.embeddingQueue.size > 0 && this.embeddingTimer === void 0) this.embeddingTimer = setTimeout(() => {
|
|
1261
|
+
this.flushEmbeddings();
|
|
1262
|
+
}, 2e3);
|
|
1263
|
+
}).catch(() => {});
|
|
1264
|
+
return;
|
|
1265
|
+
}
|
|
1266
|
+
if (this.embeddingTimer !== void 0) clearTimeout(this.embeddingTimer);
|
|
1267
|
+
this.embeddingTimer = setTimeout(() => {
|
|
1268
|
+
this.flushEmbeddings();
|
|
1269
|
+
}, 2e3);
|
|
1270
|
+
}
|
|
1271
|
+
/**
|
|
1272
|
+
* Flush queued embedding generation. Retries once on failure to tolerate
|
|
1273
|
+
* transient model-load contention, then marks embeddings as degraded and
|
|
1274
|
+
* logs the problem. Callers can still retrieve memos through keyword search.
|
|
1275
|
+
*/
|
|
1276
|
+
async flushEmbeddings() {
|
|
1277
|
+
if (this.embeddingFlushing || this.embeddingEngine === void 0 || !this.embeddingEngine.available) return;
|
|
1278
|
+
this.embeddingFlushing = true;
|
|
1279
|
+
const ids = [...this.embeddingQueue];
|
|
1280
|
+
this.embeddingQueue.clear();
|
|
1281
|
+
const reenqueue = () => {
|
|
1282
|
+
for (const id of ids) this.embeddingQueue.add(id);
|
|
1283
|
+
};
|
|
1284
|
+
try {
|
|
1285
|
+
await this.init();
|
|
1286
|
+
if (this.db === void 0) return;
|
|
1287
|
+
const pending = [];
|
|
1288
|
+
for (const id of ids) {
|
|
1289
|
+
if (this.db.prepare("SELECT id FROM memory_embeddings WHERE memory_id = ?").get(id) !== void 0) continue;
|
|
1290
|
+
const memo = await this.get(id);
|
|
1291
|
+
if (memo !== void 0) pending.push({
|
|
1292
|
+
id,
|
|
1293
|
+
text: buildEmbeddingText(memo)
|
|
1294
|
+
});
|
|
1295
|
+
}
|
|
1296
|
+
if (pending.length === 0) return;
|
|
1297
|
+
const vectors = await this.tryEmbedBatch(pending.map((p) => p.text));
|
|
1298
|
+
if (vectors === null || vectors.length !== pending.length) {
|
|
1299
|
+
this.markEmbeddingFailure(/* @__PURE__ */ new Error(vectors === null ? "embedBatch returned null" : "embedding count mismatch"));
|
|
1300
|
+
reenqueue();
|
|
1301
|
+
return;
|
|
1302
|
+
}
|
|
1303
|
+
this.clearEmbeddingFailure();
|
|
1304
|
+
const insert = this.db.prepare("INSERT OR REPLACE INTO memory_embeddings (memory_id, embedding_json, model, created_at) VALUES (?, ?, ?, ?)");
|
|
1305
|
+
const now = Date.now();
|
|
1306
|
+
this.db.exec("BEGIN TRANSACTION");
|
|
1307
|
+
try {
|
|
1308
|
+
for (let i = 0; i < pending.length; i++) insert.run(pending[i].id, JSON.stringify([...vectors[i]]), EMBEDDING_MODEL_NAME, now);
|
|
1309
|
+
this.db.exec("COMMIT");
|
|
1310
|
+
} catch (error) {
|
|
1311
|
+
this.db.exec("ROLLBACK");
|
|
1312
|
+
this.markEmbeddingFailure(error instanceof Error ? error : new Error(String(error)));
|
|
1313
|
+
reenqueue();
|
|
1314
|
+
}
|
|
1315
|
+
} catch (error) {
|
|
1316
|
+
this.markEmbeddingFailure(error instanceof Error ? error : new Error(String(error)));
|
|
1317
|
+
reenqueue();
|
|
1318
|
+
} finally {
|
|
1319
|
+
this.embeddingFlushing = false;
|
|
1320
|
+
this.embeddingTimer = void 0;
|
|
1321
|
+
}
|
|
1322
|
+
}
|
|
1323
|
+
async tryEmbedBatch(texts) {
|
|
1324
|
+
if (this.embeddingEngine === void 0) return null;
|
|
1325
|
+
try {
|
|
1326
|
+
const first = await this.embeddingEngine.embedBatch(texts);
|
|
1327
|
+
if (first !== null) return first;
|
|
1328
|
+
} catch {}
|
|
1329
|
+
try {
|
|
1330
|
+
return await this.embeddingEngine.embedBatch(texts);
|
|
1331
|
+
} catch {
|
|
1332
|
+
return null;
|
|
1333
|
+
}
|
|
1334
|
+
}
|
|
1335
|
+
markEmbeddingFailure(error) {
|
|
1336
|
+
this.embeddingDegraded = true;
|
|
1337
|
+
this.embeddingConsecutiveFailures += 1;
|
|
1338
|
+
this.lastEmbeddingError = error;
|
|
1339
|
+
this.log.warn?.("embedding flush failed", {
|
|
1340
|
+
error: error.message,
|
|
1341
|
+
consecutiveFailures: this.embeddingConsecutiveFailures
|
|
1342
|
+
});
|
|
1343
|
+
}
|
|
1344
|
+
clearEmbeddingFailure() {
|
|
1345
|
+
this.embeddingDegraded = false;
|
|
1346
|
+
this.embeddingConsecutiveFailures = 0;
|
|
1347
|
+
this.lastEmbeddingError = void 0;
|
|
1348
|
+
}
|
|
1349
|
+
/**
|
|
1350
|
+
* Runtime health of the embedding subsystem. When `degraded` is true, the
|
|
1351
|
+
* store still serves keyword search; only vector similarity is unavailable.
|
|
1352
|
+
*/
|
|
1353
|
+
embeddingStatus() {
|
|
1354
|
+
return {
|
|
1355
|
+
available: this.embeddingEngine !== void 0 && this.embeddingEngine.available,
|
|
1356
|
+
degraded: this.embeddingDegraded,
|
|
1357
|
+
consecutiveFailures: this.embeddingConsecutiveFailures,
|
|
1358
|
+
lastError: this.lastEmbeddingError?.message
|
|
1359
|
+
};
|
|
1360
|
+
}
|
|
1361
|
+
listAll(limit, offset, projectDir) {
|
|
1362
|
+
if (this.db === void 0) return {
|
|
1363
|
+
rows: [],
|
|
1364
|
+
total: 0
|
|
1365
|
+
};
|
|
1366
|
+
const countStmt = projectDir === void 0 ? this.db.prepare("SELECT COUNT(*) as total FROM memos") : this.db.prepare("SELECT COUNT(*) as total FROM memos WHERE project_dir = ? OR project_dir = ''");
|
|
1367
|
+
const total = (projectDir === void 0 ? countStmt.get() : countStmt.get(projectDir))?.total ?? 0;
|
|
1368
|
+
const stmt = projectDir === void 0 ? this.db.prepare("SELECT * FROM memos ORDER BY recorded_at DESC LIMIT ? OFFSET ?") : this.db.prepare("SELECT * FROM memos WHERE project_dir = ? OR project_dir = '' ORDER BY recorded_at DESC LIMIT ? OFFSET ?");
|
|
1369
|
+
return {
|
|
1370
|
+
rows: (projectDir === void 0 ? stmt.all(limit, offset) : stmt.all(projectDir, limit, offset)).map(rowToMemo),
|
|
1371
|
+
total
|
|
1372
|
+
};
|
|
1373
|
+
}
|
|
1374
|
+
async ensureDir() {
|
|
1375
|
+
await mkdir(dirname$1(this.dbPath), { recursive: true });
|
|
1376
|
+
}
|
|
1377
|
+
async withWriteLock(fn) {
|
|
1378
|
+
const next = this.writeLock.then(fn, fn);
|
|
1379
|
+
this.writeLock = next;
|
|
1380
|
+
return next;
|
|
1381
|
+
}
|
|
1382
|
+
};
|
|
1383
|
+
function rowToMemo(row) {
|
|
1384
|
+
const sourceSessionTitle = row["source_session_title"];
|
|
1385
|
+
const projectDir = row["project_dir"];
|
|
1386
|
+
return {
|
|
1387
|
+
id: String(row["id"]),
|
|
1388
|
+
sourceSessionId: String(row["source_session_id"]),
|
|
1389
|
+
sourceSessionTitle: typeof sourceSessionTitle === "string" ? sourceSessionTitle : void 0,
|
|
1390
|
+
userNeed: String(row["user_need"]),
|
|
1391
|
+
approach: String(row["approach"]),
|
|
1392
|
+
outcome: String(row["outcome"]),
|
|
1393
|
+
whatFailed: String(row["what_failed"]),
|
|
1394
|
+
whatWorked: String(row["what_worked"]),
|
|
1395
|
+
extractionSource: row["extraction_source"],
|
|
1396
|
+
recordedAt: Number(row["recorded_at"]),
|
|
1397
|
+
projectDir: typeof projectDir === "string" ? projectDir : "",
|
|
1398
|
+
tags: parseTags(row["tags"])
|
|
1399
|
+
};
|
|
1400
|
+
}
|
|
1401
|
+
function parseTags(value) {
|
|
1402
|
+
if (typeof value !== "string" || value.length === 0) return void 0;
|
|
1403
|
+
try {
|
|
1404
|
+
const parsed = JSON.parse(value);
|
|
1405
|
+
if (!Array.isArray(parsed)) return void 0;
|
|
1406
|
+
const tags = parsed.filter((t) => typeof t === "string");
|
|
1407
|
+
return tags.length > 0 ? tags : void 0;
|
|
1408
|
+
} catch {
|
|
1409
|
+
return;
|
|
1410
|
+
}
|
|
1411
|
+
}
|
|
1412
|
+
function memoMatchesSearch(memo, search) {
|
|
1413
|
+
return [
|
|
1414
|
+
memo.userNeed,
|
|
1415
|
+
memo.approach,
|
|
1416
|
+
memo.whatFailed,
|
|
1417
|
+
memo.whatWorked,
|
|
1418
|
+
memo.sourceSessionTitle ?? "",
|
|
1419
|
+
...memo.tags ?? []
|
|
1420
|
+
].join(" ").toLowerCase().replaceAll(/\s+/g, "").includes(search.replaceAll(/\s+/g, ""));
|
|
1421
|
+
}
|
|
1422
|
+
/**
|
|
1423
|
+
* Tokenize text so FTS5's unicode61 tokenizer can index mixed CJK/ASCII text.
|
|
1424
|
+
* CJK characters are split into individual characters separated by spaces, and
|
|
1425
|
+
* CJK/ASCII boundaries are also separated so "使用redis缓存" becomes searchable
|
|
1426
|
+
* by "redis" as well as by individual CJK characters.
|
|
1427
|
+
*/
|
|
1428
|
+
function toFtsText$1(text) {
|
|
1429
|
+
const parts = text.toLowerCase().replaceAll(/([一-鿿㐀-䶿])([a-z0-9])/g, "$1 $2").replaceAll(/([a-z0-9])([一-鿿㐀-䶿])/g, "$1 $2").split(/[^a-z0-9一-鿿㐀-䶿]+/);
|
|
1430
|
+
const tokens = [];
|
|
1431
|
+
for (const part of parts) {
|
|
1432
|
+
if (part.length === 0) continue;
|
|
1433
|
+
if (/^[a-z0-9]+$/.test(part)) tokens.push(part);
|
|
1434
|
+
else for (const ch of part) if (ch.length > 0) tokens.push(ch);
|
|
1435
|
+
}
|
|
1436
|
+
return tokens.join(" ");
|
|
1437
|
+
}
|
|
1438
|
+
function buildFtsQuery$1(search) {
|
|
1439
|
+
const tokens = toFtsText$1(search).split(/\s+/).filter((t) => t.length > 0);
|
|
1440
|
+
if (tokens.length === 0) return void 0;
|
|
1441
|
+
return tokens.map((t) => `"${t.replaceAll("\"", "\"\"")}"`).join(" AND ");
|
|
1442
|
+
}
|
|
1443
|
+
//#endregion
|
|
1444
|
+
//#region ../../packages/memory/src/extractor.ts
|
|
1445
|
+
/**
|
|
1446
|
+
* Parse memory-memo blocks from LLM compaction output.
|
|
1447
|
+
*/
|
|
1448
|
+
function parseMemoryMemos(text) {
|
|
1449
|
+
const memos = [];
|
|
1450
|
+
const regex = /```memory-memo[\s\S]*?\n([\s\S]*?)```/g;
|
|
1451
|
+
let match;
|
|
1452
|
+
while ((match = regex.exec(text)) !== null) {
|
|
1453
|
+
const jsonStr = match[1]?.trim();
|
|
1454
|
+
if (!jsonStr) continue;
|
|
1455
|
+
try {
|
|
1456
|
+
const parsed = JSON.parse(jsonStr);
|
|
1457
|
+
if (parsed["none"] === true) continue;
|
|
1458
|
+
const userNeed = typeof parsed["userNeed"] === "string" ? parsed["userNeed"].trim() : "";
|
|
1459
|
+
if (userNeed.length === 0) continue;
|
|
1460
|
+
const rawTags = parsed["tags"];
|
|
1461
|
+
const tags = Array.isArray(rawTags) ? normalizeTags(rawTags) : void 0;
|
|
1462
|
+
const note = typeof parsed["note"] === "string" ? parsed["note"].trim() : void 0;
|
|
1463
|
+
memos.push(createMemoryMemo({
|
|
1464
|
+
userNeed,
|
|
1465
|
+
approach: typeof parsed["approach"] === "string" ? parsed["approach"].trim() : "",
|
|
1466
|
+
outcome: typeof parsed["outcome"] === "string" ? parsed["outcome"].trim() : "",
|
|
1467
|
+
whatFailed: typeof parsed["whatFailed"] === "string" ? parsed["whatFailed"].trim() : "none",
|
|
1468
|
+
whatWorked: typeof parsed["whatWorked"] === "string" ? parsed["whatWorked"].trim() : "none",
|
|
1469
|
+
tags,
|
|
1470
|
+
note: note && note.length > 0 ? note : void 0,
|
|
1471
|
+
extractionSource: "compaction",
|
|
1472
|
+
sourceSessionId: "",
|
|
1473
|
+
sourceSessionTitle: ""
|
|
1474
|
+
}));
|
|
1475
|
+
} catch {}
|
|
1476
|
+
}
|
|
1477
|
+
return memos;
|
|
1478
|
+
}
|
|
1479
|
+
/**
|
|
1480
|
+
* Strip injected memory-memo content from text before writing it back to the
|
|
1481
|
+
* store, preventing feedback loops where recalled/compacted content gets
|
|
1482
|
+
* re-stored. Removes fenced ```memory-memo blocks (the format emitted by
|
|
1483
|
+
* compaction and exit-time extraction) and any <memories>...</memories>
|
|
1484
|
+
* wrapper an injector might use.
|
|
1485
|
+
*/
|
|
1486
|
+
function stripMemoryTags(text) {
|
|
1487
|
+
return text.replaceAll(/```memory-memo[\s\S]*?```/gi, "").replaceAll(/<memories>[\s\S]*?<\/memories>/gi, "").trim();
|
|
1488
|
+
}
|
|
1489
|
+
/** System prompt for exit-time extraction — instructs the LLM how to extract. */
|
|
1490
|
+
const EXIT_EXTRACTION_SYSTEM_PROMPT = "你是一个任务经验提取助手。任务是从对话记录中识别已完成的任务闭环,提炼出任务经验记录。用对话的主要语言输出(中文对话用中文,英文对话用英文)。只输出指定的 JSON 格式,不要调用任何工具。";
|
|
1491
|
+
/** Build the user prompt for exit-time extraction, including a conversation sample. */
|
|
1492
|
+
function buildExitExtractionPrompt(sessionId, messageCount, sampleText) {
|
|
1493
|
+
return `以下是会话 "${sessionId}"(共 ${messageCount} 条消息)的对话记录。请提取其中所有**已完成的任务闭环**:
|
|
1494
|
+
|
|
1495
|
+
判断标准:
|
|
1496
|
+
- 用户提出了明确的需求或问题
|
|
1497
|
+
- 给出了解决方案或回答
|
|
1498
|
+
- 结果明确(成功、部分完成、失败)
|
|
1499
|
+
|
|
1500
|
+
对每个已完成的任务闭环,输出一个结构化经验记录。**必须用对话的主要语言书写**:
|
|
1501
|
+
|
|
1502
|
+
\`\`\`memory-memo
|
|
1503
|
+
{
|
|
1504
|
+
"userNeed": "<用户需求/目标,一句话概括>",
|
|
1505
|
+
"approach": "<执行方案,做了什么,2-4 句话>",
|
|
1506
|
+
"outcome": "<最终结果,如'完成'、'部分完成'、'失败:原因'>",
|
|
1507
|
+
"whatFailed": "<踩坑记录:试了但不行的路,无则填 'none'>",
|
|
1508
|
+
"whatWorked": "<成功经验:最终奏效的关键动作,无则填 'none'>",
|
|
1509
|
+
"tags": ["<标签1>", "<标签2>", "<标签3>"],
|
|
1510
|
+
"note": "<可选:给 AI 的补充说明/建议——让未来理解这条经验更容易,如'这类任务先查接口可用性再写代码'。无则省略>"
|
|
1511
|
+
}
|
|
1512
|
+
\`\`\`
|
|
1513
|
+
|
|
1514
|
+
注意:
|
|
1515
|
+
- tags 是 3-5 个语义标签,概括任务领域/技术栈/动作类型,例如 ["react", "auth", "部署"]
|
|
1516
|
+
- whatFailed 记录重要的错误尝试,帮助未来避免重蹈覆辙
|
|
1517
|
+
- whatWorked 记录最终成功的关键动作,帮助未来复用经验
|
|
1518
|
+
- note 是可选字段,写一句能让未来 AI 更快理解/复用这条经验的说明或建议(软性提示,非强制规则)
|
|
1519
|
+
- 跳过未完成的工作,除非其中包含有价值的踩坑经验
|
|
1520
|
+
- 将紧密相关的子任务合并为一条记录
|
|
1521
|
+
- 严格遵守字段名和 JSON 格式(除 note 外不要添加额外字段)
|
|
1522
|
+
|
|
1523
|
+
如果没有已完成的任务闭环,输出:
|
|
1524
|
+
\`\`\`memory-memo
|
|
1525
|
+
{"none": true}
|
|
1526
|
+
\`\`\`
|
|
1527
|
+
|
|
1528
|
+
--- 对话记录(最近 30 条消息)---
|
|
1529
|
+
|
|
1530
|
+
${sampleText}
|
|
1531
|
+
|
|
1532
|
+
--- 对话记录结束 ---`;
|
|
1533
|
+
}
|
|
1534
|
+
//#endregion
|
|
1535
|
+
//#region ../../packages/memory/src/consolidator.ts
|
|
1536
|
+
const SIMILARITY_THRESHOLD = .45;
|
|
1537
|
+
const STALE_DAYS = 30;
|
|
1538
|
+
/**
|
|
1539
|
+
* Analyze all memos and produce a consolidation plan.
|
|
1540
|
+
*
|
|
1541
|
+
* Pure logic — no LLM call. Uses keyword similarity to find near-duplicate
|
|
1542
|
+
* memos, flags resolved/stale entries.
|
|
1543
|
+
*/
|
|
1544
|
+
async function buildConsolidationPlan(store, options) {
|
|
1545
|
+
const allMemos = [];
|
|
1546
|
+
for await (const memo of store.read(options)) allMemos.push(memo);
|
|
1547
|
+
const summaries = allMemos.map(toSummary);
|
|
1548
|
+
const duplicateGroups = findDuplicateGroups(summaries);
|
|
1549
|
+
const relatedGroups = findRelatedGroups(summaries, duplicateGroups);
|
|
1550
|
+
const resolved = findResolved(summaries);
|
|
1551
|
+
const stale = findStale(summaries, STALE_DAYS);
|
|
1552
|
+
const dedupedCount = duplicateGroups.reduce((acc, g) => acc + g.memos.length - 1, 0);
|
|
1553
|
+
return {
|
|
1554
|
+
duplicateGroups,
|
|
1555
|
+
relatedGroups,
|
|
1556
|
+
resolved,
|
|
1557
|
+
stale,
|
|
1558
|
+
summary: {
|
|
1559
|
+
totalMemos: allMemos.length,
|
|
1560
|
+
duplicatesFound: dedupedCount,
|
|
1561
|
+
relatedGroupsFound: relatedGroups.length,
|
|
1562
|
+
resolvedFound: resolved.length,
|
|
1563
|
+
staleFound: stale.length,
|
|
1564
|
+
memosAfterConsolidation: allMemos.length - dedupedCount - resolved.length - stale.length
|
|
1565
|
+
}
|
|
1566
|
+
};
|
|
1567
|
+
}
|
|
1568
|
+
/**
|
|
1569
|
+
* Apply a consolidation plan: delete duplicates, resolved, and stale memos,
|
|
1570
|
+
* appending merged replacements for duplicates.
|
|
1571
|
+
*/
|
|
1572
|
+
async function applyConsolidation(store, plan) {
|
|
1573
|
+
let deleted = 0;
|
|
1574
|
+
let created = 0;
|
|
1575
|
+
if (plan.resolved.length > 0) {
|
|
1576
|
+
const worked = plan.resolved.filter((m) => m.whatWorked).map((m) => m.whatWorked).join("; ");
|
|
1577
|
+
const failed = plan.resolved.filter((m) => m.whatFailed).map((m) => m.whatFailed).join("; ");
|
|
1578
|
+
if (worked || failed) {
|
|
1579
|
+
const archive = createMemoryMemo({
|
|
1580
|
+
sourceSessionId: plan.resolved[0].sourceSessionId,
|
|
1581
|
+
userNeed: "已解决任务的经验归档(dream 整理)",
|
|
1582
|
+
approach: "多次尝试后解决",
|
|
1583
|
+
outcome: "已完成",
|
|
1584
|
+
whatWorked: worked || "none",
|
|
1585
|
+
whatFailed: failed || "none",
|
|
1586
|
+
tags: normalizeTags(plan.resolved.flatMap((m) => m.tags ?? [])),
|
|
1587
|
+
projectDir: plan.resolved[0].projectDir,
|
|
1588
|
+
extractionSource: "compaction"
|
|
1589
|
+
});
|
|
1590
|
+
await store.append(archive);
|
|
1591
|
+
created++;
|
|
1592
|
+
}
|
|
1593
|
+
}
|
|
1594
|
+
if (plan.stale.length > 0) {
|
|
1595
|
+
const worked = plan.stale.filter((m) => m.whatWorked).map((m) => m.whatWorked).join("; ");
|
|
1596
|
+
const failed = plan.stale.filter((m) => m.whatFailed).map((m) => m.whatFailed).join("; ");
|
|
1597
|
+
if (worked || failed) {
|
|
1598
|
+
const archive = createMemoryMemo({
|
|
1599
|
+
sourceSessionId: plan.stale[0].sourceSessionId,
|
|
1600
|
+
userNeed: "过期记忆的经验归档(dream 整理)",
|
|
1601
|
+
approach: "距今超过30天未使用",
|
|
1602
|
+
outcome: "已归档",
|
|
1603
|
+
whatWorked: worked || "none",
|
|
1604
|
+
whatFailed: failed || "none",
|
|
1605
|
+
tags: normalizeTags(plan.stale.flatMap((m) => m.tags ?? [])),
|
|
1606
|
+
projectDir: plan.stale[0].projectDir,
|
|
1607
|
+
extractionSource: "compaction"
|
|
1608
|
+
});
|
|
1609
|
+
await store.append(archive);
|
|
1610
|
+
created++;
|
|
1611
|
+
}
|
|
1612
|
+
}
|
|
1613
|
+
for (const group of plan.duplicateGroups) {
|
|
1614
|
+
const newest = group.memos.reduce((a, b) => a.recordedAt > b.recordedAt ? a : b);
|
|
1615
|
+
const mergedTags = normalizeTags(group.memos.flatMap((m) => m.tags ?? []));
|
|
1616
|
+
const merged = createMemoryMemo({
|
|
1617
|
+
sourceSessionId: newest.sourceSessionId,
|
|
1618
|
+
sourceSessionTitle: newest.sourceSessionTitle,
|
|
1619
|
+
userNeed: group.merged.userNeed,
|
|
1620
|
+
approach: group.merged.approach,
|
|
1621
|
+
outcome: group.merged.outcome,
|
|
1622
|
+
whatFailed: group.merged.whatFailed,
|
|
1623
|
+
whatWorked: group.merged.whatWorked,
|
|
1624
|
+
tags: group.merged.tags ?? mergedTags,
|
|
1625
|
+
projectDir: newest.projectDir,
|
|
1626
|
+
extractionSource: "compaction"
|
|
1627
|
+
});
|
|
1628
|
+
await store.append(merged);
|
|
1629
|
+
created++;
|
|
1630
|
+
}
|
|
1631
|
+
for (const group of plan.duplicateGroups) for (const memo of group.memos) {
|
|
1632
|
+
await store.delete(memo.id);
|
|
1633
|
+
deleted++;
|
|
1634
|
+
}
|
|
1635
|
+
for (const memo of plan.resolved) {
|
|
1636
|
+
await store.delete(memo.id);
|
|
1637
|
+
deleted++;
|
|
1638
|
+
}
|
|
1639
|
+
for (const memo of plan.stale) {
|
|
1640
|
+
await store.delete(memo.id);
|
|
1641
|
+
deleted++;
|
|
1642
|
+
}
|
|
1643
|
+
return {
|
|
1644
|
+
deleted,
|
|
1645
|
+
created
|
|
1646
|
+
};
|
|
1647
|
+
}
|
|
1648
|
+
function findDuplicateGroups(memos) {
|
|
1649
|
+
const groups = [];
|
|
1650
|
+
const used = /* @__PURE__ */ new Set();
|
|
1651
|
+
for (let i = 0; i < memos.length; i++) {
|
|
1652
|
+
const first = memos[i];
|
|
1653
|
+
if (!first || used.has(first.id)) continue;
|
|
1654
|
+
const cluster = [first];
|
|
1655
|
+
for (let j = i + 1; j < memos.length; j++) {
|
|
1656
|
+
const candidate = memos[j];
|
|
1657
|
+
if (!candidate || used.has(candidate.id)) continue;
|
|
1658
|
+
if (cluster.some((m) => {
|
|
1659
|
+
return computeKeywordSimilarity(candidate, `${m.userNeed} ${m.approach}`) >= SIMILARITY_THRESHOLD;
|
|
1660
|
+
})) cluster.push(candidate);
|
|
1661
|
+
}
|
|
1662
|
+
if (cluster.length > 1) {
|
|
1663
|
+
for (const m of cluster) used.add(m.id);
|
|
1664
|
+
groups.push(buildDuplicateGroup(cluster));
|
|
1665
|
+
}
|
|
1666
|
+
}
|
|
1667
|
+
return groups;
|
|
1668
|
+
}
|
|
1669
|
+
/**
|
|
1670
|
+
* Split a whatFailed / whatWorked field into individual claims.
|
|
1671
|
+
* Handles both `;` and `;` separators.
|
|
1672
|
+
*/
|
|
1673
|
+
function splitClaims(text) {
|
|
1674
|
+
if (!text || text === "none" || text === "无") return [];
|
|
1675
|
+
return text.split(/[;;]/).map((s) => s.trim()).filter((s) => s.length > 0);
|
|
1676
|
+
}
|
|
1677
|
+
/**
|
|
1678
|
+
* Extract significant words for contradiction detection:
|
|
1679
|
+
* ASCII words >= 3 chars + CJK 2-grams.
|
|
1680
|
+
*/
|
|
1681
|
+
function extractSignificantWords(text) {
|
|
1682
|
+
const words = [];
|
|
1683
|
+
const lower = text.toLowerCase();
|
|
1684
|
+
for (const m of lower.matchAll(/[a-z0-9]+/g)) if (m[0].length >= 3) words.push(m[0]);
|
|
1685
|
+
for (const m of lower.matchAll(/[一-鿿]+/g)) {
|
|
1686
|
+
const run = m[0];
|
|
1687
|
+
for (let i = 0; i < run.length - 1; i++) words.push(run.slice(i, i + 2));
|
|
1688
|
+
}
|
|
1689
|
+
return words;
|
|
1690
|
+
}
|
|
1691
|
+
/**
|
|
1692
|
+
* Check whether `claim` overlaps with any claim in `against`.
|
|
1693
|
+
* 2+ shared significant words = overlap; 1 word is enough for single-word claims.
|
|
1694
|
+
*/
|
|
1695
|
+
function claimsOverlap(claim, against) {
|
|
1696
|
+
const words = extractSignificantWords(claim);
|
|
1697
|
+
if (words.length === 0) return false;
|
|
1698
|
+
for (const other of against) {
|
|
1699
|
+
const otherLower = other.toLowerCase();
|
|
1700
|
+
const matched = words.filter((w) => otherLower.includes(w)).length;
|
|
1701
|
+
if (matched >= 2 || matched >= 1 && words.length === 1) return true;
|
|
1702
|
+
}
|
|
1703
|
+
return false;
|
|
1704
|
+
}
|
|
1705
|
+
function buildDuplicateGroup(cluster) {
|
|
1706
|
+
const sorted = [...cluster].toSorted((a, b) => b.recordedAt - a.recordedAt);
|
|
1707
|
+
const newest = sorted[0];
|
|
1708
|
+
const mid = Math.ceil(sorted.length / 2);
|
|
1709
|
+
const newer = sorted.slice(0, mid);
|
|
1710
|
+
const newerFailedClaims = new Set(newer.flatMap((m) => splitClaims(m.whatFailed)));
|
|
1711
|
+
const newerWorkedClaims = new Set(newer.flatMap((m) => splitClaims(m.whatWorked)));
|
|
1712
|
+
const failures = /* @__PURE__ */ new Set();
|
|
1713
|
+
const successes = /* @__PURE__ */ new Set();
|
|
1714
|
+
for (const memo of cluster) {
|
|
1715
|
+
for (const claim of splitClaims(memo.whatFailed)) if (!claimsOverlap(claim, newerWorkedClaims)) failures.add(claim);
|
|
1716
|
+
for (const claim of splitClaims(memo.whatWorked)) if (!claimsOverlap(claim, newerFailedClaims)) successes.add(claim);
|
|
1717
|
+
}
|
|
1718
|
+
const bestOutcome = cluster.map((m) => m.outcome).some((o) => o.includes("完成") || o.toLowerCase().includes("done")) ? "完成" : newest.outcome;
|
|
1719
|
+
return {
|
|
1720
|
+
memos: cluster,
|
|
1721
|
+
merged: {
|
|
1722
|
+
userNeed: newest.userNeed,
|
|
1723
|
+
approach: `合并 ${cluster.length} 条相关记录。最新方案: ${newest.approach}`,
|
|
1724
|
+
outcome: bestOutcome,
|
|
1725
|
+
whatFailed: failures.size > 0 ? [...failures].join("; ") : "none",
|
|
1726
|
+
whatWorked: successes.size > 0 ? [...successes].join("; ") : "none"
|
|
1727
|
+
},
|
|
1728
|
+
reason: `发现 ${cluster.length} 条相似记录(关键词重叠 > ${Math.round(SIMILARITY_THRESHOLD * 100)}%)`
|
|
1729
|
+
};
|
|
1730
|
+
}
|
|
1731
|
+
/**
|
|
1732
|
+
* Extract topic anchors from memo text.
|
|
1733
|
+
*
|
|
1734
|
+
* Compound identifiers like `sample-project` are strong signals and kept whole.
|
|
1735
|
+
* ASCII words >= 3 chars are kept when not stopwords. CJK runs are tokenized
|
|
1736
|
+
* into 2-grams so that pure-Chinese themes like "用户认证" can still form
|
|
1737
|
+
* groups without relying on noisy single-character Jaccard overlap.
|
|
1738
|
+
*/
|
|
1739
|
+
function extractTopicAnchors(text) {
|
|
1740
|
+
const lower = text.toLowerCase();
|
|
1741
|
+
const anchors = /* @__PURE__ */ new Set();
|
|
1742
|
+
for (const match of lower.matchAll(/[a-z0-9]+[-_][a-z0-9]+(?:[-_][a-z0-9]+)*/g)) {
|
|
1743
|
+
const token = match[0];
|
|
1744
|
+
if (token.length >= 5) anchors.add(token);
|
|
1745
|
+
}
|
|
1746
|
+
for (const match of lower.matchAll(/[a-z0-9]+/g)) {
|
|
1747
|
+
const token = match[0];
|
|
1748
|
+
if (token.length >= 3 && !STOP_WORDS.has(token)) anchors.add(token);
|
|
1749
|
+
}
|
|
1750
|
+
for (const match of lower.matchAll(/[一-鿿㐀-䶿]+/g)) {
|
|
1751
|
+
const run = match[0];
|
|
1752
|
+
for (let i = 0; i < run.length - 1; i++) anchors.add(run.slice(i, i + 2));
|
|
1753
|
+
}
|
|
1754
|
+
return [...anchors];
|
|
1755
|
+
}
|
|
1756
|
+
function findRelatedGroups(memos, duplicateGroups) {
|
|
1757
|
+
const used = /* @__PURE__ */ new Set();
|
|
1758
|
+
for (const group of duplicateGroups) for (const memo of group.memos) used.add(memo.id);
|
|
1759
|
+
const available = memos.filter((m) => !used.has(m.id));
|
|
1760
|
+
const anchorIndex = /* @__PURE__ */ new Map();
|
|
1761
|
+
for (const memo of available) {
|
|
1762
|
+
const anchors = extractTopicAnchors(`${memo.userNeed} ${memo.approach} ${memo.whatFailed} ${memo.whatWorked}`);
|
|
1763
|
+
for (const anchor of anchors) {
|
|
1764
|
+
const list = anchorIndex.get(anchor) ?? [];
|
|
1765
|
+
if (!list.some((m) => m.id === memo.id)) list.push(memo);
|
|
1766
|
+
anchorIndex.set(anchor, list);
|
|
1767
|
+
}
|
|
1768
|
+
}
|
|
1769
|
+
const groups = [];
|
|
1770
|
+
const assigned = /* @__PURE__ */ new Set();
|
|
1771
|
+
const sortedAnchors = [...anchorIndex.entries()].filter(([, list]) => list.length >= 2).toSorted((a, b) => b[1].length - a[1].length);
|
|
1772
|
+
for (const [anchor, candidates] of sortedAnchors) {
|
|
1773
|
+
const groupMemos = candidates.filter((m) => !assigned.has(m.id));
|
|
1774
|
+
if (groupMemos.length >= 2) {
|
|
1775
|
+
groups.push({
|
|
1776
|
+
memos: groupMemos,
|
|
1777
|
+
topic: anchor,
|
|
1778
|
+
reason: `发现 ${groupMemos.length} 条围绕 ${anchor} 的记录`
|
|
1779
|
+
});
|
|
1780
|
+
for (const m of groupMemos) assigned.add(m.id);
|
|
1781
|
+
}
|
|
1782
|
+
}
|
|
1783
|
+
return groups;
|
|
1784
|
+
}
|
|
1785
|
+
function isOutcomeCompleted(outcome) {
|
|
1786
|
+
const lower = outcome.toLowerCase();
|
|
1787
|
+
return lower.includes("完成") || lower.includes("done") || lower.includes("completed") || lower.includes("成功") || lower.includes("success");
|
|
1788
|
+
}
|
|
1789
|
+
function findResolved(memos) {
|
|
1790
|
+
return memos.filter((m) => isOutcomeCompleted(m.outcome) && Date.now() - m.recordedAt > 10080 * 60 * 1e3);
|
|
1791
|
+
}
|
|
1792
|
+
function findStale(memos, staleDays) {
|
|
1793
|
+
const threshold = Date.now() - staleDays * 24 * 60 * 60 * 1e3;
|
|
1794
|
+
return memos.filter((m) => m.recordedAt < threshold && !isOutcomeCompleted(m.outcome) && !m.outcome.includes("blocked"));
|
|
1795
|
+
}
|
|
1796
|
+
//#endregion
|
|
1797
|
+
//#region ../../packages/memory/src/dream.ts
|
|
1798
|
+
const LOCK_FILE = "dream-lock.json";
|
|
1799
|
+
const MIN_HOURS_BETWEEN_DREAMS = 24;
|
|
1800
|
+
const MIN_SESSIONS_BETWEEN_DREAMS = 5;
|
|
1801
|
+
/**
|
|
1802
|
+
* Tracks dream consolidation state and decides when to suggest running
|
|
1803
|
+
* another dream. Persisted to `<screamHomeDir>/dream-lock.json`.
|
|
1804
|
+
*/
|
|
1805
|
+
var DreamTracker = class {
|
|
1806
|
+
state;
|
|
1807
|
+
lockPath;
|
|
1808
|
+
screamHomeDir;
|
|
1809
|
+
initialized = false;
|
|
1810
|
+
constructor(screamHomeDir) {
|
|
1811
|
+
this.screamHomeDir = screamHomeDir;
|
|
1812
|
+
this.lockPath = join$1(screamHomeDir, LOCK_FILE);
|
|
1813
|
+
this.state = {
|
|
1814
|
+
lastDreamAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1815
|
+
sessionsSinceLastDream: 0
|
|
1816
|
+
};
|
|
1817
|
+
}
|
|
1818
|
+
/** Load persisted state (call once at startup). */
|
|
1819
|
+
async init() {
|
|
1820
|
+
if (this.initialized) return;
|
|
1821
|
+
this.initialized = true;
|
|
1822
|
+
try {
|
|
1823
|
+
const raw = await readFile(this.lockPath, "utf8");
|
|
1824
|
+
const parsed = JSON.parse(raw);
|
|
1825
|
+
if (parsed.version === 1 && parsed.state) {
|
|
1826
|
+
this.state = parsed.state;
|
|
1827
|
+
return;
|
|
1828
|
+
}
|
|
1829
|
+
} catch {}
|
|
1830
|
+
await this.migrateLegacyLockFiles();
|
|
1831
|
+
}
|
|
1832
|
+
/** Record that a dream completed successfully. */
|
|
1833
|
+
async recordDream() {
|
|
1834
|
+
this.state = {
|
|
1835
|
+
lastDreamAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1836
|
+
sessionsSinceLastDream: 0
|
|
1837
|
+
};
|
|
1838
|
+
await this.persist();
|
|
1839
|
+
}
|
|
1840
|
+
/** Call on each new session to bump the session counter. */
|
|
1841
|
+
async recordNewSession() {
|
|
1842
|
+
if (!this.initialized) await this.init();
|
|
1843
|
+
this.state.sessionsSinceLastDream += 1;
|
|
1844
|
+
await this.persist();
|
|
1845
|
+
}
|
|
1846
|
+
/** Check whether it's time to suggest another dream. */
|
|
1847
|
+
shouldSuggest() {
|
|
1848
|
+
return (Date.now() - new Date(this.state.lastDreamAt).getTime()) / (1e3 * 60 * 60) >= MIN_HOURS_BETWEEN_DREAMS && this.state.sessionsSinceLastDream >= MIN_SESSIONS_BETWEEN_DREAMS;
|
|
1849
|
+
}
|
|
1850
|
+
/** Get a human-readable suggestion message when conditions are met. */
|
|
1851
|
+
getSuggestionMessage() {
|
|
1852
|
+
const hoursSince = (Date.now() - new Date(this.state.lastDreamAt).getTime()) / (1e3 * 60 * 60);
|
|
1853
|
+
return `距离上次记忆整理已过去 ${Math.floor(hoursSince / 24)} 天、${this.state.sessionsSinceLastDream} 个会话。建议运行 /dream 来合并重复记忆、清理过期条目、解决矛盾信息。`;
|
|
1854
|
+
}
|
|
1855
|
+
async persist() {
|
|
1856
|
+
const data = {
|
|
1857
|
+
version: 1,
|
|
1858
|
+
state: this.state
|
|
1859
|
+
};
|
|
1860
|
+
try {
|
|
1861
|
+
await mkdir(dirname$1(this.lockPath), { recursive: true });
|
|
1862
|
+
await writeFile(this.lockPath, JSON.stringify(data, null, 2), "utf8");
|
|
1863
|
+
} catch {}
|
|
1864
|
+
}
|
|
1865
|
+
/**
|
|
1866
|
+
* One-time migration for legacy dream-lock.json locations:
|
|
1867
|
+
* - <screamHomeDir>/.scream-code/dream-lock.json (buggy double directory)
|
|
1868
|
+
* - <screamHomeDir>/sessions/<sessionKey>/.scream-code/dream-lock.json (old per-session)
|
|
1869
|
+
*
|
|
1870
|
+
* Picks the most recent state, writes it to the new global location, and
|
|
1871
|
+
* deletes the legacy files/directories.
|
|
1872
|
+
*/
|
|
1873
|
+
async migrateLegacyLockFiles() {
|
|
1874
|
+
const candidates = [join$1(this.screamHomeDir, ".scream-code", LOCK_FILE)];
|
|
1875
|
+
const sessionsDir = join$1(this.screamHomeDir, "sessions");
|
|
1876
|
+
try {
|
|
1877
|
+
const entries = await readdir(sessionsDir, { withFileTypes: true });
|
|
1878
|
+
for (const entry of entries) if (entry.isDirectory()) candidates.push(join$1(sessionsDir, entry.name, ".scream-code", LOCK_FILE));
|
|
1879
|
+
} catch {}
|
|
1880
|
+
let bestState;
|
|
1881
|
+
for (const candidate of candidates) try {
|
|
1882
|
+
await stat(candidate);
|
|
1883
|
+
const raw = await readFile(candidate, "utf8");
|
|
1884
|
+
const parsed = JSON.parse(raw);
|
|
1885
|
+
if (parsed.version === 1 && parsed.state) {
|
|
1886
|
+
if (bestState === void 0 || new Date(parsed.state.lastDreamAt).getTime() > new Date(bestState.lastDreamAt).getTime()) bestState = parsed.state;
|
|
1887
|
+
}
|
|
1888
|
+
} catch {}
|
|
1889
|
+
if (bestState !== void 0) {
|
|
1890
|
+
this.state = bestState;
|
|
1891
|
+
await this.persist();
|
|
1892
|
+
}
|
|
1893
|
+
for (const candidate of candidates) try {
|
|
1894
|
+
await rm(candidate);
|
|
1895
|
+
await rmdir(dirname$1(candidate)).catch(() => {});
|
|
1896
|
+
} catch {}
|
|
1897
|
+
}
|
|
1898
|
+
};
|
|
1899
|
+
//#endregion
|
|
1900
|
+
//#region ../../packages/knowledge/src/store.ts
|
|
1901
|
+
const ENTITY_TYPES$1 = new Set([
|
|
1902
|
+
"person",
|
|
1903
|
+
"organization",
|
|
1904
|
+
"location",
|
|
1905
|
+
"time",
|
|
1906
|
+
"product",
|
|
1907
|
+
"metric",
|
|
1908
|
+
"action",
|
|
1909
|
+
"work",
|
|
1910
|
+
"group",
|
|
1911
|
+
"subject",
|
|
1912
|
+
"tags"
|
|
1913
|
+
]);
|
|
1914
|
+
function generateId(prefix) {
|
|
1915
|
+
return `${prefix}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
|
|
1916
|
+
}
|
|
1917
|
+
function normalizeName(name) {
|
|
1918
|
+
return name.trim().toLowerCase().replaceAll(/\s+/g, " ");
|
|
1919
|
+
}
|
|
1920
|
+
/** Serialize a vector as a raw Float32 BLOB — half the text of JSON, no parse on scan. */
|
|
1921
|
+
function vectorToBlob(vec) {
|
|
1922
|
+
if (vec === null) return null;
|
|
1923
|
+
return Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength);
|
|
1924
|
+
}
|
|
1925
|
+
/**
|
|
1926
|
+
* Decode a stored vector: Float32 BLOB, or legacy JSON text. Both formats are
|
|
1927
|
+
* accepted forever so a half-migrated (or not-yet-migrated) library stays
|
|
1928
|
+
* correct — the migration only shrinks storage, it is not a correctness gate.
|
|
1929
|
+
*/
|
|
1930
|
+
function blobToVector(value) {
|
|
1931
|
+
if (value === null || value === void 0) return null;
|
|
1932
|
+
if (typeof value === "string") try {
|
|
1933
|
+
const arr = JSON.parse(value);
|
|
1934
|
+
return new Float32Array(arr);
|
|
1935
|
+
} catch {
|
|
1936
|
+
return null;
|
|
1937
|
+
}
|
|
1938
|
+
if (!(value instanceof Uint8Array)) return null;
|
|
1939
|
+
if (value.byteLength % 4 !== 0) return null;
|
|
1940
|
+
const copy = value.buffer.slice(value.byteOffset, value.byteOffset + value.byteLength);
|
|
1941
|
+
return new Float32Array(copy);
|
|
1942
|
+
}
|
|
1943
|
+
function isEntityType(value) {
|
|
1944
|
+
return ENTITY_TYPES$1.has(value);
|
|
1945
|
+
}
|
|
1946
|
+
/** Meta key recording which embedding model the stored vectors came from. */
|
|
1947
|
+
const EMBEDDING_MODEL_META_KEY = "embedding_model";
|
|
1948
|
+
var KnowledgeStore = class {
|
|
1949
|
+
dbPath;
|
|
1950
|
+
db;
|
|
1951
|
+
initialized = false;
|
|
1952
|
+
initPromise = null;
|
|
1953
|
+
embeddingEngine;
|
|
1954
|
+
constructor(projectDir) {
|
|
1955
|
+
this.dbPath = join$1(projectDir, "knowledge", "knowledge.db");
|
|
1956
|
+
}
|
|
1957
|
+
async init() {
|
|
1958
|
+
if (this.initPromise) return this.initPromise;
|
|
1959
|
+
this.initPromise = this._doInit().catch((error) => {
|
|
1960
|
+
this.initPromise = null;
|
|
1961
|
+
throw error;
|
|
1962
|
+
});
|
|
1963
|
+
return this.initPromise;
|
|
1964
|
+
}
|
|
1965
|
+
async _doInit() {
|
|
1966
|
+
if (this.initialized) return;
|
|
1967
|
+
await mkdir(dirname$1(this.dbPath), { recursive: true });
|
|
1968
|
+
this.db = new DatabaseSync(this.dbPath);
|
|
1969
|
+
this.db.exec("PRAGMA journal_mode = WAL;");
|
|
1970
|
+
this.db.exec("PRAGMA foreign_keys = ON;");
|
|
1971
|
+
this.db.exec("PRAGMA busy_timeout = 5000;");
|
|
1972
|
+
this.createSchema();
|
|
1973
|
+
this.migrateFtsTokenization();
|
|
1974
|
+
this.migrateVectorsToBlob();
|
|
1975
|
+
this.initialized = true;
|
|
1976
|
+
}
|
|
1977
|
+
/**
|
|
1978
|
+
* One-time migration: FTS rows written before toFtsText pre-tokenization
|
|
1979
|
+
* indexed raw text (CJK runs became single tokens and are unsearchable).
|
|
1980
|
+
* Rebuild all three FTS tables from the content tables with toFtsText.
|
|
1981
|
+
* Tracked via PRAGMA user_version so it runs exactly once.
|
|
1982
|
+
*/
|
|
1983
|
+
migrateFtsTokenization() {
|
|
1984
|
+
if (this.db === void 0) return;
|
|
1985
|
+
if (this.db.prepare("PRAGMA user_version").get().user_version >= 1) return;
|
|
1986
|
+
this.db.exec("DELETE FROM knowledge_chunks_fts");
|
|
1987
|
+
this.db.exec("DELETE FROM knowledge_events_fts");
|
|
1988
|
+
this.db.exec("DELETE FROM knowledge_entities_fts");
|
|
1989
|
+
const chunks = this.db.prepare("SELECT rowid, heading, content FROM knowledge_chunks").all();
|
|
1990
|
+
const insChunk = this.db.prepare("INSERT INTO knowledge_chunks_fts (rowid, heading, content) VALUES (?, ?, ?)");
|
|
1991
|
+
for (const c of chunks) insChunk.run(c.rowid, toFtsText(c.heading ?? ""), toFtsText(c.content));
|
|
1992
|
+
const events = this.db.prepare("SELECT rowid, title, summary, content FROM knowledge_events").all();
|
|
1993
|
+
const insEvent = this.db.prepare("INSERT INTO knowledge_events_fts (rowid, title, summary, content) VALUES (?, ?, ?, ?)");
|
|
1994
|
+
for (const e of events) insEvent.run(e.rowid, toFtsText(e.title), toFtsText(e.summary ?? ""), toFtsText(e.content));
|
|
1995
|
+
const entities = this.db.prepare("SELECT rowid, name, description FROM knowledge_entities").all();
|
|
1996
|
+
const insEntity = this.db.prepare("INSERT INTO knowledge_entities_fts (rowid, name, description) VALUES (?, ?, ?)");
|
|
1997
|
+
for (const e of entities) insEntity.run(e.rowid, toFtsText(e.name), toFtsText(e.description ?? ""));
|
|
1998
|
+
this.db.exec("PRAGMA user_version = 1");
|
|
1999
|
+
}
|
|
2000
|
+
/**
|
|
2001
|
+
* One-time migration (user_version 2): rewrite legacy JSON vector text to
|
|
2002
|
+
* raw Float32 BLOBs. Reads accept both formats forever, so a half-migrated
|
|
2003
|
+
* or failed run stays correct — this only shrinks storage and removes
|
|
2004
|
+
* JSON.parse from the O(n) scan paths. typeof() = 'text' makes the pass
|
|
2005
|
+
* idempotent: converted rows come back as 'blob' and are skipped.
|
|
2006
|
+
*/
|
|
2007
|
+
migrateVectorsToBlob() {
|
|
2008
|
+
if (this.db === void 0) return;
|
|
2009
|
+
if (this.db.prepare("PRAGMA user_version").get().user_version >= 2) return;
|
|
2010
|
+
const columns = [
|
|
2011
|
+
{
|
|
2012
|
+
table: "knowledge_chunks",
|
|
2013
|
+
column: "embedding_json"
|
|
2014
|
+
},
|
|
2015
|
+
{
|
|
2016
|
+
table: "knowledge_events",
|
|
2017
|
+
column: "title_embedding_json"
|
|
2018
|
+
},
|
|
2019
|
+
{
|
|
2020
|
+
table: "knowledge_events",
|
|
2021
|
+
column: "content_embedding_json"
|
|
2022
|
+
},
|
|
2023
|
+
{
|
|
2024
|
+
table: "knowledge_entities",
|
|
2025
|
+
column: "embedding_json"
|
|
2026
|
+
},
|
|
2027
|
+
{
|
|
2028
|
+
table: "knowledge_event_entities",
|
|
2029
|
+
column: "embedding_json"
|
|
2030
|
+
}
|
|
2031
|
+
];
|
|
2032
|
+
this.db.exec("BEGIN");
|
|
2033
|
+
try {
|
|
2034
|
+
for (const { table, column } of columns) {
|
|
2035
|
+
const rows = this.db.prepare(`SELECT id AS pk, ${column} AS v FROM ${table}
|
|
2036
|
+
WHERE ${column} IS NOT NULL AND typeof(${column}) = 'text'`).all();
|
|
2037
|
+
if (rows.length === 0) continue;
|
|
2038
|
+
const update = this.db.prepare(`UPDATE ${table} SET ${column} = ? WHERE id = ?`);
|
|
2039
|
+
for (const row of rows) {
|
|
2040
|
+
const vec = blobToVector(row.v);
|
|
2041
|
+
if (vec === null) continue;
|
|
2042
|
+
update.run(vectorToBlob(vec), row.pk);
|
|
2043
|
+
}
|
|
2044
|
+
}
|
|
2045
|
+
this.db.exec("PRAGMA user_version = 2");
|
|
2046
|
+
this.db.exec("COMMIT");
|
|
2047
|
+
} catch (error) {
|
|
2048
|
+
this.db.exec("ROLLBACK");
|
|
2049
|
+
throw error;
|
|
2050
|
+
}
|
|
2051
|
+
}
|
|
2052
|
+
close() {
|
|
2053
|
+
if (this.db !== void 0) {
|
|
2054
|
+
this.db.exec("PRAGMA wal_checkpoint(TRUNCATE)");
|
|
2055
|
+
this.db.close();
|
|
2056
|
+
this.db = void 0;
|
|
2057
|
+
}
|
|
2058
|
+
this.initialized = false;
|
|
2059
|
+
this.initPromise = null;
|
|
2060
|
+
}
|
|
2061
|
+
setEmbeddingEngine(engine) {
|
|
2062
|
+
this.embeddingEngine = engine;
|
|
2063
|
+
}
|
|
2064
|
+
getEmbeddingEngine() {
|
|
2065
|
+
return this.embeddingEngine;
|
|
2066
|
+
}
|
|
2067
|
+
/**
|
|
2068
|
+
* Begin a write transaction. Use commitTransaction / rollbackTransaction
|
|
2069
|
+
* to close it. Used by ingest to keep chunks/events/entities/edges atomic —
|
|
2070
|
+
* a mid-ingest failure leaves no partial rows.
|
|
2071
|
+
*/
|
|
2072
|
+
beginTransaction() {
|
|
2073
|
+
if (this.db === void 0) throw new Error("knowledge store not initialized");
|
|
2074
|
+
this.db.exec("BEGIN");
|
|
2075
|
+
}
|
|
2076
|
+
commitTransaction() {
|
|
2077
|
+
if (this.db === void 0) throw new Error("knowledge store not initialized");
|
|
2078
|
+
this.db.exec("COMMIT");
|
|
2079
|
+
}
|
|
2080
|
+
rollbackTransaction() {
|
|
2081
|
+
if (this.db === void 0) return;
|
|
2082
|
+
try {
|
|
2083
|
+
this.db.exec("ROLLBACK");
|
|
2084
|
+
} catch {}
|
|
2085
|
+
}
|
|
2086
|
+
/**
|
|
2087
|
+
* Read a metadata value (e.g. the embedding model identity the library was
|
|
2088
|
+
* built with). Returns null when the key is absent.
|
|
2089
|
+
*/
|
|
2090
|
+
async getMeta(key) {
|
|
2091
|
+
await this.init();
|
|
2092
|
+
if (this.db === void 0) return null;
|
|
2093
|
+
const row = this.db.prepare("SELECT value FROM knowledge_meta WHERE key = ?").get(key);
|
|
2094
|
+
return row === void 0 ? null : asString(row["value"]);
|
|
2095
|
+
}
|
|
2096
|
+
/** Write a metadata value (upsert). */
|
|
2097
|
+
async setMeta(key, value) {
|
|
2098
|
+
await this.init();
|
|
2099
|
+
if (this.db === void 0) throw new Error("knowledge store not initialized");
|
|
2100
|
+
this.db.prepare("INSERT INTO knowledge_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value").run(key, value);
|
|
2101
|
+
}
|
|
2102
|
+
createSchema() {
|
|
2103
|
+
if (this.db === void 0) return;
|
|
2104
|
+
this.db.exec(`
|
|
2105
|
+
CREATE TABLE IF NOT EXISTS knowledge_sources (
|
|
2106
|
+
id TEXT PRIMARY KEY,
|
|
2107
|
+
name TEXT NOT NULL,
|
|
2108
|
+
file_path TEXT,
|
|
2109
|
+
description TEXT,
|
|
2110
|
+
created_at INTEGER NOT NULL
|
|
2111
|
+
);
|
|
2112
|
+
|
|
2113
|
+
CREATE TABLE IF NOT EXISTS knowledge_documents (
|
|
2114
|
+
id TEXT PRIMARY KEY,
|
|
2115
|
+
source_id TEXT NOT NULL REFERENCES knowledge_sources(id) ON DELETE CASCADE,
|
|
2116
|
+
title TEXT NOT NULL,
|
|
2117
|
+
content TEXT,
|
|
2118
|
+
status TEXT NOT NULL DEFAULT 'pending',
|
|
2119
|
+
chunk_count INTEGER DEFAULT 0,
|
|
2120
|
+
created_at INTEGER NOT NULL
|
|
2121
|
+
);
|
|
2122
|
+
|
|
2123
|
+
CREATE INDEX IF NOT EXISTS idx_knowledge_documents_source ON knowledge_documents(source_id);
|
|
2124
|
+
|
|
2125
|
+
CREATE TABLE IF NOT EXISTS knowledge_chunks (
|
|
2126
|
+
id TEXT PRIMARY KEY,
|
|
2127
|
+
source_id TEXT NOT NULL REFERENCES knowledge_sources(id) ON DELETE CASCADE,
|
|
2128
|
+
document_id TEXT NOT NULL REFERENCES knowledge_documents(id) ON DELETE CASCADE,
|
|
2129
|
+
rank INTEGER NOT NULL,
|
|
2130
|
+
heading TEXT,
|
|
2131
|
+
content TEXT NOT NULL,
|
|
2132
|
+
raw_content TEXT,
|
|
2133
|
+
embedding_json BLOB,
|
|
2134
|
+
created_at INTEGER NOT NULL
|
|
2135
|
+
);
|
|
2136
|
+
|
|
2137
|
+
CREATE INDEX IF NOT EXISTS idx_knowledge_chunks_doc ON knowledge_chunks(document_id);
|
|
2138
|
+
CREATE INDEX IF NOT EXISTS idx_knowledge_chunks_source ON knowledge_chunks(source_id);
|
|
2139
|
+
|
|
2140
|
+
CREATE TABLE IF NOT EXISTS knowledge_events (
|
|
2141
|
+
id TEXT PRIMARY KEY,
|
|
2142
|
+
source_id TEXT NOT NULL REFERENCES knowledge_sources(id) ON DELETE CASCADE,
|
|
2143
|
+
document_id TEXT NOT NULL REFERENCES knowledge_documents(id) ON DELETE CASCADE,
|
|
2144
|
+
chunk_id TEXT NOT NULL REFERENCES knowledge_chunks(id) ON DELETE CASCADE,
|
|
2145
|
+
rank INTEGER NOT NULL,
|
|
2146
|
+
title TEXT NOT NULL,
|
|
2147
|
+
summary TEXT,
|
|
2148
|
+
content TEXT NOT NULL,
|
|
2149
|
+
category TEXT,
|
|
2150
|
+
keywords TEXT,
|
|
2151
|
+
title_embedding_json BLOB,
|
|
2152
|
+
content_embedding_json BLOB,
|
|
2153
|
+
created_at INTEGER NOT NULL
|
|
2154
|
+
);
|
|
2155
|
+
|
|
2156
|
+
CREATE INDEX IF NOT EXISTS idx_knowledge_events_chunk ON knowledge_events(chunk_id);
|
|
2157
|
+
CREATE INDEX IF NOT EXISTS idx_knowledge_events_doc ON knowledge_events(document_id);
|
|
2158
|
+
CREATE INDEX IF NOT EXISTS idx_knowledge_events_source ON knowledge_events(source_id);
|
|
2159
|
+
|
|
2160
|
+
CREATE TABLE IF NOT EXISTS knowledge_entities (
|
|
2161
|
+
id TEXT PRIMARY KEY,
|
|
2162
|
+
source_id TEXT NOT NULL REFERENCES knowledge_sources(id) ON DELETE CASCADE,
|
|
2163
|
+
type TEXT NOT NULL,
|
|
2164
|
+
name TEXT NOT NULL,
|
|
2165
|
+
normalized_name TEXT NOT NULL,
|
|
2166
|
+
description TEXT,
|
|
2167
|
+
embedding_json BLOB,
|
|
2168
|
+
created_at INTEGER NOT NULL,
|
|
2169
|
+
UNIQUE(source_id, type, normalized_name)
|
|
2170
|
+
);
|
|
2171
|
+
|
|
2172
|
+
CREATE INDEX IF NOT EXISTS idx_knowledge_entities_source ON knowledge_entities(source_id);
|
|
2173
|
+
CREATE INDEX IF NOT EXISTS idx_knowledge_entities_name ON knowledge_entities(normalized_name);
|
|
2174
|
+
|
|
2175
|
+
CREATE TABLE IF NOT EXISTS knowledge_event_entities (
|
|
2176
|
+
id TEXT PRIMARY KEY,
|
|
2177
|
+
event_id TEXT NOT NULL REFERENCES knowledge_events(id) ON DELETE CASCADE,
|
|
2178
|
+
entity_id TEXT NOT NULL REFERENCES knowledge_entities(id) ON DELETE CASCADE,
|
|
2179
|
+
weight REAL DEFAULT 1.0,
|
|
2180
|
+
description TEXT,
|
|
2181
|
+
embedding_json BLOB,
|
|
2182
|
+
UNIQUE(event_id, entity_id)
|
|
2183
|
+
);
|
|
2184
|
+
|
|
2185
|
+
CREATE INDEX IF NOT EXISTS idx_knowledge_event_entities_event ON knowledge_event_entities(event_id);
|
|
2186
|
+
CREATE INDEX IF NOT EXISTS idx_knowledge_event_entities_entity ON knowledge_event_entities(entity_id);
|
|
2187
|
+
|
|
2188
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS knowledge_chunks_fts USING fts5(
|
|
2189
|
+
heading, content, content='knowledge_chunks', content_rowid='rowid'
|
|
2190
|
+
);
|
|
2191
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS knowledge_events_fts USING fts5(
|
|
2192
|
+
title, summary, content, content='knowledge_events', content_rowid='rowid'
|
|
2193
|
+
);
|
|
2194
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS knowledge_entities_fts USING fts5(
|
|
2195
|
+
name, description, content='knowledge_entities', content_rowid='rowid'
|
|
2196
|
+
);
|
|
2197
|
+
|
|
2198
|
+
CREATE TABLE IF NOT EXISTS knowledge_meta (
|
|
2199
|
+
key TEXT PRIMARY KEY,
|
|
2200
|
+
value TEXT NOT NULL
|
|
2201
|
+
);
|
|
2202
|
+
`);
|
|
2203
|
+
try {
|
|
2204
|
+
this.db.exec("ALTER TABLE knowledge_sources ADD COLUMN content_hash TEXT");
|
|
2205
|
+
} catch (error) {
|
|
2206
|
+
if (!(error instanceof Error ? error.message : String(error)).includes("duplicate column")) throw error;
|
|
2207
|
+
}
|
|
2208
|
+
}
|
|
2209
|
+
async createSource(params) {
|
|
2210
|
+
await this.init();
|
|
2211
|
+
if (this.db === void 0) throw new Error("knowledge store not initialized");
|
|
2212
|
+
const id = generateId("src");
|
|
2213
|
+
const createdAt = Date.now();
|
|
2214
|
+
this.db.prepare("INSERT INTO knowledge_sources (id, name, file_path, description, content_hash, created_at) VALUES (?, ?, ?, ?, ?, ?)").run(id, params.name, params.filePath ?? null, params.description ?? null, params.contentHash ?? null, createdAt);
|
|
2215
|
+
return {
|
|
2216
|
+
id,
|
|
2217
|
+
name: params.name,
|
|
2218
|
+
filePath: params.filePath ?? null,
|
|
2219
|
+
description: params.description ?? null,
|
|
2220
|
+
contentHash: params.contentHash ?? null,
|
|
2221
|
+
createdAt
|
|
2222
|
+
};
|
|
2223
|
+
}
|
|
2224
|
+
/** Backfill or refresh a source's content fingerprint (legacy rows / replacements). */
|
|
2225
|
+
async updateSourceContentHash(id, contentHash) {
|
|
2226
|
+
await this.init();
|
|
2227
|
+
if (this.db === void 0) throw new Error("knowledge store not initialized");
|
|
2228
|
+
this.db.prepare("UPDATE knowledge_sources SET content_hash = ? WHERE id = ?").run(contentHash, id);
|
|
2229
|
+
}
|
|
2230
|
+
async findSourceByFilePath(filePath) {
|
|
2231
|
+
await this.init();
|
|
2232
|
+
if (this.db === void 0) return void 0;
|
|
2233
|
+
const row = this.db.prepare("SELECT * FROM knowledge_sources WHERE file_path = ?").get(filePath);
|
|
2234
|
+
return row === void 0 ? void 0 : rowToSource(row);
|
|
2235
|
+
}
|
|
2236
|
+
async listSources() {
|
|
2237
|
+
await this.init();
|
|
2238
|
+
if (this.db === void 0) return [];
|
|
2239
|
+
return this.db.prepare("SELECT * FROM knowledge_sources ORDER BY created_at DESC").all().map(rowToSource);
|
|
2240
|
+
}
|
|
2241
|
+
async getSource(id) {
|
|
2242
|
+
await this.init();
|
|
2243
|
+
if (this.db === void 0) return void 0;
|
|
2244
|
+
const row = this.db.prepare("SELECT * FROM knowledge_sources WHERE id = ?").get(id);
|
|
2245
|
+
return row === void 0 ? void 0 : rowToSource(row);
|
|
2246
|
+
}
|
|
2247
|
+
async deleteSource(id) {
|
|
2248
|
+
await this.init();
|
|
2249
|
+
if (this.db === void 0) return false;
|
|
2250
|
+
const chunkRows = this.db.prepare("SELECT rowid, heading, content FROM knowledge_chunks WHERE source_id = ?").all(id);
|
|
2251
|
+
const eventRows = this.db.prepare("SELECT rowid, title, summary, content FROM knowledge_events WHERE source_id = ?").all(id);
|
|
2252
|
+
const entityRows = this.db.prepare("SELECT rowid, name, description FROM knowledge_entities WHERE source_id = ?").all(id);
|
|
2253
|
+
const result = this.db.prepare("DELETE FROM knowledge_sources WHERE id = ?").run(id);
|
|
2254
|
+
if (result.changes > 0) {
|
|
2255
|
+
const delChunk = this.db.prepare("INSERT INTO knowledge_chunks_fts (knowledge_chunks_fts, rowid, heading, content) VALUES ('delete', ?, ?, ?)");
|
|
2256
|
+
for (const row of chunkRows) delChunk.run(row.rowid, toFtsText(row.heading ?? ""), toFtsText(row.content));
|
|
2257
|
+
const delEvent = this.db.prepare("INSERT INTO knowledge_events_fts (knowledge_events_fts, rowid, title, summary, content) VALUES ('delete', ?, ?, ?, ?)");
|
|
2258
|
+
for (const row of eventRows) delEvent.run(row.rowid, toFtsText(row.title), toFtsText(row.summary ?? ""), toFtsText(row.content));
|
|
2259
|
+
const delEntity = this.db.prepare("INSERT INTO knowledge_entities_fts (knowledge_entities_fts, rowid, name, description) VALUES ('delete', ?, ?, ?)");
|
|
2260
|
+
for (const row of entityRows) delEntity.run(row.rowid, toFtsText(row.name), toFtsText(row.description ?? ""));
|
|
2261
|
+
}
|
|
2262
|
+
return result.changes > 0;
|
|
2263
|
+
}
|
|
2264
|
+
async createDocument(params) {
|
|
2265
|
+
if (this.db === void 0) throw new Error("knowledge store not initialized");
|
|
2266
|
+
const id = generateId("doc");
|
|
2267
|
+
const createdAt = Date.now();
|
|
2268
|
+
this.db.prepare("INSERT INTO knowledge_documents (id, source_id, title, content, status, chunk_count, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)").run(id, params.sourceId, params.title, params.content ?? null, "pending", 0, createdAt);
|
|
2269
|
+
return {
|
|
2270
|
+
id,
|
|
2271
|
+
sourceId: params.sourceId,
|
|
2272
|
+
title: params.title,
|
|
2273
|
+
content: params.content ?? null,
|
|
2274
|
+
status: "pending",
|
|
2275
|
+
chunkCount: 0,
|
|
2276
|
+
createdAt
|
|
2277
|
+
};
|
|
2278
|
+
}
|
|
2279
|
+
async updateDocumentStatus(documentId, status, chunkCount) {
|
|
2280
|
+
if (this.db === void 0) return;
|
|
2281
|
+
if (chunkCount !== void 0) this.db.prepare("UPDATE knowledge_documents SET status = ?, chunk_count = ? WHERE id = ?").run(status, chunkCount, documentId);
|
|
2282
|
+
else this.db.prepare("UPDATE knowledge_documents SET status = ? WHERE id = ?").run(status, documentId);
|
|
2283
|
+
}
|
|
2284
|
+
async listDocuments() {
|
|
2285
|
+
await this.init();
|
|
2286
|
+
if (this.db === void 0) return [];
|
|
2287
|
+
return this.db.prepare(`SELECT d.*, s.name AS source_name
|
|
2288
|
+
FROM knowledge_documents d
|
|
2289
|
+
JOIN knowledge_sources s ON s.id = d.source_id
|
|
2290
|
+
ORDER BY d.created_at DESC`).all().map((row) => ({
|
|
2291
|
+
...rowToDocument(row),
|
|
2292
|
+
sourceName: String(row["source_name"])
|
|
2293
|
+
}));
|
|
2294
|
+
}
|
|
2295
|
+
async getDocument(id) {
|
|
2296
|
+
await this.init();
|
|
2297
|
+
if (this.db === void 0) return void 0;
|
|
2298
|
+
const row = this.db.prepare("SELECT * FROM knowledge_documents WHERE id = ?").get(id);
|
|
2299
|
+
return row === void 0 ? void 0 : rowToDocument(row);
|
|
2300
|
+
}
|
|
2301
|
+
async insertChunk(params) {
|
|
2302
|
+
if (this.db === void 0) throw new Error("knowledge store not initialized");
|
|
2303
|
+
const id = generateId("chk");
|
|
2304
|
+
const createdAt = Date.now();
|
|
2305
|
+
const embeddingBlob = vectorToBlob(params.embedding);
|
|
2306
|
+
this.db.prepare(`INSERT INTO knowledge_chunks
|
|
2307
|
+
(id, source_id, document_id, rank, heading, content, raw_content, embedding_json, created_at)
|
|
2308
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(id, params.sourceId, params.documentId, params.rank, params.heading, params.content, params.rawContent, embeddingBlob, createdAt);
|
|
2309
|
+
this.db.prepare(`INSERT INTO knowledge_chunks_fts (rowid, heading, content)
|
|
2310
|
+
VALUES ((SELECT rowid FROM knowledge_chunks WHERE id = ?), ?, ?)`).run(id, toFtsText(params.heading ?? ""), toFtsText(params.content));
|
|
2311
|
+
return {
|
|
2312
|
+
id,
|
|
2313
|
+
sourceId: params.sourceId,
|
|
2314
|
+
documentId: params.documentId,
|
|
2315
|
+
rank: params.rank,
|
|
2316
|
+
heading: params.heading,
|
|
2317
|
+
content: params.content,
|
|
2318
|
+
rawContent: params.rawContent,
|
|
2319
|
+
embedding: params.embedding,
|
|
2320
|
+
createdAt
|
|
2321
|
+
};
|
|
2322
|
+
}
|
|
2323
|
+
async getChunk(id) {
|
|
2324
|
+
await this.init();
|
|
2325
|
+
if (this.db === void 0) return void 0;
|
|
2326
|
+
const row = this.db.prepare("SELECT * FROM knowledge_chunks WHERE id = ?").get(id);
|
|
2327
|
+
return row === void 0 ? void 0 : rowToChunk(row);
|
|
2328
|
+
}
|
|
2329
|
+
async listChunksByDocument(documentId) {
|
|
2330
|
+
await this.init();
|
|
2331
|
+
if (this.db === void 0) return [];
|
|
2332
|
+
return this.db.prepare("SELECT * FROM knowledge_chunks WHERE document_id = ? ORDER BY rank ASC").all(documentId).map(rowToChunk);
|
|
2333
|
+
}
|
|
2334
|
+
/** Load all chunk embeddings + ids for vector search. */
|
|
2335
|
+
async loadAllChunkEmbeddings() {
|
|
2336
|
+
await this.init();
|
|
2337
|
+
if (this.db === void 0) return [];
|
|
2338
|
+
const rows = this.db.prepare("SELECT id, embedding_json FROM knowledge_chunks WHERE embedding_json IS NOT NULL").all();
|
|
2339
|
+
const out = [];
|
|
2340
|
+
for (const row of rows) {
|
|
2341
|
+
const vec = blobToVector(row.embedding_json);
|
|
2342
|
+
if (vec !== null) out.push({
|
|
2343
|
+
id: row.id,
|
|
2344
|
+
embedding: vec
|
|
2345
|
+
});
|
|
2346
|
+
}
|
|
2347
|
+
return out;
|
|
2348
|
+
}
|
|
2349
|
+
async insertEvent(params) {
|
|
2350
|
+
if (this.db === void 0) throw new Error("knowledge store not initialized");
|
|
2351
|
+
const id = generateId("evt");
|
|
2352
|
+
const createdAt = Date.now();
|
|
2353
|
+
this.db.prepare(`INSERT INTO knowledge_events
|
|
2354
|
+
(id, source_id, document_id, chunk_id, rank, title, summary, content, category, keywords,
|
|
2355
|
+
title_embedding_json, content_embedding_json, created_at)
|
|
2356
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(id, params.sourceId, params.documentId, params.chunkId, params.rank, params.title, params.summary, params.content, params.category, JSON.stringify(params.keywords), vectorToBlob(params.titleEmbedding), vectorToBlob(params.contentEmbedding), createdAt);
|
|
2357
|
+
this.db.prepare(`INSERT INTO knowledge_events_fts (rowid, title, summary, content)
|
|
2358
|
+
VALUES ((SELECT rowid FROM knowledge_events WHERE id = ?), ?, ?, ?)`).run(id, toFtsText(params.title), toFtsText(params.summary ?? ""), toFtsText(params.content));
|
|
2359
|
+
return {
|
|
2360
|
+
id,
|
|
2361
|
+
sourceId: params.sourceId,
|
|
2362
|
+
documentId: params.documentId,
|
|
2363
|
+
chunkId: params.chunkId,
|
|
2364
|
+
rank: params.rank,
|
|
2365
|
+
title: params.title,
|
|
2366
|
+
summary: params.summary,
|
|
2367
|
+
content: params.content,
|
|
2368
|
+
category: params.category,
|
|
2369
|
+
keywords: params.keywords,
|
|
2370
|
+
titleEmbedding: params.titleEmbedding,
|
|
2371
|
+
contentEmbedding: params.contentEmbedding,
|
|
2372
|
+
createdAt
|
|
2373
|
+
};
|
|
2374
|
+
}
|
|
2375
|
+
async getEvent(id) {
|
|
2376
|
+
await this.init();
|
|
2377
|
+
if (this.db === void 0) return void 0;
|
|
2378
|
+
const row = this.db.prepare("SELECT * FROM knowledge_events WHERE id = ?").get(id);
|
|
2379
|
+
return row === void 0 ? void 0 : rowToEvent(row);
|
|
2380
|
+
}
|
|
2381
|
+
async listEventsByDocument(documentId) {
|
|
2382
|
+
await this.init();
|
|
2383
|
+
if (this.db === void 0) return [];
|
|
2384
|
+
return this.db.prepare("SELECT * FROM knowledge_events WHERE document_id = ? ORDER BY rank ASC").all(documentId).map(rowToEvent);
|
|
2385
|
+
}
|
|
2386
|
+
/** Find events by title vector similarity. */
|
|
2387
|
+
async findEventsByTitleVector(queryVec, options = {}) {
|
|
2388
|
+
await this.init();
|
|
2389
|
+
if (this.db === void 0) return [];
|
|
2390
|
+
const limit = options.limit ?? 20;
|
|
2391
|
+
const threshold = options.threshold ?? 0;
|
|
2392
|
+
const rows = this.db.prepare("SELECT id, title_embedding_json FROM knowledge_events WHERE title_embedding_json IS NOT NULL").all();
|
|
2393
|
+
const scored = [];
|
|
2394
|
+
for (const row of rows) {
|
|
2395
|
+
const vec = blobToVector(row.title_embedding_json);
|
|
2396
|
+
if (vec === null) continue;
|
|
2397
|
+
const score = this.embeddingEngine?.cosineSimilarity(queryVec, vec) ?? 0;
|
|
2398
|
+
if (score >= threshold) scored.push({
|
|
2399
|
+
id: row.id,
|
|
2400
|
+
score
|
|
2401
|
+
});
|
|
2402
|
+
}
|
|
2403
|
+
scored.sort((a, b) => b.score - a.score);
|
|
2404
|
+
const top = scored.slice(0, limit);
|
|
2405
|
+
const out = [];
|
|
2406
|
+
for (const item of top) {
|
|
2407
|
+
const event = await this.getEvent(item.id);
|
|
2408
|
+
if (event !== void 0) out.push({
|
|
2409
|
+
event,
|
|
2410
|
+
score: item.score
|
|
2411
|
+
});
|
|
2412
|
+
}
|
|
2413
|
+
return out;
|
|
2414
|
+
}
|
|
2415
|
+
async upsertEntity(params) {
|
|
2416
|
+
if (this.db === void 0) throw new Error("knowledge store not initialized");
|
|
2417
|
+
if (!isEntityType(params.type)) throw new Error(`invalid entity type: ${params.type}`);
|
|
2418
|
+
const normalizedName = normalizeName(params.name);
|
|
2419
|
+
const existing = this.db.prepare("SELECT * FROM knowledge_entities WHERE source_id = ? AND type = ? AND normalized_name = ?").get(params.sourceId, params.type, normalizedName);
|
|
2420
|
+
if (existing !== void 0) {
|
|
2421
|
+
const existingId = String(existing["id"]);
|
|
2422
|
+
let descriptionChanged = false;
|
|
2423
|
+
if (params.description !== null && existing["description"] === null) {
|
|
2424
|
+
this.db.prepare("UPDATE knowledge_entities SET description = ? WHERE id = ?").run(params.description, existingId);
|
|
2425
|
+
existing["description"] = params.description;
|
|
2426
|
+
descriptionChanged = true;
|
|
2427
|
+
}
|
|
2428
|
+
if (params.embedding !== null && existing["embedding_json"] === null) {
|
|
2429
|
+
this.db.prepare("UPDATE knowledge_entities SET embedding_json = ? WHERE id = ?").run(vectorToBlob(params.embedding), existingId);
|
|
2430
|
+
existing["embedding_json"] = vectorToBlob(params.embedding);
|
|
2431
|
+
}
|
|
2432
|
+
if (descriptionChanged) {
|
|
2433
|
+
const ftsRow = this.db.prepare("SELECT rowid, name, description FROM knowledge_entities WHERE id = ?").get(existingId);
|
|
2434
|
+
if (ftsRow !== void 0) {
|
|
2435
|
+
this.db.prepare("INSERT INTO knowledge_entities_fts (knowledge_entities_fts, rowid, name, description) VALUES ('delete', ?, ?, ?)").run(ftsRow.rowid, toFtsText(ftsRow.name), "");
|
|
2436
|
+
this.db.prepare(`INSERT INTO knowledge_entities_fts (rowid, name, description) VALUES (?, ?, ?)`).run(ftsRow.rowid, toFtsText(ftsRow.name), toFtsText(ftsRow.description ?? ""));
|
|
2437
|
+
}
|
|
2438
|
+
}
|
|
2439
|
+
return rowToEntity(existing);
|
|
2440
|
+
}
|
|
2441
|
+
const id = generateId("ent");
|
|
2442
|
+
const createdAt = Date.now();
|
|
2443
|
+
this.db.prepare(`INSERT INTO knowledge_entities
|
|
2444
|
+
(id, source_id, type, name, normalized_name, description, embedding_json, created_at)
|
|
2445
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`).run(id, params.sourceId, params.type, params.name, normalizedName, params.description, vectorToBlob(params.embedding), createdAt);
|
|
2446
|
+
this.db.prepare(`INSERT INTO knowledge_entities_fts (rowid, name, description)
|
|
2447
|
+
VALUES ((SELECT rowid FROM knowledge_entities WHERE id = ?), ?, ?)`).run(id, toFtsText(params.name), toFtsText(params.description ?? ""));
|
|
2448
|
+
return {
|
|
2449
|
+
id,
|
|
2450
|
+
sourceId: params.sourceId,
|
|
2451
|
+
type: params.type,
|
|
2452
|
+
name: params.name,
|
|
2453
|
+
normalizedName,
|
|
2454
|
+
description: params.description,
|
|
2455
|
+
embedding: params.embedding,
|
|
2456
|
+
createdAt
|
|
2457
|
+
};
|
|
2458
|
+
}
|
|
2459
|
+
async findEntitiesByName(name, sourceId) {
|
|
2460
|
+
await this.init();
|
|
2461
|
+
if (this.db === void 0) return [];
|
|
2462
|
+
const normalized = normalizeName(name);
|
|
2463
|
+
return (sourceId === void 0 ? this.db.prepare("SELECT * FROM knowledge_entities WHERE normalized_name = ?").all(normalized) : this.db.prepare("SELECT * FROM knowledge_entities WHERE normalized_name = ? AND source_id = ?").all(normalized, sourceId)).map(rowToEntity);
|
|
2464
|
+
}
|
|
2465
|
+
/** Find entities by name vector similarity. */
|
|
2466
|
+
async findEntitiesByVector(queryVec, options = {}) {
|
|
2467
|
+
await this.init();
|
|
2468
|
+
if (this.db === void 0) return [];
|
|
2469
|
+
const limit = options.limit ?? 20;
|
|
2470
|
+
const threshold = options.threshold ?? 0;
|
|
2471
|
+
const rows = this.db.prepare("SELECT id, embedding_json FROM knowledge_entities WHERE embedding_json IS NOT NULL").all();
|
|
2472
|
+
const scored = [];
|
|
2473
|
+
for (const row of rows) {
|
|
2474
|
+
const vec = blobToVector(row.embedding_json);
|
|
2475
|
+
if (vec === null) continue;
|
|
2476
|
+
const score = this.embeddingEngine?.cosineSimilarity(queryVec, vec) ?? 0;
|
|
2477
|
+
if (score >= threshold) scored.push({
|
|
2478
|
+
id: row.id,
|
|
2479
|
+
score
|
|
2480
|
+
});
|
|
2481
|
+
}
|
|
2482
|
+
scored.sort((a, b) => b.score - a.score);
|
|
2483
|
+
const top = scored.slice(0, limit);
|
|
2484
|
+
const out = [];
|
|
2485
|
+
for (const item of top) {
|
|
2486
|
+
const row = this.db.prepare("SELECT * FROM knowledge_entities WHERE id = ?").get(item.id);
|
|
2487
|
+
if (row !== void 0) out.push({
|
|
2488
|
+
entity: rowToEntity(row),
|
|
2489
|
+
score: item.score
|
|
2490
|
+
});
|
|
2491
|
+
}
|
|
2492
|
+
return out;
|
|
2493
|
+
}
|
|
2494
|
+
async insertEventEntity(params) {
|
|
2495
|
+
if (this.db === void 0) throw new Error("knowledge store not initialized");
|
|
2496
|
+
const id = generateId("ee");
|
|
2497
|
+
this.db.prepare(`INSERT OR IGNORE INTO knowledge_event_entities
|
|
2498
|
+
(id, event_id, entity_id, weight, description, embedding_json)
|
|
2499
|
+
VALUES (?, ?, ?, ?, ?, ?)`).run(id, params.eventId, params.entityId, params.weight ?? 1, params.description, vectorToBlob(params.embedding));
|
|
2500
|
+
return {
|
|
2501
|
+
id,
|
|
2502
|
+
eventId: params.eventId,
|
|
2503
|
+
entityId: params.entityId,
|
|
2504
|
+
weight: params.weight ?? 1,
|
|
2505
|
+
description: params.description,
|
|
2506
|
+
embedding: params.embedding
|
|
2507
|
+
};
|
|
2508
|
+
}
|
|
2509
|
+
async findEventsByEntity(entityId) {
|
|
2510
|
+
await this.init();
|
|
2511
|
+
if (this.db === void 0) return [];
|
|
2512
|
+
return this.db.prepare(`SELECT e.* FROM knowledge_events e
|
|
2513
|
+
JOIN knowledge_event_entities ee ON ee.event_id = e.id
|
|
2514
|
+
WHERE ee.entity_id = ?`).all(entityId).map(rowToEvent);
|
|
2515
|
+
}
|
|
2516
|
+
async findEntitiesByEvent(eventId) {
|
|
2517
|
+
await this.init();
|
|
2518
|
+
if (this.db === void 0) return [];
|
|
2519
|
+
return this.db.prepare(`SELECT en.* FROM knowledge_entities en
|
|
2520
|
+
JOIN knowledge_event_entities ee ON ee.entity_id = en.id
|
|
2521
|
+
WHERE ee.event_id = ?`).all(eventId).map(rowToEntity);
|
|
2522
|
+
}
|
|
2523
|
+
async listEntities(sourceId) {
|
|
2524
|
+
await this.init();
|
|
2525
|
+
if (this.db === void 0) return [];
|
|
2526
|
+
const sql = sourceId ? `SELECT e.*, COUNT(ee.id) AS event_count
|
|
2527
|
+
FROM knowledge_entities e
|
|
2528
|
+
LEFT JOIN knowledge_event_entities ee ON ee.entity_id = e.id
|
|
2529
|
+
WHERE e.source_id = ?
|
|
2530
|
+
GROUP BY e.id
|
|
2531
|
+
ORDER BY event_count DESC` : `SELECT e.*, COUNT(ee.id) AS event_count
|
|
2532
|
+
FROM knowledge_entities e
|
|
2533
|
+
LEFT JOIN knowledge_event_entities ee ON ee.entity_id = e.id
|
|
2534
|
+
GROUP BY e.id
|
|
2535
|
+
ORDER BY event_count DESC`;
|
|
2536
|
+
return (sourceId ? this.db.prepare(sql).all(sourceId) : this.db.prepare(sql).all()).map((row) => ({
|
|
2537
|
+
...rowToEntity(row),
|
|
2538
|
+
eventCount: Number(row["event_count"] ?? 0)
|
|
2539
|
+
}));
|
|
2540
|
+
}
|
|
2541
|
+
async listEvents(sourceId) {
|
|
2542
|
+
await this.init();
|
|
2543
|
+
if (this.db === void 0) return [];
|
|
2544
|
+
const sql = sourceId ? "SELECT * FROM knowledge_events WHERE source_id = ? ORDER BY rank ASC" : "SELECT * FROM knowledge_events ORDER BY created_at DESC";
|
|
2545
|
+
return (sourceId ? this.db.prepare(sql).all(sourceId) : this.db.prepare(sql).all()).map(rowToEvent);
|
|
2546
|
+
}
|
|
2547
|
+
async listEventEntities(sourceId) {
|
|
2548
|
+
await this.init();
|
|
2549
|
+
if (this.db === void 0) return [];
|
|
2550
|
+
const sql = sourceId ? `SELECT ee.event_id, ee.entity_id
|
|
2551
|
+
FROM knowledge_event_entities ee
|
|
2552
|
+
JOIN knowledge_events e ON e.id = ee.event_id
|
|
2553
|
+
WHERE e.source_id = ?` : "SELECT event_id, entity_id FROM knowledge_event_entities";
|
|
2554
|
+
return (sourceId ? this.db.prepare(sql).all(sourceId) : this.db.prepare(sql).all()).map((row) => ({
|
|
2555
|
+
eventId: asString(row["event_id"]),
|
|
2556
|
+
entityId: asString(row["entity_id"])
|
|
2557
|
+
}));
|
|
2558
|
+
}
|
|
2559
|
+
/** FTS hits ranked by bm25 relevance — score = -bm25 (larger is better). */
|
|
2560
|
+
async ftsSearchChunks(query, limit = 50) {
|
|
2561
|
+
await this.init();
|
|
2562
|
+
if (this.db === void 0) return [];
|
|
2563
|
+
const ftsQuery = buildFtsQuery(query);
|
|
2564
|
+
if (ftsQuery === void 0) return [];
|
|
2565
|
+
return this.db.prepare(`SELECT c.*, bm25(knowledge_chunks_fts) AS bm
|
|
2566
|
+
FROM knowledge_chunks c
|
|
2567
|
+
JOIN knowledge_chunks_fts f ON c.rowid = f.rowid
|
|
2568
|
+
WHERE f.knowledge_chunks_fts MATCH ?
|
|
2569
|
+
ORDER BY bm ASC
|
|
2570
|
+
LIMIT ?`).all(ftsQuery, limit).map((row) => ({
|
|
2571
|
+
chunk: rowToChunk(row),
|
|
2572
|
+
score: -Number(row["bm"] ?? 0)
|
|
2573
|
+
}));
|
|
2574
|
+
}
|
|
2575
|
+
async searchChunksByVector(queryVec, options = {}) {
|
|
2576
|
+
await this.init();
|
|
2577
|
+
if (this.db === void 0) return [];
|
|
2578
|
+
const limit = options.limit ?? 50;
|
|
2579
|
+
const threshold = options.threshold ?? 0;
|
|
2580
|
+
const rows = this.db.prepare("SELECT id, embedding_json FROM knowledge_chunks WHERE embedding_json IS NOT NULL").all();
|
|
2581
|
+
const scored = [];
|
|
2582
|
+
for (const row of rows) {
|
|
2583
|
+
const vec = blobToVector(row.embedding_json);
|
|
2584
|
+
if (vec === null) continue;
|
|
2585
|
+
const score = this.embeddingEngine?.cosineSimilarity(queryVec, vec) ?? 0;
|
|
2586
|
+
if (score >= threshold) scored.push({
|
|
2587
|
+
id: row.id,
|
|
2588
|
+
score
|
|
2589
|
+
});
|
|
2590
|
+
}
|
|
2591
|
+
scored.sort((a, b) => b.score - a.score);
|
|
2592
|
+
const top = scored.slice(0, limit);
|
|
2593
|
+
const out = [];
|
|
2594
|
+
for (const item of top) {
|
|
2595
|
+
const chunk = await this.getChunk(item.id);
|
|
2596
|
+
if (chunk !== void 0) out.push({
|
|
2597
|
+
chunk,
|
|
2598
|
+
score: item.score
|
|
2599
|
+
});
|
|
2600
|
+
}
|
|
2601
|
+
return out;
|
|
2602
|
+
}
|
|
2603
|
+
/**
|
|
2604
|
+
* Recompute every embedding for one source from the already-stored texts
|
|
2605
|
+
* (no LLM extraction). Repairs libraries whose vectors are missing
|
|
2606
|
+
* (ingested while the embedding model was unavailable) or stale (produced
|
|
2607
|
+
* by a different model). All texts are embedded up front — any batch
|
|
2608
|
+
* failure aborts before the first write, so the library is never left
|
|
2609
|
+
* half-updated. On success the embedding-model meta is stamped with the
|
|
2610
|
+
* engine's identity, unblocking future ingests after a model change.
|
|
2611
|
+
*/
|
|
2612
|
+
async reembedSource(sourceId, engine) {
|
|
2613
|
+
await this.init();
|
|
2614
|
+
if (this.db === void 0) throw new Error("knowledge store not initialized");
|
|
2615
|
+
if (!engine.available) throw new Error("embedding engine unavailable");
|
|
2616
|
+
const chunkRows = this.db.prepare("SELECT id, heading, content FROM knowledge_chunks WHERE source_id = ? ORDER BY rank ASC").all(sourceId);
|
|
2617
|
+
const chunkTexts = chunkRows.map((c) => c.heading !== null ? `${c.heading}\n${c.content}` : c.content);
|
|
2618
|
+
const eventRows = this.db.prepare("SELECT id, title, content FROM knowledge_events WHERE source_id = ? ORDER BY rank ASC").all(sourceId);
|
|
2619
|
+
const eventTitleTexts = eventRows.map((e) => e.title);
|
|
2620
|
+
const eventContentTexts = eventRows.map((e) => `${e.title}\n\n${e.content}`);
|
|
2621
|
+
const entityRows = this.db.prepare("SELECT id, name FROM knowledge_entities WHERE source_id = ?").all(sourceId);
|
|
2622
|
+
const relationRows = this.db.prepare(`SELECT ee.id, ee.description, e.title, en.name
|
|
2623
|
+
FROM knowledge_event_entities ee
|
|
2624
|
+
JOIN knowledge_events e ON e.id = ee.event_id
|
|
2625
|
+
JOIN knowledge_entities en ON en.id = ee.entity_id
|
|
2626
|
+
WHERE e.source_id = ?`).all(sourceId);
|
|
2627
|
+
const relationTexts = relationRows.map((r) => r.description !== null && r.description.length > 0 ? r.description : `${r.title} ${r.name}`);
|
|
2628
|
+
const embedAll = async (texts) => {
|
|
2629
|
+
if (texts.length === 0) return [];
|
|
2630
|
+
const vectors = await engine.embedBatch(texts);
|
|
2631
|
+
if (vectors === null || vectors.length !== texts.length) throw new Error("向量嵌入失败,已取消本次重新嵌入");
|
|
2632
|
+
return vectors;
|
|
2633
|
+
};
|
|
2634
|
+
const chunkVecs = await embedAll(chunkTexts);
|
|
2635
|
+
const eventTitleVecs = await embedAll(eventTitleTexts);
|
|
2636
|
+
const eventContentVecs = await embedAll(eventContentTexts);
|
|
2637
|
+
const entityVecs = await embedAll(entityRows.map((e) => e.name));
|
|
2638
|
+
const relationVecs = await embedAll(relationTexts);
|
|
2639
|
+
this.beginTransaction();
|
|
2640
|
+
try {
|
|
2641
|
+
const updChunk = this.db.prepare("UPDATE knowledge_chunks SET embedding_json = ? WHERE id = ?");
|
|
2642
|
+
for (let i = 0; i < chunkRows.length; i++) updChunk.run(vectorToBlob(chunkVecs[i] ?? null), chunkRows[i].id);
|
|
2643
|
+
const updEvent = this.db.prepare("UPDATE knowledge_events SET title_embedding_json = ?, content_embedding_json = ? WHERE id = ?");
|
|
2644
|
+
for (let i = 0; i < eventRows.length; i++) updEvent.run(vectorToBlob(eventTitleVecs[i] ?? null), vectorToBlob(eventContentVecs[i] ?? null), eventRows[i].id);
|
|
2645
|
+
const updEntity = this.db.prepare("UPDATE knowledge_entities SET embedding_json = ? WHERE id = ?");
|
|
2646
|
+
for (let i = 0; i < entityRows.length; i++) updEntity.run(vectorToBlob(entityVecs[i] ?? null), entityRows[i].id);
|
|
2647
|
+
const updRelation = this.db.prepare("UPDATE knowledge_event_entities SET embedding_json = ? WHERE id = ?");
|
|
2648
|
+
for (let i = 0; i < relationRows.length; i++) updRelation.run(vectorToBlob(relationVecs[i] ?? null), relationRows[i].id);
|
|
2649
|
+
await this.setMeta(EMBEDDING_MODEL_META_KEY, engine.modelName);
|
|
2650
|
+
this.commitTransaction();
|
|
2651
|
+
} catch (error) {
|
|
2652
|
+
this.rollbackTransaction();
|
|
2653
|
+
throw error;
|
|
2654
|
+
}
|
|
2655
|
+
return {
|
|
2656
|
+
chunks: chunkRows.length,
|
|
2657
|
+
events: eventRows.length,
|
|
2658
|
+
entities: entityRows.length,
|
|
2659
|
+
relations: relationRows.length
|
|
2660
|
+
};
|
|
2661
|
+
}
|
|
2662
|
+
/**
|
|
2663
|
+
* Chunk vector coverage — how many of the stored chunks carry an embedding.
|
|
2664
|
+
* Used by the TUI to surface partial/skipped embedding after an ingest.
|
|
2665
|
+
*/
|
|
2666
|
+
async embeddingCoverage() {
|
|
2667
|
+
await this.init();
|
|
2668
|
+
if (this.db === void 0) return {
|
|
2669
|
+
total: 0,
|
|
2670
|
+
embedded: 0
|
|
2671
|
+
};
|
|
2672
|
+
const row = this.db.prepare(`SELECT COUNT(*) AS total,
|
|
2673
|
+
SUM(CASE WHEN embedding_json IS NOT NULL THEN 1 ELSE 0 END) AS embedded
|
|
2674
|
+
FROM knowledge_chunks`).get();
|
|
2675
|
+
if (row === void 0) return {
|
|
2676
|
+
total: 0,
|
|
2677
|
+
embedded: 0
|
|
2678
|
+
};
|
|
2679
|
+
return {
|
|
2680
|
+
total: Number(row["total"] ?? 0),
|
|
2681
|
+
embedded: Number(row["embedded"] ?? 0)
|
|
2682
|
+
};
|
|
2683
|
+
}
|
|
2684
|
+
async stats() {
|
|
2685
|
+
await this.init();
|
|
2686
|
+
if (this.db === void 0) return {
|
|
2687
|
+
sources: 0,
|
|
2688
|
+
documents: 0,
|
|
2689
|
+
chunks: 0,
|
|
2690
|
+
events: 0,
|
|
2691
|
+
entities: 0
|
|
2692
|
+
};
|
|
2693
|
+
const count = (table) => {
|
|
2694
|
+
return this.db.prepare(`SELECT COUNT(*) AS c FROM ${table}`).get()?.c ?? 0;
|
|
2695
|
+
};
|
|
2696
|
+
return {
|
|
2697
|
+
sources: count("knowledge_sources"),
|
|
2698
|
+
documents: count("knowledge_documents"),
|
|
2699
|
+
chunks: count("knowledge_chunks"),
|
|
2700
|
+
events: count("knowledge_events"),
|
|
2701
|
+
entities: count("knowledge_entities")
|
|
2702
|
+
};
|
|
2703
|
+
}
|
|
2704
|
+
async buildSearchResult(chunkId, score, eventId = null) {
|
|
2705
|
+
if (this.db === void 0) return void 0;
|
|
2706
|
+
const chunk = await this.getChunk(chunkId);
|
|
2707
|
+
if (chunk === void 0) return void 0;
|
|
2708
|
+
if (this.db.prepare("SELECT * FROM knowledge_documents WHERE id = ?").get(chunk.documentId) === void 0) return void 0;
|
|
2709
|
+
const sourceRow = this.db.prepare("SELECT * FROM knowledge_sources WHERE id = ?").get(chunk.sourceId);
|
|
2710
|
+
if (sourceRow === void 0) return void 0;
|
|
2711
|
+
let eventTitle = null;
|
|
2712
|
+
if (eventId !== null) eventTitle = (await this.getEvent(eventId))?.title ?? null;
|
|
2713
|
+
return {
|
|
2714
|
+
chunkId: chunk.id,
|
|
2715
|
+
documentId: chunk.documentId,
|
|
2716
|
+
sourceId: chunk.sourceId,
|
|
2717
|
+
sourceName: String(sourceRow["name"]),
|
|
2718
|
+
heading: chunk.heading,
|
|
2719
|
+
content: chunk.content,
|
|
2720
|
+
score,
|
|
2721
|
+
eventId,
|
|
2722
|
+
eventTitle
|
|
2723
|
+
};
|
|
2724
|
+
}
|
|
2725
|
+
};
|
|
2726
|
+
function asString(value) {
|
|
2727
|
+
if (typeof value === "string") return value;
|
|
2728
|
+
if (typeof value === "number" || typeof value === "bigint" || typeof value === "boolean") return String(value);
|
|
2729
|
+
return "";
|
|
2730
|
+
}
|
|
2731
|
+
function asNullableString(value) {
|
|
2732
|
+
if (value === null || value === void 0) return null;
|
|
2733
|
+
if (typeof value === "string") return value;
|
|
2734
|
+
if (typeof value === "number" || typeof value === "bigint" || typeof value === "boolean") return String(value);
|
|
2735
|
+
return null;
|
|
2736
|
+
}
|
|
2737
|
+
function rowToSource(row) {
|
|
2738
|
+
return {
|
|
2739
|
+
id: asString(row["id"]),
|
|
2740
|
+
name: asString(row["name"]),
|
|
2741
|
+
filePath: asNullableString(row["file_path"]),
|
|
2742
|
+
description: asNullableString(row["description"]),
|
|
2743
|
+
contentHash: asNullableString(row["content_hash"]),
|
|
2744
|
+
createdAt: Number(row["created_at"])
|
|
2745
|
+
};
|
|
2746
|
+
}
|
|
2747
|
+
function rowToDocument(row) {
|
|
2748
|
+
return {
|
|
2749
|
+
id: asString(row["id"]),
|
|
2750
|
+
sourceId: asString(row["source_id"]),
|
|
2751
|
+
title: asString(row["title"]),
|
|
2752
|
+
content: asNullableString(row["content"]),
|
|
2753
|
+
status: asString(row["status"]),
|
|
2754
|
+
chunkCount: Number(row["chunk_count"] ?? 0),
|
|
2755
|
+
createdAt: Number(row["created_at"])
|
|
2756
|
+
};
|
|
2757
|
+
}
|
|
2758
|
+
function rowToChunk(row) {
|
|
2759
|
+
return {
|
|
2760
|
+
id: asString(row["id"]),
|
|
2761
|
+
sourceId: asString(row["source_id"]),
|
|
2762
|
+
documentId: asString(row["document_id"]),
|
|
2763
|
+
rank: Number(row["rank"]),
|
|
2764
|
+
heading: asNullableString(row["heading"]),
|
|
2765
|
+
content: asString(row["content"]),
|
|
2766
|
+
rawContent: asNullableString(row["raw_content"]),
|
|
2767
|
+
embedding: blobToVector(row["embedding_json"]),
|
|
2768
|
+
createdAt: Number(row["created_at"])
|
|
2769
|
+
};
|
|
2770
|
+
}
|
|
2771
|
+
function rowToEvent(row) {
|
|
2772
|
+
let keywords = [];
|
|
2773
|
+
try {
|
|
2774
|
+
const parsed = JSON.parse(asString(row["keywords"] ?? "[]"));
|
|
2775
|
+
if (Array.isArray(parsed)) keywords = parsed.filter((k) => typeof k === "string");
|
|
2776
|
+
} catch {}
|
|
2777
|
+
return {
|
|
2778
|
+
id: asString(row["id"]),
|
|
2779
|
+
sourceId: asString(row["source_id"]),
|
|
2780
|
+
documentId: asString(row["document_id"]),
|
|
2781
|
+
chunkId: asString(row["chunk_id"]),
|
|
2782
|
+
rank: Number(row["rank"]),
|
|
2783
|
+
title: asString(row["title"]),
|
|
2784
|
+
summary: asNullableString(row["summary"]),
|
|
2785
|
+
content: asString(row["content"]),
|
|
2786
|
+
category: asNullableString(row["category"]),
|
|
2787
|
+
keywords,
|
|
2788
|
+
titleEmbedding: blobToVector(row["title_embedding_json"]),
|
|
2789
|
+
contentEmbedding: blobToVector(row["content_embedding_json"]),
|
|
2790
|
+
createdAt: Number(row["created_at"])
|
|
2791
|
+
};
|
|
2792
|
+
}
|
|
2793
|
+
function rowToEntity(row) {
|
|
2794
|
+
return {
|
|
2795
|
+
id: asString(row["id"]),
|
|
2796
|
+
sourceId: asString(row["source_id"]),
|
|
2797
|
+
type: asString(row["type"]),
|
|
2798
|
+
name: asString(row["name"]),
|
|
2799
|
+
normalizedName: asString(row["normalized_name"]),
|
|
2800
|
+
description: asNullableString(row["description"]),
|
|
2801
|
+
embedding: blobToVector(row["embedding_json"]),
|
|
2802
|
+
createdAt: Number(row["created_at"])
|
|
2803
|
+
};
|
|
2804
|
+
}
|
|
2805
|
+
/**
|
|
2806
|
+
* Tokenize text so FTS5's unicode61 tokenizer can index mixed CJK/ASCII text.
|
|
2807
|
+
* Mirrors the memory package's `toFtsText`.
|
|
2808
|
+
*/
|
|
2809
|
+
function toFtsText(text) {
|
|
2810
|
+
const parts = text.toLowerCase().replaceAll(/([一-鿿㐀-䶿])([a-z0-9])/g, "$1 $2").replaceAll(/([a-z0-9])([一-鿿㐀-䶿])/g, "$1 $2").split(/[^a-z0-9一-鿿㐀-䶿]+/);
|
|
2811
|
+
const tokens = [];
|
|
2812
|
+
for (const part of parts) {
|
|
2813
|
+
if (part.length === 0) continue;
|
|
2814
|
+
if (/^[a-z0-9]+$/.test(part)) tokens.push(part);
|
|
2815
|
+
else for (const ch of part) if (ch.length > 0) tokens.push(ch);
|
|
2816
|
+
}
|
|
2817
|
+
return tokens.join(" ");
|
|
2818
|
+
}
|
|
2819
|
+
function buildFtsQuery(search) {
|
|
2820
|
+
const tokens = toFtsText(search).split(/\s+/).filter((t) => t.length > 0);
|
|
2821
|
+
if (tokens.length === 0) return void 0;
|
|
2822
|
+
return tokens.map((t) => `"${t.replaceAll("\"", "\"\"")}"`).join(" AND ");
|
|
2823
|
+
}
|
|
2824
|
+
//#endregion
|
|
2825
|
+
//#region ../../packages/knowledge/src/shared-store.ts
|
|
2826
|
+
const entries = /* @__PURE__ */ new Map();
|
|
2827
|
+
function entryFor(homeDir) {
|
|
2828
|
+
const existing = entries.get(homeDir);
|
|
2829
|
+
if (existing !== void 0) return existing;
|
|
2830
|
+
const store = new KnowledgeStore(homeDir);
|
|
2831
|
+
const entry = {
|
|
2832
|
+
store,
|
|
2833
|
+
ready: (async () => {
|
|
2834
|
+
try {
|
|
2835
|
+
await store.init();
|
|
2836
|
+
const cacheDir = join$1(homeDir, "cache", "fastembed");
|
|
2837
|
+
mkdirSync(cacheDir, { recursive: true });
|
|
2838
|
+
store.setEmbeddingEngine(createFastEmbedEngine(cacheDir));
|
|
2839
|
+
} catch (error) {
|
|
2840
|
+
entries.delete(homeDir);
|
|
2841
|
+
throw error;
|
|
2842
|
+
}
|
|
2843
|
+
})()
|
|
2844
|
+
};
|
|
2845
|
+
entries.set(homeDir, entry);
|
|
2846
|
+
return entry;
|
|
2847
|
+
}
|
|
2848
|
+
/**
|
|
2849
|
+
* Process-level KnowledgeStore per homeDir, plus the embedding engine wired
|
|
2850
|
+
* onto it — one handle and one status source instead of one per host (TUI
|
|
2851
|
+
* command vs agent session).
|
|
2852
|
+
*
|
|
2853
|
+
* The map key is the raw homeDir string so the engine cache key
|
|
2854
|
+
* join(homeDir, 'cache', 'fastembed') stays byte-identical with every other
|
|
2855
|
+
* createFastEmbedEngine call site: same string → same engine instance.
|
|
2856
|
+
*/
|
|
2857
|
+
function sharedKnowledgeStore(homeDir) {
|
|
2858
|
+
return entryFor(homeDir).store;
|
|
2859
|
+
}
|
|
2860
|
+
/** Resolves once init + engine wiring finished; rejects once per failed build. */
|
|
2861
|
+
function sharedKnowledgeStoreReady(homeDir) {
|
|
2862
|
+
return entryFor(homeDir).ready;
|
|
2863
|
+
}
|
|
2864
|
+
//#endregion
|
|
2865
|
+
//#region ../../packages/knowledge/src/chunking.ts
|
|
2866
|
+
const HEADING_RE = /^(#{1,6})\s+(.+?)\s*$/;
|
|
2867
|
+
const CODE_FENCE_RE = /^(\s*)(`{3,}|~{3,})/;
|
|
2868
|
+
/**
|
|
2869
|
+
* Soft cap on chunk token count. BGE-small-zh-v1.5 (the fastembed model used
|
|
2870
|
+
* here) has a 512-token sequence limit; chunks longer than that get truncated
|
|
2871
|
+
* at the embedding layer, hurting retrieval quality. We split at ~480 to leave
|
|
2872
|
+
* headroom for the heading line that gets prepended at embed time.
|
|
2873
|
+
*/
|
|
2874
|
+
const MAX_CHUNK_TOKENS = 480;
|
|
2875
|
+
/**
|
|
2876
|
+
* Split a markdown document into sections using heading_strict strategy:
|
|
2877
|
+
* each section begins at a heading and contains all body content until the
|
|
2878
|
+
* next heading. Consecutive headings collapse to the lowest-level one as the
|
|
2879
|
+
* main heading. Content before the first heading is grouped as "Introduction".
|
|
2880
|
+
*
|
|
2881
|
+
* Mirrors SAG's `buildHeadingStrictSections`.
|
|
2882
|
+
*/
|
|
2883
|
+
function chunkMarkdown(content) {
|
|
2884
|
+
const lines = content.split("\n");
|
|
2885
|
+
const sections = [];
|
|
2886
|
+
let buffer = [];
|
|
2887
|
+
let currentHeading = null;
|
|
2888
|
+
let currentLevel = null;
|
|
2889
|
+
let rank = 0;
|
|
2890
|
+
let inFence = false;
|
|
2891
|
+
const flush = () => {
|
|
2892
|
+
const raw = buffer.join("\n").trim();
|
|
2893
|
+
const heading = currentHeading;
|
|
2894
|
+
const level = currentLevel;
|
|
2895
|
+
buffer = [];
|
|
2896
|
+
if (raw.length === 0) return;
|
|
2897
|
+
const body = stripMarkdown(raw);
|
|
2898
|
+
if (body.length === 0 && heading === null) return;
|
|
2899
|
+
const subSections = splitLargeSection({
|
|
2900
|
+
heading,
|
|
2901
|
+
headingLevel: level,
|
|
2902
|
+
content: body,
|
|
2903
|
+
rawContent: raw,
|
|
2904
|
+
rank
|
|
2905
|
+
});
|
|
2906
|
+
for (const section of subSections) {
|
|
2907
|
+
sections.push({
|
|
2908
|
+
...section,
|
|
2909
|
+
rank
|
|
2910
|
+
});
|
|
2911
|
+
rank += 1;
|
|
2912
|
+
}
|
|
2913
|
+
};
|
|
2914
|
+
for (const line of lines) {
|
|
2915
|
+
if (CODE_FENCE_RE.exec(line)) {
|
|
2916
|
+
inFence = !inFence;
|
|
2917
|
+
buffer.push(line);
|
|
2918
|
+
continue;
|
|
2919
|
+
}
|
|
2920
|
+
if (inFence) {
|
|
2921
|
+
buffer.push(line);
|
|
2922
|
+
continue;
|
|
2923
|
+
}
|
|
2924
|
+
const headingMatch = HEADING_RE.exec(line);
|
|
2925
|
+
if (headingMatch) {
|
|
2926
|
+
flush();
|
|
2927
|
+
currentHeading = headingMatch[2].trim();
|
|
2928
|
+
currentLevel = headingMatch[1].length;
|
|
2929
|
+
} else buffer.push(line);
|
|
2930
|
+
}
|
|
2931
|
+
flush();
|
|
2932
|
+
return sections;
|
|
2933
|
+
}
|
|
2934
|
+
/**
|
|
2935
|
+
* Strip markdown decorations that hurt embedding quality:
|
|
2936
|
+
* code fences, inline code, images, links, emphasis markers.
|
|
2937
|
+
*
|
|
2938
|
+
* Mirrors SAG's `stripMarkdown`.
|
|
2939
|
+
*/
|
|
2940
|
+
function stripMarkdown(text) {
|
|
2941
|
+
return text.replaceAll(/^(\s*)```[^\n]*$\n?/gm, "$1").replaceAll(/^(\s*)```[^\n]*$/gm, "").replaceAll(/`([^`]+)`/g, "$1").replaceAll(/!\[([^\]]*)\]\([^)]+\)/g, "$1").replaceAll(/\[([^\]]+)\]\([^)]+\)/g, "$1").replaceAll(/\*\*([^*]+)\*\*/g, "$1").replaceAll(/__([^_]+)__/g, "$1").replaceAll(/(?<!\w)\*([^*\n]+)\*(?!\w)/g, "$1").replaceAll(/(?<!\w)_([^_\n]+)_(?!\w)/g, "$1").replaceAll(/^#{1,6}\s+/gm, "").replaceAll(/\n{3,}/g, "\n\n").trim();
|
|
2942
|
+
}
|
|
2943
|
+
/**
|
|
2944
|
+
* Split a plain text file into chunks by blank-line paragraphs.
|
|
2945
|
+
* Each non-empty paragraph becomes one chunk with no heading.
|
|
2946
|
+
* Long paragraphs are split at sentence boundaries to stay under the
|
|
2947
|
+
* embedding model's token cap (same MAX_CHUNK_TOKENS as markdown).
|
|
2948
|
+
*/
|
|
2949
|
+
function chunkText(content) {
|
|
2950
|
+
const MAX_PARA_TOKENS = MAX_CHUNK_TOKENS;
|
|
2951
|
+
const MAX_PARA_CHARS = MAX_PARA_TOKENS * 4;
|
|
2952
|
+
const paragraphs = content.split(/\n\s*\n/).map((p) => p.trim()).filter((p) => p.length > 0);
|
|
2953
|
+
const sections = [];
|
|
2954
|
+
let rank = 0;
|
|
2955
|
+
for (const para of paragraphs) {
|
|
2956
|
+
if (estimateTokens(para) <= MAX_PARA_TOKENS) {
|
|
2957
|
+
sections.push({
|
|
2958
|
+
heading: null,
|
|
2959
|
+
headingLevel: null,
|
|
2960
|
+
content: para,
|
|
2961
|
+
rawContent: para,
|
|
2962
|
+
rank: rank++
|
|
2963
|
+
});
|
|
2964
|
+
continue;
|
|
2965
|
+
}
|
|
2966
|
+
const sentences = para.match(/[^.!?。!?\n]+[.!?。!?\n]+|[^.!?。!?\n]+$/g) ?? [para];
|
|
2967
|
+
let buffer = "";
|
|
2968
|
+
const flushBuffer = () => {
|
|
2969
|
+
const text = buffer.trim();
|
|
2970
|
+
if (text.length === 0) return;
|
|
2971
|
+
sections.push({
|
|
2972
|
+
heading: null,
|
|
2973
|
+
headingLevel: null,
|
|
2974
|
+
content: text,
|
|
2975
|
+
rawContent: text,
|
|
2976
|
+
rank: rank++
|
|
2977
|
+
});
|
|
2978
|
+
buffer = "";
|
|
2979
|
+
};
|
|
2980
|
+
for (const sentence of sentences) {
|
|
2981
|
+
const candidate = (buffer + " " + sentence).trim();
|
|
2982
|
+
if (buffer.length > 0 && candidate.length > MAX_PARA_CHARS) flushBuffer();
|
|
2983
|
+
buffer = buffer.length === 0 ? sentence : `${buffer} ${sentence}`;
|
|
2984
|
+
}
|
|
2985
|
+
flushBuffer();
|
|
2986
|
+
}
|
|
2987
|
+
return sections;
|
|
2988
|
+
}
|
|
2989
|
+
/** Rough token estimate — chars / 4 (English-leaning; CJK underestimates). */
|
|
2990
|
+
function estimateTokens(text) {
|
|
2991
|
+
return Math.ceil(text.length / 4);
|
|
2992
|
+
}
|
|
2993
|
+
/**
|
|
2994
|
+
* Split a section that exceeds MAX_CHUNK_TOKENS into smaller sub-sections.
|
|
2995
|
+
* Strategy (mirrors SAG's `splitLargeSection`):
|
|
2996
|
+
* 1. Split into paragraphs by blank lines.
|
|
2997
|
+
* 2. Greedily pack paragraphs into chunks under the token cap.
|
|
2998
|
+
* 3. A paragraph that alone exceeds the cap is split at sentence boundaries;
|
|
2999
|
+
* a sentence that itself exceeds the cap is split by character limit.
|
|
3000
|
+
*
|
|
3001
|
+
* Sections under the cap are returned unchanged (single-element array).
|
|
3002
|
+
* The heading is preserved across all sub-sections; rank is reassigned by the
|
|
3003
|
+
* caller. rawContent is rebuilt from the sub-chunk's content paragraphs.
|
|
3004
|
+
*/
|
|
3005
|
+
function splitLargeSection(section) {
|
|
3006
|
+
if (estimateTokens(section.content) <= MAX_CHUNK_TOKENS) return [section];
|
|
3007
|
+
const paragraphs = section.rawContent.split(/\n{2,}/).map((p) => p.trim()).filter((p) => p.length > 0);
|
|
3008
|
+
if (paragraphs.length <= 1) return splitBySentences(section, section.rawContent);
|
|
3009
|
+
const result = [];
|
|
3010
|
+
let buffer = [];
|
|
3011
|
+
let bufferTokens = 0;
|
|
3012
|
+
const flushBuffer = () => {
|
|
3013
|
+
if (buffer.length === 0) return;
|
|
3014
|
+
const raw = buffer.join("\n\n").trim();
|
|
3015
|
+
if (raw.length === 0) {
|
|
3016
|
+
buffer = [];
|
|
3017
|
+
bufferTokens = 0;
|
|
3018
|
+
return;
|
|
3019
|
+
}
|
|
3020
|
+
result.push({
|
|
3021
|
+
heading: section.heading,
|
|
3022
|
+
headingLevel: section.headingLevel,
|
|
3023
|
+
content: stripMarkdown(raw),
|
|
3024
|
+
rawContent: raw,
|
|
3025
|
+
rank: 0
|
|
3026
|
+
});
|
|
3027
|
+
buffer = [];
|
|
3028
|
+
bufferTokens = 0;
|
|
3029
|
+
};
|
|
3030
|
+
for (const para of paragraphs) {
|
|
3031
|
+
const paraTokens = estimateTokens(para);
|
|
3032
|
+
if (paraTokens > MAX_CHUNK_TOKENS) {
|
|
3033
|
+
flushBuffer();
|
|
3034
|
+
for (const frag of splitBySentences(section, para)) result.push(frag);
|
|
3035
|
+
continue;
|
|
3036
|
+
}
|
|
3037
|
+
if (buffer.length > 0 && bufferTokens + paraTokens > MAX_CHUNK_TOKENS) flushBuffer();
|
|
3038
|
+
buffer.push(para);
|
|
3039
|
+
bufferTokens += paraTokens;
|
|
3040
|
+
}
|
|
3041
|
+
flushBuffer();
|
|
3042
|
+
return result.length > 0 ? result : [section];
|
|
3043
|
+
}
|
|
3044
|
+
/**
|
|
3045
|
+
* Split a long text into sub-sections at sentence boundaries.
|
|
3046
|
+
* Falls back to a hard character cut when a single sentence exceeds the cap.
|
|
3047
|
+
*/
|
|
3048
|
+
function splitBySentences(section, text) {
|
|
3049
|
+
const sentences = splitSentences(text);
|
|
3050
|
+
const result = [];
|
|
3051
|
+
let buffer = "";
|
|
3052
|
+
let bufferTokens = 0;
|
|
3053
|
+
const flush = () => {
|
|
3054
|
+
const trimmed = buffer.trim();
|
|
3055
|
+
if (trimmed.length === 0) {
|
|
3056
|
+
buffer = "";
|
|
3057
|
+
bufferTokens = 0;
|
|
3058
|
+
return;
|
|
3059
|
+
}
|
|
3060
|
+
result.push({
|
|
3061
|
+
heading: section.heading,
|
|
3062
|
+
headingLevel: section.headingLevel,
|
|
3063
|
+
content: stripMarkdown(trimmed),
|
|
3064
|
+
rawContent: trimmed,
|
|
3065
|
+
rank: 0
|
|
3066
|
+
});
|
|
3067
|
+
buffer = "";
|
|
3068
|
+
bufferTokens = 0;
|
|
3069
|
+
};
|
|
3070
|
+
for (const sentence of sentences) {
|
|
3071
|
+
const sentTokens = estimateTokens(sentence);
|
|
3072
|
+
if (sentTokens > MAX_CHUNK_TOKENS) {
|
|
3073
|
+
flush();
|
|
3074
|
+
for (const frag of splitByCharLimit(sentence)) result.push({
|
|
3075
|
+
heading: section.heading,
|
|
3076
|
+
headingLevel: section.headingLevel,
|
|
3077
|
+
content: stripMarkdown(frag),
|
|
3078
|
+
rawContent: frag,
|
|
3079
|
+
rank: 0
|
|
3080
|
+
});
|
|
3081
|
+
continue;
|
|
3082
|
+
}
|
|
3083
|
+
if (buffer.length > 0 && bufferTokens + sentTokens > MAX_CHUNK_TOKENS) flush();
|
|
3084
|
+
buffer = buffer.length === 0 ? sentence : `${buffer} ${sentence}`;
|
|
3085
|
+
bufferTokens += sentTokens;
|
|
3086
|
+
}
|
|
3087
|
+
flush();
|
|
3088
|
+
return result.length > 0 ? result : [{
|
|
3089
|
+
heading: section.heading,
|
|
3090
|
+
headingLevel: section.headingLevel,
|
|
3091
|
+
content: section.content,
|
|
3092
|
+
rawContent: section.rawContent,
|
|
3093
|
+
rank: 0
|
|
3094
|
+
}];
|
|
3095
|
+
}
|
|
3096
|
+
/** Split text into sentences using CJK + ASCII punctuation. */
|
|
3097
|
+
function splitSentences(text) {
|
|
3098
|
+
return text.match(/[^。!?!?.\n]+[。!?!?.\n]+|[^。!?!?.\n]+$/g) ?? [text];
|
|
3099
|
+
}
|
|
3100
|
+
/** Hard character cut for pathological single sentences longer than the cap. */
|
|
3101
|
+
function splitByCharLimit(text) {
|
|
3102
|
+
const maxChars = MAX_CHUNK_TOKENS * 4;
|
|
3103
|
+
const chunks = [];
|
|
3104
|
+
let remaining = text.trim();
|
|
3105
|
+
while (remaining.length > maxChars) {
|
|
3106
|
+
chunks.push(remaining.slice(0, maxChars));
|
|
3107
|
+
remaining = remaining.slice(maxChars).trimStart();
|
|
3108
|
+
}
|
|
3109
|
+
if (remaining.length > 0) chunks.push(remaining);
|
|
3110
|
+
return chunks;
|
|
3111
|
+
}
|
|
3112
|
+
//#endregion
|
|
3113
|
+
//#region ../../packages/knowledge/src/extractor.ts
|
|
3114
|
+
/** Max LLM call attempts for extraction — fails fall back to a heading-derived event. */
|
|
3115
|
+
const EXTRACTION_MAX_ATTEMPTS = 3;
|
|
3116
|
+
/** Base delay (ms) for exponential backoff: 100, 200, 400. */
|
|
3117
|
+
const EXTRACTION_BACKOFF_BASE_MS = 100;
|
|
3118
|
+
function backoffDelay(attempt) {
|
|
3119
|
+
return EXTRACTION_BACKOFF_BASE_MS * 2 ** Math.max(0, attempt - 1);
|
|
3120
|
+
}
|
|
3121
|
+
async function callLlmWithBackoff(llm, systemPrompt, userPrompt) {
|
|
3122
|
+
let lastError;
|
|
3123
|
+
for (let attempt = 1; attempt <= EXTRACTION_MAX_ATTEMPTS; attempt += 1) try {
|
|
3124
|
+
return await llm.generate(systemPrompt, userPrompt);
|
|
3125
|
+
} catch (error) {
|
|
3126
|
+
lastError = error;
|
|
3127
|
+
if (attempt < EXTRACTION_MAX_ATTEMPTS) await new Promise((resolve) => setTimeout(resolve, backoffDelay(attempt)));
|
|
3128
|
+
}
|
|
3129
|
+
throw lastError;
|
|
3130
|
+
}
|
|
3131
|
+
const ENTITY_TYPES = [
|
|
3132
|
+
"person",
|
|
3133
|
+
"organization",
|
|
3134
|
+
"location",
|
|
3135
|
+
"time",
|
|
3136
|
+
"product",
|
|
3137
|
+
"metric",
|
|
3138
|
+
"action",
|
|
3139
|
+
"work",
|
|
3140
|
+
"group",
|
|
3141
|
+
"subject",
|
|
3142
|
+
"tags"
|
|
3143
|
+
];
|
|
3144
|
+
const EXTRACTION_SYSTEM_PROMPT = `You are a professional knowledge content extractor for a SAG-style event-centric knowledge graph.
|
|
3145
|
+
|
|
3146
|
+
Your task: given a markdown section (one chunk), extract exactly ONE fused event that captures the section's core informational content, plus the entities mentioned.
|
|
3147
|
+
|
|
3148
|
+
# Output Format
|
|
3149
|
+
|
|
3150
|
+
Return a JSON object with this exact shape:
|
|
3151
|
+
|
|
3152
|
+
{
|
|
3153
|
+
"items": [
|
|
3154
|
+
{
|
|
3155
|
+
"title": "short event title (≤80 chars, noun phrase or short sentence)",
|
|
3156
|
+
"summary": "one-sentence summary of what this section is about",
|
|
3157
|
+
"content": "the full informational content of this section, rephrased as a self-contained event description (1-4 sentences)",
|
|
3158
|
+
"category": "one of: definition, process, comparison, decision, history, reference, other",
|
|
3159
|
+
"keywords": ["3-5 short keywords or phrases"],
|
|
3160
|
+
"entities": [
|
|
3161
|
+
{ "type": "one of ENTITY_TYPES below", "name": "entity name", "description": "one-line context for the entity" }
|
|
3162
|
+
]
|
|
3163
|
+
}
|
|
3164
|
+
]
|
|
3165
|
+
}
|
|
3166
|
+
|
|
3167
|
+
# Entity Types
|
|
3168
|
+
|
|
3169
|
+
Use only these types:
|
|
3170
|
+
- person — named individuals
|
|
3171
|
+
- organization — companies, teams, institutions
|
|
3172
|
+
- location — places, geographic regions
|
|
3173
|
+
- time — dates, time periods, eras
|
|
3174
|
+
- product — products, services, software, tools
|
|
3175
|
+
- metric — numbers, statistics, measurements
|
|
3176
|
+
- action — processes, operations, activities
|
|
3177
|
+
- work — books, papers, projects, artworks
|
|
3178
|
+
- group — categories, classes, groups of things
|
|
3179
|
+
- subject — abstract topics, concepts, fields
|
|
3180
|
+
- tags — use sparingly for cross-cutting labels
|
|
3181
|
+
|
|
3182
|
+
# Rules
|
|
3183
|
+
|
|
3184
|
+
1. Produce EXACTLY ONE item in the "items" array.
|
|
3185
|
+
2. The event title should be specific enough to disambiguate from sibling sections.
|
|
3186
|
+
3. The content must be self-contained — a reader should understand it without the surrounding context.
|
|
3187
|
+
4. Extract 2-6 entities per chunk. Only include entities that are clearly referenced in the text.
|
|
3188
|
+
5. Use the entity's most specific name (e.g. "Anthropic" rather than "the company").
|
|
3189
|
+
6. Output language must follow the main input language. Chinese input requires Chinese title, summary, content, category, keywords, and entity descriptions. English input requires English fields.
|
|
3190
|
+
7. Output ONLY the JSON object — no prose, no markdown fences, no commentary.
|
|
3191
|
+
|
|
3192
|
+
# Example
|
|
3193
|
+
|
|
3194
|
+
Input section:
|
|
3195
|
+
"""
|
|
3196
|
+
## Model Context Protocol
|
|
3197
|
+
|
|
3198
|
+
The Model Context Protocol (MCP) is an open standard introduced by Anthropic in 2024. It defines a JSON-RPC interface between AI assistants and external tools. MCP servers expose resources, prompts, and tools that clients like Claude Desktop can consume.
|
|
3199
|
+
"""
|
|
3200
|
+
|
|
3201
|
+
Output:
|
|
3202
|
+
{
|
|
3203
|
+
"items": [
|
|
3204
|
+
{
|
|
3205
|
+
"title": "Model Context Protocol introduction",
|
|
3206
|
+
"summary": "MCP is an open standard for AI assistant / tool integration introduced by Anthropic in 2024.",
|
|
3207
|
+
"content": "The Model Context Protocol (MCP) is an open standard introduced by Anthropic in 2024 that defines a JSON-RPC interface between AI assistants and external tools. MCP servers expose resources, prompts, and tools that clients like Claude Desktop can consume.",
|
|
3208
|
+
"category": "definition",
|
|
3209
|
+
"keywords": ["MCP", "JSON-RPC", "Anthropic", "AI tools"],
|
|
3210
|
+
"entities": [
|
|
3211
|
+
{ "type": "product", "name": "Model Context Protocol", "description": "Open standard for AI assistant / tool integration" },
|
|
3212
|
+
{ "type": "organization", "name": "Anthropic", "description": "Company that introduced MCP in 2024" },
|
|
3213
|
+
{ "type": "time", "name": "2024", "description": "Year MCP was introduced" },
|
|
3214
|
+
{ "type": "product", "name": "Claude Desktop", "description": "An MCP client application" }
|
|
3215
|
+
]
|
|
3216
|
+
}
|
|
3217
|
+
]
|
|
3218
|
+
}`;
|
|
3219
|
+
/**
|
|
3220
|
+
* Build the user prompt for extracting events/entities from a chunk.
|
|
3221
|
+
* Includes a one-shot example to anchor the expected output shape.
|
|
3222
|
+
*/
|
|
3223
|
+
function buildExtractionUserPrompt(chunk) {
|
|
3224
|
+
return [
|
|
3225
|
+
"Extract the fused event and entities from the following markdown section.",
|
|
3226
|
+
"",
|
|
3227
|
+
`Section heading: ${chunk.heading ?? "(no heading)"}`,
|
|
3228
|
+
"",
|
|
3229
|
+
"Section content:",
|
|
3230
|
+
"\"\"\"",
|
|
3231
|
+
chunk.content,
|
|
3232
|
+
"\"\"\"",
|
|
3233
|
+
"",
|
|
3234
|
+
"Return only the JSON object per the schema."
|
|
3235
|
+
].join("\n");
|
|
3236
|
+
}
|
|
3237
|
+
/** Extract the first {...} JSON block from a text response. */
|
|
3238
|
+
function extractJsonFromText(text) {
|
|
3239
|
+
const match = text.match(/\{[\s\S]*\}/);
|
|
3240
|
+
return match === null ? null : match[0];
|
|
3241
|
+
}
|
|
3242
|
+
/** Parse LLM extraction output into structured events. Returns at most 1 event. */
|
|
3243
|
+
function parseExtractionResponse(text, fallbackChunk) {
|
|
3244
|
+
const jsonText = extractJsonFromText(text);
|
|
3245
|
+
if (jsonText === null) return fallbackEvent(fallbackChunk);
|
|
3246
|
+
try {
|
|
3247
|
+
const items = JSON.parse(jsonText)["items"];
|
|
3248
|
+
if (!Array.isArray(items) || items.length === 0) return fallbackEvent(fallbackChunk);
|
|
3249
|
+
const first = items[0];
|
|
3250
|
+
if (first === null || typeof first !== "object") return fallbackEvent(fallbackChunk);
|
|
3251
|
+
const obj = first;
|
|
3252
|
+
return {
|
|
3253
|
+
title: typeof obj["title"] === "string" && obj["title"].trim().length > 0 ? obj["title"].trim() : fallbackChunk.heading ?? "Untitled",
|
|
3254
|
+
summary: typeof obj["summary"] === "string" ? obj["summary"].trim() : "",
|
|
3255
|
+
content: typeof obj["content"] === "string" && obj["content"].trim().length > 0 ? obj["content"].trim() : fallbackChunk.content,
|
|
3256
|
+
category: typeof obj["category"] === "string" ? obj["category"].trim() : "other",
|
|
3257
|
+
keywords: parseKeywords(obj["keywords"]),
|
|
3258
|
+
entities: parseEntities(obj["entities"])
|
|
3259
|
+
};
|
|
3260
|
+
} catch {
|
|
3261
|
+
return fallbackEvent(fallbackChunk);
|
|
3262
|
+
}
|
|
3263
|
+
}
|
|
3264
|
+
function parseKeywords(value) {
|
|
3265
|
+
if (!Array.isArray(value)) return [];
|
|
3266
|
+
const out = [];
|
|
3267
|
+
for (const item of value) if (typeof item === "string" && item.trim().length > 0) out.push(item.trim());
|
|
3268
|
+
return out.slice(0, 10);
|
|
3269
|
+
}
|
|
3270
|
+
function parseEntities(value) {
|
|
3271
|
+
if (!Array.isArray(value)) return [];
|
|
3272
|
+
const out = [];
|
|
3273
|
+
const seen = /* @__PURE__ */ new Set();
|
|
3274
|
+
for (const item of value) {
|
|
3275
|
+
if (item === null || typeof item !== "object") continue;
|
|
3276
|
+
const obj = item;
|
|
3277
|
+
const type = typeof obj["type"] === "string" ? obj["type"] : "";
|
|
3278
|
+
const name = typeof obj["name"] === "string" ? obj["name"].trim() : "";
|
|
3279
|
+
const description = typeof obj["description"] === "string" ? obj["description"].trim() : "";
|
|
3280
|
+
if (name.length === 0) continue;
|
|
3281
|
+
if (!ENTITY_TYPES.includes(type)) continue;
|
|
3282
|
+
const key = `${type}|${name.toLowerCase()}`;
|
|
3283
|
+
if (seen.has(key)) continue;
|
|
3284
|
+
seen.add(key);
|
|
3285
|
+
out.push({
|
|
3286
|
+
type,
|
|
3287
|
+
name,
|
|
3288
|
+
description
|
|
3289
|
+
});
|
|
3290
|
+
}
|
|
3291
|
+
return out.slice(0, 20);
|
|
3292
|
+
}
|
|
3293
|
+
function fallbackEvent(chunk) {
|
|
3294
|
+
return {
|
|
3295
|
+
title: chunk.heading ?? "Untitled section",
|
|
3296
|
+
summary: "",
|
|
3297
|
+
content: chunk.content,
|
|
3298
|
+
category: "other",
|
|
3299
|
+
keywords: [],
|
|
3300
|
+
entities: []
|
|
3301
|
+
};
|
|
3302
|
+
}
|
|
3303
|
+
/**
|
|
3304
|
+
* Run extraction on a single chunk via the LLM caller.
|
|
3305
|
+
* Retries with exponential backoff on transient failures; falls back to a
|
|
3306
|
+
* heading-derived event only after all attempts fail.
|
|
3307
|
+
*/
|
|
3308
|
+
async function extractEventFromChunk(llm, chunk) {
|
|
3309
|
+
try {
|
|
3310
|
+
return parseExtractionResponse(await callLlmWithBackoff(llm, EXTRACTION_SYSTEM_PROMPT, buildExtractionUserPrompt(chunk)), chunk);
|
|
3311
|
+
} catch {
|
|
3312
|
+
return fallbackEvent(chunk);
|
|
3313
|
+
}
|
|
3314
|
+
}
|
|
3315
|
+
/**
|
|
3316
|
+
* Ask the LLM to rerank candidate events by relevance to a query.
|
|
3317
|
+
* Returns the candidate ids in reranked order (most relevant first).
|
|
3318
|
+
*
|
|
3319
|
+
* If the LLM fails or returns unparseable output, returns the input order.
|
|
3320
|
+
*/
|
|
3321
|
+
async function rerankEventsWithLlm(llm, query, candidates, topK) {
|
|
3322
|
+
if (candidates.length === 0) return [];
|
|
3323
|
+
if (candidates.length <= topK) return candidates.map((c) => c.id);
|
|
3324
|
+
const candidateList = candidates.map((c, i) => `[${i}] id=${c.id}\n title: ${c.title}\n summary: ${c.summary}`).join("\n");
|
|
3325
|
+
const system = `You are a precise relevance judge for a knowledge base retrieval system.
|
|
3326
|
+
|
|
3327
|
+
Given a user query and a list of candidate events, select the ${topK} event ids most useful for answering the query.
|
|
3328
|
+
|
|
3329
|
+
Return ONLY a JSON object: {"ids": ["id1", "id2", ...]}
|
|
3330
|
+
|
|
3331
|
+
The ids must come from the candidate list. Output at most ${topK} ids, most relevant first. No prose, no markdown.`;
|
|
3332
|
+
const user = `Query: ${query}
|
|
3333
|
+
|
|
3334
|
+
Candidates:
|
|
3335
|
+
${candidateList}
|
|
3336
|
+
|
|
3337
|
+
Return the JSON object now.`;
|
|
3338
|
+
try {
|
|
3339
|
+
const jsonText = extractJsonFromText(await llm.generate(system, user));
|
|
3340
|
+
if (jsonText === null) return candidates.slice(0, topK).map((c) => c.id);
|
|
3341
|
+
const parsed = JSON.parse(jsonText);
|
|
3342
|
+
if (!Array.isArray(parsed["ids"])) return candidates.slice(0, topK).map((c) => c.id);
|
|
3343
|
+
const validIds = new Set(candidates.map((c) => c.id));
|
|
3344
|
+
const result = [];
|
|
3345
|
+
for (const id of parsed["ids"]) {
|
|
3346
|
+
if (typeof id === "string" && validIds.has(id) && !result.includes(id)) result.push(id);
|
|
3347
|
+
if (result.length >= topK) break;
|
|
3348
|
+
}
|
|
3349
|
+
if (result.length === 0) return candidates.slice(0, topK).map((c) => c.id);
|
|
3350
|
+
return result;
|
|
3351
|
+
} catch {
|
|
3352
|
+
return candidates.slice(0, topK).map((c) => c.id);
|
|
3353
|
+
}
|
|
3354
|
+
}
|
|
3355
|
+
/**
|
|
3356
|
+
* Extract named entities from a query for entity-recall retrieval.
|
|
3357
|
+
* Returns a list of {type, name} pairs. Best-effort — falls back to empty.
|
|
3358
|
+
*/
|
|
3359
|
+
async function extractQueryEntities(llm, query) {
|
|
3360
|
+
const system = `Extract named entities from the user query. Return ONLY a JSON object: {"entities": [{"type": "...", "name": "..."}]}
|
|
3361
|
+
|
|
3362
|
+
Use these entity types: ${ENTITY_TYPES.join(", ")}.
|
|
3363
|
+
|
|
3364
|
+
Only include entities that explicitly appear in the query. If no clear entities, return an empty array.`;
|
|
3365
|
+
try {
|
|
3366
|
+
const jsonText = extractJsonFromText(await llm.generate(system, `Query: ${query}`));
|
|
3367
|
+
if (jsonText === null) return [];
|
|
3368
|
+
const parsed = JSON.parse(jsonText);
|
|
3369
|
+
if (!Array.isArray(parsed["entities"])) return [];
|
|
3370
|
+
const out = [];
|
|
3371
|
+
for (const item of parsed["entities"]) {
|
|
3372
|
+
if (item === null || typeof item !== "object") continue;
|
|
3373
|
+
const obj = item;
|
|
3374
|
+
const type = typeof obj["type"] === "string" ? obj["type"] : "";
|
|
3375
|
+
const name = typeof obj["name"] === "string" ? obj["name"].trim() : "";
|
|
3376
|
+
if (name.length === 0) continue;
|
|
3377
|
+
if (!ENTITY_TYPES.includes(type)) continue;
|
|
3378
|
+
out.push({
|
|
3379
|
+
type,
|
|
3380
|
+
name
|
|
3381
|
+
});
|
|
3382
|
+
}
|
|
3383
|
+
return out;
|
|
3384
|
+
} catch {
|
|
3385
|
+
return [];
|
|
3386
|
+
}
|
|
3387
|
+
}
|
|
3388
|
+
//#endregion
|
|
3389
|
+
//#region ../../packages/knowledge/src/ingest.ts
|
|
3390
|
+
/**
|
|
3391
|
+
* Embed texts, returning null-array if embeddings are skipped or on failure.
|
|
3392
|
+
* Caller should set skipEmbedding=true when the model didn't load in time,
|
|
3393
|
+
* so the entire ingestion is consistent (all or nothing for embeddings).
|
|
3394
|
+
* Search falls back to FTS5 keyword match for chunks without embeddings.
|
|
3395
|
+
*/
|
|
3396
|
+
async function embedOrNulls(engine, texts, skipEmbedding) {
|
|
3397
|
+
if (engine === void 0 || texts.length === 0 || skipEmbedding) return texts.map(() => null);
|
|
3398
|
+
if (!engine.available) return texts.map(() => null);
|
|
3399
|
+
const vectors = await engine.embedBatch(texts);
|
|
3400
|
+
if (vectors === null || vectors.length !== texts.length) return texts.map(() => null);
|
|
3401
|
+
return vectors;
|
|
3402
|
+
}
|
|
3403
|
+
/** LLM concurrency for extraction — callers and comments below refer to this constant. */
|
|
3404
|
+
const LLM_CONCURRENCY = 20;
|
|
3405
|
+
const SUPPORTED_EXTENSIONS = new Set([
|
|
3406
|
+
".md",
|
|
3407
|
+
".markdown",
|
|
3408
|
+
".txt"
|
|
3409
|
+
]);
|
|
3410
|
+
/** Read a file and return its content. */
|
|
3411
|
+
async function readFileContent(filePath) {
|
|
3412
|
+
return readFile(filePath, "utf-8");
|
|
3413
|
+
}
|
|
3414
|
+
function getFileExtension(filePath) {
|
|
3415
|
+
const lastDot = filePath.lastIndexOf(".");
|
|
3416
|
+
return lastDot > 0 ? filePath.slice(lastDot).toLowerCase() : "";
|
|
3417
|
+
}
|
|
3418
|
+
function chunkContent(content, ext) {
|
|
3419
|
+
return ext === ".txt" ? chunkText(content) : chunkMarkdown(content);
|
|
3420
|
+
}
|
|
3421
|
+
function isSupportedFile(filePath) {
|
|
3422
|
+
return SUPPORTED_EXTENSIONS.has(getFileExtension(filePath));
|
|
3423
|
+
}
|
|
3424
|
+
/** Map async iterator values with a concurrency limit. */
|
|
3425
|
+
async function mapWithConcurrency(items, limit, fn, onProgress) {
|
|
3426
|
+
const results = Array.from({ length: items.length });
|
|
3427
|
+
let cursor = 0;
|
|
3428
|
+
let completed = 0;
|
|
3429
|
+
const total = items.length;
|
|
3430
|
+
async function worker() {
|
|
3431
|
+
while (cursor < items.length) {
|
|
3432
|
+
const idx = cursor;
|
|
3433
|
+
cursor += 1;
|
|
3434
|
+
const item = items[idx];
|
|
3435
|
+
const result = await fn(item, idx);
|
|
3436
|
+
results[idx] = result;
|
|
3437
|
+
completed += 1;
|
|
3438
|
+
onProgress?.(completed, total, result);
|
|
3439
|
+
}
|
|
3440
|
+
}
|
|
3441
|
+
const workers = Array.from({ length: Math.min(limit, items.length) }, () => worker());
|
|
3442
|
+
await Promise.all(workers);
|
|
3443
|
+
return results;
|
|
3444
|
+
}
|
|
3445
|
+
/**
|
|
3446
|
+
* Stamp or verify the embedding-model identity recorded in the library.
|
|
3447
|
+
* A library written by a different model would mix vector spaces — block the
|
|
3448
|
+
* ingest with a clear message instead of silently storing incompatible
|
|
3449
|
+
* embeddings. A legacy library without the marker is stamped with the current
|
|
3450
|
+
* model (all existing data was written by it).
|
|
3451
|
+
*/
|
|
3452
|
+
async function ensureEmbeddingModelMatches(store, engine) {
|
|
3453
|
+
if (engine === void 0) return;
|
|
3454
|
+
const recorded = await store.getMeta(EMBEDDING_MODEL_META_KEY);
|
|
3455
|
+
if (recorded === null) {
|
|
3456
|
+
await store.setMeta(EMBEDDING_MODEL_META_KEY, engine.modelName);
|
|
3457
|
+
return;
|
|
3458
|
+
}
|
|
3459
|
+
if (recorded !== engine.modelName) throw new Error(`知识库由模型 ${recorded} 嵌入,当前模型为 ${engine.modelName},请先在 /knowledge 中重新嵌入知识库`);
|
|
3460
|
+
}
|
|
3461
|
+
/**
|
|
3462
|
+
* Ingest a markdown file into the knowledge base.
|
|
3463
|
+
*
|
|
3464
|
+
* Steps:
|
|
3465
|
+
* 1. Dedupe by file_path — if a source already exists for this file, error.
|
|
3466
|
+
* 2. Read file → chunkMarkdown (heading_strict).
|
|
3467
|
+
* 3. Embed chunks ("{heading}\n{content}") → store knowledge_chunks.
|
|
3468
|
+
* 4. For each chunk (concurrency LLM_CONCURRENCY): LLM extract 1 event + N entities.
|
|
3469
|
+
* 5. Embed event title + content ("{title}\n\n{content}") → store knowledge_events.
|
|
3470
|
+
* 6. Upsert entities (dedupe by source_id+type+normalizedName) + embed entity name → store.
|
|
3471
|
+
* 7. For each (event, entity) pair: embed relation (entity.description || "{eventTitle} {entityName}") → store knowledge_event_entities.
|
|
3472
|
+
* 8. Update document status = 'completed'.
|
|
3473
|
+
*
|
|
3474
|
+
* All writes run inside a single transaction — a mid-ingest failure rolls back
|
|
3475
|
+
* every partial row (source/document/chunks/events/entities/edges).
|
|
3476
|
+
*
|
|
3477
|
+
* Reports progress via callback.
|
|
3478
|
+
*/
|
|
3479
|
+
async function ingestFile(store, llm, filePath, onProgress) {
|
|
3480
|
+
const engine = store.getEmbeddingEngine();
|
|
3481
|
+
const skipEmbed = engine === void 0 || !engine.available;
|
|
3482
|
+
if (skipEmbed) onProgress?.({
|
|
3483
|
+
stage: "embedding-check",
|
|
3484
|
+
message: "向量模型未就绪,本次摄入跳过向量嵌入"
|
|
3485
|
+
});
|
|
3486
|
+
await ensureEmbeddingModelMatches(store, engine);
|
|
3487
|
+
onProgress?.({
|
|
3488
|
+
stage: "chunking",
|
|
3489
|
+
message: `读取并切分文件: ${basename$1(filePath)}`
|
|
3490
|
+
});
|
|
3491
|
+
const content = await readFileContent(filePath);
|
|
3492
|
+
const contentHash = createHash("sha256").update(content).digest("hex");
|
|
3493
|
+
const existing = await store.findSourceByFilePath(filePath);
|
|
3494
|
+
let replacedOldVersion = false;
|
|
3495
|
+
if (existing !== void 0) {
|
|
3496
|
+
if (existing.contentHash === contentHash || existing.contentHash === null) {
|
|
3497
|
+
if (existing.contentHash === null) await store.updateSourceContentHash(existing.id, contentHash);
|
|
3498
|
+
onProgress?.({
|
|
3499
|
+
stage: "completed",
|
|
3500
|
+
message: "内容未变化,已跳过"
|
|
3501
|
+
});
|
|
3502
|
+
return {
|
|
3503
|
+
documentId: null,
|
|
3504
|
+
chunkCount: 0,
|
|
3505
|
+
eventCount: 0,
|
|
3506
|
+
entityCount: 0,
|
|
3507
|
+
outcome: "unchanged"
|
|
3508
|
+
};
|
|
3509
|
+
}
|
|
3510
|
+
onProgress?.({
|
|
3511
|
+
stage: "chunking",
|
|
3512
|
+
message: `检测到内容变更,替换旧版本: ${existing.name}`
|
|
3513
|
+
});
|
|
3514
|
+
await store.deleteSource(existing.id);
|
|
3515
|
+
replacedOldVersion = true;
|
|
3516
|
+
}
|
|
3517
|
+
const sections = chunkContent(content, getFileExtension(filePath));
|
|
3518
|
+
store.beginTransaction();
|
|
3519
|
+
try {
|
|
3520
|
+
if (sections.length === 0) {
|
|
3521
|
+
onProgress?.({
|
|
3522
|
+
stage: "error",
|
|
3523
|
+
message: "文件无有效内容"
|
|
3524
|
+
});
|
|
3525
|
+
throw new Error("文件无有效内容可摄入");
|
|
3526
|
+
}
|
|
3527
|
+
const fileName = basename$1(filePath);
|
|
3528
|
+
const source = await store.createSource({
|
|
3529
|
+
name: fileName,
|
|
3530
|
+
filePath,
|
|
3531
|
+
description: null,
|
|
3532
|
+
contentHash
|
|
3533
|
+
});
|
|
3534
|
+
const document = await store.createDocument({
|
|
3535
|
+
sourceId: source.id,
|
|
3536
|
+
title: fileName,
|
|
3537
|
+
content
|
|
3538
|
+
});
|
|
3539
|
+
const { chunks, events, uniqueEntities } = await ingestSections(store, llm, {
|
|
3540
|
+
source,
|
|
3541
|
+
document,
|
|
3542
|
+
sections,
|
|
3543
|
+
engine,
|
|
3544
|
+
skipEmbed
|
|
3545
|
+
}, onProgress);
|
|
3546
|
+
return {
|
|
3547
|
+
documentId: document.id,
|
|
3548
|
+
chunkCount: chunks.length,
|
|
3549
|
+
eventCount: events.length,
|
|
3550
|
+
entityCount: uniqueEntities.length,
|
|
3551
|
+
outcome: replacedOldVersion ? "updated" : "created"
|
|
3552
|
+
};
|
|
3553
|
+
} catch (error) {
|
|
3554
|
+
store.rollbackTransaction();
|
|
3555
|
+
if (replacedOldVersion) {
|
|
3556
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
3557
|
+
throw new Error(`旧版本已删除、新版本摄入失败,请重新摄入该文件: ${filePath}(${message})`);
|
|
3558
|
+
}
|
|
3559
|
+
throw error;
|
|
3560
|
+
}
|
|
3561
|
+
}
|
|
3562
|
+
/**
|
|
3563
|
+
* Shared steps 4-9 of both ingest paths: embed + store chunks, extract events
|
|
3564
|
+
* via LLM, embed + store events, upsert entities, store relations, commit the
|
|
3565
|
+
* caller-opened transaction, and emit the completed progress. The file and
|
|
3566
|
+
* content pipelines were identical here modulo formatting — one body keeps
|
|
3567
|
+
* them from drifting apart again (content had already diverged: it dropped
|
|
3568
|
+
* the per-chunk embedding variable and the entity-embedding temp). The caller
|
|
3569
|
+
* owns beginTransaction and the catch/rollback.
|
|
3570
|
+
*/
|
|
3571
|
+
async function ingestSections(store, llm, args, onProgress) {
|
|
3572
|
+
const { source, document, sections, engine, skipEmbed } = args;
|
|
3573
|
+
onProgress?.({
|
|
3574
|
+
stage: "embedding-chunks",
|
|
3575
|
+
chunkIndex: 0,
|
|
3576
|
+
totalChunks: sections.length,
|
|
3577
|
+
message: `嵌入 chunks: 0/${sections.length}`
|
|
3578
|
+
});
|
|
3579
|
+
const chunkEmbeddings = await embedOrNulls(engine, sections.map((s) => s.heading !== null ? `${s.heading}\n${s.content}` : s.content), skipEmbed);
|
|
3580
|
+
const chunks = [];
|
|
3581
|
+
for (let i = 0; i < sections.length; i++) {
|
|
3582
|
+
const section = sections[i];
|
|
3583
|
+
const embedding = chunkEmbeddings[i] ?? null;
|
|
3584
|
+
const chunk = await store.insertChunk({
|
|
3585
|
+
sourceId: source.id,
|
|
3586
|
+
documentId: document.id,
|
|
3587
|
+
rank: section.rank,
|
|
3588
|
+
heading: section.heading,
|
|
3589
|
+
content: section.content,
|
|
3590
|
+
rawContent: section.rawContent,
|
|
3591
|
+
embedding
|
|
3592
|
+
});
|
|
3593
|
+
chunks.push(chunk);
|
|
3594
|
+
onProgress?.({
|
|
3595
|
+
stage: "embedding-chunks",
|
|
3596
|
+
chunkIndex: i + 1,
|
|
3597
|
+
totalChunks: sections.length,
|
|
3598
|
+
message: `嵌入 chunks: ${i + 1}/${sections.length}`
|
|
3599
|
+
});
|
|
3600
|
+
}
|
|
3601
|
+
onProgress?.({
|
|
3602
|
+
stage: "extracting",
|
|
3603
|
+
chunkIndex: 0,
|
|
3604
|
+
totalChunks: sections.length,
|
|
3605
|
+
message: `抽取事件: 0/${sections.length}`
|
|
3606
|
+
});
|
|
3607
|
+
const extractedEvents = await mapWithConcurrency(sections, LLM_CONCURRENCY, (section) => extractEventFromChunk(llm, section), (completed, total) => {
|
|
3608
|
+
onProgress?.({
|
|
3609
|
+
stage: "extracting",
|
|
3610
|
+
chunkIndex: completed,
|
|
3611
|
+
totalChunks: total,
|
|
3612
|
+
message: `抽取事件: ${completed}/${total}`
|
|
3613
|
+
});
|
|
3614
|
+
});
|
|
3615
|
+
onProgress?.({
|
|
3616
|
+
stage: "embedding-events",
|
|
3617
|
+
chunkIndex: 0,
|
|
3618
|
+
totalChunks: sections.length,
|
|
3619
|
+
message: `嵌入 events: 0/${sections.length}`
|
|
3620
|
+
});
|
|
3621
|
+
const titleTexts = extractedEvents.map((e) => e.title);
|
|
3622
|
+
const contentTexts = extractedEvents.map((e) => `${e.title}\n\n${e.content}`);
|
|
3623
|
+
const [titleEmbeddings, contentEmbeddings] = await Promise.all([embedOrNulls(engine, titleTexts, skipEmbed), embedOrNulls(engine, contentTexts, skipEmbed)]);
|
|
3624
|
+
const events = [];
|
|
3625
|
+
for (let i = 0; i < extractedEvents.length; i++) {
|
|
3626
|
+
const extracted = extractedEvents[i];
|
|
3627
|
+
const chunk = chunks[i];
|
|
3628
|
+
const event = await store.insertEvent({
|
|
3629
|
+
sourceId: source.id,
|
|
3630
|
+
documentId: document.id,
|
|
3631
|
+
chunkId: chunk.id,
|
|
3632
|
+
rank: i,
|
|
3633
|
+
title: extracted.title,
|
|
3634
|
+
summary: extracted.summary.length > 0 ? extracted.summary : null,
|
|
3635
|
+
content: extracted.content,
|
|
3636
|
+
category: extracted.category.length > 0 ? extracted.category : null,
|
|
3637
|
+
keywords: extracted.keywords,
|
|
3638
|
+
titleEmbedding: titleEmbeddings[i] ?? null,
|
|
3639
|
+
contentEmbedding: contentEmbeddings[i] ?? null
|
|
3640
|
+
});
|
|
3641
|
+
events.push(event);
|
|
3642
|
+
onProgress?.({
|
|
3643
|
+
stage: "embedding-events",
|
|
3644
|
+
chunkIndex: i + 1,
|
|
3645
|
+
totalChunks: sections.length,
|
|
3646
|
+
message: `嵌入 events: ${i + 1}/${sections.length}`
|
|
3647
|
+
});
|
|
3648
|
+
}
|
|
3649
|
+
onProgress?.({
|
|
3650
|
+
stage: "embedding-entities",
|
|
3651
|
+
message: "嵌入 entities..."
|
|
3652
|
+
});
|
|
3653
|
+
const entityMap = /* @__PURE__ */ new Map();
|
|
3654
|
+
for (const extracted of extractedEvents) for (const entity of extracted.entities) {
|
|
3655
|
+
const key = `${entity.type}|${entity.name.toLowerCase()}`;
|
|
3656
|
+
if (!entityMap.has(key)) entityMap.set(key, entity);
|
|
3657
|
+
}
|
|
3658
|
+
const uniqueEntities = Array.from(entityMap.values());
|
|
3659
|
+
const entityEmbeddings = uniqueEntities.length > 0 ? await embedOrNulls(engine, uniqueEntities.map((e) => e.name), skipEmbed) : [];
|
|
3660
|
+
const entityIdByEntityKey = /* @__PURE__ */ new Map();
|
|
3661
|
+
for (let i = 0; i < uniqueEntities.length; i++) {
|
|
3662
|
+
const entity = uniqueEntities[i];
|
|
3663
|
+
const embedding = entityEmbeddings[i] ?? null;
|
|
3664
|
+
const stored = await store.upsertEntity({
|
|
3665
|
+
sourceId: source.id,
|
|
3666
|
+
type: entity.type,
|
|
3667
|
+
name: entity.name,
|
|
3668
|
+
description: entity.description.length > 0 ? entity.description : null,
|
|
3669
|
+
embedding
|
|
3670
|
+
});
|
|
3671
|
+
entityIdByEntityKey.set(`${entity.type}|${entity.name.toLowerCase()}`, stored.id);
|
|
3672
|
+
}
|
|
3673
|
+
onProgress?.({
|
|
3674
|
+
stage: "embedding-relations",
|
|
3675
|
+
message: "嵌入关系..."
|
|
3676
|
+
});
|
|
3677
|
+
const relationPairs = [];
|
|
3678
|
+
for (let i = 0; i < extractedEvents.length; i++) {
|
|
3679
|
+
const extracted = extractedEvents[i];
|
|
3680
|
+
for (const entity of extracted.entities) relationPairs.push({
|
|
3681
|
+
eventIndex: i,
|
|
3682
|
+
entity
|
|
3683
|
+
});
|
|
3684
|
+
}
|
|
3685
|
+
const relationTexts = relationPairs.map(({ eventIndex, entity }) => {
|
|
3686
|
+
const event = events[eventIndex];
|
|
3687
|
+
return entity.description.length > 0 ? entity.description : `${event.title} ${entity.name}`;
|
|
3688
|
+
});
|
|
3689
|
+
const relationEmbeddings = relationPairs.length > 0 ? await embedOrNulls(engine, relationTexts, skipEmbed) : [];
|
|
3690
|
+
for (let i = 0; i < relationPairs.length; i++) {
|
|
3691
|
+
const pair = relationPairs[i];
|
|
3692
|
+
const event = events[pair.eventIndex];
|
|
3693
|
+
const entityKey = `${pair.entity.type}|${pair.entity.name.toLowerCase()}`;
|
|
3694
|
+
const entityId = entityIdByEntityKey.get(entityKey);
|
|
3695
|
+
if (entityId === void 0) continue;
|
|
3696
|
+
const embedding = relationEmbeddings?.[i] ?? null;
|
|
3697
|
+
await store.insertEventEntity({
|
|
3698
|
+
eventId: event.id,
|
|
3699
|
+
entityId,
|
|
3700
|
+
weight: 1,
|
|
3701
|
+
description: pair.entity.description.length > 0 ? pair.entity.description : null,
|
|
3702
|
+
embedding
|
|
3703
|
+
});
|
|
3704
|
+
}
|
|
3705
|
+
await store.updateDocumentStatus(document.id, "completed", chunks.length);
|
|
3706
|
+
store.commitTransaction();
|
|
3707
|
+
onProgress?.({
|
|
3708
|
+
stage: "completed",
|
|
3709
|
+
message: `摄入完成:${chunks.length} chunks, ${events.length} events, ${uniqueEntities.length} entities`
|
|
3710
|
+
});
|
|
3711
|
+
return {
|
|
3712
|
+
chunks,
|
|
3713
|
+
events,
|
|
3714
|
+
uniqueEntities
|
|
3715
|
+
};
|
|
3716
|
+
}
|
|
3717
|
+
async function collectSupportedFiles(dirPath) {
|
|
3718
|
+
const files = [];
|
|
3719
|
+
const entries = await readdir(dirPath, { withFileTypes: true });
|
|
3720
|
+
for (const entry of entries) {
|
|
3721
|
+
const fullPath = join$1(dirPath, entry.name);
|
|
3722
|
+
if (entry.isDirectory()) files.push(...await collectSupportedFiles(fullPath));
|
|
3723
|
+
else if (entry.isFile() && isSupportedFile(fullPath)) files.push(fullPath);
|
|
3724
|
+
}
|
|
3725
|
+
return files.sort();
|
|
3726
|
+
}
|
|
3727
|
+
async function ingestDirectory(store, llm, dirPath, onProgress) {
|
|
3728
|
+
const files = await collectSupportedFiles(dirPath);
|
|
3729
|
+
if (files.length === 0) throw new Error(`目录下没有支持的文件 (.md, .markdown, .txt): ${dirPath}`);
|
|
3730
|
+
const errors = [];
|
|
3731
|
+
let succeeded = 0;
|
|
3732
|
+
let created = 0;
|
|
3733
|
+
let updated = 0;
|
|
3734
|
+
let unchanged = 0;
|
|
3735
|
+
let totalChunks = 0;
|
|
3736
|
+
let totalEvents = 0;
|
|
3737
|
+
let totalEntities = 0;
|
|
3738
|
+
for (let i = 0; i < files.length; i++) {
|
|
3739
|
+
const filePath = files[i];
|
|
3740
|
+
const fileProgress = (progress) => {
|
|
3741
|
+
const prefix = `批量摄入 ${i + 1}/${files.length}: ${basename$1(filePath)}`;
|
|
3742
|
+
onProgress?.({
|
|
3743
|
+
...progress,
|
|
3744
|
+
message: `${prefix} · ${progress.message}`
|
|
3745
|
+
});
|
|
3746
|
+
};
|
|
3747
|
+
try {
|
|
3748
|
+
const result = await ingestFile(store, llm, filePath, fileProgress);
|
|
3749
|
+
succeeded += 1;
|
|
3750
|
+
if (result.outcome === "created") created += 1;
|
|
3751
|
+
else if (result.outcome === "updated") updated += 1;
|
|
3752
|
+
else unchanged += 1;
|
|
3753
|
+
totalChunks += result.chunkCount;
|
|
3754
|
+
totalEvents += result.eventCount;
|
|
3755
|
+
totalEntities += result.entityCount;
|
|
3756
|
+
} catch (error) {
|
|
3757
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
3758
|
+
errors.push({
|
|
3759
|
+
filePath,
|
|
3760
|
+
message
|
|
3761
|
+
});
|
|
3762
|
+
onProgress?.({
|
|
3763
|
+
stage: "error",
|
|
3764
|
+
message: `批量摄入 ${i + 1}/${files.length}: ${basename$1(filePath)} · 失败: ${message}`
|
|
3765
|
+
});
|
|
3766
|
+
}
|
|
3767
|
+
}
|
|
3768
|
+
onProgress?.({
|
|
3769
|
+
stage: "completed",
|
|
3770
|
+
message: `批量摄入完成:新增 ${created} · 更新 ${updated} · 未变化 ${unchanged} · 失败 ${errors.length}(${totalChunks} chunks, ${totalEvents} events, ${totalEntities} entities)`
|
|
3771
|
+
});
|
|
3772
|
+
return {
|
|
3773
|
+
succeeded,
|
|
3774
|
+
failed: errors.length,
|
|
3775
|
+
created,
|
|
3776
|
+
updated,
|
|
3777
|
+
unchanged,
|
|
3778
|
+
totalChunks,
|
|
3779
|
+
totalEvents,
|
|
3780
|
+
totalEntities,
|
|
3781
|
+
errors
|
|
3782
|
+
};
|
|
3783
|
+
}
|
|
3784
|
+
//#endregion
|
|
3785
|
+
//#region ../../packages/knowledge/src/search.ts
|
|
3786
|
+
const DEFAULT_TOP_K = 5;
|
|
3787
|
+
const MAX_TOP_K = 20;
|
|
3788
|
+
const SEED_EVENT_LIMIT = 30;
|
|
3789
|
+
const EXPANDED_EVENT_LIMIT = 100;
|
|
3790
|
+
const COARSE_RANK_LIMIT = 50;
|
|
3791
|
+
const ENTITY_VECTOR_THRESHOLD = .7;
|
|
3792
|
+
const TITLE_VECTOR_THRESHOLD = .4;
|
|
3793
|
+
async function timed(onStep, step, detail, fn, payload) {
|
|
3794
|
+
const start = performance.now();
|
|
3795
|
+
try {
|
|
3796
|
+
const result = await fn();
|
|
3797
|
+
const traceStep = {
|
|
3798
|
+
step,
|
|
3799
|
+
detail,
|
|
3800
|
+
durationMs: Math.round((performance.now() - start) * 100) / 100,
|
|
3801
|
+
...payload !== void 0 ? { payload: payload(result) } : {}
|
|
3802
|
+
};
|
|
3803
|
+
onStep?.(traceStep);
|
|
3804
|
+
return result;
|
|
3805
|
+
} catch (error) {
|
|
3806
|
+
const durationMs = Math.round((performance.now() - start) * 100) / 100;
|
|
3807
|
+
onStep?.({
|
|
3808
|
+
step,
|
|
3809
|
+
detail: `${detail} 失败:${error instanceof Error ? error.message : String(error)}`,
|
|
3810
|
+
durationMs
|
|
3811
|
+
});
|
|
3812
|
+
throw error;
|
|
3813
|
+
}
|
|
3814
|
+
}
|
|
3815
|
+
/**
|
|
3816
|
+
* Multi-hop retrieval:
|
|
3817
|
+
* 1. Vectorize query.
|
|
3818
|
+
* 2. Entity recall — LLM extracts entities from query → match by name + vector.
|
|
3819
|
+
* 3. Seed events — events linked to recalled entities + events by title vector similarity.
|
|
3820
|
+
* 4. BFS expand (1 hop) — from seed events, walk to entities, then to other events.
|
|
3821
|
+
* 5. Coarse rank — score graph-reachable events by content embedding similarity.
|
|
3822
|
+
* Graph associativity is the gate; content vector orders within it.
|
|
3823
|
+
* 6. LLM rerank — pick top-K most relevant.
|
|
3824
|
+
* 7. Return corresponding chunks (deduped by chunk_id) with scores and provenance.
|
|
3825
|
+
* If rerank yields fewer than topK chunks, backfill by direct chunk vector search.
|
|
3826
|
+
*
|
|
3827
|
+
* Optional `onStep` callback receives a trace step at each phase — used by the
|
|
3828
|
+
* KnowledgeLookup tool to expose retrieval diagnostics to the agent.
|
|
3829
|
+
*/
|
|
3830
|
+
async function multiSearch(store, llm, query, options = {}) {
|
|
3831
|
+
return (await multiSearchWithTrace(store, llm, query, options)).results;
|
|
3832
|
+
}
|
|
3833
|
+
/** Like multiSearch but also returns the retrieval trace for diagnostics. */
|
|
3834
|
+
async function multiSearchWithTrace(store, llm, query, options = {}) {
|
|
3835
|
+
const topK = Math.min(options.topK ?? DEFAULT_TOP_K, MAX_TOP_K);
|
|
3836
|
+
const skipLlm = options.skipLlm === true;
|
|
3837
|
+
const skipRerank = skipLlm || options.skipRerank === true;
|
|
3838
|
+
const steps = [];
|
|
3839
|
+
const onStep = (step) => steps.push(step);
|
|
3840
|
+
const rerankedEventTitles = [];
|
|
3841
|
+
let fallbackReason = null;
|
|
3842
|
+
const engine = store.getEmbeddingEngine();
|
|
3843
|
+
if (engine === void 0 || !engine.available) {
|
|
3844
|
+
fallbackReason = "embedding engine unavailable; used FTS5 keyword fallback";
|
|
3845
|
+
return {
|
|
3846
|
+
results: await ftsFallback(store, query, topK),
|
|
3847
|
+
trace: {
|
|
3848
|
+
steps,
|
|
3849
|
+
rerankedEventTitles,
|
|
3850
|
+
fallbackReason
|
|
3851
|
+
}
|
|
3852
|
+
};
|
|
3853
|
+
}
|
|
3854
|
+
const recordedModel = await store.getMeta(EMBEDDING_MODEL_META_KEY);
|
|
3855
|
+
if (recordedModel !== null && recordedModel !== engine.modelName) {
|
|
3856
|
+
fallbackReason = `embedding model mismatch (library: ${recordedModel}, current: ${engine.modelName}); used FTS5 keyword fallback`;
|
|
3857
|
+
return {
|
|
3858
|
+
results: await ftsFallback(store, query, topK),
|
|
3859
|
+
trace: {
|
|
3860
|
+
steps,
|
|
3861
|
+
rerankedEventTitles,
|
|
3862
|
+
fallbackReason
|
|
3863
|
+
}
|
|
3864
|
+
};
|
|
3865
|
+
}
|
|
3866
|
+
const queryEmbeddings = await timed(onStep, "queryEmbedding", "把用户问题转成向量,用于召回相关事件和切片。", async () => engine.embedBatch([query]));
|
|
3867
|
+
if (queryEmbeddings === null || queryEmbeddings.length === 0) {
|
|
3868
|
+
fallbackReason = "query embedding returned null; used FTS5 fallback";
|
|
3869
|
+
return {
|
|
3870
|
+
results: await ftsFallback(store, query, topK),
|
|
3871
|
+
trace: {
|
|
3872
|
+
steps,
|
|
3873
|
+
rerankedEventTitles,
|
|
3874
|
+
fallbackReason
|
|
3875
|
+
}
|
|
3876
|
+
};
|
|
3877
|
+
}
|
|
3878
|
+
const queryVec = queryEmbeddings[0];
|
|
3879
|
+
const recalledEntities = await timed(onStep, "entityRecall", skipLlm ? "本地向量实体召回(skipLlm,不调 LLM)。" : "LLM 抽 query 实体 + 名字精确匹配 + 向量召回。", () => recallEntities(store, llm, query, queryVec, !skipLlm), (entities) => ({ count: entities.length }));
|
|
3880
|
+
const seedEventIds = /* @__PURE__ */ new Set();
|
|
3881
|
+
for (const entity of recalledEntities) {
|
|
3882
|
+
const events = await store.findEventsByEntity(entity.id);
|
|
3883
|
+
for (const event of events) seedEventIds.add(event.id);
|
|
3884
|
+
}
|
|
3885
|
+
const titleMatches = await timed(onStep, "seedEventsByTitle", "按查询向量在事件标题向量上召回 seed events。", () => store.findEventsByTitleVector(queryVec, {
|
|
3886
|
+
limit: SEED_EVENT_LIMIT,
|
|
3887
|
+
threshold: TITLE_VECTOR_THRESHOLD
|
|
3888
|
+
}), (matches) => ({ count: matches.length }));
|
|
3889
|
+
for (const { event } of titleMatches) seedEventIds.add(event.id);
|
|
3890
|
+
if (seedEventIds.size === 0) {
|
|
3891
|
+
const results = await chunkVectorFallback(store, queryVec, topK);
|
|
3892
|
+
if (results.length > 0) {
|
|
3893
|
+
fallbackReason = "no seed events; used direct chunk vector search";
|
|
3894
|
+
return {
|
|
3895
|
+
results,
|
|
3896
|
+
trace: {
|
|
3897
|
+
steps,
|
|
3898
|
+
rerankedEventTitles,
|
|
3899
|
+
fallbackReason
|
|
3900
|
+
}
|
|
3901
|
+
};
|
|
3902
|
+
}
|
|
3903
|
+
fallbackReason = "no seed events and direct chunk vector search returned no results; used FTS5 keyword fallback";
|
|
3904
|
+
return {
|
|
3905
|
+
results: await ftsFallback(store, query, topK),
|
|
3906
|
+
trace: {
|
|
3907
|
+
steps,
|
|
3908
|
+
rerankedEventTitles,
|
|
3909
|
+
fallbackReason
|
|
3910
|
+
}
|
|
3911
|
+
};
|
|
3912
|
+
}
|
|
3913
|
+
const expandedEventIds = await timed(onStep, "bfsExpand", "从 seed events 沿实体关系 1 跳扩展候选事件。", async () => {
|
|
3914
|
+
const ids = new Set(seedEventIds);
|
|
3915
|
+
let capped = false;
|
|
3916
|
+
for (const eventId of seedEventIds) {
|
|
3917
|
+
const entities = await store.findEntitiesByEvent(eventId);
|
|
3918
|
+
for (const entity of entities) {
|
|
3919
|
+
const neighborEvents = await store.findEventsByEntity(entity.id);
|
|
3920
|
+
for (const neighbor of neighborEvents) {
|
|
3921
|
+
ids.add(neighbor.id);
|
|
3922
|
+
if (ids.size >= EXPANDED_EVENT_LIMIT) {
|
|
3923
|
+
capped = true;
|
|
3924
|
+
break;
|
|
3925
|
+
}
|
|
3926
|
+
}
|
|
3927
|
+
if (ids.size >= EXPANDED_EVENT_LIMIT) break;
|
|
3928
|
+
}
|
|
3929
|
+
if (ids.size >= EXPANDED_EVENT_LIMIT) break;
|
|
3930
|
+
}
|
|
3931
|
+
return {
|
|
3932
|
+
ids,
|
|
3933
|
+
capped
|
|
3934
|
+
};
|
|
3935
|
+
}, (result) => ({
|
|
3936
|
+
expandedCount: result.ids.size,
|
|
3937
|
+
capped: result.capped
|
|
3938
|
+
}));
|
|
3939
|
+
const { candidates, topCandidates } = await timed(onStep, "coarseRank", "按 content 向量相似度对候选事件排序,取前 COARSE_RANK_LIMIT 个。", async () => {
|
|
3940
|
+
const cands = [];
|
|
3941
|
+
for (const eventId of expandedEventIds.ids) {
|
|
3942
|
+
const event = await store.getEvent(eventId);
|
|
3943
|
+
if (event === void 0) continue;
|
|
3944
|
+
const vec = event.contentEmbedding;
|
|
3945
|
+
const score = vec === null ? 0 : engine.cosineSimilarity(queryVec, vec);
|
|
3946
|
+
cands.push({
|
|
3947
|
+
id: event.id,
|
|
3948
|
+
title: event.title,
|
|
3949
|
+
summary: event.summary ?? "",
|
|
3950
|
+
score,
|
|
3951
|
+
chunkId: event.chunkId
|
|
3952
|
+
});
|
|
3953
|
+
}
|
|
3954
|
+
cands.sort((a, b) => b.score - a.score);
|
|
3955
|
+
return {
|
|
3956
|
+
candidates: cands,
|
|
3957
|
+
topCandidates: cands.slice(0, COARSE_RANK_LIMIT)
|
|
3958
|
+
};
|
|
3959
|
+
}, (result) => ({
|
|
3960
|
+
totalCandidates: result.candidates.length,
|
|
3961
|
+
kept: result.topCandidates.length,
|
|
3962
|
+
topScore: result.topCandidates[0]?.score ?? 0
|
|
3963
|
+
}));
|
|
3964
|
+
if (candidates.length === 0) {
|
|
3965
|
+
const results = await chunkVectorFallback(store, queryVec, topK);
|
|
3966
|
+
if (results.length > 0) {
|
|
3967
|
+
fallbackReason = "no graph-reachable events with content; used chunk vector fallback";
|
|
3968
|
+
return {
|
|
3969
|
+
results,
|
|
3970
|
+
trace: {
|
|
3971
|
+
steps,
|
|
3972
|
+
rerankedEventTitles,
|
|
3973
|
+
fallbackReason
|
|
3974
|
+
}
|
|
3975
|
+
};
|
|
3976
|
+
}
|
|
3977
|
+
fallbackReason = "no graph-reachable events with content and chunk vector search returned no results; used FTS5 keyword fallback";
|
|
3978
|
+
return {
|
|
3979
|
+
results: await ftsFallback(store, query, topK),
|
|
3980
|
+
trace: {
|
|
3981
|
+
steps,
|
|
3982
|
+
rerankedEventTitles,
|
|
3983
|
+
fallbackReason
|
|
3984
|
+
}
|
|
3985
|
+
};
|
|
3986
|
+
}
|
|
3987
|
+
let rankedIds;
|
|
3988
|
+
if (skipRerank) {
|
|
3989
|
+
rankedIds = topCandidates.map((c) => c.id);
|
|
3990
|
+
onStep({
|
|
3991
|
+
step: "rerank",
|
|
3992
|
+
detail: "跳过 LLM rerank(skipRerank/skipLlm),直接用 coarse rank 顺序。",
|
|
3993
|
+
durationMs: 0,
|
|
3994
|
+
payload: { count: rankedIds.length }
|
|
3995
|
+
});
|
|
3996
|
+
} else if (topCandidates.length <= topK) {
|
|
3997
|
+
rankedIds = topCandidates.map((c) => c.id);
|
|
3998
|
+
onStep({
|
|
3999
|
+
step: "rerank",
|
|
4000
|
+
detail: `候选数 ${topCandidates.length} ≤ topK ${topK},无需 LLM rerank。`,
|
|
4001
|
+
durationMs: 0,
|
|
4002
|
+
payload: { count: rankedIds.length }
|
|
4003
|
+
});
|
|
4004
|
+
} else rankedIds = await timed(onStep, "rerank", `LLM 从 ${topCandidates.length} 个候选中选最相关的 ${topK} 个。`, () => rerankEventsWithLlm(llm, query, topCandidates.map((c) => ({
|
|
4005
|
+
id: c.id,
|
|
4006
|
+
title: c.title,
|
|
4007
|
+
summary: c.summary
|
|
4008
|
+
})), topK), (ids) => ({ count: ids.length }));
|
|
4009
|
+
for (const id of rankedIds) {
|
|
4010
|
+
const c = topCandidates.find((x) => x.id === id);
|
|
4011
|
+
if (c !== void 0) rerankedEventTitles.push(c.title);
|
|
4012
|
+
}
|
|
4013
|
+
const seen = /* @__PURE__ */ new Set();
|
|
4014
|
+
const results = [];
|
|
4015
|
+
for (const eventId of rankedIds) {
|
|
4016
|
+
const candidate = topCandidates.find((c) => c.id === eventId);
|
|
4017
|
+
if (candidate === void 0) continue;
|
|
4018
|
+
if (seen.has(candidate.chunkId)) continue;
|
|
4019
|
+
seen.add(candidate.chunkId);
|
|
4020
|
+
const result = await store.buildSearchResult(candidate.chunkId, candidate.score, eventId);
|
|
4021
|
+
if (result !== void 0) results.push(result);
|
|
4022
|
+
if (results.length >= topK) break;
|
|
4023
|
+
}
|
|
4024
|
+
if (results.length < topK) {
|
|
4025
|
+
let supplemented = 0;
|
|
4026
|
+
for (const candidate of topCandidates) {
|
|
4027
|
+
if (results.length >= topK) break;
|
|
4028
|
+
if (rankedIds.includes(candidate.id)) continue;
|
|
4029
|
+
if (seen.has(candidate.chunkId)) continue;
|
|
4030
|
+
seen.add(candidate.chunkId);
|
|
4031
|
+
const result = await store.buildSearchResult(candidate.chunkId, candidate.score, candidate.id);
|
|
4032
|
+
if (result !== void 0) {
|
|
4033
|
+
results.push(result);
|
|
4034
|
+
supplemented += 1;
|
|
4035
|
+
}
|
|
4036
|
+
}
|
|
4037
|
+
if (supplemented > 0) onStep({
|
|
4038
|
+
step: "supplementFromCoarse",
|
|
4039
|
+
detail: `rerank 结果不足,从 coarse rank 残余补 ${supplemented} 个。`,
|
|
4040
|
+
durationMs: 0,
|
|
4041
|
+
payload: { supplemented }
|
|
4042
|
+
});
|
|
4043
|
+
}
|
|
4044
|
+
if (results.length < topK) {
|
|
4045
|
+
const remaining = topK - results.length;
|
|
4046
|
+
const backfill = await timed(onStep, "backfill", `rerank 结果不足,回退到 chunk 向量搜索补 ${remaining} 个。`, () => store.searchChunksByVector(queryVec, { limit: remaining * 2 }), (matches) => ({ backfillCandidates: matches.length }));
|
|
4047
|
+
for (const { chunk, score } of backfill) {
|
|
4048
|
+
if (results.length >= topK) break;
|
|
4049
|
+
if (seen.has(chunk.id)) continue;
|
|
4050
|
+
seen.add(chunk.id);
|
|
4051
|
+
const result = await store.buildSearchResult(chunk.id, score);
|
|
4052
|
+
if (result !== void 0) results.push(result);
|
|
4053
|
+
}
|
|
4054
|
+
}
|
|
4055
|
+
if (results.length === 0) {
|
|
4056
|
+
fallbackReason = "vector retrieval returned no results; used FTS5 keyword fallback";
|
|
4057
|
+
return {
|
|
4058
|
+
results: await ftsFallback(store, query, topK),
|
|
4059
|
+
trace: {
|
|
4060
|
+
steps,
|
|
4061
|
+
rerankedEventTitles,
|
|
4062
|
+
fallbackReason
|
|
4063
|
+
}
|
|
4064
|
+
};
|
|
4065
|
+
}
|
|
4066
|
+
return {
|
|
4067
|
+
results,
|
|
4068
|
+
trace: {
|
|
4069
|
+
steps,
|
|
4070
|
+
rerankedEventTitles,
|
|
4071
|
+
fallbackReason
|
|
4072
|
+
}
|
|
4073
|
+
};
|
|
4074
|
+
}
|
|
4075
|
+
/** Recall entities by name (LLM-extracted) and by vector similarity. */
|
|
4076
|
+
async function recallEntities(store, llm, query, queryVec, useLlm) {
|
|
4077
|
+
const out = /* @__PURE__ */ new Map();
|
|
4078
|
+
if (useLlm) {
|
|
4079
|
+
const queryEntities = await extractQueryEntities(llm, query);
|
|
4080
|
+
for (const { name } of queryEntities) {
|
|
4081
|
+
const matches = await store.findEntitiesByName(name);
|
|
4082
|
+
for (const entity of matches) out.set(entity.id, { id: entity.id });
|
|
4083
|
+
}
|
|
4084
|
+
}
|
|
4085
|
+
const vectorMatches = await store.findEntitiesByVector(queryVec, {
|
|
4086
|
+
limit: 20,
|
|
4087
|
+
threshold: ENTITY_VECTOR_THRESHOLD
|
|
4088
|
+
});
|
|
4089
|
+
for (const { entity } of vectorMatches) out.set(entity.id, { id: entity.id });
|
|
4090
|
+
return Array.from(out.values());
|
|
4091
|
+
}
|
|
4092
|
+
/** FTS5 keyword fallback when embeddings are unavailable. */
|
|
4093
|
+
async function ftsFallback(store, query, topK) {
|
|
4094
|
+
const hits = await store.ftsSearchChunks(query, topK * 2);
|
|
4095
|
+
const results = [];
|
|
4096
|
+
for (const { chunk, score } of hits) {
|
|
4097
|
+
const result = await store.buildSearchResult(chunk.id, score);
|
|
4098
|
+
if (result !== void 0) results.push(result);
|
|
4099
|
+
if (results.length >= topK) break;
|
|
4100
|
+
}
|
|
4101
|
+
return results;
|
|
4102
|
+
}
|
|
4103
|
+
/** Direct chunk vector search fallback. */
|
|
4104
|
+
async function chunkVectorFallback(store, queryVec, topK) {
|
|
4105
|
+
const matches = await store.searchChunksByVector(queryVec, { limit: topK * 2 });
|
|
4106
|
+
const results = [];
|
|
4107
|
+
for (const { chunk, score } of matches) {
|
|
4108
|
+
const result = await store.buildSearchResult(chunk.id, score);
|
|
4109
|
+
if (result !== void 0) results.push(result);
|
|
4110
|
+
if (results.length >= topK) break;
|
|
4111
|
+
}
|
|
4112
|
+
return results;
|
|
4113
|
+
}
|
|
4114
|
+
//#endregion
|
|
4115
|
+
export { classifyEmbeddingFailure as A, rankMemos as B, applyConsolidation as C, parseMemoryMemos as D, buildExitExtractionPrompt as E, createMemoryMemo as F, normalize$1 as G, dirname$1 as H, toSummary as I, resolve$1 as J, parse as K, generateTags as L, createFastEmbedEngine as M, hasEmbeddingModelCache as N, stripMemoryTags as O, probeLocalEmbeddingSupport as P, normalizeTags as R, DreamTracker as S, EXIT_EXTRACTION_SYSTEM_PROMPT as T, isAbsolute$1 as U, basename$1 as V, join$1 as W, splitLargeSection as _, isSupportedFile as a, sharedKnowledgeStoreReady as b, buildExtractionUserPrompt as c, extractQueryEntities as d, parseExtractionResponse as f, estimateTokens as g, chunkText as h, ingestFile as i, clearEmbeddingModelCache as j, MemoryMemoStore as k, extractEventFromChunk as l, chunkMarkdown as m, multiSearchWithTrace as n, ENTITY_TYPES as o, rerankEventsWithLlm as p, relative$1 as q, ingestDirectory as r, EXTRACTION_SYSTEM_PROMPT as s, multiSearch as t, extractJsonFromText as u, stripMarkdown as v, buildConsolidationPlan as w, KnowledgeStore as x, sharedKnowledgeStore as y, buildProjectTagCloud as z };
|