scream-code 0.16.6 → 0.16.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -3
- package/dist/{app-D9oJ-Y8W.mjs → app-GeQQHNLx.mjs} +335 -88
- package/dist/{dispatch-fjwbqaYI.mjs → dispatch-CNitkOqX.mjs} +1934 -2022
- package/dist/{dispatch-dDbUBD7J.mjs → dispatch-Du8ddILK.mjs} +1 -1
- package/dist/main.mjs +1 -1
- package/dist/public/assets/index-DMGV8LA-.js +240 -0
- package/dist/public/assets/index-DMGV8LA-.js.map +1 -0
- package/dist/public/assets/index-DdPmkwIL.css +1 -0
- package/dist/public/index.html +2 -2
- package/dist/src-CXmZg1VT.mjs +4115 -0
- package/dist/{src-h6ZExh4A.mjs → src-DIAKpkXO.mjs} +1 -1
- package/dist/{text-input-dialog-DBlx8pMR.mjs → text-input-dialog-6v2667rm.mjs} +140 -6
- package/dist/{text-input-dialog-Cj0egbsU.mjs → text-input-dialog-DDUarM0_.mjs} +1 -1
- package/package.json +3 -2
- package/dist/public/assets/index-DQBhBKK1.css +0 -1
- package/dist/public/assets/index-DakPeNBt.js +0 -240
- package/dist/public/assets/index-DakPeNBt.js.map +0 -1
- package/dist/src-tDEINaMV.mjs +0 -2249
package/dist/src-tDEINaMV.mjs
DELETED
|
@@ -1,2249 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
import { fileURLToPath as __cjsShimFileURLToPath } from 'node:url';
|
|
3
|
-
import { dirname as __cjsShimDirname } from 'node:path';
|
|
4
|
-
const __filename = __cjsShimFileURLToPath(import.meta.url);
|
|
5
|
-
const __dirname = __cjsShimDirname(__filename);
|
|
6
|
-
import { createHash } from "node:crypto";
|
|
7
|
-
import { mkdir, readFile, readdir } from "node:fs/promises";
|
|
8
|
-
import { DatabaseSync } from "node:sqlite";
|
|
9
|
-
//#region ../../node_modules/.pnpm/pathe@2.0.3/node_modules/pathe/dist/shared/pathe.M-eThtNZ.mjs
|
|
10
|
-
const _DRIVE_LETTER_START_RE = /^[A-Za-z]:\//;
|
|
11
|
-
function normalizeWindowsPath(input = "") {
|
|
12
|
-
if (!input) return input;
|
|
13
|
-
return input.replace(/\\/g, "/").replace(_DRIVE_LETTER_START_RE, (r) => r.toUpperCase());
|
|
14
|
-
}
|
|
15
|
-
const _UNC_REGEX = /^[/\\]{2}/;
|
|
16
|
-
const _IS_ABSOLUTE_RE = /^[/\\](?![/\\])|^[/\\]{2}(?!\.)|^[A-Za-z]:[/\\]/;
|
|
17
|
-
const _DRIVE_LETTER_RE = /^[A-Za-z]:$/;
|
|
18
|
-
const _ROOT_FOLDER_RE = /^\/([A-Za-z]:)?$/;
|
|
19
|
-
const _EXTNAME_RE = /.(\.[^./]+|\.)$/;
|
|
20
|
-
const _PATH_ROOT_RE = /^[/\\]|^[a-zA-Z]:[/\\]/;
|
|
21
|
-
const normalize = function(path) {
|
|
22
|
-
if (path.length === 0) return ".";
|
|
23
|
-
path = normalizeWindowsPath(path);
|
|
24
|
-
const isUNCPath = path.match(_UNC_REGEX);
|
|
25
|
-
const isPathAbsolute = isAbsolute(path);
|
|
26
|
-
const trailingSeparator = path[path.length - 1] === "/";
|
|
27
|
-
path = normalizeString(path, !isPathAbsolute);
|
|
28
|
-
if (path.length === 0) {
|
|
29
|
-
if (isPathAbsolute) return "/";
|
|
30
|
-
return trailingSeparator ? "./" : ".";
|
|
31
|
-
}
|
|
32
|
-
if (trailingSeparator) path += "/";
|
|
33
|
-
if (_DRIVE_LETTER_RE.test(path)) path += "/";
|
|
34
|
-
if (isUNCPath) {
|
|
35
|
-
if (!isPathAbsolute) return `//./${path}`;
|
|
36
|
-
return `//${path}`;
|
|
37
|
-
}
|
|
38
|
-
return isPathAbsolute && !isAbsolute(path) ? `/${path}` : path;
|
|
39
|
-
};
|
|
40
|
-
const join = function(...segments) {
|
|
41
|
-
let path = "";
|
|
42
|
-
for (const seg of segments) {
|
|
43
|
-
if (!seg) continue;
|
|
44
|
-
if (path.length > 0) {
|
|
45
|
-
const pathTrailing = path[path.length - 1] === "/";
|
|
46
|
-
const segLeading = seg[0] === "/";
|
|
47
|
-
if (pathTrailing && segLeading) path += seg.slice(1);
|
|
48
|
-
else path += pathTrailing || segLeading ? seg : `/${seg}`;
|
|
49
|
-
} else path += seg;
|
|
50
|
-
}
|
|
51
|
-
return normalize(path);
|
|
52
|
-
};
|
|
53
|
-
function cwd() {
|
|
54
|
-
if (typeof process !== "undefined" && typeof process.cwd === "function") return process.cwd().replace(/\\/g, "/");
|
|
55
|
-
return "/";
|
|
56
|
-
}
|
|
57
|
-
const resolve = function(...arguments_) {
|
|
58
|
-
arguments_ = arguments_.map((argument) => normalizeWindowsPath(argument));
|
|
59
|
-
let resolvedPath = "";
|
|
60
|
-
let resolvedAbsolute = false;
|
|
61
|
-
for (let index = arguments_.length - 1; index >= -1 && !resolvedAbsolute; index--) {
|
|
62
|
-
const path = index >= 0 ? arguments_[index] : cwd();
|
|
63
|
-
if (!path || path.length === 0) continue;
|
|
64
|
-
resolvedPath = `${path}/${resolvedPath}`;
|
|
65
|
-
resolvedAbsolute = isAbsolute(path);
|
|
66
|
-
}
|
|
67
|
-
resolvedPath = normalizeString(resolvedPath, !resolvedAbsolute);
|
|
68
|
-
if (resolvedAbsolute && !isAbsolute(resolvedPath)) return `/${resolvedPath}`;
|
|
69
|
-
return resolvedPath.length > 0 ? resolvedPath : ".";
|
|
70
|
-
};
|
|
71
|
-
function normalizeString(path, allowAboveRoot) {
|
|
72
|
-
let res = "";
|
|
73
|
-
let lastSegmentLength = 0;
|
|
74
|
-
let lastSlash = -1;
|
|
75
|
-
let dots = 0;
|
|
76
|
-
let char = null;
|
|
77
|
-
for (let index = 0; index <= path.length; ++index) {
|
|
78
|
-
if (index < path.length) char = path[index];
|
|
79
|
-
else if (char === "/") break;
|
|
80
|
-
else char = "/";
|
|
81
|
-
if (char === "/") {
|
|
82
|
-
if (lastSlash === index - 1 || dots === 1);
|
|
83
|
-
else if (dots === 2) {
|
|
84
|
-
if (res.length < 2 || lastSegmentLength !== 2 || res[res.length - 1] !== "." || res[res.length - 2] !== ".") {
|
|
85
|
-
if (res.length > 2) {
|
|
86
|
-
const lastSlashIndex = res.lastIndexOf("/");
|
|
87
|
-
if (lastSlashIndex === -1) {
|
|
88
|
-
res = "";
|
|
89
|
-
lastSegmentLength = 0;
|
|
90
|
-
} else {
|
|
91
|
-
res = res.slice(0, lastSlashIndex);
|
|
92
|
-
lastSegmentLength = res.length - 1 - res.lastIndexOf("/");
|
|
93
|
-
}
|
|
94
|
-
lastSlash = index;
|
|
95
|
-
dots = 0;
|
|
96
|
-
continue;
|
|
97
|
-
} else if (res.length > 0) {
|
|
98
|
-
res = "";
|
|
99
|
-
lastSegmentLength = 0;
|
|
100
|
-
lastSlash = index;
|
|
101
|
-
dots = 0;
|
|
102
|
-
continue;
|
|
103
|
-
}
|
|
104
|
-
}
|
|
105
|
-
if (allowAboveRoot) {
|
|
106
|
-
res += res.length > 0 ? "/.." : "..";
|
|
107
|
-
lastSegmentLength = 2;
|
|
108
|
-
}
|
|
109
|
-
} else {
|
|
110
|
-
if (res.length > 0) res += `/${path.slice(lastSlash + 1, index)}`;
|
|
111
|
-
else res = path.slice(lastSlash + 1, index);
|
|
112
|
-
lastSegmentLength = index - lastSlash - 1;
|
|
113
|
-
}
|
|
114
|
-
lastSlash = index;
|
|
115
|
-
dots = 0;
|
|
116
|
-
} else if (char === "." && dots !== -1) ++dots;
|
|
117
|
-
else dots = -1;
|
|
118
|
-
}
|
|
119
|
-
return res;
|
|
120
|
-
}
|
|
121
|
-
const isAbsolute = function(p) {
|
|
122
|
-
return _IS_ABSOLUTE_RE.test(p);
|
|
123
|
-
};
|
|
124
|
-
const extname = function(p) {
|
|
125
|
-
if (p === "..") return "";
|
|
126
|
-
const match = _EXTNAME_RE.exec(normalizeWindowsPath(p));
|
|
127
|
-
return match && match[1] || "";
|
|
128
|
-
};
|
|
129
|
-
const relative = function(from, to) {
|
|
130
|
-
const _from = resolve(from).replace(_ROOT_FOLDER_RE, "$1").split("/");
|
|
131
|
-
const _to = resolve(to).replace(_ROOT_FOLDER_RE, "$1").split("/");
|
|
132
|
-
if (_to[0][1] === ":" && _from[0][1] === ":" && _from[0] !== _to[0]) return _to.join("/");
|
|
133
|
-
const _fromCopy = [..._from];
|
|
134
|
-
for (const segment of _fromCopy) {
|
|
135
|
-
if (_to[0] !== segment) break;
|
|
136
|
-
_from.shift();
|
|
137
|
-
_to.shift();
|
|
138
|
-
}
|
|
139
|
-
return [..._from.map(() => ".."), ..._to].join("/");
|
|
140
|
-
};
|
|
141
|
-
const dirname = function(p) {
|
|
142
|
-
const segments = normalizeWindowsPath(p).replace(/\/$/, "").split("/").slice(0, -1);
|
|
143
|
-
if (segments.length === 1 && _DRIVE_LETTER_RE.test(segments[0])) segments[0] += "/";
|
|
144
|
-
return segments.join("/") || (isAbsolute(p) ? "/" : ".");
|
|
145
|
-
};
|
|
146
|
-
const basename = function(p, extension) {
|
|
147
|
-
const segments = normalizeWindowsPath(p).split("/");
|
|
148
|
-
let lastSegment = "";
|
|
149
|
-
for (let i = segments.length - 1; i >= 0; i--) {
|
|
150
|
-
const val = segments[i];
|
|
151
|
-
if (val) {
|
|
152
|
-
lastSegment = val;
|
|
153
|
-
break;
|
|
154
|
-
}
|
|
155
|
-
}
|
|
156
|
-
return extension && lastSegment.endsWith(extension) ? lastSegment.slice(0, -extension.length) : lastSegment;
|
|
157
|
-
};
|
|
158
|
-
const parse = function(p) {
|
|
159
|
-
const root = _PATH_ROOT_RE.exec(p)?.[0]?.replace(/\\/g, "/") || "";
|
|
160
|
-
const base = basename(p);
|
|
161
|
-
const extension = extname(base);
|
|
162
|
-
return {
|
|
163
|
-
root,
|
|
164
|
-
dir: dirname(p),
|
|
165
|
-
base,
|
|
166
|
-
ext: extension,
|
|
167
|
-
name: base.slice(0, base.length - extension.length)
|
|
168
|
-
};
|
|
169
|
-
};
|
|
170
|
-
//#endregion
|
|
171
|
-
//#region ../../packages/knowledge/src/store.ts
|
|
172
|
-
const ENTITY_TYPES$1 = new Set([
|
|
173
|
-
"person",
|
|
174
|
-
"organization",
|
|
175
|
-
"location",
|
|
176
|
-
"time",
|
|
177
|
-
"product",
|
|
178
|
-
"metric",
|
|
179
|
-
"action",
|
|
180
|
-
"work",
|
|
181
|
-
"group",
|
|
182
|
-
"subject",
|
|
183
|
-
"tags"
|
|
184
|
-
]);
|
|
185
|
-
function generateId(prefix) {
|
|
186
|
-
return `${prefix}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
|
|
187
|
-
}
|
|
188
|
-
function normalizeName(name) {
|
|
189
|
-
return name.trim().toLowerCase().replaceAll(/\s+/g, " ");
|
|
190
|
-
}
|
|
191
|
-
function vectorToJson(vec) {
|
|
192
|
-
if (vec === null) return null;
|
|
193
|
-
return JSON.stringify(Array.from(vec));
|
|
194
|
-
}
|
|
195
|
-
function jsonToVector(json) {
|
|
196
|
-
if (json === null) return null;
|
|
197
|
-
try {
|
|
198
|
-
const arr = JSON.parse(json);
|
|
199
|
-
return new Float32Array(arr);
|
|
200
|
-
} catch {
|
|
201
|
-
return null;
|
|
202
|
-
}
|
|
203
|
-
}
|
|
204
|
-
function isEntityType(value) {
|
|
205
|
-
return ENTITY_TYPES$1.has(value);
|
|
206
|
-
}
|
|
207
|
-
/** Meta key recording which embedding model the stored vectors came from. */
|
|
208
|
-
const EMBEDDING_MODEL_META_KEY = "embedding_model";
|
|
209
|
-
var KnowledgeStore = class {
|
|
210
|
-
dbPath;
|
|
211
|
-
db;
|
|
212
|
-
initialized = false;
|
|
213
|
-
initPromise = null;
|
|
214
|
-
embeddingEngine;
|
|
215
|
-
constructor(projectDir) {
|
|
216
|
-
this.dbPath = join(projectDir, "knowledge", "knowledge.db");
|
|
217
|
-
}
|
|
218
|
-
async init() {
|
|
219
|
-
if (this.initPromise) return this.initPromise;
|
|
220
|
-
this.initPromise = this._doInit().catch((error) => {
|
|
221
|
-
this.initPromise = null;
|
|
222
|
-
throw error;
|
|
223
|
-
});
|
|
224
|
-
return this.initPromise;
|
|
225
|
-
}
|
|
226
|
-
async _doInit() {
|
|
227
|
-
if (this.initialized) return;
|
|
228
|
-
await mkdir(dirname(this.dbPath), { recursive: true });
|
|
229
|
-
this.db = new DatabaseSync(this.dbPath);
|
|
230
|
-
this.db.exec("PRAGMA journal_mode = WAL;");
|
|
231
|
-
this.db.exec("PRAGMA foreign_keys = ON;");
|
|
232
|
-
this.db.exec("PRAGMA busy_timeout = 5000;");
|
|
233
|
-
this.createSchema();
|
|
234
|
-
this.migrateFtsTokenization();
|
|
235
|
-
this.initialized = true;
|
|
236
|
-
}
|
|
237
|
-
/**
|
|
238
|
-
* One-time migration: FTS rows written before toFtsText pre-tokenization
|
|
239
|
-
* indexed raw text (CJK runs became single tokens and are unsearchable).
|
|
240
|
-
* Rebuild all three FTS tables from the content tables with toFtsText.
|
|
241
|
-
* Tracked via PRAGMA user_version so it runs exactly once.
|
|
242
|
-
*/
|
|
243
|
-
migrateFtsTokenization() {
|
|
244
|
-
if (this.db === void 0) return;
|
|
245
|
-
if (this.db.prepare("PRAGMA user_version").get().user_version >= 1) return;
|
|
246
|
-
this.db.exec("DELETE FROM knowledge_chunks_fts");
|
|
247
|
-
this.db.exec("DELETE FROM knowledge_events_fts");
|
|
248
|
-
this.db.exec("DELETE FROM knowledge_entities_fts");
|
|
249
|
-
const chunks = this.db.prepare("SELECT rowid, heading, content FROM knowledge_chunks").all();
|
|
250
|
-
const insChunk = this.db.prepare("INSERT INTO knowledge_chunks_fts (rowid, heading, content) VALUES (?, ?, ?)");
|
|
251
|
-
for (const c of chunks) insChunk.run(c.rowid, toFtsText(c.heading ?? ""), toFtsText(c.content));
|
|
252
|
-
const events = this.db.prepare("SELECT rowid, title, summary, content FROM knowledge_events").all();
|
|
253
|
-
const insEvent = this.db.prepare("INSERT INTO knowledge_events_fts (rowid, title, summary, content) VALUES (?, ?, ?, ?)");
|
|
254
|
-
for (const e of events) insEvent.run(e.rowid, toFtsText(e.title), toFtsText(e.summary ?? ""), toFtsText(e.content));
|
|
255
|
-
const entities = this.db.prepare("SELECT rowid, name, description FROM knowledge_entities").all();
|
|
256
|
-
const insEntity = this.db.prepare("INSERT INTO knowledge_entities_fts (rowid, name, description) VALUES (?, ?, ?)");
|
|
257
|
-
for (const e of entities) insEntity.run(e.rowid, toFtsText(e.name), toFtsText(e.description ?? ""));
|
|
258
|
-
this.db.exec("PRAGMA user_version = 1");
|
|
259
|
-
}
|
|
260
|
-
close() {
|
|
261
|
-
if (this.db !== void 0) {
|
|
262
|
-
this.db.exec("PRAGMA wal_checkpoint(TRUNCATE)");
|
|
263
|
-
this.db.close();
|
|
264
|
-
this.db = void 0;
|
|
265
|
-
}
|
|
266
|
-
this.initialized = false;
|
|
267
|
-
this.initPromise = null;
|
|
268
|
-
}
|
|
269
|
-
setEmbeddingEngine(engine) {
|
|
270
|
-
this.embeddingEngine = engine;
|
|
271
|
-
}
|
|
272
|
-
getEmbeddingEngine() {
|
|
273
|
-
return this.embeddingEngine;
|
|
274
|
-
}
|
|
275
|
-
/**
|
|
276
|
-
* Begin a write transaction. Use commitTransaction / rollbackTransaction
|
|
277
|
-
* to close it. Used by ingest to keep chunks/events/entities/edges atomic —
|
|
278
|
-
* a mid-ingest failure leaves no partial rows.
|
|
279
|
-
*/
|
|
280
|
-
beginTransaction() {
|
|
281
|
-
if (this.db === void 0) throw new Error("knowledge store not initialized");
|
|
282
|
-
this.db.exec("BEGIN");
|
|
283
|
-
}
|
|
284
|
-
commitTransaction() {
|
|
285
|
-
if (this.db === void 0) throw new Error("knowledge store not initialized");
|
|
286
|
-
this.db.exec("COMMIT");
|
|
287
|
-
}
|
|
288
|
-
rollbackTransaction() {
|
|
289
|
-
if (this.db === void 0) return;
|
|
290
|
-
try {
|
|
291
|
-
this.db.exec("ROLLBACK");
|
|
292
|
-
} catch {}
|
|
293
|
-
}
|
|
294
|
-
/**
|
|
295
|
-
* Read a metadata value (e.g. the embedding model identity the library was
|
|
296
|
-
* built with). Returns null when the key is absent.
|
|
297
|
-
*/
|
|
298
|
-
async getMeta(key) {
|
|
299
|
-
await this.init();
|
|
300
|
-
if (this.db === void 0) return null;
|
|
301
|
-
const row = this.db.prepare("SELECT value FROM knowledge_meta WHERE key = ?").get(key);
|
|
302
|
-
return row === void 0 ? null : asString(row["value"]);
|
|
303
|
-
}
|
|
304
|
-
/** Write a metadata value (upsert). */
|
|
305
|
-
async setMeta(key, value) {
|
|
306
|
-
await this.init();
|
|
307
|
-
if (this.db === void 0) throw new Error("knowledge store not initialized");
|
|
308
|
-
this.db.prepare("INSERT INTO knowledge_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value").run(key, value);
|
|
309
|
-
}
|
|
310
|
-
createSchema() {
|
|
311
|
-
if (this.db === void 0) return;
|
|
312
|
-
this.db.exec(`
|
|
313
|
-
CREATE TABLE IF NOT EXISTS knowledge_sources (
|
|
314
|
-
id TEXT PRIMARY KEY,
|
|
315
|
-
name TEXT NOT NULL,
|
|
316
|
-
file_path TEXT,
|
|
317
|
-
description TEXT,
|
|
318
|
-
created_at INTEGER NOT NULL
|
|
319
|
-
);
|
|
320
|
-
|
|
321
|
-
CREATE TABLE IF NOT EXISTS knowledge_documents (
|
|
322
|
-
id TEXT PRIMARY KEY,
|
|
323
|
-
source_id TEXT NOT NULL REFERENCES knowledge_sources(id) ON DELETE CASCADE,
|
|
324
|
-
title TEXT NOT NULL,
|
|
325
|
-
content TEXT,
|
|
326
|
-
status TEXT NOT NULL DEFAULT 'pending',
|
|
327
|
-
chunk_count INTEGER DEFAULT 0,
|
|
328
|
-
created_at INTEGER NOT NULL
|
|
329
|
-
);
|
|
330
|
-
|
|
331
|
-
CREATE INDEX IF NOT EXISTS idx_knowledge_documents_source ON knowledge_documents(source_id);
|
|
332
|
-
|
|
333
|
-
CREATE TABLE IF NOT EXISTS knowledge_chunks (
|
|
334
|
-
id TEXT PRIMARY KEY,
|
|
335
|
-
source_id TEXT NOT NULL REFERENCES knowledge_sources(id) ON DELETE CASCADE,
|
|
336
|
-
document_id TEXT NOT NULL REFERENCES knowledge_documents(id) ON DELETE CASCADE,
|
|
337
|
-
rank INTEGER NOT NULL,
|
|
338
|
-
heading TEXT,
|
|
339
|
-
content TEXT NOT NULL,
|
|
340
|
-
raw_content TEXT,
|
|
341
|
-
embedding_json TEXT,
|
|
342
|
-
created_at INTEGER NOT NULL
|
|
343
|
-
);
|
|
344
|
-
|
|
345
|
-
CREATE INDEX IF NOT EXISTS idx_knowledge_chunks_doc ON knowledge_chunks(document_id);
|
|
346
|
-
CREATE INDEX IF NOT EXISTS idx_knowledge_chunks_source ON knowledge_chunks(source_id);
|
|
347
|
-
|
|
348
|
-
CREATE TABLE IF NOT EXISTS knowledge_events (
|
|
349
|
-
id TEXT PRIMARY KEY,
|
|
350
|
-
source_id TEXT NOT NULL REFERENCES knowledge_sources(id) ON DELETE CASCADE,
|
|
351
|
-
document_id TEXT NOT NULL REFERENCES knowledge_documents(id) ON DELETE CASCADE,
|
|
352
|
-
chunk_id TEXT NOT NULL REFERENCES knowledge_chunks(id) ON DELETE CASCADE,
|
|
353
|
-
rank INTEGER NOT NULL,
|
|
354
|
-
title TEXT NOT NULL,
|
|
355
|
-
summary TEXT,
|
|
356
|
-
content TEXT NOT NULL,
|
|
357
|
-
category TEXT,
|
|
358
|
-
keywords TEXT,
|
|
359
|
-
title_embedding_json TEXT,
|
|
360
|
-
content_embedding_json TEXT,
|
|
361
|
-
created_at INTEGER NOT NULL
|
|
362
|
-
);
|
|
363
|
-
|
|
364
|
-
CREATE INDEX IF NOT EXISTS idx_knowledge_events_chunk ON knowledge_events(chunk_id);
|
|
365
|
-
CREATE INDEX IF NOT EXISTS idx_knowledge_events_doc ON knowledge_events(document_id);
|
|
366
|
-
CREATE INDEX IF NOT EXISTS idx_knowledge_events_source ON knowledge_events(source_id);
|
|
367
|
-
|
|
368
|
-
CREATE TABLE IF NOT EXISTS knowledge_entities (
|
|
369
|
-
id TEXT PRIMARY KEY,
|
|
370
|
-
source_id TEXT NOT NULL REFERENCES knowledge_sources(id) ON DELETE CASCADE,
|
|
371
|
-
type TEXT NOT NULL,
|
|
372
|
-
name TEXT NOT NULL,
|
|
373
|
-
normalized_name TEXT NOT NULL,
|
|
374
|
-
description TEXT,
|
|
375
|
-
embedding_json TEXT,
|
|
376
|
-
created_at INTEGER NOT NULL,
|
|
377
|
-
UNIQUE(source_id, type, normalized_name)
|
|
378
|
-
);
|
|
379
|
-
|
|
380
|
-
CREATE INDEX IF NOT EXISTS idx_knowledge_entities_source ON knowledge_entities(source_id);
|
|
381
|
-
CREATE INDEX IF NOT EXISTS idx_knowledge_entities_name ON knowledge_entities(normalized_name);
|
|
382
|
-
|
|
383
|
-
CREATE TABLE IF NOT EXISTS knowledge_event_entities (
|
|
384
|
-
id TEXT PRIMARY KEY,
|
|
385
|
-
event_id TEXT NOT NULL REFERENCES knowledge_events(id) ON DELETE CASCADE,
|
|
386
|
-
entity_id TEXT NOT NULL REFERENCES knowledge_entities(id) ON DELETE CASCADE,
|
|
387
|
-
weight REAL DEFAULT 1.0,
|
|
388
|
-
description TEXT,
|
|
389
|
-
embedding_json TEXT,
|
|
390
|
-
UNIQUE(event_id, entity_id)
|
|
391
|
-
);
|
|
392
|
-
|
|
393
|
-
CREATE INDEX IF NOT EXISTS idx_knowledge_event_entities_event ON knowledge_event_entities(event_id);
|
|
394
|
-
CREATE INDEX IF NOT EXISTS idx_knowledge_event_entities_entity ON knowledge_event_entities(entity_id);
|
|
395
|
-
|
|
396
|
-
CREATE VIRTUAL TABLE IF NOT EXISTS knowledge_chunks_fts USING fts5(
|
|
397
|
-
heading, content, content='knowledge_chunks', content_rowid='rowid'
|
|
398
|
-
);
|
|
399
|
-
CREATE VIRTUAL TABLE IF NOT EXISTS knowledge_events_fts USING fts5(
|
|
400
|
-
title, summary, content, content='knowledge_events', content_rowid='rowid'
|
|
401
|
-
);
|
|
402
|
-
CREATE VIRTUAL TABLE IF NOT EXISTS knowledge_entities_fts USING fts5(
|
|
403
|
-
name, description, content='knowledge_entities', content_rowid='rowid'
|
|
404
|
-
);
|
|
405
|
-
|
|
406
|
-
CREATE TABLE IF NOT EXISTS knowledge_meta (
|
|
407
|
-
key TEXT PRIMARY KEY,
|
|
408
|
-
value TEXT NOT NULL
|
|
409
|
-
);
|
|
410
|
-
`);
|
|
411
|
-
try {
|
|
412
|
-
this.db.exec("ALTER TABLE knowledge_sources ADD COLUMN content_hash TEXT");
|
|
413
|
-
} catch (error) {
|
|
414
|
-
if (!(error instanceof Error ? error.message : String(error)).includes("duplicate column")) throw error;
|
|
415
|
-
}
|
|
416
|
-
}
|
|
417
|
-
async createSource(params) {
|
|
418
|
-
await this.init();
|
|
419
|
-
if (this.db === void 0) throw new Error("knowledge store not initialized");
|
|
420
|
-
const id = generateId("src");
|
|
421
|
-
const createdAt = Date.now();
|
|
422
|
-
this.db.prepare("INSERT INTO knowledge_sources (id, name, file_path, description, content_hash, created_at) VALUES (?, ?, ?, ?, ?, ?)").run(id, params.name, params.filePath ?? null, params.description ?? null, params.contentHash ?? null, createdAt);
|
|
423
|
-
return {
|
|
424
|
-
id,
|
|
425
|
-
name: params.name,
|
|
426
|
-
filePath: params.filePath ?? null,
|
|
427
|
-
description: params.description ?? null,
|
|
428
|
-
contentHash: params.contentHash ?? null,
|
|
429
|
-
createdAt
|
|
430
|
-
};
|
|
431
|
-
}
|
|
432
|
-
/** Backfill or refresh a source's content fingerprint (legacy rows / replacements). */
|
|
433
|
-
async updateSourceContentHash(id, contentHash) {
|
|
434
|
-
await this.init();
|
|
435
|
-
if (this.db === void 0) throw new Error("knowledge store not initialized");
|
|
436
|
-
this.db.prepare("UPDATE knowledge_sources SET content_hash = ? WHERE id = ?").run(contentHash, id);
|
|
437
|
-
}
|
|
438
|
-
async findSourceByFilePath(filePath) {
|
|
439
|
-
await this.init();
|
|
440
|
-
if (this.db === void 0) return void 0;
|
|
441
|
-
const row = this.db.prepare("SELECT * FROM knowledge_sources WHERE file_path = ?").get(filePath);
|
|
442
|
-
return row === void 0 ? void 0 : rowToSource(row);
|
|
443
|
-
}
|
|
444
|
-
async listSources() {
|
|
445
|
-
await this.init();
|
|
446
|
-
if (this.db === void 0) return [];
|
|
447
|
-
return this.db.prepare("SELECT * FROM knowledge_sources ORDER BY created_at DESC").all().map(rowToSource);
|
|
448
|
-
}
|
|
449
|
-
async getSource(id) {
|
|
450
|
-
await this.init();
|
|
451
|
-
if (this.db === void 0) return void 0;
|
|
452
|
-
const row = this.db.prepare("SELECT * FROM knowledge_sources WHERE id = ?").get(id);
|
|
453
|
-
return row === void 0 ? void 0 : rowToSource(row);
|
|
454
|
-
}
|
|
455
|
-
async deleteSource(id) {
|
|
456
|
-
await this.init();
|
|
457
|
-
if (this.db === void 0) return false;
|
|
458
|
-
const chunkRows = this.db.prepare("SELECT rowid, heading, content FROM knowledge_chunks WHERE source_id = ?").all(id);
|
|
459
|
-
const eventRows = this.db.prepare("SELECT rowid, title, summary, content FROM knowledge_events WHERE source_id = ?").all(id);
|
|
460
|
-
const entityRows = this.db.prepare("SELECT rowid, name, description FROM knowledge_entities WHERE source_id = ?").all(id);
|
|
461
|
-
const result = this.db.prepare("DELETE FROM knowledge_sources WHERE id = ?").run(id);
|
|
462
|
-
if (result.changes > 0) {
|
|
463
|
-
const delChunk = this.db.prepare("INSERT INTO knowledge_chunks_fts (knowledge_chunks_fts, rowid, heading, content) VALUES ('delete', ?, ?, ?)");
|
|
464
|
-
for (const row of chunkRows) delChunk.run(row.rowid, toFtsText(row.heading ?? ""), toFtsText(row.content));
|
|
465
|
-
const delEvent = this.db.prepare("INSERT INTO knowledge_events_fts (knowledge_events_fts, rowid, title, summary, content) VALUES ('delete', ?, ?, ?, ?)");
|
|
466
|
-
for (const row of eventRows) delEvent.run(row.rowid, toFtsText(row.title), toFtsText(row.summary ?? ""), toFtsText(row.content));
|
|
467
|
-
const delEntity = this.db.prepare("INSERT INTO knowledge_entities_fts (knowledge_entities_fts, rowid, name, description) VALUES ('delete', ?, ?, ?)");
|
|
468
|
-
for (const row of entityRows) delEntity.run(row.rowid, toFtsText(row.name), toFtsText(row.description ?? ""));
|
|
469
|
-
}
|
|
470
|
-
return result.changes > 0;
|
|
471
|
-
}
|
|
472
|
-
async createDocument(params) {
|
|
473
|
-
if (this.db === void 0) throw new Error("knowledge store not initialized");
|
|
474
|
-
const id = generateId("doc");
|
|
475
|
-
const createdAt = Date.now();
|
|
476
|
-
this.db.prepare("INSERT INTO knowledge_documents (id, source_id, title, content, status, chunk_count, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)").run(id, params.sourceId, params.title, params.content ?? null, "pending", 0, createdAt);
|
|
477
|
-
return {
|
|
478
|
-
id,
|
|
479
|
-
sourceId: params.sourceId,
|
|
480
|
-
title: params.title,
|
|
481
|
-
content: params.content ?? null,
|
|
482
|
-
status: "pending",
|
|
483
|
-
chunkCount: 0,
|
|
484
|
-
createdAt
|
|
485
|
-
};
|
|
486
|
-
}
|
|
487
|
-
async updateDocumentStatus(documentId, status, chunkCount) {
|
|
488
|
-
if (this.db === void 0) return;
|
|
489
|
-
if (chunkCount !== void 0) this.db.prepare("UPDATE knowledge_documents SET status = ?, chunk_count = ? WHERE id = ?").run(status, chunkCount, documentId);
|
|
490
|
-
else this.db.prepare("UPDATE knowledge_documents SET status = ? WHERE id = ?").run(status, documentId);
|
|
491
|
-
}
|
|
492
|
-
async listDocuments() {
|
|
493
|
-
await this.init();
|
|
494
|
-
if (this.db === void 0) return [];
|
|
495
|
-
return this.db.prepare(`SELECT d.*, s.name AS source_name
|
|
496
|
-
FROM knowledge_documents d
|
|
497
|
-
JOIN knowledge_sources s ON s.id = d.source_id
|
|
498
|
-
ORDER BY d.created_at DESC`).all().map((row) => ({
|
|
499
|
-
...rowToDocument(row),
|
|
500
|
-
sourceName: String(row["source_name"])
|
|
501
|
-
}));
|
|
502
|
-
}
|
|
503
|
-
async getDocument(id) {
|
|
504
|
-
await this.init();
|
|
505
|
-
if (this.db === void 0) return void 0;
|
|
506
|
-
const row = this.db.prepare("SELECT * FROM knowledge_documents WHERE id = ?").get(id);
|
|
507
|
-
return row === void 0 ? void 0 : rowToDocument(row);
|
|
508
|
-
}
|
|
509
|
-
async insertChunk(params) {
|
|
510
|
-
if (this.db === void 0) throw new Error("knowledge store not initialized");
|
|
511
|
-
const id = generateId("chk");
|
|
512
|
-
const createdAt = Date.now();
|
|
513
|
-
const embeddingJson = vectorToJson(params.embedding);
|
|
514
|
-
this.db.prepare(`INSERT INTO knowledge_chunks
|
|
515
|
-
(id, source_id, document_id, rank, heading, content, raw_content, embedding_json, created_at)
|
|
516
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(id, params.sourceId, params.documentId, params.rank, params.heading, params.content, params.rawContent, embeddingJson, createdAt);
|
|
517
|
-
this.db.prepare(`INSERT INTO knowledge_chunks_fts (rowid, heading, content)
|
|
518
|
-
VALUES ((SELECT rowid FROM knowledge_chunks WHERE id = ?), ?, ?)`).run(id, toFtsText(params.heading ?? ""), toFtsText(params.content));
|
|
519
|
-
return {
|
|
520
|
-
id,
|
|
521
|
-
sourceId: params.sourceId,
|
|
522
|
-
documentId: params.documentId,
|
|
523
|
-
rank: params.rank,
|
|
524
|
-
heading: params.heading,
|
|
525
|
-
content: params.content,
|
|
526
|
-
rawContent: params.rawContent,
|
|
527
|
-
embedding: params.embedding,
|
|
528
|
-
createdAt
|
|
529
|
-
};
|
|
530
|
-
}
|
|
531
|
-
async getChunk(id) {
|
|
532
|
-
await this.init();
|
|
533
|
-
if (this.db === void 0) return void 0;
|
|
534
|
-
const row = this.db.prepare("SELECT * FROM knowledge_chunks WHERE id = ?").get(id);
|
|
535
|
-
return row === void 0 ? void 0 : rowToChunk(row);
|
|
536
|
-
}
|
|
537
|
-
async listChunksByDocument(documentId) {
|
|
538
|
-
await this.init();
|
|
539
|
-
if (this.db === void 0) return [];
|
|
540
|
-
return this.db.prepare("SELECT * FROM knowledge_chunks WHERE document_id = ? ORDER BY rank ASC").all(documentId).map(rowToChunk);
|
|
541
|
-
}
|
|
542
|
-
/** Load all chunk embeddings + ids for vector search. */
|
|
543
|
-
async loadAllChunkEmbeddings() {
|
|
544
|
-
await this.init();
|
|
545
|
-
if (this.db === void 0) return [];
|
|
546
|
-
const rows = this.db.prepare("SELECT id, embedding_json FROM knowledge_chunks WHERE embedding_json IS NOT NULL").all();
|
|
547
|
-
const out = [];
|
|
548
|
-
for (const row of rows) {
|
|
549
|
-
const vec = jsonToVector(row.embedding_json);
|
|
550
|
-
if (vec !== null) out.push({
|
|
551
|
-
id: row.id,
|
|
552
|
-
embedding: vec
|
|
553
|
-
});
|
|
554
|
-
}
|
|
555
|
-
return out;
|
|
556
|
-
}
|
|
557
|
-
async insertEvent(params) {
|
|
558
|
-
if (this.db === void 0) throw new Error("knowledge store not initialized");
|
|
559
|
-
const id = generateId("evt");
|
|
560
|
-
const createdAt = Date.now();
|
|
561
|
-
this.db.prepare(`INSERT INTO knowledge_events
|
|
562
|
-
(id, source_id, document_id, chunk_id, rank, title, summary, content, category, keywords,
|
|
563
|
-
title_embedding_json, content_embedding_json, created_at)
|
|
564
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(id, params.sourceId, params.documentId, params.chunkId, params.rank, params.title, params.summary, params.content, params.category, JSON.stringify(params.keywords), vectorToJson(params.titleEmbedding), vectorToJson(params.contentEmbedding), createdAt);
|
|
565
|
-
this.db.prepare(`INSERT INTO knowledge_events_fts (rowid, title, summary, content)
|
|
566
|
-
VALUES ((SELECT rowid FROM knowledge_events WHERE id = ?), ?, ?, ?)`).run(id, toFtsText(params.title), toFtsText(params.summary ?? ""), toFtsText(params.content));
|
|
567
|
-
return {
|
|
568
|
-
id,
|
|
569
|
-
sourceId: params.sourceId,
|
|
570
|
-
documentId: params.documentId,
|
|
571
|
-
chunkId: params.chunkId,
|
|
572
|
-
rank: params.rank,
|
|
573
|
-
title: params.title,
|
|
574
|
-
summary: params.summary,
|
|
575
|
-
content: params.content,
|
|
576
|
-
category: params.category,
|
|
577
|
-
keywords: params.keywords,
|
|
578
|
-
titleEmbedding: params.titleEmbedding,
|
|
579
|
-
contentEmbedding: params.contentEmbedding,
|
|
580
|
-
createdAt
|
|
581
|
-
};
|
|
582
|
-
}
|
|
583
|
-
async getEvent(id) {
|
|
584
|
-
await this.init();
|
|
585
|
-
if (this.db === void 0) return void 0;
|
|
586
|
-
const row = this.db.prepare("SELECT * FROM knowledge_events WHERE id = ?").get(id);
|
|
587
|
-
return row === void 0 ? void 0 : rowToEvent(row);
|
|
588
|
-
}
|
|
589
|
-
async listEventsByDocument(documentId) {
|
|
590
|
-
await this.init();
|
|
591
|
-
if (this.db === void 0) return [];
|
|
592
|
-
return this.db.prepare("SELECT * FROM knowledge_events WHERE document_id = ? ORDER BY rank ASC").all(documentId).map(rowToEvent);
|
|
593
|
-
}
|
|
594
|
-
/** Find events by title vector similarity. */
|
|
595
|
-
async findEventsByTitleVector(queryVec, options = {}) {
|
|
596
|
-
await this.init();
|
|
597
|
-
if (this.db === void 0) return [];
|
|
598
|
-
const limit = options.limit ?? 20;
|
|
599
|
-
const threshold = options.threshold ?? 0;
|
|
600
|
-
const rows = this.db.prepare("SELECT id, title_embedding_json FROM knowledge_events WHERE title_embedding_json IS NOT NULL").all();
|
|
601
|
-
const scored = [];
|
|
602
|
-
for (const row of rows) {
|
|
603
|
-
const vec = jsonToVector(row.title_embedding_json);
|
|
604
|
-
if (vec === null) continue;
|
|
605
|
-
const score = this.embeddingEngine?.cosineSimilarity(queryVec, vec) ?? 0;
|
|
606
|
-
if (score >= threshold) scored.push({
|
|
607
|
-
id: row.id,
|
|
608
|
-
score
|
|
609
|
-
});
|
|
610
|
-
}
|
|
611
|
-
scored.sort((a, b) => b.score - a.score);
|
|
612
|
-
const top = scored.slice(0, limit);
|
|
613
|
-
const out = [];
|
|
614
|
-
for (const item of top) {
|
|
615
|
-
const event = await this.getEvent(item.id);
|
|
616
|
-
if (event !== void 0) out.push({
|
|
617
|
-
event,
|
|
618
|
-
score: item.score
|
|
619
|
-
});
|
|
620
|
-
}
|
|
621
|
-
return out;
|
|
622
|
-
}
|
|
623
|
-
async upsertEntity(params) {
|
|
624
|
-
if (this.db === void 0) throw new Error("knowledge store not initialized");
|
|
625
|
-
if (!isEntityType(params.type)) throw new Error(`invalid entity type: ${params.type}`);
|
|
626
|
-
const normalizedName = normalizeName(params.name);
|
|
627
|
-
const existing = this.db.prepare("SELECT * FROM knowledge_entities WHERE source_id = ? AND type = ? AND normalized_name = ?").get(params.sourceId, params.type, normalizedName);
|
|
628
|
-
if (existing !== void 0) {
|
|
629
|
-
const existingId = String(existing["id"]);
|
|
630
|
-
let descriptionChanged = false;
|
|
631
|
-
if (params.description !== null && existing["description"] === null) {
|
|
632
|
-
this.db.prepare("UPDATE knowledge_entities SET description = ? WHERE id = ?").run(params.description, existingId);
|
|
633
|
-
existing["description"] = params.description;
|
|
634
|
-
descriptionChanged = true;
|
|
635
|
-
}
|
|
636
|
-
if (params.embedding !== null && existing["embedding_json"] === null) {
|
|
637
|
-
this.db.prepare("UPDATE knowledge_entities SET embedding_json = ? WHERE id = ?").run(vectorToJson(params.embedding), existingId);
|
|
638
|
-
existing["embedding_json"] = vectorToJson(params.embedding);
|
|
639
|
-
}
|
|
640
|
-
if (descriptionChanged) {
|
|
641
|
-
const ftsRow = this.db.prepare("SELECT rowid, name, description FROM knowledge_entities WHERE id = ?").get(existingId);
|
|
642
|
-
if (ftsRow !== void 0) {
|
|
643
|
-
this.db.prepare("INSERT INTO knowledge_entities_fts (knowledge_entities_fts, rowid, name, description) VALUES ('delete', ?, ?, ?)").run(ftsRow.rowid, toFtsText(ftsRow.name), "");
|
|
644
|
-
this.db.prepare(`INSERT INTO knowledge_entities_fts (rowid, name, description) VALUES (?, ?, ?)`).run(ftsRow.rowid, toFtsText(ftsRow.name), toFtsText(ftsRow.description ?? ""));
|
|
645
|
-
}
|
|
646
|
-
}
|
|
647
|
-
return rowToEntity(existing);
|
|
648
|
-
}
|
|
649
|
-
const id = generateId("ent");
|
|
650
|
-
const createdAt = Date.now();
|
|
651
|
-
this.db.prepare(`INSERT INTO knowledge_entities
|
|
652
|
-
(id, source_id, type, name, normalized_name, description, embedding_json, created_at)
|
|
653
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`).run(id, params.sourceId, params.type, params.name, normalizedName, params.description, vectorToJson(params.embedding), createdAt);
|
|
654
|
-
this.db.prepare(`INSERT INTO knowledge_entities_fts (rowid, name, description)
|
|
655
|
-
VALUES ((SELECT rowid FROM knowledge_entities WHERE id = ?), ?, ?)`).run(id, toFtsText(params.name), toFtsText(params.description ?? ""));
|
|
656
|
-
return {
|
|
657
|
-
id,
|
|
658
|
-
sourceId: params.sourceId,
|
|
659
|
-
type: params.type,
|
|
660
|
-
name: params.name,
|
|
661
|
-
normalizedName,
|
|
662
|
-
description: params.description,
|
|
663
|
-
embedding: params.embedding,
|
|
664
|
-
createdAt
|
|
665
|
-
};
|
|
666
|
-
}
|
|
667
|
-
async findEntitiesByName(name, sourceId) {
|
|
668
|
-
await this.init();
|
|
669
|
-
if (this.db === void 0) return [];
|
|
670
|
-
const normalized = normalizeName(name);
|
|
671
|
-
return (sourceId === void 0 ? this.db.prepare("SELECT * FROM knowledge_entities WHERE normalized_name = ?").all(normalized) : this.db.prepare("SELECT * FROM knowledge_entities WHERE normalized_name = ? AND source_id = ?").all(normalized, sourceId)).map(rowToEntity);
|
|
672
|
-
}
|
|
673
|
-
/** Find entities by name vector similarity. */
|
|
674
|
-
async findEntitiesByVector(queryVec, options = {}) {
|
|
675
|
-
await this.init();
|
|
676
|
-
if (this.db === void 0) return [];
|
|
677
|
-
const limit = options.limit ?? 20;
|
|
678
|
-
const threshold = options.threshold ?? 0;
|
|
679
|
-
const rows = this.db.prepare("SELECT id, embedding_json FROM knowledge_entities WHERE embedding_json IS NOT NULL").all();
|
|
680
|
-
const scored = [];
|
|
681
|
-
for (const row of rows) {
|
|
682
|
-
const vec = jsonToVector(row.embedding_json);
|
|
683
|
-
if (vec === null) continue;
|
|
684
|
-
const score = this.embeddingEngine?.cosineSimilarity(queryVec, vec) ?? 0;
|
|
685
|
-
if (score >= threshold) scored.push({
|
|
686
|
-
id: row.id,
|
|
687
|
-
score
|
|
688
|
-
});
|
|
689
|
-
}
|
|
690
|
-
scored.sort((a, b) => b.score - a.score);
|
|
691
|
-
const top = scored.slice(0, limit);
|
|
692
|
-
const out = [];
|
|
693
|
-
for (const item of top) {
|
|
694
|
-
const row = this.db.prepare("SELECT * FROM knowledge_entities WHERE id = ?").get(item.id);
|
|
695
|
-
if (row !== void 0) out.push({
|
|
696
|
-
entity: rowToEntity(row),
|
|
697
|
-
score: item.score
|
|
698
|
-
});
|
|
699
|
-
}
|
|
700
|
-
return out;
|
|
701
|
-
}
|
|
702
|
-
async insertEventEntity(params) {
|
|
703
|
-
if (this.db === void 0) throw new Error("knowledge store not initialized");
|
|
704
|
-
const id = generateId("ee");
|
|
705
|
-
this.db.prepare(`INSERT OR IGNORE INTO knowledge_event_entities
|
|
706
|
-
(id, event_id, entity_id, weight, description, embedding_json)
|
|
707
|
-
VALUES (?, ?, ?, ?, ?, ?)`).run(id, params.eventId, params.entityId, params.weight ?? 1, params.description, vectorToJson(params.embedding));
|
|
708
|
-
return {
|
|
709
|
-
id,
|
|
710
|
-
eventId: params.eventId,
|
|
711
|
-
entityId: params.entityId,
|
|
712
|
-
weight: params.weight ?? 1,
|
|
713
|
-
description: params.description,
|
|
714
|
-
embedding: params.embedding
|
|
715
|
-
};
|
|
716
|
-
}
|
|
717
|
-
async findEventsByEntity(entityId) {
|
|
718
|
-
await this.init();
|
|
719
|
-
if (this.db === void 0) return [];
|
|
720
|
-
return this.db.prepare(`SELECT e.* FROM knowledge_events e
|
|
721
|
-
JOIN knowledge_event_entities ee ON ee.event_id = e.id
|
|
722
|
-
WHERE ee.entity_id = ?`).all(entityId).map(rowToEvent);
|
|
723
|
-
}
|
|
724
|
-
async findEntitiesByEvent(eventId) {
|
|
725
|
-
await this.init();
|
|
726
|
-
if (this.db === void 0) return [];
|
|
727
|
-
return this.db.prepare(`SELECT en.* FROM knowledge_entities en
|
|
728
|
-
JOIN knowledge_event_entities ee ON ee.entity_id = en.id
|
|
729
|
-
WHERE ee.event_id = ?`).all(eventId).map(rowToEntity);
|
|
730
|
-
}
|
|
731
|
-
async listEntities(sourceId) {
|
|
732
|
-
await this.init();
|
|
733
|
-
if (this.db === void 0) return [];
|
|
734
|
-
const sql = sourceId ? `SELECT e.*, COUNT(ee.id) AS event_count
|
|
735
|
-
FROM knowledge_entities e
|
|
736
|
-
LEFT JOIN knowledge_event_entities ee ON ee.entity_id = e.id
|
|
737
|
-
WHERE e.source_id = ?
|
|
738
|
-
GROUP BY e.id
|
|
739
|
-
ORDER BY event_count DESC` : `SELECT e.*, COUNT(ee.id) AS event_count
|
|
740
|
-
FROM knowledge_entities e
|
|
741
|
-
LEFT JOIN knowledge_event_entities ee ON ee.entity_id = e.id
|
|
742
|
-
GROUP BY e.id
|
|
743
|
-
ORDER BY event_count DESC`;
|
|
744
|
-
return (sourceId ? this.db.prepare(sql).all(sourceId) : this.db.prepare(sql).all()).map((row) => ({
|
|
745
|
-
...rowToEntity(row),
|
|
746
|
-
eventCount: Number(row["event_count"] ?? 0)
|
|
747
|
-
}));
|
|
748
|
-
}
|
|
749
|
-
async listEvents(sourceId) {
|
|
750
|
-
await this.init();
|
|
751
|
-
if (this.db === void 0) return [];
|
|
752
|
-
const sql = sourceId ? "SELECT * FROM knowledge_events WHERE source_id = ? ORDER BY rank ASC" : "SELECT * FROM knowledge_events ORDER BY created_at DESC";
|
|
753
|
-
return (sourceId ? this.db.prepare(sql).all(sourceId) : this.db.prepare(sql).all()).map(rowToEvent);
|
|
754
|
-
}
|
|
755
|
-
async listEventEntities(sourceId) {
|
|
756
|
-
await this.init();
|
|
757
|
-
if (this.db === void 0) return [];
|
|
758
|
-
const sql = sourceId ? `SELECT ee.event_id, ee.entity_id
|
|
759
|
-
FROM knowledge_event_entities ee
|
|
760
|
-
JOIN knowledge_events e ON e.id = ee.event_id
|
|
761
|
-
WHERE e.source_id = ?` : "SELECT event_id, entity_id FROM knowledge_event_entities";
|
|
762
|
-
return (sourceId ? this.db.prepare(sql).all(sourceId) : this.db.prepare(sql).all()).map((row) => ({
|
|
763
|
-
eventId: asString(row["event_id"]),
|
|
764
|
-
entityId: asString(row["entity_id"])
|
|
765
|
-
}));
|
|
766
|
-
}
|
|
767
|
-
async ftsSearchChunks(query, limit = 50) {
|
|
768
|
-
await this.init();
|
|
769
|
-
if (this.db === void 0) return [];
|
|
770
|
-
const ftsQuery = buildFtsQuery(query);
|
|
771
|
-
if (ftsQuery === void 0) return [];
|
|
772
|
-
return this.db.prepare(`SELECT c.* FROM knowledge_chunks c
|
|
773
|
-
JOIN knowledge_chunks_fts f ON c.rowid = f.rowid
|
|
774
|
-
WHERE f.knowledge_chunks_fts MATCH ?
|
|
775
|
-
LIMIT ?`).all(ftsQuery, limit).map(rowToChunk);
|
|
776
|
-
}
|
|
777
|
-
async searchChunksByVector(queryVec, options = {}) {
|
|
778
|
-
await this.init();
|
|
779
|
-
if (this.db === void 0) return [];
|
|
780
|
-
const limit = options.limit ?? 50;
|
|
781
|
-
const threshold = options.threshold ?? 0;
|
|
782
|
-
const rows = this.db.prepare("SELECT id, embedding_json FROM knowledge_chunks WHERE embedding_json IS NOT NULL").all();
|
|
783
|
-
const scored = [];
|
|
784
|
-
for (const row of rows) {
|
|
785
|
-
const vec = jsonToVector(row.embedding_json);
|
|
786
|
-
if (vec === null) continue;
|
|
787
|
-
const score = this.embeddingEngine?.cosineSimilarity(queryVec, vec) ?? 0;
|
|
788
|
-
if (score >= threshold) scored.push({
|
|
789
|
-
id: row.id,
|
|
790
|
-
score
|
|
791
|
-
});
|
|
792
|
-
}
|
|
793
|
-
scored.sort((a, b) => b.score - a.score);
|
|
794
|
-
const top = scored.slice(0, limit);
|
|
795
|
-
const out = [];
|
|
796
|
-
for (const item of top) {
|
|
797
|
-
const chunk = await this.getChunk(item.id);
|
|
798
|
-
if (chunk !== void 0) out.push({
|
|
799
|
-
chunk,
|
|
800
|
-
score: item.score
|
|
801
|
-
});
|
|
802
|
-
}
|
|
803
|
-
return out;
|
|
804
|
-
}
|
|
805
|
-
/**
|
|
806
|
-
* Recompute every embedding for one source from the already-stored texts
|
|
807
|
-
* (no LLM extraction). Repairs libraries whose vectors are missing
|
|
808
|
-
* (ingested while the embedding model was unavailable) or stale (produced
|
|
809
|
-
* by a different model). All texts are embedded up front — any batch
|
|
810
|
-
* failure aborts before the first write, so the library is never left
|
|
811
|
-
* half-updated. On success the embedding-model meta is stamped with the
|
|
812
|
-
* engine's identity, unblocking future ingests after a model change.
|
|
813
|
-
*/
|
|
814
|
-
async reembedSource(sourceId, engine) {
|
|
815
|
-
await this.init();
|
|
816
|
-
if (this.db === void 0) throw new Error("knowledge store not initialized");
|
|
817
|
-
if (!engine.available) throw new Error("embedding engine unavailable");
|
|
818
|
-
const chunkRows = this.db.prepare("SELECT id, heading, content FROM knowledge_chunks WHERE source_id = ? ORDER BY rank ASC").all(sourceId);
|
|
819
|
-
const chunkTexts = chunkRows.map((c) => c.heading !== null ? `${c.heading}\n${c.content}` : c.content);
|
|
820
|
-
const eventRows = this.db.prepare("SELECT id, title, content FROM knowledge_events WHERE source_id = ? ORDER BY rank ASC").all(sourceId);
|
|
821
|
-
const eventTitleTexts = eventRows.map((e) => e.title);
|
|
822
|
-
const eventContentTexts = eventRows.map((e) => `${e.title}\n\n${e.content}`);
|
|
823
|
-
const entityRows = this.db.prepare("SELECT id, name FROM knowledge_entities WHERE source_id = ?").all(sourceId);
|
|
824
|
-
const relationRows = this.db.prepare(`SELECT ee.id, ee.description, e.title, en.name
|
|
825
|
-
FROM knowledge_event_entities ee
|
|
826
|
-
JOIN knowledge_events e ON e.id = ee.event_id
|
|
827
|
-
JOIN knowledge_entities en ON en.id = ee.entity_id
|
|
828
|
-
WHERE e.source_id = ?`).all(sourceId);
|
|
829
|
-
const relationTexts = relationRows.map((r) => r.description !== null && r.description.length > 0 ? r.description : `${r.title} ${r.name}`);
|
|
830
|
-
const embedAll = async (texts) => {
|
|
831
|
-
if (texts.length === 0) return [];
|
|
832
|
-
const vectors = await engine.embedBatch(texts);
|
|
833
|
-
if (vectors === null || vectors.length !== texts.length) throw new Error("向量嵌入失败,已取消本次重新嵌入");
|
|
834
|
-
return vectors;
|
|
835
|
-
};
|
|
836
|
-
const chunkVecs = await embedAll(chunkTexts);
|
|
837
|
-
const eventTitleVecs = await embedAll(eventTitleTexts);
|
|
838
|
-
const eventContentVecs = await embedAll(eventContentTexts);
|
|
839
|
-
const entityVecs = await embedAll(entityRows.map((e) => e.name));
|
|
840
|
-
const relationVecs = await embedAll(relationTexts);
|
|
841
|
-
this.beginTransaction();
|
|
842
|
-
try {
|
|
843
|
-
const updChunk = this.db.prepare("UPDATE knowledge_chunks SET embedding_json = ? WHERE id = ?");
|
|
844
|
-
for (let i = 0; i < chunkRows.length; i++) updChunk.run(vectorToJson(chunkVecs[i] ?? null), chunkRows[i].id);
|
|
845
|
-
const updEvent = this.db.prepare("UPDATE knowledge_events SET title_embedding_json = ?, content_embedding_json = ? WHERE id = ?");
|
|
846
|
-
for (let i = 0; i < eventRows.length; i++) updEvent.run(vectorToJson(eventTitleVecs[i] ?? null), vectorToJson(eventContentVecs[i] ?? null), eventRows[i].id);
|
|
847
|
-
const updEntity = this.db.prepare("UPDATE knowledge_entities SET embedding_json = ? WHERE id = ?");
|
|
848
|
-
for (let i = 0; i < entityRows.length; i++) updEntity.run(vectorToJson(entityVecs[i] ?? null), entityRows[i].id);
|
|
849
|
-
const updRelation = this.db.prepare("UPDATE knowledge_event_entities SET embedding_json = ? WHERE id = ?");
|
|
850
|
-
for (let i = 0; i < relationRows.length; i++) updRelation.run(vectorToJson(relationVecs[i] ?? null), relationRows[i].id);
|
|
851
|
-
await this.setMeta(EMBEDDING_MODEL_META_KEY, engine.modelName);
|
|
852
|
-
this.commitTransaction();
|
|
853
|
-
} catch (error) {
|
|
854
|
-
this.rollbackTransaction();
|
|
855
|
-
throw error;
|
|
856
|
-
}
|
|
857
|
-
return {
|
|
858
|
-
chunks: chunkRows.length,
|
|
859
|
-
events: eventRows.length,
|
|
860
|
-
entities: entityRows.length,
|
|
861
|
-
relations: relationRows.length
|
|
862
|
-
};
|
|
863
|
-
}
|
|
864
|
-
/**
|
|
865
|
-
* Chunk vector coverage — how many of the stored chunks carry an embedding.
|
|
866
|
-
* Used by the TUI to surface partial/skipped embedding after an ingest.
|
|
867
|
-
*/
|
|
868
|
-
async embeddingCoverage() {
|
|
869
|
-
await this.init();
|
|
870
|
-
if (this.db === void 0) return {
|
|
871
|
-
total: 0,
|
|
872
|
-
embedded: 0
|
|
873
|
-
};
|
|
874
|
-
const row = this.db.prepare(`SELECT COUNT(*) AS total,
|
|
875
|
-
SUM(CASE WHEN embedding_json IS NOT NULL THEN 1 ELSE 0 END) AS embedded
|
|
876
|
-
FROM knowledge_chunks`).get();
|
|
877
|
-
if (row === void 0) return {
|
|
878
|
-
total: 0,
|
|
879
|
-
embedded: 0
|
|
880
|
-
};
|
|
881
|
-
return {
|
|
882
|
-
total: Number(row["total"] ?? 0),
|
|
883
|
-
embedded: Number(row["embedded"] ?? 0)
|
|
884
|
-
};
|
|
885
|
-
}
|
|
886
|
-
async stats() {
|
|
887
|
-
await this.init();
|
|
888
|
-
if (this.db === void 0) return {
|
|
889
|
-
sources: 0,
|
|
890
|
-
documents: 0,
|
|
891
|
-
chunks: 0,
|
|
892
|
-
events: 0,
|
|
893
|
-
entities: 0
|
|
894
|
-
};
|
|
895
|
-
const count = (table) => {
|
|
896
|
-
return this.db.prepare(`SELECT COUNT(*) AS c FROM ${table}`).get()?.c ?? 0;
|
|
897
|
-
};
|
|
898
|
-
return {
|
|
899
|
-
sources: count("knowledge_sources"),
|
|
900
|
-
documents: count("knowledge_documents"),
|
|
901
|
-
chunks: count("knowledge_chunks"),
|
|
902
|
-
events: count("knowledge_events"),
|
|
903
|
-
entities: count("knowledge_entities")
|
|
904
|
-
};
|
|
905
|
-
}
|
|
906
|
-
async buildSearchResult(chunkId, score, eventId = null) {
|
|
907
|
-
if (this.db === void 0) return void 0;
|
|
908
|
-
const chunk = await this.getChunk(chunkId);
|
|
909
|
-
if (chunk === void 0) return void 0;
|
|
910
|
-
if (this.db.prepare("SELECT * FROM knowledge_documents WHERE id = ?").get(chunk.documentId) === void 0) return void 0;
|
|
911
|
-
const sourceRow = this.db.prepare("SELECT * FROM knowledge_sources WHERE id = ?").get(chunk.sourceId);
|
|
912
|
-
if (sourceRow === void 0) return void 0;
|
|
913
|
-
let eventTitle = null;
|
|
914
|
-
if (eventId !== null) eventTitle = (await this.getEvent(eventId))?.title ?? null;
|
|
915
|
-
return {
|
|
916
|
-
chunkId: chunk.id,
|
|
917
|
-
documentId: chunk.documentId,
|
|
918
|
-
sourceId: chunk.sourceId,
|
|
919
|
-
sourceName: String(sourceRow["name"]),
|
|
920
|
-
heading: chunk.heading,
|
|
921
|
-
content: chunk.content,
|
|
922
|
-
score,
|
|
923
|
-
eventId,
|
|
924
|
-
eventTitle
|
|
925
|
-
};
|
|
926
|
-
}
|
|
927
|
-
};
|
|
928
|
-
function asString(value) {
|
|
929
|
-
if (typeof value === "string") return value;
|
|
930
|
-
if (typeof value === "number" || typeof value === "bigint" || typeof value === "boolean") return String(value);
|
|
931
|
-
return "";
|
|
932
|
-
}
|
|
933
|
-
function asNullableString(value) {
|
|
934
|
-
if (value === null || value === void 0) return null;
|
|
935
|
-
if (typeof value === "string") return value;
|
|
936
|
-
if (typeof value === "number" || typeof value === "bigint" || typeof value === "boolean") return String(value);
|
|
937
|
-
return null;
|
|
938
|
-
}
|
|
939
|
-
function rowToSource(row) {
|
|
940
|
-
return {
|
|
941
|
-
id: asString(row["id"]),
|
|
942
|
-
name: asString(row["name"]),
|
|
943
|
-
filePath: asNullableString(row["file_path"]),
|
|
944
|
-
description: asNullableString(row["description"]),
|
|
945
|
-
contentHash: asNullableString(row["content_hash"]),
|
|
946
|
-
createdAt: Number(row["created_at"])
|
|
947
|
-
};
|
|
948
|
-
}
|
|
949
|
-
function rowToDocument(row) {
|
|
950
|
-
return {
|
|
951
|
-
id: asString(row["id"]),
|
|
952
|
-
sourceId: asString(row["source_id"]),
|
|
953
|
-
title: asString(row["title"]),
|
|
954
|
-
content: asNullableString(row["content"]),
|
|
955
|
-
status: asString(row["status"]),
|
|
956
|
-
chunkCount: Number(row["chunk_count"] ?? 0),
|
|
957
|
-
createdAt: Number(row["created_at"])
|
|
958
|
-
};
|
|
959
|
-
}
|
|
960
|
-
function rowToChunk(row) {
|
|
961
|
-
return {
|
|
962
|
-
id: asString(row["id"]),
|
|
963
|
-
sourceId: asString(row["source_id"]),
|
|
964
|
-
documentId: asString(row["document_id"]),
|
|
965
|
-
rank: Number(row["rank"]),
|
|
966
|
-
heading: asNullableString(row["heading"]),
|
|
967
|
-
content: asString(row["content"]),
|
|
968
|
-
rawContent: asNullableString(row["raw_content"]),
|
|
969
|
-
embedding: jsonToVector(asNullableString(row["embedding_json"])),
|
|
970
|
-
createdAt: Number(row["created_at"])
|
|
971
|
-
};
|
|
972
|
-
}
|
|
973
|
-
function rowToEvent(row) {
|
|
974
|
-
let keywords = [];
|
|
975
|
-
try {
|
|
976
|
-
const parsed = JSON.parse(asString(row["keywords"] ?? "[]"));
|
|
977
|
-
if (Array.isArray(parsed)) keywords = parsed.filter((k) => typeof k === "string");
|
|
978
|
-
} catch {}
|
|
979
|
-
return {
|
|
980
|
-
id: asString(row["id"]),
|
|
981
|
-
sourceId: asString(row["source_id"]),
|
|
982
|
-
documentId: asString(row["document_id"]),
|
|
983
|
-
chunkId: asString(row["chunk_id"]),
|
|
984
|
-
rank: Number(row["rank"]),
|
|
985
|
-
title: asString(row["title"]),
|
|
986
|
-
summary: asNullableString(row["summary"]),
|
|
987
|
-
content: asString(row["content"]),
|
|
988
|
-
category: asNullableString(row["category"]),
|
|
989
|
-
keywords,
|
|
990
|
-
titleEmbedding: jsonToVector(asNullableString(row["title_embedding_json"])),
|
|
991
|
-
contentEmbedding: jsonToVector(asNullableString(row["content_embedding_json"])),
|
|
992
|
-
createdAt: Number(row["created_at"])
|
|
993
|
-
};
|
|
994
|
-
}
|
|
995
|
-
function rowToEntity(row) {
|
|
996
|
-
return {
|
|
997
|
-
id: asString(row["id"]),
|
|
998
|
-
sourceId: asString(row["source_id"]),
|
|
999
|
-
type: asString(row["type"]),
|
|
1000
|
-
name: asString(row["name"]),
|
|
1001
|
-
normalizedName: asString(row["normalized_name"]),
|
|
1002
|
-
description: asNullableString(row["description"]),
|
|
1003
|
-
embedding: jsonToVector(asNullableString(row["embedding_json"])),
|
|
1004
|
-
createdAt: Number(row["created_at"])
|
|
1005
|
-
};
|
|
1006
|
-
}
|
|
1007
|
-
/**
|
|
1008
|
-
* Tokenize text so FTS5's unicode61 tokenizer can index mixed CJK/ASCII text.
|
|
1009
|
-
* Mirrors the memory package's `toFtsText`.
|
|
1010
|
-
*/
|
|
1011
|
-
function toFtsText(text) {
|
|
1012
|
-
const parts = text.toLowerCase().replaceAll(/([一-鿿㐀-䶿])([a-z0-9])/g, "$1 $2").replaceAll(/([a-z0-9])([一-鿿㐀-䶿])/g, "$1 $2").split(/[^a-z0-9一-鿿㐀-䶿]+/);
|
|
1013
|
-
const tokens = [];
|
|
1014
|
-
for (const part of parts) {
|
|
1015
|
-
if (part.length === 0) continue;
|
|
1016
|
-
if (/^[a-z0-9]+$/.test(part)) tokens.push(part);
|
|
1017
|
-
else for (const ch of part) if (ch.length > 0) tokens.push(ch);
|
|
1018
|
-
}
|
|
1019
|
-
return tokens.join(" ");
|
|
1020
|
-
}
|
|
1021
|
-
function buildFtsQuery(search) {
|
|
1022
|
-
const tokens = toFtsText(search).split(/\s+/).filter((t) => t.length > 0);
|
|
1023
|
-
if (tokens.length === 0) return void 0;
|
|
1024
|
-
return tokens.map((t) => `"${t.replaceAll("\"", "\"\"")}"`).join(" AND ");
|
|
1025
|
-
}
|
|
1026
|
-
//#endregion
|
|
1027
|
-
//#region ../../packages/knowledge/src/chunking.ts
|
|
1028
|
-
const HEADING_RE = /^(#{1,6})\s+(.+?)\s*$/;
|
|
1029
|
-
const CODE_FENCE_RE = /^(\s*)(`{3,}|~{3,})/;
|
|
1030
|
-
/**
|
|
1031
|
-
* Soft cap on chunk token count. BGE-small-zh-v1.5 (the fastembed model used
|
|
1032
|
-
* here) has a 512-token sequence limit; chunks longer than that get truncated
|
|
1033
|
-
* at the embedding layer, hurting retrieval quality. We split at ~480 to leave
|
|
1034
|
-
* headroom for the heading line that gets prepended at embed time.
|
|
1035
|
-
*/
|
|
1036
|
-
const MAX_CHUNK_TOKENS = 480;
|
|
1037
|
-
/**
|
|
1038
|
-
* Split a markdown document into sections using heading_strict strategy:
|
|
1039
|
-
* each section begins at a heading and contains all body content until the
|
|
1040
|
-
* next heading. Consecutive headings collapse to the lowest-level one as the
|
|
1041
|
-
* main heading. Content before the first heading is grouped as "Introduction".
|
|
1042
|
-
*
|
|
1043
|
-
* Mirrors SAG's `buildHeadingStrictSections`.
|
|
1044
|
-
*/
|
|
1045
|
-
function chunkMarkdown(content) {
|
|
1046
|
-
const lines = content.split("\n");
|
|
1047
|
-
const sections = [];
|
|
1048
|
-
let buffer = [];
|
|
1049
|
-
let currentHeading = null;
|
|
1050
|
-
let currentLevel = null;
|
|
1051
|
-
let rank = 0;
|
|
1052
|
-
let inFence = false;
|
|
1053
|
-
const flush = () => {
|
|
1054
|
-
const raw = buffer.join("\n").trim();
|
|
1055
|
-
const heading = currentHeading;
|
|
1056
|
-
const level = currentLevel;
|
|
1057
|
-
buffer = [];
|
|
1058
|
-
if (raw.length === 0) return;
|
|
1059
|
-
const body = stripMarkdown(raw);
|
|
1060
|
-
if (body.length === 0 && heading === null) return;
|
|
1061
|
-
const subSections = splitLargeSection({
|
|
1062
|
-
heading,
|
|
1063
|
-
headingLevel: level,
|
|
1064
|
-
content: body,
|
|
1065
|
-
rawContent: raw,
|
|
1066
|
-
rank
|
|
1067
|
-
});
|
|
1068
|
-
for (const section of subSections) {
|
|
1069
|
-
sections.push({
|
|
1070
|
-
...section,
|
|
1071
|
-
rank
|
|
1072
|
-
});
|
|
1073
|
-
rank += 1;
|
|
1074
|
-
}
|
|
1075
|
-
};
|
|
1076
|
-
for (const line of lines) {
|
|
1077
|
-
if (CODE_FENCE_RE.exec(line)) {
|
|
1078
|
-
inFence = !inFence;
|
|
1079
|
-
buffer.push(line);
|
|
1080
|
-
continue;
|
|
1081
|
-
}
|
|
1082
|
-
if (inFence) {
|
|
1083
|
-
buffer.push(line);
|
|
1084
|
-
continue;
|
|
1085
|
-
}
|
|
1086
|
-
const headingMatch = HEADING_RE.exec(line);
|
|
1087
|
-
if (headingMatch) {
|
|
1088
|
-
flush();
|
|
1089
|
-
currentHeading = headingMatch[2].trim();
|
|
1090
|
-
currentLevel = headingMatch[1].length;
|
|
1091
|
-
} else buffer.push(line);
|
|
1092
|
-
}
|
|
1093
|
-
flush();
|
|
1094
|
-
return sections;
|
|
1095
|
-
}
|
|
1096
|
-
/**
|
|
1097
|
-
* Strip markdown decorations that hurt embedding quality:
|
|
1098
|
-
* code fences, inline code, images, links, emphasis markers.
|
|
1099
|
-
*
|
|
1100
|
-
* Mirrors SAG's `stripMarkdown`.
|
|
1101
|
-
*/
|
|
1102
|
-
function stripMarkdown(text) {
|
|
1103
|
-
return text.replaceAll(/^(\s*)```[^\n]*$\n?/gm, "$1").replaceAll(/^(\s*)```[^\n]*$/gm, "").replaceAll(/`([^`]+)`/g, "$1").replaceAll(/!\[([^\]]*)\]\([^)]+\)/g, "$1").replaceAll(/\[([^\]]+)\]\([^)]+\)/g, "$1").replaceAll(/\*\*([^*]+)\*\*/g, "$1").replaceAll(/__([^_]+)__/g, "$1").replaceAll(/(?<!\w)\*([^*\n]+)\*(?!\w)/g, "$1").replaceAll(/(?<!\w)_([^_\n]+)_(?!\w)/g, "$1").replaceAll(/^#{1,6}\s+/gm, "").replaceAll(/\n{3,}/g, "\n\n").trim();
|
|
1104
|
-
}
|
|
1105
|
-
/**
|
|
1106
|
-
* Split a plain text file into chunks by blank-line paragraphs.
|
|
1107
|
-
* Each non-empty paragraph becomes one chunk with no heading.
|
|
1108
|
-
* Long paragraphs are split at sentence boundaries to stay under the
|
|
1109
|
-
* embedding model's token cap (same MAX_CHUNK_TOKENS as markdown).
|
|
1110
|
-
*/
|
|
1111
|
-
function chunkText(content) {
|
|
1112
|
-
const MAX_PARA_TOKENS = MAX_CHUNK_TOKENS;
|
|
1113
|
-
const MAX_PARA_CHARS = MAX_PARA_TOKENS * 4;
|
|
1114
|
-
const paragraphs = content.split(/\n\s*\n/).map((p) => p.trim()).filter((p) => p.length > 0);
|
|
1115
|
-
const sections = [];
|
|
1116
|
-
let rank = 0;
|
|
1117
|
-
for (const para of paragraphs) {
|
|
1118
|
-
if (estimateTokens(para) <= MAX_PARA_TOKENS) {
|
|
1119
|
-
sections.push({
|
|
1120
|
-
heading: null,
|
|
1121
|
-
headingLevel: null,
|
|
1122
|
-
content: para,
|
|
1123
|
-
rawContent: para,
|
|
1124
|
-
rank: rank++
|
|
1125
|
-
});
|
|
1126
|
-
continue;
|
|
1127
|
-
}
|
|
1128
|
-
const sentences = para.match(/[^.!?。!?\n]+[.!?。!?\n]+|[^.!?。!?\n]+$/g) ?? [para];
|
|
1129
|
-
let buffer = "";
|
|
1130
|
-
const flushBuffer = () => {
|
|
1131
|
-
const text = buffer.trim();
|
|
1132
|
-
if (text.length === 0) return;
|
|
1133
|
-
sections.push({
|
|
1134
|
-
heading: null,
|
|
1135
|
-
headingLevel: null,
|
|
1136
|
-
content: text,
|
|
1137
|
-
rawContent: text,
|
|
1138
|
-
rank: rank++
|
|
1139
|
-
});
|
|
1140
|
-
buffer = "";
|
|
1141
|
-
};
|
|
1142
|
-
for (const sentence of sentences) {
|
|
1143
|
-
const candidate = (buffer + " " + sentence).trim();
|
|
1144
|
-
if (buffer.length > 0 && candidate.length > MAX_PARA_CHARS) flushBuffer();
|
|
1145
|
-
buffer = buffer.length === 0 ? sentence : `${buffer} ${sentence}`;
|
|
1146
|
-
}
|
|
1147
|
-
flushBuffer();
|
|
1148
|
-
}
|
|
1149
|
-
return sections;
|
|
1150
|
-
}
|
|
1151
|
-
/** Rough token estimate — chars / 4 (English-leaning; CJK underestimates). */
|
|
1152
|
-
function estimateTokens(text) {
|
|
1153
|
-
return Math.ceil(text.length / 4);
|
|
1154
|
-
}
|
|
1155
|
-
/**
|
|
1156
|
-
* Split a section that exceeds MAX_CHUNK_TOKENS into smaller sub-sections.
|
|
1157
|
-
* Strategy (mirrors SAG's `splitLargeSection`):
|
|
1158
|
-
* 1. Split into paragraphs by blank lines.
|
|
1159
|
-
* 2. Greedily pack paragraphs into chunks under the token cap.
|
|
1160
|
-
* 3. A paragraph that alone exceeds the cap is split at sentence boundaries;
|
|
1161
|
-
* a sentence that itself exceeds the cap is split by character limit.
|
|
1162
|
-
*
|
|
1163
|
-
* Sections under the cap are returned unchanged (single-element array).
|
|
1164
|
-
* The heading is preserved across all sub-sections; rank is reassigned by the
|
|
1165
|
-
* caller. rawContent is rebuilt from the sub-chunk's content paragraphs.
|
|
1166
|
-
*/
|
|
1167
|
-
function splitLargeSection(section) {
|
|
1168
|
-
if (estimateTokens(section.content) <= MAX_CHUNK_TOKENS) return [section];
|
|
1169
|
-
const paragraphs = section.rawContent.split(/\n{2,}/).map((p) => p.trim()).filter((p) => p.length > 0);
|
|
1170
|
-
if (paragraphs.length <= 1) return splitBySentences(section, section.rawContent);
|
|
1171
|
-
const result = [];
|
|
1172
|
-
let buffer = [];
|
|
1173
|
-
let bufferTokens = 0;
|
|
1174
|
-
const flushBuffer = () => {
|
|
1175
|
-
if (buffer.length === 0) return;
|
|
1176
|
-
const raw = buffer.join("\n\n").trim();
|
|
1177
|
-
if (raw.length === 0) {
|
|
1178
|
-
buffer = [];
|
|
1179
|
-
bufferTokens = 0;
|
|
1180
|
-
return;
|
|
1181
|
-
}
|
|
1182
|
-
result.push({
|
|
1183
|
-
heading: section.heading,
|
|
1184
|
-
headingLevel: section.headingLevel,
|
|
1185
|
-
content: stripMarkdown(raw),
|
|
1186
|
-
rawContent: raw,
|
|
1187
|
-
rank: 0
|
|
1188
|
-
});
|
|
1189
|
-
buffer = [];
|
|
1190
|
-
bufferTokens = 0;
|
|
1191
|
-
};
|
|
1192
|
-
for (const para of paragraphs) {
|
|
1193
|
-
const paraTokens = estimateTokens(para);
|
|
1194
|
-
if (paraTokens > MAX_CHUNK_TOKENS) {
|
|
1195
|
-
flushBuffer();
|
|
1196
|
-
for (const frag of splitBySentences(section, para)) result.push(frag);
|
|
1197
|
-
continue;
|
|
1198
|
-
}
|
|
1199
|
-
if (buffer.length > 0 && bufferTokens + paraTokens > MAX_CHUNK_TOKENS) flushBuffer();
|
|
1200
|
-
buffer.push(para);
|
|
1201
|
-
bufferTokens += paraTokens;
|
|
1202
|
-
}
|
|
1203
|
-
flushBuffer();
|
|
1204
|
-
return result.length > 0 ? result : [section];
|
|
1205
|
-
}
|
|
1206
|
-
/**
|
|
1207
|
-
* Split a long text into sub-sections at sentence boundaries.
|
|
1208
|
-
* Falls back to a hard character cut when a single sentence exceeds the cap.
|
|
1209
|
-
*/
|
|
1210
|
-
function splitBySentences(section, text) {
|
|
1211
|
-
const sentences = splitSentences(text);
|
|
1212
|
-
const result = [];
|
|
1213
|
-
let buffer = "";
|
|
1214
|
-
let bufferTokens = 0;
|
|
1215
|
-
const flush = () => {
|
|
1216
|
-
const trimmed = buffer.trim();
|
|
1217
|
-
if (trimmed.length === 0) {
|
|
1218
|
-
buffer = "";
|
|
1219
|
-
bufferTokens = 0;
|
|
1220
|
-
return;
|
|
1221
|
-
}
|
|
1222
|
-
result.push({
|
|
1223
|
-
heading: section.heading,
|
|
1224
|
-
headingLevel: section.headingLevel,
|
|
1225
|
-
content: stripMarkdown(trimmed),
|
|
1226
|
-
rawContent: trimmed,
|
|
1227
|
-
rank: 0
|
|
1228
|
-
});
|
|
1229
|
-
buffer = "";
|
|
1230
|
-
bufferTokens = 0;
|
|
1231
|
-
};
|
|
1232
|
-
for (const sentence of sentences) {
|
|
1233
|
-
const sentTokens = estimateTokens(sentence);
|
|
1234
|
-
if (sentTokens > MAX_CHUNK_TOKENS) {
|
|
1235
|
-
flush();
|
|
1236
|
-
for (const frag of splitByCharLimit(sentence)) result.push({
|
|
1237
|
-
heading: section.heading,
|
|
1238
|
-
headingLevel: section.headingLevel,
|
|
1239
|
-
content: stripMarkdown(frag),
|
|
1240
|
-
rawContent: frag,
|
|
1241
|
-
rank: 0
|
|
1242
|
-
});
|
|
1243
|
-
continue;
|
|
1244
|
-
}
|
|
1245
|
-
if (buffer.length > 0 && bufferTokens + sentTokens > MAX_CHUNK_TOKENS) flush();
|
|
1246
|
-
buffer = buffer.length === 0 ? sentence : `${buffer} ${sentence}`;
|
|
1247
|
-
bufferTokens += sentTokens;
|
|
1248
|
-
}
|
|
1249
|
-
flush();
|
|
1250
|
-
return result.length > 0 ? result : [{
|
|
1251
|
-
heading: section.heading,
|
|
1252
|
-
headingLevel: section.headingLevel,
|
|
1253
|
-
content: section.content,
|
|
1254
|
-
rawContent: section.rawContent,
|
|
1255
|
-
rank: 0
|
|
1256
|
-
}];
|
|
1257
|
-
}
|
|
1258
|
-
/** Split text into sentences using CJK + ASCII punctuation. */
|
|
1259
|
-
function splitSentences(text) {
|
|
1260
|
-
return text.match(/[^。!?!?.\n]+[。!?!?.\n]+|[^。!?!?.\n]+$/g) ?? [text];
|
|
1261
|
-
}
|
|
1262
|
-
/** Hard character cut for pathological single sentences longer than the cap. */
|
|
1263
|
-
function splitByCharLimit(text) {
|
|
1264
|
-
const maxChars = MAX_CHUNK_TOKENS * 4;
|
|
1265
|
-
const chunks = [];
|
|
1266
|
-
let remaining = text.trim();
|
|
1267
|
-
while (remaining.length > maxChars) {
|
|
1268
|
-
chunks.push(remaining.slice(0, maxChars));
|
|
1269
|
-
remaining = remaining.slice(maxChars).trimStart();
|
|
1270
|
-
}
|
|
1271
|
-
if (remaining.length > 0) chunks.push(remaining);
|
|
1272
|
-
return chunks;
|
|
1273
|
-
}
|
|
1274
|
-
//#endregion
|
|
1275
|
-
//#region ../../packages/knowledge/src/extractor.ts
|
|
1276
|
-
/** Max LLM call attempts for extraction — fails fall back to a heading-derived event. */
|
|
1277
|
-
const EXTRACTION_MAX_ATTEMPTS = 3;
|
|
1278
|
-
/** Base delay (ms) for exponential backoff: 100, 200, 400. */
|
|
1279
|
-
const EXTRACTION_BACKOFF_BASE_MS = 100;
|
|
1280
|
-
function backoffDelay(attempt) {
|
|
1281
|
-
return EXTRACTION_BACKOFF_BASE_MS * 2 ** Math.max(0, attempt - 1);
|
|
1282
|
-
}
|
|
1283
|
-
async function callLlmWithBackoff(llm, systemPrompt, userPrompt) {
|
|
1284
|
-
let lastError;
|
|
1285
|
-
for (let attempt = 1; attempt <= EXTRACTION_MAX_ATTEMPTS; attempt += 1) try {
|
|
1286
|
-
return await llm.generate(systemPrompt, userPrompt);
|
|
1287
|
-
} catch (error) {
|
|
1288
|
-
lastError = error;
|
|
1289
|
-
if (attempt < EXTRACTION_MAX_ATTEMPTS) await new Promise((resolve) => setTimeout(resolve, backoffDelay(attempt)));
|
|
1290
|
-
}
|
|
1291
|
-
throw lastError;
|
|
1292
|
-
}
|
|
1293
|
-
const ENTITY_TYPES = [
|
|
1294
|
-
"person",
|
|
1295
|
-
"organization",
|
|
1296
|
-
"location",
|
|
1297
|
-
"time",
|
|
1298
|
-
"product",
|
|
1299
|
-
"metric",
|
|
1300
|
-
"action",
|
|
1301
|
-
"work",
|
|
1302
|
-
"group",
|
|
1303
|
-
"subject",
|
|
1304
|
-
"tags"
|
|
1305
|
-
];
|
|
1306
|
-
const EXTRACTION_SYSTEM_PROMPT = `You are a professional knowledge content extractor for a SAG-style event-centric knowledge graph.
|
|
1307
|
-
|
|
1308
|
-
Your task: given a markdown section (one chunk), extract exactly ONE fused event that captures the section's core informational content, plus the entities mentioned.
|
|
1309
|
-
|
|
1310
|
-
# Output Format
|
|
1311
|
-
|
|
1312
|
-
Return a JSON object with this exact shape:
|
|
1313
|
-
|
|
1314
|
-
{
|
|
1315
|
-
"items": [
|
|
1316
|
-
{
|
|
1317
|
-
"title": "short event title (≤80 chars, noun phrase or short sentence)",
|
|
1318
|
-
"summary": "one-sentence summary of what this section is about",
|
|
1319
|
-
"content": "the full informational content of this section, rephrased as a self-contained event description (1-4 sentences)",
|
|
1320
|
-
"category": "one of: definition, process, comparison, decision, history, reference, other",
|
|
1321
|
-
"keywords": ["3-5 short keywords or phrases"],
|
|
1322
|
-
"entities": [
|
|
1323
|
-
{ "type": "one of ENTITY_TYPES below", "name": "entity name", "description": "one-line context for the entity" }
|
|
1324
|
-
]
|
|
1325
|
-
}
|
|
1326
|
-
]
|
|
1327
|
-
}
|
|
1328
|
-
|
|
1329
|
-
# Entity Types
|
|
1330
|
-
|
|
1331
|
-
Use only these types:
|
|
1332
|
-
- person — named individuals
|
|
1333
|
-
- organization — companies, teams, institutions
|
|
1334
|
-
- location — places, geographic regions
|
|
1335
|
-
- time — dates, time periods, eras
|
|
1336
|
-
- product — products, services, software, tools
|
|
1337
|
-
- metric — numbers, statistics, measurements
|
|
1338
|
-
- action — processes, operations, activities
|
|
1339
|
-
- work — books, papers, projects, artworks
|
|
1340
|
-
- group — categories, classes, groups of things
|
|
1341
|
-
- subject — abstract topics, concepts, fields
|
|
1342
|
-
- tags — use sparingly for cross-cutting labels
|
|
1343
|
-
|
|
1344
|
-
# Rules
|
|
1345
|
-
|
|
1346
|
-
1. Produce EXACTLY ONE item in the "items" array.
|
|
1347
|
-
2. The event title should be specific enough to disambiguate from sibling sections.
|
|
1348
|
-
3. The content must be self-contained — a reader should understand it without the surrounding context.
|
|
1349
|
-
4. Extract 2-6 entities per chunk. Only include entities that are clearly referenced in the text.
|
|
1350
|
-
5. Use the entity's most specific name (e.g. "Anthropic" rather than "the company").
|
|
1351
|
-
6. Output language must follow the main input language. Chinese input requires Chinese title, summary, content, category, keywords, and entity descriptions. English input requires English fields.
|
|
1352
|
-
7. Output ONLY the JSON object — no prose, no markdown fences, no commentary.
|
|
1353
|
-
|
|
1354
|
-
# Example
|
|
1355
|
-
|
|
1356
|
-
Input section:
|
|
1357
|
-
"""
|
|
1358
|
-
## Model Context Protocol
|
|
1359
|
-
|
|
1360
|
-
The Model Context Protocol (MCP) is an open standard introduced by Anthropic in 2024. It defines a JSON-RPC interface between AI assistants and external tools. MCP servers expose resources, prompts, and tools that clients like Claude Desktop can consume.
|
|
1361
|
-
"""
|
|
1362
|
-
|
|
1363
|
-
Output:
|
|
1364
|
-
{
|
|
1365
|
-
"items": [
|
|
1366
|
-
{
|
|
1367
|
-
"title": "Model Context Protocol introduction",
|
|
1368
|
-
"summary": "MCP is an open standard for AI assistant / tool integration introduced by Anthropic in 2024.",
|
|
1369
|
-
"content": "The Model Context Protocol (MCP) is an open standard introduced by Anthropic in 2024 that defines a JSON-RPC interface between AI assistants and external tools. MCP servers expose resources, prompts, and tools that clients like Claude Desktop can consume.",
|
|
1370
|
-
"category": "definition",
|
|
1371
|
-
"keywords": ["MCP", "JSON-RPC", "Anthropic", "AI tools"],
|
|
1372
|
-
"entities": [
|
|
1373
|
-
{ "type": "product", "name": "Model Context Protocol", "description": "Open standard for AI assistant / tool integration" },
|
|
1374
|
-
{ "type": "organization", "name": "Anthropic", "description": "Company that introduced MCP in 2024" },
|
|
1375
|
-
{ "type": "time", "name": "2024", "description": "Year MCP was introduced" },
|
|
1376
|
-
{ "type": "product", "name": "Claude Desktop", "description": "An MCP client application" }
|
|
1377
|
-
]
|
|
1378
|
-
}
|
|
1379
|
-
]
|
|
1380
|
-
}`;
|
|
1381
|
-
/**
|
|
1382
|
-
* Build the user prompt for extracting events/entities from a chunk.
|
|
1383
|
-
* Includes a one-shot example to anchor the expected output shape.
|
|
1384
|
-
*/
|
|
1385
|
-
function buildExtractionUserPrompt(chunk) {
|
|
1386
|
-
return [
|
|
1387
|
-
"Extract the fused event and entities from the following markdown section.",
|
|
1388
|
-
"",
|
|
1389
|
-
`Section heading: ${chunk.heading ?? "(no heading)"}`,
|
|
1390
|
-
"",
|
|
1391
|
-
"Section content:",
|
|
1392
|
-
"\"\"\"",
|
|
1393
|
-
chunk.content,
|
|
1394
|
-
"\"\"\"",
|
|
1395
|
-
"",
|
|
1396
|
-
"Return only the JSON object per the schema."
|
|
1397
|
-
].join("\n");
|
|
1398
|
-
}
|
|
1399
|
-
/** Extract the first {...} JSON block from a text response. */
|
|
1400
|
-
function extractJsonFromText(text) {
|
|
1401
|
-
const match = text.match(/\{[\s\S]*\}/);
|
|
1402
|
-
return match === null ? null : match[0];
|
|
1403
|
-
}
|
|
1404
|
-
/** Parse LLM extraction output into structured events. Returns at most 1 event. */
|
|
1405
|
-
function parseExtractionResponse(text, fallbackChunk) {
|
|
1406
|
-
const jsonText = extractJsonFromText(text);
|
|
1407
|
-
if (jsonText === null) return fallbackEvent(fallbackChunk);
|
|
1408
|
-
try {
|
|
1409
|
-
const items = JSON.parse(jsonText)["items"];
|
|
1410
|
-
if (!Array.isArray(items) || items.length === 0) return fallbackEvent(fallbackChunk);
|
|
1411
|
-
const first = items[0];
|
|
1412
|
-
if (first === null || typeof first !== "object") return fallbackEvent(fallbackChunk);
|
|
1413
|
-
const obj = first;
|
|
1414
|
-
return {
|
|
1415
|
-
title: typeof obj["title"] === "string" && obj["title"].trim().length > 0 ? obj["title"].trim() : fallbackChunk.heading ?? "Untitled",
|
|
1416
|
-
summary: typeof obj["summary"] === "string" ? obj["summary"].trim() : "",
|
|
1417
|
-
content: typeof obj["content"] === "string" && obj["content"].trim().length > 0 ? obj["content"].trim() : fallbackChunk.content,
|
|
1418
|
-
category: typeof obj["category"] === "string" ? obj["category"].trim() : "other",
|
|
1419
|
-
keywords: parseKeywords(obj["keywords"]),
|
|
1420
|
-
entities: parseEntities(obj["entities"])
|
|
1421
|
-
};
|
|
1422
|
-
} catch {
|
|
1423
|
-
return fallbackEvent(fallbackChunk);
|
|
1424
|
-
}
|
|
1425
|
-
}
|
|
1426
|
-
function parseKeywords(value) {
|
|
1427
|
-
if (!Array.isArray(value)) return [];
|
|
1428
|
-
const out = [];
|
|
1429
|
-
for (const item of value) if (typeof item === "string" && item.trim().length > 0) out.push(item.trim());
|
|
1430
|
-
return out.slice(0, 10);
|
|
1431
|
-
}
|
|
1432
|
-
function parseEntities(value) {
|
|
1433
|
-
if (!Array.isArray(value)) return [];
|
|
1434
|
-
const out = [];
|
|
1435
|
-
const seen = /* @__PURE__ */ new Set();
|
|
1436
|
-
for (const item of value) {
|
|
1437
|
-
if (item === null || typeof item !== "object") continue;
|
|
1438
|
-
const obj = item;
|
|
1439
|
-
const type = typeof obj["type"] === "string" ? obj["type"] : "";
|
|
1440
|
-
const name = typeof obj["name"] === "string" ? obj["name"].trim() : "";
|
|
1441
|
-
const description = typeof obj["description"] === "string" ? obj["description"].trim() : "";
|
|
1442
|
-
if (name.length === 0) continue;
|
|
1443
|
-
if (!ENTITY_TYPES.includes(type)) continue;
|
|
1444
|
-
const key = `${type}|${name.toLowerCase()}`;
|
|
1445
|
-
if (seen.has(key)) continue;
|
|
1446
|
-
seen.add(key);
|
|
1447
|
-
out.push({
|
|
1448
|
-
type,
|
|
1449
|
-
name,
|
|
1450
|
-
description
|
|
1451
|
-
});
|
|
1452
|
-
}
|
|
1453
|
-
return out.slice(0, 20);
|
|
1454
|
-
}
|
|
1455
|
-
function fallbackEvent(chunk) {
|
|
1456
|
-
return {
|
|
1457
|
-
title: chunk.heading ?? "Untitled section",
|
|
1458
|
-
summary: "",
|
|
1459
|
-
content: chunk.content,
|
|
1460
|
-
category: "other",
|
|
1461
|
-
keywords: [],
|
|
1462
|
-
entities: []
|
|
1463
|
-
};
|
|
1464
|
-
}
|
|
1465
|
-
/**
|
|
1466
|
-
* Run extraction on a single chunk via the LLM caller.
|
|
1467
|
-
* Retries with exponential backoff on transient failures; falls back to a
|
|
1468
|
-
* heading-derived event only after all attempts fail.
|
|
1469
|
-
*/
|
|
1470
|
-
async function extractEventFromChunk(llm, chunk) {
|
|
1471
|
-
try {
|
|
1472
|
-
return parseExtractionResponse(await callLlmWithBackoff(llm, EXTRACTION_SYSTEM_PROMPT, buildExtractionUserPrompt(chunk)), chunk);
|
|
1473
|
-
} catch {
|
|
1474
|
-
return fallbackEvent(chunk);
|
|
1475
|
-
}
|
|
1476
|
-
}
|
|
1477
|
-
/**
|
|
1478
|
-
* Ask the LLM to rerank candidate events by relevance to a query.
|
|
1479
|
-
* Returns the candidate ids in reranked order (most relevant first).
|
|
1480
|
-
*
|
|
1481
|
-
* If the LLM fails or returns unparseable output, returns the input order.
|
|
1482
|
-
*/
|
|
1483
|
-
async function rerankEventsWithLlm(llm, query, candidates, topK) {
|
|
1484
|
-
if (candidates.length === 0) return [];
|
|
1485
|
-
if (candidates.length <= topK) return candidates.map((c) => c.id);
|
|
1486
|
-
const candidateList = candidates.map((c, i) => `[${i}] id=${c.id}\n title: ${c.title}\n summary: ${c.summary}`).join("\n");
|
|
1487
|
-
const system = `You are a precise relevance judge for a knowledge base retrieval system.
|
|
1488
|
-
|
|
1489
|
-
Given a user query and a list of candidate events, select the ${topK} event ids most useful for answering the query.
|
|
1490
|
-
|
|
1491
|
-
Return ONLY a JSON object: {"ids": ["id1", "id2", ...]}
|
|
1492
|
-
|
|
1493
|
-
The ids must come from the candidate list. Output at most ${topK} ids, most relevant first. No prose, no markdown.`;
|
|
1494
|
-
const user = `Query: ${query}
|
|
1495
|
-
|
|
1496
|
-
Candidates:
|
|
1497
|
-
${candidateList}
|
|
1498
|
-
|
|
1499
|
-
Return the JSON object now.`;
|
|
1500
|
-
try {
|
|
1501
|
-
const jsonText = extractJsonFromText(await llm.generate(system, user));
|
|
1502
|
-
if (jsonText === null) return candidates.slice(0, topK).map((c) => c.id);
|
|
1503
|
-
const parsed = JSON.parse(jsonText);
|
|
1504
|
-
if (!Array.isArray(parsed["ids"])) return candidates.slice(0, topK).map((c) => c.id);
|
|
1505
|
-
const validIds = new Set(candidates.map((c) => c.id));
|
|
1506
|
-
const result = [];
|
|
1507
|
-
for (const id of parsed["ids"]) {
|
|
1508
|
-
if (typeof id === "string" && validIds.has(id) && !result.includes(id)) result.push(id);
|
|
1509
|
-
if (result.length >= topK) break;
|
|
1510
|
-
}
|
|
1511
|
-
if (result.length === 0) return candidates.slice(0, topK).map((c) => c.id);
|
|
1512
|
-
return result;
|
|
1513
|
-
} catch {
|
|
1514
|
-
return candidates.slice(0, topK).map((c) => c.id);
|
|
1515
|
-
}
|
|
1516
|
-
}
|
|
1517
|
-
/**
|
|
1518
|
-
* Extract named entities from a query for entity-recall retrieval.
|
|
1519
|
-
* Returns a list of {type, name} pairs. Best-effort — falls back to empty.
|
|
1520
|
-
*/
|
|
1521
|
-
async function extractQueryEntities(llm, query) {
|
|
1522
|
-
const system = `Extract named entities from the user query. Return ONLY a JSON object: {"entities": [{"type": "...", "name": "..."}]}
|
|
1523
|
-
|
|
1524
|
-
Use these entity types: ${ENTITY_TYPES.join(", ")}.
|
|
1525
|
-
|
|
1526
|
-
Only include entities that explicitly appear in the query. If no clear entities, return an empty array.`;
|
|
1527
|
-
try {
|
|
1528
|
-
const jsonText = extractJsonFromText(await llm.generate(system, `Query: ${query}`));
|
|
1529
|
-
if (jsonText === null) return [];
|
|
1530
|
-
const parsed = JSON.parse(jsonText);
|
|
1531
|
-
if (!Array.isArray(parsed["entities"])) return [];
|
|
1532
|
-
const out = [];
|
|
1533
|
-
for (const item of parsed["entities"]) {
|
|
1534
|
-
if (item === null || typeof item !== "object") continue;
|
|
1535
|
-
const obj = item;
|
|
1536
|
-
const type = typeof obj["type"] === "string" ? obj["type"] : "";
|
|
1537
|
-
const name = typeof obj["name"] === "string" ? obj["name"].trim() : "";
|
|
1538
|
-
if (name.length === 0) continue;
|
|
1539
|
-
if (!ENTITY_TYPES.includes(type)) continue;
|
|
1540
|
-
out.push({
|
|
1541
|
-
type,
|
|
1542
|
-
name
|
|
1543
|
-
});
|
|
1544
|
-
}
|
|
1545
|
-
return out;
|
|
1546
|
-
} catch {
|
|
1547
|
-
return [];
|
|
1548
|
-
}
|
|
1549
|
-
}
|
|
1550
|
-
//#endregion
|
|
1551
|
-
//#region ../../packages/knowledge/src/ingest.ts
|
|
1552
|
-
/**
|
|
1553
|
-
* Embed texts, returning null-array if embeddings are skipped or on failure.
|
|
1554
|
-
* Caller should set skipEmbedding=true when the model didn't load in time,
|
|
1555
|
-
* so the entire ingestion is consistent (all or nothing for embeddings).
|
|
1556
|
-
* Search falls back to FTS5 keyword match for chunks without embeddings.
|
|
1557
|
-
*/
|
|
1558
|
-
async function embedOrNulls(engine, texts, skipEmbedding) {
|
|
1559
|
-
if (engine === void 0 || texts.length === 0 || skipEmbedding) return texts.map(() => null);
|
|
1560
|
-
if (!engine.available) return texts.map(() => null);
|
|
1561
|
-
const vectors = await engine.embedBatch(texts);
|
|
1562
|
-
if (vectors === null || vectors.length !== texts.length) return texts.map(() => null);
|
|
1563
|
-
return vectors;
|
|
1564
|
-
}
|
|
1565
|
-
/** LLM concurrency for extraction — kept low to avoid rate-limit bursts. */
|
|
1566
|
-
const LLM_CONCURRENCY = 20;
|
|
1567
|
-
const SUPPORTED_EXTENSIONS = new Set([
|
|
1568
|
-
".md",
|
|
1569
|
-
".markdown",
|
|
1570
|
-
".txt"
|
|
1571
|
-
]);
|
|
1572
|
-
/** Read a file and return its content. */
|
|
1573
|
-
async function readFileContent(filePath) {
|
|
1574
|
-
return readFile(filePath, "utf-8");
|
|
1575
|
-
}
|
|
1576
|
-
function getFileExtension(filePath) {
|
|
1577
|
-
const lastDot = filePath.lastIndexOf(".");
|
|
1578
|
-
return lastDot > 0 ? filePath.slice(lastDot).toLowerCase() : "";
|
|
1579
|
-
}
|
|
1580
|
-
function chunkContent(content, ext) {
|
|
1581
|
-
return ext === ".txt" ? chunkText(content) : chunkMarkdown(content);
|
|
1582
|
-
}
|
|
1583
|
-
function isSupportedFile(filePath) {
|
|
1584
|
-
return SUPPORTED_EXTENSIONS.has(getFileExtension(filePath));
|
|
1585
|
-
}
|
|
1586
|
-
/** Map async iterator values with a concurrency limit. */
|
|
1587
|
-
async function mapWithConcurrency(items, limit, fn, onProgress) {
|
|
1588
|
-
const results = Array.from({ length: items.length });
|
|
1589
|
-
let cursor = 0;
|
|
1590
|
-
let completed = 0;
|
|
1591
|
-
const total = items.length;
|
|
1592
|
-
async function worker() {
|
|
1593
|
-
while (cursor < items.length) {
|
|
1594
|
-
const idx = cursor;
|
|
1595
|
-
cursor += 1;
|
|
1596
|
-
const item = items[idx];
|
|
1597
|
-
const result = await fn(item, idx);
|
|
1598
|
-
results[idx] = result;
|
|
1599
|
-
completed += 1;
|
|
1600
|
-
onProgress?.(completed, total, result);
|
|
1601
|
-
}
|
|
1602
|
-
}
|
|
1603
|
-
const workers = Array.from({ length: Math.min(limit, items.length) }, () => worker());
|
|
1604
|
-
await Promise.all(workers);
|
|
1605
|
-
return results;
|
|
1606
|
-
}
|
|
1607
|
-
/**
|
|
1608
|
-
* Stamp or verify the embedding-model identity recorded in the library.
|
|
1609
|
-
* A library written by a different model would mix vector spaces — block the
|
|
1610
|
-
* ingest with a clear message instead of silently storing incompatible
|
|
1611
|
-
* embeddings. A legacy library without the marker is stamped with the current
|
|
1612
|
-
* model (all existing data was written by it).
|
|
1613
|
-
*/
|
|
1614
|
-
async function ensureEmbeddingModelMatches(store, engine) {
|
|
1615
|
-
if (engine === void 0) return;
|
|
1616
|
-
const recorded = await store.getMeta(EMBEDDING_MODEL_META_KEY);
|
|
1617
|
-
if (recorded === null) {
|
|
1618
|
-
await store.setMeta(EMBEDDING_MODEL_META_KEY, engine.modelName);
|
|
1619
|
-
return;
|
|
1620
|
-
}
|
|
1621
|
-
if (recorded !== engine.modelName) throw new Error(`知识库由模型 ${recorded} 嵌入,当前模型为 ${engine.modelName},请先在 /knowledge 中重新嵌入知识库`);
|
|
1622
|
-
}
|
|
1623
|
-
/**
|
|
1624
|
-
* Ingest a markdown file into the knowledge base.
|
|
1625
|
-
*
|
|
1626
|
-
* Steps:
|
|
1627
|
-
* 1. Dedupe by file_path — if a source already exists for this file, error.
|
|
1628
|
-
* 2. Read file → chunkMarkdown (heading_strict).
|
|
1629
|
-
* 3. Embed chunks ("{heading}\n{content}") → store knowledge_chunks.
|
|
1630
|
-
* 4. For each chunk (concurrency 3): LLM extract 1 event + N entities.
|
|
1631
|
-
* 5. Embed event title + content ("{title}\n\n{content}") → store knowledge_events.
|
|
1632
|
-
* 6. Upsert entities (dedupe by source_id+type+normalizedName) + embed entity name → store.
|
|
1633
|
-
* 7. For each (event, entity) pair: embed relation (entity.description || "{eventTitle} {entityName}") → store knowledge_event_entities.
|
|
1634
|
-
* 8. Update document status = 'completed'.
|
|
1635
|
-
*
|
|
1636
|
-
* All writes run inside a single transaction — a mid-ingest failure rolls back
|
|
1637
|
-
* every partial row (source/document/chunks/events/entities/edges).
|
|
1638
|
-
*
|
|
1639
|
-
* Reports progress via callback.
|
|
1640
|
-
*/
|
|
1641
|
-
async function ingestFile(store, llm, filePath, onProgress) {
|
|
1642
|
-
const engine = store.getEmbeddingEngine();
|
|
1643
|
-
const skipEmbed = engine === void 0 || !engine.available;
|
|
1644
|
-
if (skipEmbed) onProgress?.({
|
|
1645
|
-
stage: "embedding-check",
|
|
1646
|
-
message: "向量模型未就绪,本次摄入跳过向量嵌入"
|
|
1647
|
-
});
|
|
1648
|
-
await ensureEmbeddingModelMatches(store, engine);
|
|
1649
|
-
onProgress?.({
|
|
1650
|
-
stage: "chunking",
|
|
1651
|
-
message: `读取并切分文件: ${basename(filePath)}`
|
|
1652
|
-
});
|
|
1653
|
-
const content = await readFileContent(filePath);
|
|
1654
|
-
const contentHash = createHash("sha256").update(content).digest("hex");
|
|
1655
|
-
const existing = await store.findSourceByFilePath(filePath);
|
|
1656
|
-
let replacedOldVersion = false;
|
|
1657
|
-
if (existing !== void 0) {
|
|
1658
|
-
if (existing.contentHash === contentHash || existing.contentHash === null) {
|
|
1659
|
-
if (existing.contentHash === null) await store.updateSourceContentHash(existing.id, contentHash);
|
|
1660
|
-
onProgress?.({
|
|
1661
|
-
stage: "completed",
|
|
1662
|
-
message: "内容未变化,已跳过"
|
|
1663
|
-
});
|
|
1664
|
-
return {
|
|
1665
|
-
documentId: null,
|
|
1666
|
-
chunkCount: 0,
|
|
1667
|
-
eventCount: 0,
|
|
1668
|
-
entityCount: 0,
|
|
1669
|
-
outcome: "unchanged"
|
|
1670
|
-
};
|
|
1671
|
-
}
|
|
1672
|
-
onProgress?.({
|
|
1673
|
-
stage: "chunking",
|
|
1674
|
-
message: `检测到内容变更,替换旧版本: ${existing.name}`
|
|
1675
|
-
});
|
|
1676
|
-
await store.deleteSource(existing.id);
|
|
1677
|
-
replacedOldVersion = true;
|
|
1678
|
-
}
|
|
1679
|
-
const sections = chunkContent(content, getFileExtension(filePath));
|
|
1680
|
-
store.beginTransaction();
|
|
1681
|
-
try {
|
|
1682
|
-
if (sections.length === 0) {
|
|
1683
|
-
onProgress?.({
|
|
1684
|
-
stage: "error",
|
|
1685
|
-
message: "文件无有效内容"
|
|
1686
|
-
});
|
|
1687
|
-
throw new Error("文件无有效内容可摄入");
|
|
1688
|
-
}
|
|
1689
|
-
const fileName = basename(filePath);
|
|
1690
|
-
const source = await store.createSource({
|
|
1691
|
-
name: fileName,
|
|
1692
|
-
filePath,
|
|
1693
|
-
description: null,
|
|
1694
|
-
contentHash
|
|
1695
|
-
});
|
|
1696
|
-
const document = await store.createDocument({
|
|
1697
|
-
sourceId: source.id,
|
|
1698
|
-
title: fileName,
|
|
1699
|
-
content
|
|
1700
|
-
});
|
|
1701
|
-
onProgress?.({
|
|
1702
|
-
stage: "embedding-chunks",
|
|
1703
|
-
chunkIndex: 0,
|
|
1704
|
-
totalChunks: sections.length,
|
|
1705
|
-
message: `嵌入 chunks: 0/${sections.length}`
|
|
1706
|
-
});
|
|
1707
|
-
const chunkEmbeddings = await embedOrNulls(engine, sections.map((s) => s.heading !== null ? `${s.heading}\n${s.content}` : s.content), skipEmbed);
|
|
1708
|
-
const chunks = [];
|
|
1709
|
-
for (let i = 0; i < sections.length; i++) {
|
|
1710
|
-
const section = sections[i];
|
|
1711
|
-
const embedding = chunkEmbeddings[i] ?? null;
|
|
1712
|
-
const chunk = await store.insertChunk({
|
|
1713
|
-
sourceId: source.id,
|
|
1714
|
-
documentId: document.id,
|
|
1715
|
-
rank: section.rank,
|
|
1716
|
-
heading: section.heading,
|
|
1717
|
-
content: section.content,
|
|
1718
|
-
rawContent: section.rawContent,
|
|
1719
|
-
embedding
|
|
1720
|
-
});
|
|
1721
|
-
chunks.push(chunk);
|
|
1722
|
-
onProgress?.({
|
|
1723
|
-
stage: "embedding-chunks",
|
|
1724
|
-
chunkIndex: i + 1,
|
|
1725
|
-
totalChunks: sections.length,
|
|
1726
|
-
message: `嵌入 chunks: ${i + 1}/${sections.length}`
|
|
1727
|
-
});
|
|
1728
|
-
}
|
|
1729
|
-
onProgress?.({
|
|
1730
|
-
stage: "extracting",
|
|
1731
|
-
chunkIndex: 0,
|
|
1732
|
-
totalChunks: sections.length,
|
|
1733
|
-
message: `抽取事件: 0/${sections.length}`
|
|
1734
|
-
});
|
|
1735
|
-
const extractedEvents = await mapWithConcurrency(sections, LLM_CONCURRENCY, (section) => extractEventFromChunk(llm, section), (completed, total) => {
|
|
1736
|
-
onProgress?.({
|
|
1737
|
-
stage: "extracting",
|
|
1738
|
-
chunkIndex: completed,
|
|
1739
|
-
totalChunks: total,
|
|
1740
|
-
message: `抽取事件: ${completed}/${total}`
|
|
1741
|
-
});
|
|
1742
|
-
});
|
|
1743
|
-
onProgress?.({
|
|
1744
|
-
stage: "embedding-events",
|
|
1745
|
-
chunkIndex: 0,
|
|
1746
|
-
totalChunks: sections.length,
|
|
1747
|
-
message: `嵌入 events: 0/${sections.length}`
|
|
1748
|
-
});
|
|
1749
|
-
const titleTexts = extractedEvents.map((e) => e.title);
|
|
1750
|
-
const contentTexts = extractedEvents.map((e) => `${e.title}\n\n${e.content}`);
|
|
1751
|
-
const [titleEmbeddings, contentEmbeddings] = await Promise.all([embedOrNulls(engine, titleTexts, skipEmbed), embedOrNulls(engine, contentTexts, skipEmbed)]);
|
|
1752
|
-
const events = [];
|
|
1753
|
-
for (let i = 0; i < extractedEvents.length; i++) {
|
|
1754
|
-
const extracted = extractedEvents[i];
|
|
1755
|
-
const chunk = chunks[i];
|
|
1756
|
-
const event = await store.insertEvent({
|
|
1757
|
-
sourceId: source.id,
|
|
1758
|
-
documentId: document.id,
|
|
1759
|
-
chunkId: chunk.id,
|
|
1760
|
-
rank: i,
|
|
1761
|
-
title: extracted.title,
|
|
1762
|
-
summary: extracted.summary.length > 0 ? extracted.summary : null,
|
|
1763
|
-
content: extracted.content,
|
|
1764
|
-
category: extracted.category.length > 0 ? extracted.category : null,
|
|
1765
|
-
keywords: extracted.keywords,
|
|
1766
|
-
titleEmbedding: titleEmbeddings[i] ?? null,
|
|
1767
|
-
contentEmbedding: contentEmbeddings[i] ?? null
|
|
1768
|
-
});
|
|
1769
|
-
events.push(event);
|
|
1770
|
-
onProgress?.({
|
|
1771
|
-
stage: "embedding-events",
|
|
1772
|
-
chunkIndex: i + 1,
|
|
1773
|
-
totalChunks: sections.length,
|
|
1774
|
-
message: `嵌入 events: ${i + 1}/${sections.length}`
|
|
1775
|
-
});
|
|
1776
|
-
}
|
|
1777
|
-
onProgress?.({
|
|
1778
|
-
stage: "embedding-entities",
|
|
1779
|
-
message: "嵌入 entities..."
|
|
1780
|
-
});
|
|
1781
|
-
const entityMap = /* @__PURE__ */ new Map();
|
|
1782
|
-
for (const extracted of extractedEvents) for (const entity of extracted.entities) {
|
|
1783
|
-
const key = `${entity.type}|${entity.name.toLowerCase()}`;
|
|
1784
|
-
if (!entityMap.has(key)) entityMap.set(key, entity);
|
|
1785
|
-
}
|
|
1786
|
-
const uniqueEntities = Array.from(entityMap.values());
|
|
1787
|
-
const entityEmbeddings = uniqueEntities.length > 0 ? await embedOrNulls(engine, uniqueEntities.map((e) => e.name), skipEmbed) : [];
|
|
1788
|
-
const entityIdByEntityKey = /* @__PURE__ */ new Map();
|
|
1789
|
-
for (let i = 0; i < uniqueEntities.length; i++) {
|
|
1790
|
-
const entity = uniqueEntities[i];
|
|
1791
|
-
const embedding = entityEmbeddings[i] ?? null;
|
|
1792
|
-
const stored = await store.upsertEntity({
|
|
1793
|
-
sourceId: source.id,
|
|
1794
|
-
type: entity.type,
|
|
1795
|
-
name: entity.name,
|
|
1796
|
-
description: entity.description.length > 0 ? entity.description : null,
|
|
1797
|
-
embedding
|
|
1798
|
-
});
|
|
1799
|
-
entityIdByEntityKey.set(`${entity.type}|${entity.name.toLowerCase()}`, stored.id);
|
|
1800
|
-
}
|
|
1801
|
-
onProgress?.({
|
|
1802
|
-
stage: "embedding-relations",
|
|
1803
|
-
message: "嵌入关系..."
|
|
1804
|
-
});
|
|
1805
|
-
const relationPairs = [];
|
|
1806
|
-
for (let i = 0; i < extractedEvents.length; i++) {
|
|
1807
|
-
const extracted = extractedEvents[i];
|
|
1808
|
-
for (const entity of extracted.entities) relationPairs.push({
|
|
1809
|
-
eventIndex: i,
|
|
1810
|
-
entity
|
|
1811
|
-
});
|
|
1812
|
-
}
|
|
1813
|
-
const relationTexts = relationPairs.map(({ eventIndex, entity }) => {
|
|
1814
|
-
const event = events[eventIndex];
|
|
1815
|
-
return entity.description.length > 0 ? entity.description : `${event.title} ${entity.name}`;
|
|
1816
|
-
});
|
|
1817
|
-
const relationEmbeddings = relationPairs.length > 0 ? await embedOrNulls(engine, relationTexts, skipEmbed) : [];
|
|
1818
|
-
for (let i = 0; i < relationPairs.length; i++) {
|
|
1819
|
-
const pair = relationPairs[i];
|
|
1820
|
-
const event = events[pair.eventIndex];
|
|
1821
|
-
const entityKey = `${pair.entity.type}|${pair.entity.name.toLowerCase()}`;
|
|
1822
|
-
const entityId = entityIdByEntityKey.get(entityKey);
|
|
1823
|
-
if (entityId === void 0) continue;
|
|
1824
|
-
const embedding = relationEmbeddings?.[i] ?? null;
|
|
1825
|
-
await store.insertEventEntity({
|
|
1826
|
-
eventId: event.id,
|
|
1827
|
-
entityId,
|
|
1828
|
-
weight: 1,
|
|
1829
|
-
description: pair.entity.description.length > 0 ? pair.entity.description : null,
|
|
1830
|
-
embedding
|
|
1831
|
-
});
|
|
1832
|
-
}
|
|
1833
|
-
await store.updateDocumentStatus(document.id, "completed", chunks.length);
|
|
1834
|
-
store.commitTransaction();
|
|
1835
|
-
onProgress?.({
|
|
1836
|
-
stage: "completed",
|
|
1837
|
-
message: `摄入完成:${chunks.length} chunks, ${events.length} events, ${uniqueEntities.length} entities`
|
|
1838
|
-
});
|
|
1839
|
-
return {
|
|
1840
|
-
documentId: document.id,
|
|
1841
|
-
chunkCount: chunks.length,
|
|
1842
|
-
eventCount: events.length,
|
|
1843
|
-
entityCount: uniqueEntities.length,
|
|
1844
|
-
outcome: replacedOldVersion ? "updated" : "created"
|
|
1845
|
-
};
|
|
1846
|
-
} catch (error) {
|
|
1847
|
-
store.rollbackTransaction();
|
|
1848
|
-
if (replacedOldVersion) {
|
|
1849
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
1850
|
-
throw new Error(`旧版本已删除、新版本摄入失败,请重新摄入该文件: ${filePath}(${message})`);
|
|
1851
|
-
}
|
|
1852
|
-
throw error;
|
|
1853
|
-
}
|
|
1854
|
-
}
|
|
1855
|
-
async function collectSupportedFiles(dirPath) {
|
|
1856
|
-
const files = [];
|
|
1857
|
-
const entries = await readdir(dirPath, { withFileTypes: true });
|
|
1858
|
-
for (const entry of entries) {
|
|
1859
|
-
const fullPath = join(dirPath, entry.name);
|
|
1860
|
-
if (entry.isDirectory()) files.push(...await collectSupportedFiles(fullPath));
|
|
1861
|
-
else if (entry.isFile() && isSupportedFile(fullPath)) files.push(fullPath);
|
|
1862
|
-
}
|
|
1863
|
-
return files.sort();
|
|
1864
|
-
}
|
|
1865
|
-
async function ingestDirectory(store, llm, dirPath, onProgress) {
|
|
1866
|
-
const files = await collectSupportedFiles(dirPath);
|
|
1867
|
-
if (files.length === 0) throw new Error(`目录下没有支持的文件 (.md, .markdown, .txt): ${dirPath}`);
|
|
1868
|
-
const errors = [];
|
|
1869
|
-
let succeeded = 0;
|
|
1870
|
-
let created = 0;
|
|
1871
|
-
let updated = 0;
|
|
1872
|
-
let unchanged = 0;
|
|
1873
|
-
let totalChunks = 0;
|
|
1874
|
-
let totalEvents = 0;
|
|
1875
|
-
let totalEntities = 0;
|
|
1876
|
-
for (let i = 0; i < files.length; i++) {
|
|
1877
|
-
const filePath = files[i];
|
|
1878
|
-
const fileProgress = (progress) => {
|
|
1879
|
-
const prefix = `批量摄入 ${i + 1}/${files.length}: ${basename(filePath)}`;
|
|
1880
|
-
onProgress?.({
|
|
1881
|
-
...progress,
|
|
1882
|
-
message: `${prefix} · ${progress.message}`
|
|
1883
|
-
});
|
|
1884
|
-
};
|
|
1885
|
-
try {
|
|
1886
|
-
const result = await ingestFile(store, llm, filePath, fileProgress);
|
|
1887
|
-
succeeded += 1;
|
|
1888
|
-
if (result.outcome === "created") created += 1;
|
|
1889
|
-
else if (result.outcome === "updated") updated += 1;
|
|
1890
|
-
else unchanged += 1;
|
|
1891
|
-
totalChunks += result.chunkCount;
|
|
1892
|
-
totalEvents += result.eventCount;
|
|
1893
|
-
totalEntities += result.entityCount;
|
|
1894
|
-
} catch (error) {
|
|
1895
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
1896
|
-
errors.push({
|
|
1897
|
-
filePath,
|
|
1898
|
-
message
|
|
1899
|
-
});
|
|
1900
|
-
onProgress?.({
|
|
1901
|
-
stage: "error",
|
|
1902
|
-
message: `批量摄入 ${i + 1}/${files.length}: ${basename(filePath)} · 失败: ${message}`
|
|
1903
|
-
});
|
|
1904
|
-
}
|
|
1905
|
-
}
|
|
1906
|
-
onProgress?.({
|
|
1907
|
-
stage: "completed",
|
|
1908
|
-
message: `批量摄入完成:新增 ${created} · 更新 ${updated} · 未变化 ${unchanged} · 失败 ${errors.length}(${totalChunks} chunks, ${totalEvents} events, ${totalEntities} entities)`
|
|
1909
|
-
});
|
|
1910
|
-
return {
|
|
1911
|
-
succeeded,
|
|
1912
|
-
failed: errors.length,
|
|
1913
|
-
created,
|
|
1914
|
-
updated,
|
|
1915
|
-
unchanged,
|
|
1916
|
-
totalChunks,
|
|
1917
|
-
totalEvents,
|
|
1918
|
-
totalEntities,
|
|
1919
|
-
errors
|
|
1920
|
-
};
|
|
1921
|
-
}
|
|
1922
|
-
//#endregion
|
|
1923
|
-
//#region ../../packages/knowledge/src/search.ts
|
|
1924
|
-
const DEFAULT_TOP_K = 5;
|
|
1925
|
-
const MAX_TOP_K = 20;
|
|
1926
|
-
const SEED_EVENT_LIMIT = 30;
|
|
1927
|
-
const EXPANDED_EVENT_LIMIT = 100;
|
|
1928
|
-
const COARSE_RANK_LIMIT = 50;
|
|
1929
|
-
const ENTITY_VECTOR_THRESHOLD = .7;
|
|
1930
|
-
const TITLE_VECTOR_THRESHOLD = .4;
|
|
1931
|
-
async function timed(onStep, step, detail, fn, payload) {
|
|
1932
|
-
const start = performance.now();
|
|
1933
|
-
try {
|
|
1934
|
-
const result = await fn();
|
|
1935
|
-
const traceStep = {
|
|
1936
|
-
step,
|
|
1937
|
-
detail,
|
|
1938
|
-
durationMs: Math.round((performance.now() - start) * 100) / 100,
|
|
1939
|
-
...payload !== void 0 ? { payload: payload(result) } : {}
|
|
1940
|
-
};
|
|
1941
|
-
onStep?.(traceStep);
|
|
1942
|
-
return result;
|
|
1943
|
-
} catch (error) {
|
|
1944
|
-
const durationMs = Math.round((performance.now() - start) * 100) / 100;
|
|
1945
|
-
onStep?.({
|
|
1946
|
-
step,
|
|
1947
|
-
detail: `${detail} 失败:${error instanceof Error ? error.message : String(error)}`,
|
|
1948
|
-
durationMs
|
|
1949
|
-
});
|
|
1950
|
-
throw error;
|
|
1951
|
-
}
|
|
1952
|
-
}
|
|
1953
|
-
/**
|
|
1954
|
-
* Multi-hop retrieval:
|
|
1955
|
-
* 1. Vectorize query.
|
|
1956
|
-
* 2. Entity recall — LLM extracts entities from query → match by name + vector.
|
|
1957
|
-
* 3. Seed events — events linked to recalled entities + events by title vector similarity.
|
|
1958
|
-
* 4. BFS expand (1 hop) — from seed events, walk to entities, then to other events.
|
|
1959
|
-
* 5. Coarse rank — score graph-reachable events by content embedding similarity.
|
|
1960
|
-
* Graph associativity is the gate; content vector orders within it.
|
|
1961
|
-
* 6. LLM rerank — pick top-K most relevant.
|
|
1962
|
-
* 7. Return corresponding chunks (deduped by chunk_id) with scores and provenance.
|
|
1963
|
-
* If rerank yields fewer than topK chunks, backfill by direct chunk vector search.
|
|
1964
|
-
*
|
|
1965
|
-
* Optional `onStep` callback receives a trace step at each phase — used by the
|
|
1966
|
-
* KnowledgeLookup tool to expose retrieval diagnostics to the agent.
|
|
1967
|
-
*/
|
|
1968
|
-
async function multiSearch(store, llm, query, options = {}) {
|
|
1969
|
-
return (await multiSearchWithTrace(store, llm, query, options)).results;
|
|
1970
|
-
}
|
|
1971
|
-
/** Like multiSearch but also returns the retrieval trace for diagnostics. */
|
|
1972
|
-
async function multiSearchWithTrace(store, llm, query, options = {}) {
|
|
1973
|
-
const topK = Math.min(options.topK ?? DEFAULT_TOP_K, MAX_TOP_K);
|
|
1974
|
-
const steps = [];
|
|
1975
|
-
const onStep = (step) => steps.push(step);
|
|
1976
|
-
const rerankedEventTitles = [];
|
|
1977
|
-
let fallbackReason = null;
|
|
1978
|
-
const engine = store.getEmbeddingEngine();
|
|
1979
|
-
if (engine === void 0 || !engine.available) {
|
|
1980
|
-
fallbackReason = "embedding engine unavailable; used FTS5 keyword fallback";
|
|
1981
|
-
return {
|
|
1982
|
-
results: await ftsFallback(store, query, topK),
|
|
1983
|
-
trace: {
|
|
1984
|
-
steps,
|
|
1985
|
-
rerankedEventTitles,
|
|
1986
|
-
fallbackReason
|
|
1987
|
-
}
|
|
1988
|
-
};
|
|
1989
|
-
}
|
|
1990
|
-
const recordedModel = await store.getMeta(EMBEDDING_MODEL_META_KEY);
|
|
1991
|
-
if (recordedModel !== null && recordedModel !== engine.modelName) {
|
|
1992
|
-
fallbackReason = `embedding model mismatch (library: ${recordedModel}, current: ${engine.modelName}); used FTS5 keyword fallback`;
|
|
1993
|
-
return {
|
|
1994
|
-
results: await ftsFallback(store, query, topK),
|
|
1995
|
-
trace: {
|
|
1996
|
-
steps,
|
|
1997
|
-
rerankedEventTitles,
|
|
1998
|
-
fallbackReason
|
|
1999
|
-
}
|
|
2000
|
-
};
|
|
2001
|
-
}
|
|
2002
|
-
const queryEmbeddings = await timed(onStep, "queryEmbedding", "把用户问题转成向量,用于召回相关事件和切片。", async () => engine.embedBatch([query]));
|
|
2003
|
-
if (queryEmbeddings === null || queryEmbeddings.length === 0) {
|
|
2004
|
-
fallbackReason = "query embedding returned null; used FTS5 fallback";
|
|
2005
|
-
return {
|
|
2006
|
-
results: await ftsFallback(store, query, topK),
|
|
2007
|
-
trace: {
|
|
2008
|
-
steps,
|
|
2009
|
-
rerankedEventTitles,
|
|
2010
|
-
fallbackReason
|
|
2011
|
-
}
|
|
2012
|
-
};
|
|
2013
|
-
}
|
|
2014
|
-
const queryVec = queryEmbeddings[0];
|
|
2015
|
-
const recalledEntities = await timed(onStep, "entityRecall", "LLM 抽 query 实体 + 名字精确匹配 + 向量召回。", () => recallEntities(store, llm, query, queryVec), (entities) => ({ count: entities.length }));
|
|
2016
|
-
const seedEventIds = /* @__PURE__ */ new Set();
|
|
2017
|
-
for (const entity of recalledEntities) {
|
|
2018
|
-
const events = await store.findEventsByEntity(entity.id);
|
|
2019
|
-
for (const event of events) seedEventIds.add(event.id);
|
|
2020
|
-
}
|
|
2021
|
-
const titleMatches = await timed(onStep, "seedEventsByTitle", "按查询向量在事件标题向量上召回 seed events。", () => store.findEventsByTitleVector(queryVec, {
|
|
2022
|
-
limit: SEED_EVENT_LIMIT,
|
|
2023
|
-
threshold: TITLE_VECTOR_THRESHOLD
|
|
2024
|
-
}), (matches) => ({ count: matches.length }));
|
|
2025
|
-
for (const { event } of titleMatches) seedEventIds.add(event.id);
|
|
2026
|
-
if (seedEventIds.size === 0) {
|
|
2027
|
-
const results = await chunkVectorFallback(store, queryVec, topK);
|
|
2028
|
-
if (results.length > 0) {
|
|
2029
|
-
fallbackReason = "no seed events; used direct chunk vector search";
|
|
2030
|
-
return {
|
|
2031
|
-
results,
|
|
2032
|
-
trace: {
|
|
2033
|
-
steps,
|
|
2034
|
-
rerankedEventTitles,
|
|
2035
|
-
fallbackReason
|
|
2036
|
-
}
|
|
2037
|
-
};
|
|
2038
|
-
}
|
|
2039
|
-
fallbackReason = "no seed events and direct chunk vector search returned no results; used FTS5 keyword fallback";
|
|
2040
|
-
return {
|
|
2041
|
-
results: await ftsFallback(store, query, topK),
|
|
2042
|
-
trace: {
|
|
2043
|
-
steps,
|
|
2044
|
-
rerankedEventTitles,
|
|
2045
|
-
fallbackReason
|
|
2046
|
-
}
|
|
2047
|
-
};
|
|
2048
|
-
}
|
|
2049
|
-
const expandedEventIds = await timed(onStep, "bfsExpand", "从 seed events 沿实体关系 1 跳扩展候选事件。", async () => {
|
|
2050
|
-
const ids = new Set(seedEventIds);
|
|
2051
|
-
let capped = false;
|
|
2052
|
-
for (const eventId of seedEventIds) {
|
|
2053
|
-
const entities = await store.findEntitiesByEvent(eventId);
|
|
2054
|
-
for (const entity of entities) {
|
|
2055
|
-
const neighborEvents = await store.findEventsByEntity(entity.id);
|
|
2056
|
-
for (const neighbor of neighborEvents) {
|
|
2057
|
-
ids.add(neighbor.id);
|
|
2058
|
-
if (ids.size >= EXPANDED_EVENT_LIMIT) {
|
|
2059
|
-
capped = true;
|
|
2060
|
-
break;
|
|
2061
|
-
}
|
|
2062
|
-
}
|
|
2063
|
-
if (ids.size >= EXPANDED_EVENT_LIMIT) break;
|
|
2064
|
-
}
|
|
2065
|
-
if (ids.size >= EXPANDED_EVENT_LIMIT) break;
|
|
2066
|
-
}
|
|
2067
|
-
return {
|
|
2068
|
-
ids,
|
|
2069
|
-
capped
|
|
2070
|
-
};
|
|
2071
|
-
}, (result) => ({
|
|
2072
|
-
expandedCount: result.ids.size,
|
|
2073
|
-
capped: result.capped
|
|
2074
|
-
}));
|
|
2075
|
-
const { candidates, topCandidates } = await timed(onStep, "coarseRank", "按 content 向量相似度对候选事件排序,取前 COARSE_RANK_LIMIT 个。", async () => {
|
|
2076
|
-
const cands = [];
|
|
2077
|
-
for (const eventId of expandedEventIds.ids) {
|
|
2078
|
-
const event = await store.getEvent(eventId);
|
|
2079
|
-
if (event === void 0) continue;
|
|
2080
|
-
const vec = event.contentEmbedding;
|
|
2081
|
-
const score = vec === null ? 0 : engine.cosineSimilarity(queryVec, vec);
|
|
2082
|
-
cands.push({
|
|
2083
|
-
id: event.id,
|
|
2084
|
-
title: event.title,
|
|
2085
|
-
summary: event.summary ?? "",
|
|
2086
|
-
score,
|
|
2087
|
-
chunkId: event.chunkId
|
|
2088
|
-
});
|
|
2089
|
-
}
|
|
2090
|
-
cands.sort((a, b) => b.score - a.score);
|
|
2091
|
-
return {
|
|
2092
|
-
candidates: cands,
|
|
2093
|
-
topCandidates: cands.slice(0, COARSE_RANK_LIMIT)
|
|
2094
|
-
};
|
|
2095
|
-
}, (result) => ({
|
|
2096
|
-
totalCandidates: result.candidates.length,
|
|
2097
|
-
kept: result.topCandidates.length,
|
|
2098
|
-
topScore: result.topCandidates[0]?.score ?? 0
|
|
2099
|
-
}));
|
|
2100
|
-
if (candidates.length === 0) {
|
|
2101
|
-
const results = await chunkVectorFallback(store, queryVec, topK);
|
|
2102
|
-
if (results.length > 0) {
|
|
2103
|
-
fallbackReason = "no graph-reachable events with content; used chunk vector fallback";
|
|
2104
|
-
return {
|
|
2105
|
-
results,
|
|
2106
|
-
trace: {
|
|
2107
|
-
steps,
|
|
2108
|
-
rerankedEventTitles,
|
|
2109
|
-
fallbackReason
|
|
2110
|
-
}
|
|
2111
|
-
};
|
|
2112
|
-
}
|
|
2113
|
-
fallbackReason = "no graph-reachable events with content and chunk vector search returned no results; used FTS5 keyword fallback";
|
|
2114
|
-
return {
|
|
2115
|
-
results: await ftsFallback(store, query, topK),
|
|
2116
|
-
trace: {
|
|
2117
|
-
steps,
|
|
2118
|
-
rerankedEventTitles,
|
|
2119
|
-
fallbackReason
|
|
2120
|
-
}
|
|
2121
|
-
};
|
|
2122
|
-
}
|
|
2123
|
-
let rankedIds;
|
|
2124
|
-
if (options.skipRerank === true) {
|
|
2125
|
-
rankedIds = topCandidates.map((c) => c.id);
|
|
2126
|
-
onStep({
|
|
2127
|
-
step: "rerank",
|
|
2128
|
-
detail: "跳过 LLM rerank(skipRerank=true),直接用 coarse rank 顺序。",
|
|
2129
|
-
durationMs: 0,
|
|
2130
|
-
payload: { count: rankedIds.length }
|
|
2131
|
-
});
|
|
2132
|
-
} else if (topCandidates.length <= topK) {
|
|
2133
|
-
rankedIds = topCandidates.map((c) => c.id);
|
|
2134
|
-
onStep({
|
|
2135
|
-
step: "rerank",
|
|
2136
|
-
detail: `候选数 ${topCandidates.length} ≤ topK ${topK},无需 LLM rerank。`,
|
|
2137
|
-
durationMs: 0,
|
|
2138
|
-
payload: { count: rankedIds.length }
|
|
2139
|
-
});
|
|
2140
|
-
} else rankedIds = await timed(onStep, "rerank", `LLM 从 ${topCandidates.length} 个候选中选最相关的 ${topK} 个。`, () => rerankEventsWithLlm(llm, query, topCandidates.map((c) => ({
|
|
2141
|
-
id: c.id,
|
|
2142
|
-
title: c.title,
|
|
2143
|
-
summary: c.summary
|
|
2144
|
-
})), topK), (ids) => ({ count: ids.length }));
|
|
2145
|
-
for (const id of rankedIds) {
|
|
2146
|
-
const c = topCandidates.find((x) => x.id === id);
|
|
2147
|
-
if (c !== void 0) rerankedEventTitles.push(c.title);
|
|
2148
|
-
}
|
|
2149
|
-
const seen = /* @__PURE__ */ new Set();
|
|
2150
|
-
const results = [];
|
|
2151
|
-
for (const eventId of rankedIds) {
|
|
2152
|
-
const candidate = topCandidates.find((c) => c.id === eventId);
|
|
2153
|
-
if (candidate === void 0) continue;
|
|
2154
|
-
if (seen.has(candidate.chunkId)) continue;
|
|
2155
|
-
seen.add(candidate.chunkId);
|
|
2156
|
-
const result = await store.buildSearchResult(candidate.chunkId, candidate.score, eventId);
|
|
2157
|
-
if (result !== void 0) results.push(result);
|
|
2158
|
-
if (results.length >= topK) break;
|
|
2159
|
-
}
|
|
2160
|
-
if (results.length < topK) {
|
|
2161
|
-
let supplemented = 0;
|
|
2162
|
-
for (const candidate of topCandidates) {
|
|
2163
|
-
if (results.length >= topK) break;
|
|
2164
|
-
if (rankedIds.includes(candidate.id)) continue;
|
|
2165
|
-
if (seen.has(candidate.chunkId)) continue;
|
|
2166
|
-
seen.add(candidate.chunkId);
|
|
2167
|
-
const result = await store.buildSearchResult(candidate.chunkId, candidate.score, candidate.id);
|
|
2168
|
-
if (result !== void 0) {
|
|
2169
|
-
results.push(result);
|
|
2170
|
-
supplemented += 1;
|
|
2171
|
-
}
|
|
2172
|
-
}
|
|
2173
|
-
if (supplemented > 0) onStep({
|
|
2174
|
-
step: "supplementFromCoarse",
|
|
2175
|
-
detail: `rerank 结果不足,从 coarse rank 残余补 ${supplemented} 个。`,
|
|
2176
|
-
durationMs: 0,
|
|
2177
|
-
payload: { supplemented }
|
|
2178
|
-
});
|
|
2179
|
-
}
|
|
2180
|
-
if (results.length < topK) {
|
|
2181
|
-
const remaining = topK - results.length;
|
|
2182
|
-
const backfill = await timed(onStep, "backfill", `rerank 结果不足,回退到 chunk 向量搜索补 ${remaining} 个。`, () => store.searchChunksByVector(queryVec, { limit: remaining * 2 }), (matches) => ({ backfillCandidates: matches.length }));
|
|
2183
|
-
for (const { chunk, score } of backfill) {
|
|
2184
|
-
if (results.length >= topK) break;
|
|
2185
|
-
if (seen.has(chunk.id)) continue;
|
|
2186
|
-
seen.add(chunk.id);
|
|
2187
|
-
const result = await store.buildSearchResult(chunk.id, score);
|
|
2188
|
-
if (result !== void 0) results.push(result);
|
|
2189
|
-
}
|
|
2190
|
-
}
|
|
2191
|
-
if (results.length === 0) {
|
|
2192
|
-
fallbackReason = "vector retrieval returned no results; used FTS5 keyword fallback";
|
|
2193
|
-
return {
|
|
2194
|
-
results: await ftsFallback(store, query, topK),
|
|
2195
|
-
trace: {
|
|
2196
|
-
steps,
|
|
2197
|
-
rerankedEventTitles,
|
|
2198
|
-
fallbackReason
|
|
2199
|
-
}
|
|
2200
|
-
};
|
|
2201
|
-
}
|
|
2202
|
-
return {
|
|
2203
|
-
results,
|
|
2204
|
-
trace: {
|
|
2205
|
-
steps,
|
|
2206
|
-
rerankedEventTitles,
|
|
2207
|
-
fallbackReason
|
|
2208
|
-
}
|
|
2209
|
-
};
|
|
2210
|
-
}
|
|
2211
|
-
/** Recall entities by name (LLM-extracted) and by vector similarity. */
|
|
2212
|
-
async function recallEntities(store, llm, query, queryVec) {
|
|
2213
|
-
const out = /* @__PURE__ */ new Map();
|
|
2214
|
-
const queryEntities = await extractQueryEntities(llm, query);
|
|
2215
|
-
for (const { name } of queryEntities) {
|
|
2216
|
-
const matches = await store.findEntitiesByName(name);
|
|
2217
|
-
for (const entity of matches) out.set(entity.id, { id: entity.id });
|
|
2218
|
-
}
|
|
2219
|
-
const vectorMatches = await store.findEntitiesByVector(queryVec, {
|
|
2220
|
-
limit: 20,
|
|
2221
|
-
threshold: ENTITY_VECTOR_THRESHOLD
|
|
2222
|
-
});
|
|
2223
|
-
for (const { entity } of vectorMatches) out.set(entity.id, { id: entity.id });
|
|
2224
|
-
return Array.from(out.values());
|
|
2225
|
-
}
|
|
2226
|
-
/** FTS5 keyword fallback when embeddings are unavailable. */
|
|
2227
|
-
async function ftsFallback(store, query, topK) {
|
|
2228
|
-
const chunks = await store.ftsSearchChunks(query, topK * 2);
|
|
2229
|
-
const results = [];
|
|
2230
|
-
for (const chunk of chunks) {
|
|
2231
|
-
const result = await store.buildSearchResult(chunk.id, 0);
|
|
2232
|
-
if (result !== void 0) results.push(result);
|
|
2233
|
-
if (results.length >= topK) break;
|
|
2234
|
-
}
|
|
2235
|
-
return results;
|
|
2236
|
-
}
|
|
2237
|
-
/** Direct chunk vector search fallback. */
|
|
2238
|
-
async function chunkVectorFallback(store, queryVec, topK) {
|
|
2239
|
-
const matches = await store.searchChunksByVector(queryVec, { limit: topK * 2 });
|
|
2240
|
-
const results = [];
|
|
2241
|
-
for (const { chunk, score } of matches) {
|
|
2242
|
-
const result = await store.buildSearchResult(chunk.id, score);
|
|
2243
|
-
if (result !== void 0) results.push(result);
|
|
2244
|
-
if (results.length >= topK) break;
|
|
2245
|
-
}
|
|
2246
|
-
return results;
|
|
2247
|
-
}
|
|
2248
|
-
//#endregion
|
|
2249
|
-
export { join as C, resolve as D, relative as E, isAbsolute as S, parse as T, splitLargeSection as _, isSupportedFile as a, basename as b, buildExtractionUserPrompt as c, extractQueryEntities as d, parseExtractionResponse as f, estimateTokens as g, chunkText as h, ingestFile as i, extractEventFromChunk as l, chunkMarkdown as m, multiSearchWithTrace as n, ENTITY_TYPES as o, rerankEventsWithLlm as p, ingestDirectory as r, EXTRACTION_SYSTEM_PROMPT as s, multiSearch as t, extractJsonFromText as u, stripMarkdown as v, normalize as w, dirname as x, KnowledgeStore as y };
|