pi-knowledge 0.4.1 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/README.md +29 -8
- package/dist/index.d.ts +38 -0
- package/dist/index.js +435 -0
- package/dist/src/diagnostics/health.d.ts +19 -0
- package/dist/src/diagnostics/health.js +110 -0
- package/dist/src/embedding/provider.d.ts +5 -0
- package/dist/src/embedding/provider.js +99 -0
- package/dist/src/embedding/vectors.d.ts +15 -0
- package/dist/src/embedding/vectors.js +111 -0
- package/dist/src/engine.d.ts +90 -0
- package/dist/src/engine.js +1370 -0
- package/dist/src/indexer/chunker.d.ts +53 -0
- package/dist/src/indexer/chunker.js +498 -0
- package/dist/src/indexer/chunkers/code-ast.d.ts +3 -0
- package/dist/src/indexer/chunkers/code-ast.js +179 -0
- package/dist/src/model-worker-client.d.ts +10 -0
- package/dist/src/model-worker-client.js +138 -0
- package/dist/src/model-worker.d.ts +1 -0
- package/dist/src/model-worker.js +61 -0
- package/dist/src/search/bm25.d.ts +8 -0
- package/dist/src/search/bm25.js +38 -0
- package/dist/src/search/fusion.d.ts +19 -0
- package/dist/src/search/fusion.js +34 -0
- package/dist/src/search/query.d.ts +5 -0
- package/dist/src/search/query.js +57 -0
- package/dist/src/search/ranking.d.ts +27 -0
- package/dist/src/search/ranking.js +180 -0
- package/dist/src/search/reranker.d.ts +10 -0
- package/dist/src/search/reranker.js +66 -0
- package/dist/src/search/vector.d.ts +14 -0
- package/dist/src/search/vector.js +58 -0
- package/dist/src/storage/sqlite.d.ts +99 -0
- package/dist/src/storage/sqlite.js +330 -0
- package/dist/src/watcher/file-watcher.d.ts +4 -0
- package/dist/src/watcher/file-watcher.js +108 -0
- package/extension.js +9 -0
- package/package.json +20 -6
- package/index.ts +0 -500
- package/src/diagnostics/health.ts +0 -145
- package/src/embedding/provider.ts +0 -106
- package/src/embedding/vectors.ts +0 -119
- package/src/engine.ts +0 -1592
- package/src/indexer/chunker.ts +0 -612
- package/src/indexer/chunkers/code-ast.ts +0 -209
- package/src/model-worker-client.ts +0 -146
- package/src/model-worker.ts +0 -88
- package/src/search/bm25.ts +0 -55
- package/src/search/fusion.ts +0 -48
- package/src/search/query.ts +0 -59
- package/src/search/ranking.ts +0 -201
- package/src/search/reranker.ts +0 -74
- package/src/search/vector.ts +0 -81
- package/src/storage/sqlite.ts +0 -489
- package/src/watcher/file-watcher.ts +0 -116
|
@@ -0,0 +1,1370 @@
|
|
|
1
|
+
import { existsSync, renameSync, rmSync, statSync } from "node:fs";
|
|
2
|
+
import { join, resolve } from "node:path";
|
|
3
|
+
import { diagnoseKB } from "./diagnostics/health.js";
|
|
4
|
+
import { dispose as disposeEmbedding, embedDocuments, embedQuery, prepareForShutdown as prepareEmbeddingForShutdown, } from "./embedding/provider.js";
|
|
5
|
+
import { openVectorReader, openVectorWriter } from "./embedding/vectors.js";
|
|
6
|
+
import { buildChunkEmbeddingText, chunkFile, chunkIdentityHash, createSkippedScanStats, isReadableTextFile, iterateScannableFiles, iterateScannedFiles, preTokenizeForFTS, summarizeSkippedScan, } from "./indexer/chunker.js";
|
|
7
|
+
import { shutdownModelWorker } from "./model-worker-client.js";
|
|
8
|
+
import { searchBM25 } from "./search/bm25.js";
|
|
9
|
+
import { weightedScoreFusion } from "./search/fusion.js";
|
|
10
|
+
import { normalizedQueryText, tokenizeForSearch } from "./search/query.js";
|
|
11
|
+
import { hasAnyLexicalEvidence, hasEnoughLexicalEvidence, MIN_HYBRID_SCORE, normalizeFileTypeFilter, queryCoverage, scoreChunkForQuery, } from "./search/ranking.js";
|
|
12
|
+
import { disposeReranker, prepareRerankerForShutdown, rerank } from "./search/reranker.js";
|
|
13
|
+
import { searchVectorFile } from "./search/vector.js";
|
|
14
|
+
import { createKB, deleteChunksByIds, deleteKB, finishIndexingJob, getChunkById, getChunkCount, getChunksByFile, getChunksByKB, getFileCount, getKB, getKBByName, insertChunks, iterateChunkIdsByKB, iterateChunksByKB, listKBs, openDatabase, startIndexingJob, updateIndexingJob, updateKBCounts, updateKBStatus, } from "./storage/sqlite.js";
|
|
15
|
+
export const CURRENT_EMBEDDING_MODEL = "multilingual-e5-small";
|
|
16
|
+
const ADAPTIVE_CONTEXT_LINES = 80;
|
|
17
|
+
const ADAPTIVE_MAX_CONTEXT_CHARS = 6_000;
|
|
18
|
+
const ADAPTIVE_NEIGHBOR_TARGET = 5;
|
|
19
|
+
const INDEX_EMBED_BATCH_SIZE = 64;
|
|
20
|
+
const VECTOR_REDUNDANCY_WEIGHT = 0.35;
|
|
21
|
+
function isCancellationError(error) {
|
|
22
|
+
return error instanceof Error && error.message === "Cancelled";
|
|
23
|
+
}
|
|
24
|
+
function tempVectorPath(vectorPath) {
|
|
25
|
+
return `${vectorPath}.${Date.now()}.${Math.random().toString(16).slice(2)}.tmp`;
|
|
26
|
+
}
|
|
27
|
+
function toScanOptions(options = {}) {
|
|
28
|
+
return {
|
|
29
|
+
includeSuggestedText: options.include_suggested_text === true,
|
|
30
|
+
includePaths: options.include_paths,
|
|
31
|
+
excludePaths: options.exclude_paths,
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
function serializeAddOptions(options = {}) {
|
|
35
|
+
const normalized = {};
|
|
36
|
+
if (options.include_suggested_text === true)
|
|
37
|
+
normalized.include_suggested_text = true;
|
|
38
|
+
if (options.include_paths && options.include_paths.length > 0)
|
|
39
|
+
normalized.include_paths = options.include_paths;
|
|
40
|
+
if (options.exclude_paths && options.exclude_paths.length > 0)
|
|
41
|
+
normalized.exclude_paths = options.exclude_paths;
|
|
42
|
+
return Object.keys(normalized).length > 0 ? JSON.stringify(normalized) : undefined;
|
|
43
|
+
}
|
|
44
|
+
function parseAddOptions(raw) {
|
|
45
|
+
if (!raw)
|
|
46
|
+
return {};
|
|
47
|
+
try {
|
|
48
|
+
const parsed = JSON.parse(raw);
|
|
49
|
+
return {
|
|
50
|
+
include_suggested_text: parsed.include_suggested_text === true,
|
|
51
|
+
include_paths: Array.isArray(parsed.include_paths)
|
|
52
|
+
? parsed.include_paths.filter((item) => typeof item === "string")
|
|
53
|
+
: undefined,
|
|
54
|
+
exclude_paths: Array.isArray(parsed.exclude_paths)
|
|
55
|
+
? parsed.exclude_paths.filter((item) => typeof item === "string")
|
|
56
|
+
: undefined,
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
catch {
|
|
60
|
+
return {};
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
function planDirectoryScan(dirPath, options = {}) {
|
|
64
|
+
const skipped = createSkippedScanStats();
|
|
65
|
+
let files = 0;
|
|
66
|
+
let bytes = 0;
|
|
67
|
+
for (const file of iterateScannableFiles(dirPath, skipped, options)) {
|
|
68
|
+
files++;
|
|
69
|
+
bytes += file.size;
|
|
70
|
+
}
|
|
71
|
+
return {
|
|
72
|
+
files,
|
|
73
|
+
bytes,
|
|
74
|
+
skippedTotal: skipped.total,
|
|
75
|
+
skippedSummary: summarizeSkippedScan(skipped),
|
|
76
|
+
skipped,
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
function formatBytes(bytes) {
|
|
80
|
+
if (bytes < 1024)
|
|
81
|
+
return `${bytes} B`;
|
|
82
|
+
const kb = bytes / 1024;
|
|
83
|
+
if (kb < 1024)
|
|
84
|
+
return `${kb.toFixed(1)} KB`;
|
|
85
|
+
const mb = kb / 1024;
|
|
86
|
+
if (mb < 1024)
|
|
87
|
+
return `${mb.toFixed(1)} MB`;
|
|
88
|
+
return `${(mb / 1024).toFixed(2)} GB`;
|
|
89
|
+
}
|
|
90
|
+
function cosineSimilarity(a, b) {
|
|
91
|
+
if (!a || !b || a.length !== b.length)
|
|
92
|
+
return 0;
|
|
93
|
+
let dot = 0;
|
|
94
|
+
for (let i = 0; i < a.length; i++)
|
|
95
|
+
dot += a[i] * b[i];
|
|
96
|
+
return dot;
|
|
97
|
+
}
|
|
98
|
+
function tokenizeForSimilarity(text) {
|
|
99
|
+
return tokenizeForSearch(text);
|
|
100
|
+
}
|
|
101
|
+
function jaccardSimilarity(a, b) {
|
|
102
|
+
if (a.size === 0 || b.size === 0)
|
|
103
|
+
return 0;
|
|
104
|
+
let intersection = 0;
|
|
105
|
+
for (const token of a) {
|
|
106
|
+
if (b.has(token))
|
|
107
|
+
intersection++;
|
|
108
|
+
}
|
|
109
|
+
return intersection / (a.size + b.size - intersection);
|
|
110
|
+
}
|
|
111
|
+
function lineProximity(a, b) {
|
|
112
|
+
if (a.chunk.file_path !== b.chunk.file_path || a.chunk.kb_id !== b.chunk.kb_id)
|
|
113
|
+
return 0;
|
|
114
|
+
if (a.sourceChunkIds.some((id) => b.sourceChunkIds.includes(id)))
|
|
115
|
+
return 1;
|
|
116
|
+
const overlap = Math.max(0, Math.min(a.endLine, b.endLine) - Math.max(a.startLine, b.startLine) + 1);
|
|
117
|
+
if (overlap > 0)
|
|
118
|
+
return 1;
|
|
119
|
+
const gap = Math.max(a.startLine, b.startLine) - Math.min(a.endLine, b.endLine);
|
|
120
|
+
if (gap <= 20)
|
|
121
|
+
return 0.8;
|
|
122
|
+
if (gap <= 80)
|
|
123
|
+
return 0.45;
|
|
124
|
+
return 0.2;
|
|
125
|
+
}
|
|
126
|
+
function normalizeScores(candidates) {
|
|
127
|
+
const scores = candidates.map((candidate) => candidate.score);
|
|
128
|
+
const min = Math.min(...scores);
|
|
129
|
+
const max = Math.max(...scores);
|
|
130
|
+
const normalized = new Map();
|
|
131
|
+
for (const candidate of candidates) {
|
|
132
|
+
const value = max === min ? 1 : (candidate.score - min) / (max - min);
|
|
133
|
+
normalized.set(candidate.chunk.id, value);
|
|
134
|
+
}
|
|
135
|
+
return normalized;
|
|
136
|
+
}
|
|
137
|
+
function diversifyRankedChunks(candidates, diversity, vectorsByChunkId = new Map()) {
|
|
138
|
+
if (diversity === "off" || candidates.length <= 2)
|
|
139
|
+
return candidates;
|
|
140
|
+
const lambda = diversity === "strong" ? 0.62 : 0.76;
|
|
141
|
+
const normalized = normalizeScores(candidates);
|
|
142
|
+
const tokenSets = new Map(candidates.map((candidate) => [candidate.chunk.id, tokenizeForSimilarity(candidate.content)]));
|
|
143
|
+
const selected = [];
|
|
144
|
+
const remaining = [...candidates];
|
|
145
|
+
while (remaining.length > 0) {
|
|
146
|
+
let bestIndex = 0;
|
|
147
|
+
let bestScore = Number.NEGATIVE_INFINITY;
|
|
148
|
+
for (let i = 0; i < remaining.length; i++) {
|
|
149
|
+
const candidate = remaining[i];
|
|
150
|
+
let redundancy = 0;
|
|
151
|
+
for (const chosen of selected) {
|
|
152
|
+
const lexical = jaccardSimilarity(tokenSets.get(candidate.chunk.id) ?? new Set(), tokenSets.get(chosen.chunk.id) ?? new Set());
|
|
153
|
+
const vector = Math.max(0, cosineSimilarity(vectorsByChunkId.get(candidate.chunk.id), vectorsByChunkId.get(chosen.chunk.id)));
|
|
154
|
+
redundancy = Math.max(redundancy, Math.max(lexical, lineProximity(candidate, chosen), vector * VECTOR_REDUNDANCY_WEIGHT));
|
|
155
|
+
}
|
|
156
|
+
const relevance = normalized.get(candidate.chunk.id) ?? 0;
|
|
157
|
+
const mmrScore = lambda * relevance - (1 - lambda) * redundancy;
|
|
158
|
+
if (mmrScore > bestScore) {
|
|
159
|
+
bestScore = mmrScore;
|
|
160
|
+
bestIndex = i;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
selected.push(remaining.splice(bestIndex, 1)[0]);
|
|
164
|
+
}
|
|
165
|
+
return selected;
|
|
166
|
+
}
|
|
167
|
+
function interleaveByFile(candidates, diversity) {
|
|
168
|
+
if (diversity === "off" || candidates.length <= 2)
|
|
169
|
+
return candidates;
|
|
170
|
+
const buckets = new Map();
|
|
171
|
+
const fileOrder = [];
|
|
172
|
+
for (const candidate of candidates) {
|
|
173
|
+
const key = `${candidate.chunk.kb_id}:${candidate.chunk.file_path}`;
|
|
174
|
+
if (!buckets.has(key)) {
|
|
175
|
+
buckets.set(key, []);
|
|
176
|
+
fileOrder.push(key);
|
|
177
|
+
}
|
|
178
|
+
buckets.get(key)?.push(candidate);
|
|
179
|
+
}
|
|
180
|
+
if (fileOrder.length <= 1)
|
|
181
|
+
return candidates;
|
|
182
|
+
const result = [];
|
|
183
|
+
let round = 0;
|
|
184
|
+
while (result.length < candidates.length) {
|
|
185
|
+
let added = false;
|
|
186
|
+
for (const key of fileOrder) {
|
|
187
|
+
const bucket = buckets.get(key);
|
|
188
|
+
const item = bucket?.[round];
|
|
189
|
+
if (!item)
|
|
190
|
+
continue;
|
|
191
|
+
result.push(item);
|
|
192
|
+
added = true;
|
|
193
|
+
}
|
|
194
|
+
if (!added)
|
|
195
|
+
break;
|
|
196
|
+
round++;
|
|
197
|
+
}
|
|
198
|
+
return result;
|
|
199
|
+
}
|
|
200
|
+
function buildQuerySnippet(content, query, maxLength = 240) {
|
|
201
|
+
const terms = [...tokenizeForSimilarity(query)].sort((a, b) => b.length - a.length);
|
|
202
|
+
const lower = content.toLowerCase();
|
|
203
|
+
let matchIndex = -1;
|
|
204
|
+
for (const term of terms) {
|
|
205
|
+
matchIndex = lower.indexOf(term.toLowerCase());
|
|
206
|
+
if (matchIndex >= 0)
|
|
207
|
+
break;
|
|
208
|
+
}
|
|
209
|
+
if (matchIndex < 0)
|
|
210
|
+
return content.slice(0, maxLength);
|
|
211
|
+
const half = Math.floor(maxLength / 2);
|
|
212
|
+
const start = Math.max(0, matchIndex - half);
|
|
213
|
+
const end = Math.min(content.length, start + maxLength);
|
|
214
|
+
const prefix = start > 0 ? "..." : "";
|
|
215
|
+
const suffix = end < content.length ? "..." : "";
|
|
216
|
+
return `${prefix}${content.slice(start, end)}${suffix}`;
|
|
217
|
+
}
|
|
218
|
+
function buildAdaptiveContext(seed, chunks, queryTokens) {
|
|
219
|
+
const scored = chunks
|
|
220
|
+
.map((chunk) => {
|
|
221
|
+
const distance = chunk.id === seed.id
|
|
222
|
+
? 0
|
|
223
|
+
: Math.min(Math.abs(chunk.start_line - seed.start_line), Math.abs(chunk.end_line - seed.end_line));
|
|
224
|
+
const proximity = 1 / (1 + distance / 20);
|
|
225
|
+
const coverage = queryCoverage(chunk.content, queryTokens);
|
|
226
|
+
const seedBoost = chunk.id === seed.id ? 2 : 0;
|
|
227
|
+
return { chunk, score: seedBoost + proximity + coverage };
|
|
228
|
+
})
|
|
229
|
+
.sort((a, b) => b.score - a.score)
|
|
230
|
+
.slice(0, ADAPTIVE_NEIGHBOR_TARGET);
|
|
231
|
+
if (!scored.some((item) => item.chunk.id === seed.id))
|
|
232
|
+
scored.push({ chunk: seed, score: Number.MAX_SAFE_INTEGER });
|
|
233
|
+
const selectedIds = new Set(scored.map((item) => item.chunk.id));
|
|
234
|
+
const ordered = chunks
|
|
235
|
+
.filter((chunk) => selectedIds.has(chunk.id))
|
|
236
|
+
.sort((a, b) => a.start_line - b.start_line || a.end_line - b.end_line);
|
|
237
|
+
const parts = [];
|
|
238
|
+
const sourceChunkIds = [];
|
|
239
|
+
let totalLength = 0;
|
|
240
|
+
for (const chunk of ordered) {
|
|
241
|
+
if (totalLength >= ADAPTIVE_MAX_CONTEXT_CHARS)
|
|
242
|
+
break;
|
|
243
|
+
const remaining = ADAPTIVE_MAX_CONTEXT_CHARS - totalLength;
|
|
244
|
+
const text = chunk.content.slice(0, remaining);
|
|
245
|
+
parts.push(text);
|
|
246
|
+
sourceChunkIds.push(chunk.id);
|
|
247
|
+
totalLength += text.length + 2;
|
|
248
|
+
}
|
|
249
|
+
return {
|
|
250
|
+
content: parts.join("\n\n"),
|
|
251
|
+
startLine: Math.min(...ordered.map((chunk) => chunk.start_line)),
|
|
252
|
+
endLine: Math.max(...ordered.map((chunk) => chunk.end_line)),
|
|
253
|
+
sourceChunkIds,
|
|
254
|
+
};
|
|
255
|
+
}
|
|
256
|
+
function pushAdaptiveCandidate(candidates, candidate) {
|
|
257
|
+
const overlappingIndex = candidates.findIndex((existing) => existing.chunk.kb_id === candidate.chunk.kb_id &&
|
|
258
|
+
existing.chunk.file_path === candidate.chunk.file_path &&
|
|
259
|
+
existing.sourceChunkIds.some((id) => candidate.sourceChunkIds.includes(id)));
|
|
260
|
+
if (overlappingIndex < 0) {
|
|
261
|
+
candidates.push(candidate);
|
|
262
|
+
return;
|
|
263
|
+
}
|
|
264
|
+
const existing = candidates[overlappingIndex];
|
|
265
|
+
if (candidate.score > existing.score || candidate.sourceChunkIds.length > existing.sourceChunkIds.length) {
|
|
266
|
+
candidates[overlappingIndex] = candidate;
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
function formatDuration(ms) {
|
|
270
|
+
if (!Number.isFinite(ms) || ms <= 0)
|
|
271
|
+
return "0s";
|
|
272
|
+
const seconds = Math.ceil(ms / 1000);
|
|
273
|
+
if (seconds < 60)
|
|
274
|
+
return `${seconds}s`;
|
|
275
|
+
const minutes = Math.floor(seconds / 60);
|
|
276
|
+
const rest = seconds % 60;
|
|
277
|
+
return rest === 0 ? `${minutes}m` : `${minutes}m ${rest}s`;
|
|
278
|
+
}
|
|
279
|
+
function isExactLookupQuery(query) {
|
|
280
|
+
const trimmed = query.trim();
|
|
281
|
+
if (/["'`]/.test(trimmed))
|
|
282
|
+
return true;
|
|
283
|
+
if (/[./\\][\w.-]+/.test(trimmed))
|
|
284
|
+
return true;
|
|
285
|
+
if (/\b[A-Z][A-Za-z0-9_]*[A-Z][A-Za-z0-9_]*\b/.test(trimmed))
|
|
286
|
+
return true;
|
|
287
|
+
if (/\b[A-Z]{2,}[_-]?[A-Z0-9]*\b/.test(trimmed))
|
|
288
|
+
return true;
|
|
289
|
+
if (/\b[a-zA-Z_][\w-]*\.(ts|tsx|js|jsx|go|rs|py|java|md|json|ya?ml|toml)\b/.test(trimmed))
|
|
290
|
+
return true;
|
|
291
|
+
if (/\b[A-Z]+-\d+\b/.test(trimmed))
|
|
292
|
+
return true;
|
|
293
|
+
return false;
|
|
294
|
+
}
|
|
295
|
+
function chooseAutoMode(query) {
|
|
296
|
+
const normalized = query.trim();
|
|
297
|
+
if (isExactLookupQuery(normalized))
|
|
298
|
+
return "fast";
|
|
299
|
+
const wordCount = normalized.split(/\s+/).filter(Boolean).length;
|
|
300
|
+
if (wordCount >= 10 || /how|why|explain|concept|architecture|design|流程|架構|概念|為什麼|如何/i.test(normalized)) {
|
|
301
|
+
return "semantic";
|
|
302
|
+
}
|
|
303
|
+
return "hybrid";
|
|
304
|
+
}
|
|
305
|
+
function fallbackModesFor(mode) {
|
|
306
|
+
if (mode === "fast")
|
|
307
|
+
return ["hybrid", "semantic"];
|
|
308
|
+
if (mode === "semantic")
|
|
309
|
+
return ["hybrid", "adaptive"];
|
|
310
|
+
if (mode === "adaptive")
|
|
311
|
+
return ["hybrid", "deep"];
|
|
312
|
+
if (mode === "deep")
|
|
313
|
+
return ["hybrid", "adaptive"];
|
|
314
|
+
return ["fast", "semantic", "adaptive"];
|
|
315
|
+
}
|
|
316
|
+
function isWeakAutoResponse(query, response, primaryMode, attemptMode) {
|
|
317
|
+
if (response.results.length === 0)
|
|
318
|
+
return true;
|
|
319
|
+
if (primaryMode === "fast") {
|
|
320
|
+
const exactNeedle = query
|
|
321
|
+
.trim()
|
|
322
|
+
.replace(/^["'`]|["'`]$/g, "")
|
|
323
|
+
.toLowerCase();
|
|
324
|
+
const hasExactHit = response.results.some((result) => result.file_path.toLowerCase().includes(exactNeedle) || result.content.toLowerCase().includes(exactNeedle));
|
|
325
|
+
if (!hasExactHit && response.results.every((result) => (result.ranking?.coverage ?? 0) < 0.67))
|
|
326
|
+
return true;
|
|
327
|
+
}
|
|
328
|
+
if (primaryMode !== "semantic" && attemptMode === "semantic") {
|
|
329
|
+
return response.results.every((result) => (result.ranking?.coverage ?? 0) === 0);
|
|
330
|
+
}
|
|
331
|
+
return false;
|
|
332
|
+
}
|
|
333
|
+
async function chunkUrl(source, signal) {
|
|
334
|
+
const res = await fetch(source, { signal });
|
|
335
|
+
if (!res.ok)
|
|
336
|
+
throw new Error(`Fetch failed: ${res.status}`);
|
|
337
|
+
const html = await res.text();
|
|
338
|
+
const text = html
|
|
339
|
+
.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "")
|
|
340
|
+
.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "")
|
|
341
|
+
.replace(/<[^>]+>/g, " ")
|
|
342
|
+
.replace(/&[a-z]+;/gi, " ")
|
|
343
|
+
.replace(/\s+/g, " ")
|
|
344
|
+
.trim();
|
|
345
|
+
return chunkFile(text, source);
|
|
346
|
+
}
|
|
347
|
+
function normalizeExtractedText(text) {
|
|
348
|
+
return Array.isArray(text) ? text.join("\n\n") : text;
|
|
349
|
+
}
|
|
350
|
+
export class KnowledgeEngine {
|
|
351
|
+
db = null;
|
|
352
|
+
knowledgeDir = "";
|
|
353
|
+
async initialize(knowledgeDir) {
|
|
354
|
+
this.knowledgeDir = knowledgeDir;
|
|
355
|
+
this.db = openDatabase(knowledgeDir);
|
|
356
|
+
}
|
|
357
|
+
plan(source, options = {}) {
|
|
358
|
+
const resolvedSource = resolve(source);
|
|
359
|
+
const isUrl = source.startsWith("http://") || source.startsWith("https://");
|
|
360
|
+
const isDir = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isDirectory();
|
|
361
|
+
const isFile = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isFile();
|
|
362
|
+
const sourceType = isUrl ? "url" : isDir ? "directory" : isFile ? "file" : "text";
|
|
363
|
+
if (isDir) {
|
|
364
|
+
const plan = planDirectoryScan(resolvedSource, toScanOptions(options));
|
|
365
|
+
return {
|
|
366
|
+
source_type: "directory",
|
|
367
|
+
scannable_files: plan.files,
|
|
368
|
+
scannable_bytes: plan.bytes,
|
|
369
|
+
skipped: plan.skipped,
|
|
370
|
+
summary: `Directory plan: ${plan.files} scannable files, ${formatBytes(plan.bytes)} scannable text, skipped ${plan.skippedTotal} (${plan.skippedSummary})`,
|
|
371
|
+
};
|
|
372
|
+
}
|
|
373
|
+
if (isFile) {
|
|
374
|
+
const skipped = createSkippedScanStats();
|
|
375
|
+
const supportedDocument = resolvedSource.endsWith(".pdf") || resolvedSource.endsWith(".docx") || resolvedSource.endsWith(".doc");
|
|
376
|
+
if (!supportedDocument && !isReadableTextFile(resolvedSource)) {
|
|
377
|
+
skipped.total = 1;
|
|
378
|
+
skipped.by_reason.binary = 1;
|
|
379
|
+
skipped.samples.push({ path: resolvedSource, reason: "binary" });
|
|
380
|
+
return {
|
|
381
|
+
source_type: "file",
|
|
382
|
+
scannable_files: 0,
|
|
383
|
+
scannable_bytes: 0,
|
|
384
|
+
skipped,
|
|
385
|
+
summary: "File plan: 0 scannable files; file is unsupported binary/non-text",
|
|
386
|
+
};
|
|
387
|
+
}
|
|
388
|
+
const size = statSync(resolvedSource).size;
|
|
389
|
+
return {
|
|
390
|
+
source_type: "file",
|
|
391
|
+
scannable_files: 1,
|
|
392
|
+
scannable_bytes: size,
|
|
393
|
+
skipped,
|
|
394
|
+
summary: `File plan: 1 scannable file, ${formatBytes(size)} source size`,
|
|
395
|
+
};
|
|
396
|
+
}
|
|
397
|
+
const skipped = createSkippedScanStats();
|
|
398
|
+
return {
|
|
399
|
+
source_type: sourceType,
|
|
400
|
+
scannable_files: 1,
|
|
401
|
+
scannable_bytes: Buffer.byteLength(source),
|
|
402
|
+
skipped,
|
|
403
|
+
summary: `${sourceType === "url" ? "URL" : "Inline text"} plan: 1 source`,
|
|
404
|
+
};
|
|
405
|
+
}
|
|
406
|
+
async add(source, name, onProgress, signal, options = {}) {
|
|
407
|
+
if (!this.db)
|
|
408
|
+
throw new Error("Engine not initialized");
|
|
409
|
+
const db = this.db;
|
|
410
|
+
const resolvedSource = resolve(source);
|
|
411
|
+
const isUrl = source.startsWith("http://") || source.startsWith("https://");
|
|
412
|
+
const isDir = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isDirectory();
|
|
413
|
+
const isFile = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isFile();
|
|
414
|
+
const sourceType = isUrl ? "url" : isDir ? "directory" : isFile ? "file" : "text";
|
|
415
|
+
const scanOptions = toScanOptions(options);
|
|
416
|
+
const existingKB = getKBByName(db, name);
|
|
417
|
+
if (existingKB) {
|
|
418
|
+
throw new Error(`Knowledge base "${name}" already exists. Use knowledge_update to refresh it, or knowledge_remove before adding a replacement.`);
|
|
419
|
+
}
|
|
420
|
+
const kb = createKB(db, {
|
|
421
|
+
name,
|
|
422
|
+
source_path: isDir || isFile ? resolvedSource : isUrl ? source : undefined,
|
|
423
|
+
source_type: sourceType,
|
|
424
|
+
source_options: serializeAddOptions(options),
|
|
425
|
+
});
|
|
426
|
+
updateKBStatus(db, kb.id, "indexing");
|
|
427
|
+
startIndexingJob(db, kb.id, "add", `Starting indexing for "${name}"`);
|
|
428
|
+
let vectorWriter;
|
|
429
|
+
let tempVectorFile;
|
|
430
|
+
try {
|
|
431
|
+
const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
|
|
432
|
+
tempVectorFile = tempVectorPath(vectorPath);
|
|
433
|
+
const writer = openVectorWriter(tempVectorFile);
|
|
434
|
+
vectorWriter = writer;
|
|
435
|
+
let chunkCount = 0;
|
|
436
|
+
let fileCount = 0;
|
|
437
|
+
const pendingChunks = [];
|
|
438
|
+
const startedAt = Date.now();
|
|
439
|
+
let latestSkippedTotal = 0;
|
|
440
|
+
const reportProgress = (phase, processedFiles, totalFiles, skippedTotal = 0) => {
|
|
441
|
+
latestSkippedTotal = skippedTotal;
|
|
442
|
+
const elapsed = Date.now() - startedAt;
|
|
443
|
+
const chunkRate = chunkCount / Math.max(1, elapsed / 1000);
|
|
444
|
+
let suffix = `${chunkCount} chunks, ${chunkRate.toFixed(1)} chunks/s, elapsed ${formatDuration(elapsed)}`;
|
|
445
|
+
if (processedFiles !== undefined && totalFiles !== undefined && totalFiles > 0) {
|
|
446
|
+
const rate = processedFiles / Math.max(1, elapsed / 1000);
|
|
447
|
+
const remainingFiles = Math.max(0, totalFiles - processedFiles);
|
|
448
|
+
const etaMs = rate > 0 ? (remainingFiles / rate) * 1000 : 0;
|
|
449
|
+
suffix = `${processedFiles}/${totalFiles} files, ${suffix}, file ETA ${formatDuration(etaMs)}`;
|
|
450
|
+
}
|
|
451
|
+
else if (processedFiles !== undefined) {
|
|
452
|
+
suffix = `${processedFiles} files scanned, ${suffix}`;
|
|
453
|
+
}
|
|
454
|
+
if (skippedTotal > 0)
|
|
455
|
+
suffix = `${suffix}, skipped ${skippedTotal}`;
|
|
456
|
+
const message = `${phase}: ${suffix}`;
|
|
457
|
+
updateIndexingJob(db, kb.id, {
|
|
458
|
+
phase,
|
|
459
|
+
message,
|
|
460
|
+
processed_files: processedFiles,
|
|
461
|
+
processed_chunks: chunkCount,
|
|
462
|
+
total_files: totalFiles,
|
|
463
|
+
skipped_total: skippedTotal,
|
|
464
|
+
added_chunks: chunkCount,
|
|
465
|
+
});
|
|
466
|
+
onProgress?.(message);
|
|
467
|
+
};
|
|
468
|
+
const flushPending = async (processedFiles, totalFiles) => {
|
|
469
|
+
if (pendingChunks.length === 0)
|
|
470
|
+
return;
|
|
471
|
+
if (signal?.aborted)
|
|
472
|
+
throw new Error("Cancelled");
|
|
473
|
+
const batch = pendingChunks.splice(0, INDEX_EMBED_BATCH_SIZE);
|
|
474
|
+
reportProgress(`Embedding batch of ${batch.length}`, processedFiles, totalFiles);
|
|
475
|
+
const vectors = await embedDocuments(batch.map((chunk) => buildChunkEmbeddingText(chunk)), signal);
|
|
476
|
+
if (signal?.aborted)
|
|
477
|
+
throw new Error("Cancelled");
|
|
478
|
+
insertChunks(db, kb.id, batch);
|
|
479
|
+
writer.append(vectors);
|
|
480
|
+
chunkCount += batch.length;
|
|
481
|
+
updateKBCounts(db, kb.id, chunkCount, fileCount);
|
|
482
|
+
reportProgress("Stored batch", processedFiles, totalFiles);
|
|
483
|
+
};
|
|
484
|
+
const addChunks = async (chunks, processedFiles, totalFiles) => {
|
|
485
|
+
pendingChunks.push(...chunks);
|
|
486
|
+
while (pendingChunks.length >= INDEX_EMBED_BATCH_SIZE) {
|
|
487
|
+
await flushPending(processedFiles, totalFiles);
|
|
488
|
+
}
|
|
489
|
+
};
|
|
490
|
+
if (isUrl) {
|
|
491
|
+
const message = `Fetching ${source}...`;
|
|
492
|
+
updateIndexingJob(this.db, kb.id, { phase: "fetching", message });
|
|
493
|
+
onProgress?.(message);
|
|
494
|
+
const chunks = await chunkUrl(source, signal);
|
|
495
|
+
fileCount = 1;
|
|
496
|
+
await addChunks(chunks);
|
|
497
|
+
}
|
|
498
|
+
else if (isFile && resolvedSource.endsWith(".pdf")) {
|
|
499
|
+
updateIndexingJob(this.db, kb.id, { phase: "extracting", message: "Extracting text from PDF..." });
|
|
500
|
+
onProgress?.("Extracting text from PDF...");
|
|
501
|
+
const { extractText } = await import("unpdf");
|
|
502
|
+
const buf = (await import("node:fs")).readFileSync(resolvedSource);
|
|
503
|
+
const { text } = await extractText(new Uint8Array(buf));
|
|
504
|
+
fileCount = 1;
|
|
505
|
+
await addChunks(await chunkFile(normalizeExtractedText(text), resolvedSource));
|
|
506
|
+
}
|
|
507
|
+
else if (isFile && (resolvedSource.endsWith(".docx") || resolvedSource.endsWith(".doc"))) {
|
|
508
|
+
updateIndexingJob(this.db, kb.id, { phase: "extracting", message: "Extracting text from DOCX..." });
|
|
509
|
+
onProgress?.("Extracting text from DOCX...");
|
|
510
|
+
const mammoth = await import("mammoth");
|
|
511
|
+
const result = await mammoth.extractRawText({ path: resolvedSource });
|
|
512
|
+
fileCount = 1;
|
|
513
|
+
await addChunks(await chunkFile(result.value, resolvedSource));
|
|
514
|
+
}
|
|
515
|
+
else if (isDir) {
|
|
516
|
+
const plan = planDirectoryScan(resolvedSource, scanOptions);
|
|
517
|
+
const planningMessage = `Planned directory scan: ${plan.files} files, ${formatBytes(plan.bytes)} scannable text, skipped ${plan.skippedTotal} (${plan.skippedSummary})`;
|
|
518
|
+
updateIndexingJob(this.db, kb.id, {
|
|
519
|
+
phase: "planning",
|
|
520
|
+
message: planningMessage,
|
|
521
|
+
total_files: plan.files,
|
|
522
|
+
skipped_total: plan.skippedTotal,
|
|
523
|
+
});
|
|
524
|
+
onProgress?.(planningMessage);
|
|
525
|
+
const scanningMessage = `Scanning ${resolvedSource}...`;
|
|
526
|
+
updateIndexingJob(this.db, kb.id, {
|
|
527
|
+
phase: "scanning",
|
|
528
|
+
message: scanningMessage,
|
|
529
|
+
total_files: plan.files,
|
|
530
|
+
skipped_total: plan.skippedTotal,
|
|
531
|
+
});
|
|
532
|
+
onProgress?.(scanningMessage);
|
|
533
|
+
const skipped = createSkippedScanStats();
|
|
534
|
+
let processedFiles = 0;
|
|
535
|
+
for (const file of iterateScannedFiles(resolvedSource, skipped, scanOptions)) {
|
|
536
|
+
if (signal?.aborted)
|
|
537
|
+
throw new Error("Cancelled");
|
|
538
|
+
const chunks = await chunkFile(file.content, file.relPath);
|
|
539
|
+
processedFiles++;
|
|
540
|
+
latestSkippedTotal = skipped.total;
|
|
541
|
+
if (chunks.length > 0)
|
|
542
|
+
fileCount++;
|
|
543
|
+
await addChunks(chunks, processedFiles, plan.files);
|
|
544
|
+
if (processedFiles % 25 === 0)
|
|
545
|
+
reportProgress("Chunking", processedFiles, plan.files, skipped.total);
|
|
546
|
+
}
|
|
547
|
+
const finalizingMessage = `Scanned ${processedFiles} files, skipped ${skipped.total} (${summarizeSkippedScan(skipped)}), finalizing...`;
|
|
548
|
+
updateIndexingJob(this.db, kb.id, {
|
|
549
|
+
phase: "finalizing",
|
|
550
|
+
message: finalizingMessage,
|
|
551
|
+
processed_files: processedFiles,
|
|
552
|
+
processed_chunks: chunkCount,
|
|
553
|
+
total_files: plan.files,
|
|
554
|
+
skipped_total: skipped.total,
|
|
555
|
+
added_chunks: chunkCount,
|
|
556
|
+
});
|
|
557
|
+
onProgress?.(finalizingMessage);
|
|
558
|
+
}
|
|
559
|
+
else if (isFile) {
|
|
560
|
+
if (!isReadableTextFile(resolvedSource)) {
|
|
561
|
+
throw new Error(`File is not readable text and has no supported extractor: ${resolvedSource}`);
|
|
562
|
+
}
|
|
563
|
+
const { readFileSync } = await import("node:fs");
|
|
564
|
+
const content = readFileSync(resolvedSource, "utf-8");
|
|
565
|
+
fileCount = 1;
|
|
566
|
+
await addChunks(await chunkFile(content, resolvedSource));
|
|
567
|
+
}
|
|
568
|
+
else {
|
|
569
|
+
fileCount = 1;
|
|
570
|
+
await addChunks(await chunkFile(source, "inline-text"));
|
|
571
|
+
}
|
|
572
|
+
await flushPending();
|
|
573
|
+
vectorWriter.close();
|
|
574
|
+
vectorWriter = undefined;
|
|
575
|
+
renameSync(tempVectorFile, vectorPath);
|
|
576
|
+
tempVectorFile = undefined;
|
|
577
|
+
const savedFileCount = isDir ? getFileCount(this.db, kb.id) : fileCount;
|
|
578
|
+
updateKBCounts(this.db, kb.id, chunkCount, savedFileCount);
|
|
579
|
+
updateKBStatus(this.db, kb.id, "ready");
|
|
580
|
+
const savedKB = getKB(this.db, kb.id);
|
|
581
|
+
if (!savedKB)
|
|
582
|
+
throw new Error(`Knowledge base disappeared after add: ${kb.id}`);
|
|
583
|
+
const readyMessage = `Ready: ${chunkCount} chunks from ${savedFileCount} files in ${formatDuration(Date.now() - startedAt)}`;
|
|
584
|
+
updateIndexingJob(this.db, kb.id, {
|
|
585
|
+
phase: "ready",
|
|
586
|
+
message: readyMessage,
|
|
587
|
+
processed_files: savedFileCount,
|
|
588
|
+
processed_chunks: chunkCount,
|
|
589
|
+
skipped_total: latestSkippedTotal,
|
|
590
|
+
added_chunks: chunkCount,
|
|
591
|
+
});
|
|
592
|
+
finishIndexingJob(this.db, kb.id, "succeeded", readyMessage);
|
|
593
|
+
onProgress?.(readyMessage);
|
|
594
|
+
return { kb: savedKB, chunkCount };
|
|
595
|
+
}
|
|
596
|
+
catch (e) {
|
|
597
|
+
vectorWriter?.close();
|
|
598
|
+
if (tempVectorFile)
|
|
599
|
+
rmSync(tempVectorFile, { force: true });
|
|
600
|
+
if (this.db) {
|
|
601
|
+
finishIndexingJob(this.db, kb.id, isCancellationError(e) ? "cancelled" : "failed", isCancellationError(e) ? "Indexing cancelled." : "Indexing failed.", e instanceof Error ? e.message : String(e));
|
|
602
|
+
}
|
|
603
|
+
// Clean up partial state on failure
|
|
604
|
+
deleteKB(this.db, kb.id);
|
|
605
|
+
throw e;
|
|
606
|
+
}
|
|
607
|
+
}
|
|
608
|
+
async update(nameOrId, onProgress, signal) {
|
|
609
|
+
if (!this.db)
|
|
610
|
+
throw new Error("Engine not initialized");
|
|
611
|
+
const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
|
|
612
|
+
if (!kb)
|
|
613
|
+
throw new Error(`Knowledge base not found: ${nameOrId}`);
|
|
614
|
+
if (!kb.source_path || (kb.source_type !== "url" && !existsSync(kb.source_path))) {
|
|
615
|
+
throw new Error(`Source path not available or missing: ${kb.source_path}`);
|
|
616
|
+
}
|
|
617
|
+
updateKBStatus(this.db, kb.id, "indexing");
|
|
618
|
+
startIndexingJob(this.db, kb.id, "update", `Starting update for "${kb.name}"`);
|
|
619
|
+
const scanOptions = toScanOptions(parseAddOptions(kb.source_options));
|
|
620
|
+
let replacementVectorPath;
|
|
621
|
+
let addedVectorPath;
|
|
622
|
+
let addedVectorWriter;
|
|
623
|
+
try {
|
|
624
|
+
const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
|
|
625
|
+
addedVectorPath = tempVectorPath(`${vectorPath}.added`);
|
|
626
|
+
addedVectorWriter = openVectorWriter(addedVectorPath);
|
|
627
|
+
const existingHashes = new Map();
|
|
628
|
+
let existingIndex = 0;
|
|
629
|
+
for (const chunk of iterateChunksByKB(this.db, kb.id)) {
|
|
630
|
+
const entries = existingHashes.get(chunk.content_hash) ?? [];
|
|
631
|
+
entries.push({ id: chunk.id, vectorIndex: existingIndex });
|
|
632
|
+
existingHashes.set(chunk.content_hash, entries);
|
|
633
|
+
existingIndex++;
|
|
634
|
+
}
|
|
635
|
+
const oldVectorIndexByHash = new Map();
|
|
636
|
+
const newVectorIndexByHash = new Map();
|
|
637
|
+
const pendingChunks = [];
|
|
638
|
+
let addedVectorCount = 0;
|
|
639
|
+
let addedCount = 0;
|
|
640
|
+
let unchanged = 0;
|
|
641
|
+
let scannedFiles = 0;
|
|
642
|
+
let scannedChunks = 0;
|
|
643
|
+
let plannedTotalFiles;
|
|
644
|
+
const startedAt = Date.now();
|
|
645
|
+
const flushPending = async () => {
|
|
646
|
+
if (!this.db || !addedVectorWriter || pendingChunks.length === 0)
|
|
647
|
+
return;
|
|
648
|
+
if (signal?.aborted)
|
|
649
|
+
throw new Error("Cancelled");
|
|
650
|
+
const batch = pendingChunks.splice(0, INDEX_EMBED_BATCH_SIZE);
|
|
651
|
+
const elapsed = Date.now() - startedAt;
|
|
652
|
+
const message = `Embedding update batch: ${addedCount} new chunks stored, ${scannedFiles} files scanned, elapsed ${formatDuration(elapsed)}`;
|
|
653
|
+
updateIndexingJob(this.db, kb.id, {
|
|
654
|
+
phase: "embedding",
|
|
655
|
+
message,
|
|
656
|
+
processed_files: scannedFiles,
|
|
657
|
+
processed_chunks: scannedChunks,
|
|
658
|
+
total_files: plannedTotalFiles,
|
|
659
|
+
added_chunks: addedCount,
|
|
660
|
+
unchanged_chunks: unchanged,
|
|
661
|
+
});
|
|
662
|
+
onProgress?.(message);
|
|
663
|
+
const newVectors = await embedDocuments(batch.map((c) => buildChunkEmbeddingText(c)), signal);
|
|
664
|
+
if (signal?.aborted)
|
|
665
|
+
throw new Error("Cancelled");
|
|
666
|
+
addedVectorWriter.append(newVectors);
|
|
667
|
+
for (let i = 0; i < batch.length; i++) {
|
|
668
|
+
const indexes = newVectorIndexByHash.get(batch[i].content_hash) ?? [];
|
|
669
|
+
indexes.push(addedVectorCount + i);
|
|
670
|
+
newVectorIndexByHash.set(batch[i].content_hash, indexes);
|
|
671
|
+
}
|
|
672
|
+
addedVectorCount += newVectors.length;
|
|
673
|
+
insertChunks(this.db, kb.id, batch);
|
|
674
|
+
addedCount += batch.length;
|
|
675
|
+
updateKBCounts(this.db, kb.id, getChunkCount(this.db, kb.id), getFileCount(this.db, kb.id));
|
|
676
|
+
const storedMessage = `Stored update batch: +${addedCount} chunks, =${unchanged} unchanged`;
|
|
677
|
+
updateIndexingJob(this.db, kb.id, {
|
|
678
|
+
phase: "storing",
|
|
679
|
+
message: storedMessage,
|
|
680
|
+
processed_files: scannedFiles,
|
|
681
|
+
processed_chunks: scannedChunks,
|
|
682
|
+
total_files: plannedTotalFiles,
|
|
683
|
+
added_chunks: addedCount,
|
|
684
|
+
unchanged_chunks: unchanged,
|
|
685
|
+
});
|
|
686
|
+
onProgress?.(storedMessage);
|
|
687
|
+
};
|
|
688
|
+
const processChunks = async (chunks) => {
|
|
689
|
+
for (const chunk of chunks) {
|
|
690
|
+
scannedChunks++;
|
|
691
|
+
const existing = existingHashes.get(chunk.content_hash);
|
|
692
|
+
if (existing && existing.length > 0) {
|
|
693
|
+
const retained = existing.shift();
|
|
694
|
+
if (retained) {
|
|
695
|
+
const indexes = oldVectorIndexByHash.get(chunk.content_hash) ?? [];
|
|
696
|
+
indexes.push(retained.vectorIndex);
|
|
697
|
+
oldVectorIndexByHash.set(chunk.content_hash, indexes);
|
|
698
|
+
}
|
|
699
|
+
unchanged++;
|
|
700
|
+
continue;
|
|
701
|
+
}
|
|
702
|
+
pendingChunks.push(chunk);
|
|
703
|
+
if (pendingChunks.length >= INDEX_EMBED_BATCH_SIZE)
|
|
704
|
+
await flushPending();
|
|
705
|
+
}
|
|
706
|
+
};
|
|
707
|
+
updateIndexingJob(this.db, kb.id, { phase: "scanning", message: "Scanning source..." });
|
|
708
|
+
onProgress?.("Scanning source...");
|
|
709
|
+
if (kb.source_type === "url") {
|
|
710
|
+
const message = `Fetching ${kb.source_path}...`;
|
|
711
|
+
updateIndexingJob(this.db, kb.id, { phase: "fetching", message });
|
|
712
|
+
onProgress?.(message);
|
|
713
|
+
scannedFiles = 1;
|
|
714
|
+
await processChunks(await chunkUrl(kb.source_path, signal));
|
|
715
|
+
}
|
|
716
|
+
else if (statSync(kb.source_path).isDirectory()) {
|
|
717
|
+
const plan = planDirectoryScan(kb.source_path, scanOptions);
|
|
718
|
+
plannedTotalFiles = plan.files;
|
|
719
|
+
const planningMessage = `Planned directory scan: ${plan.files} files, ${formatBytes(plan.bytes)} scannable text, skipped ${plan.skippedTotal} (${plan.skippedSummary})`;
|
|
720
|
+
updateIndexingJob(this.db, kb.id, {
|
|
721
|
+
phase: "planning",
|
|
722
|
+
message: planningMessage,
|
|
723
|
+
total_files: plan.files,
|
|
724
|
+
skipped_total: plan.skippedTotal,
|
|
725
|
+
});
|
|
726
|
+
onProgress?.(planningMessage);
|
|
727
|
+
const skipped = createSkippedScanStats();
|
|
728
|
+
for (const file of iterateScannedFiles(kb.source_path, skipped, scanOptions)) {
|
|
729
|
+
if (signal?.aborted)
|
|
730
|
+
throw new Error("Cancelled");
|
|
731
|
+
scannedFiles++;
|
|
732
|
+
await processChunks(await chunkFile(file.content, file.relPath));
|
|
733
|
+
if (scannedFiles % 25 === 0) {
|
|
734
|
+
const message = `Scanned ${scannedFiles} files, ${scannedChunks} chunks, skipped ${skipped.total}, +${addedCount} =${unchanged}`;
|
|
735
|
+
updateIndexingJob(this.db, kb.id, {
|
|
736
|
+
phase: "scanning",
|
|
737
|
+
message,
|
|
738
|
+
processed_files: scannedFiles,
|
|
739
|
+
processed_chunks: scannedChunks,
|
|
740
|
+
total_files: plan.files,
|
|
741
|
+
skipped_total: skipped.total,
|
|
742
|
+
added_chunks: addedCount,
|
|
743
|
+
unchanged_chunks: unchanged,
|
|
744
|
+
});
|
|
745
|
+
onProgress?.(message);
|
|
746
|
+
}
|
|
747
|
+
}
|
|
748
|
+
const message = `Scanned ${scannedFiles} files, skipped ${skipped.total} (${summarizeSkippedScan(skipped)}), reconciling deletes...`;
|
|
749
|
+
updateIndexingJob(this.db, kb.id, {
|
|
750
|
+
phase: "reconciling",
|
|
751
|
+
message,
|
|
752
|
+
processed_files: scannedFiles,
|
|
753
|
+
processed_chunks: scannedChunks,
|
|
754
|
+
total_files: plan.files,
|
|
755
|
+
skipped_total: skipped.total,
|
|
756
|
+
added_chunks: addedCount,
|
|
757
|
+
unchanged_chunks: unchanged,
|
|
758
|
+
});
|
|
759
|
+
onProgress?.(message);
|
|
760
|
+
}
|
|
761
|
+
else {
|
|
762
|
+
const { readFileSync } = await import("node:fs");
|
|
763
|
+
scannedFiles = 1;
|
|
764
|
+
await processChunks(await chunkFile(readFileSync(kb.source_path, "utf-8"), kb.source_path));
|
|
765
|
+
}
|
|
766
|
+
await flushPending();
|
|
767
|
+
addedVectorWriter.close();
|
|
768
|
+
addedVectorWriter = undefined;
|
|
769
|
+
const idsToRemove = [];
|
|
770
|
+
for (const entries of existingHashes.values()) {
|
|
771
|
+
idsToRemove.push(...entries.map((entry) => entry.id));
|
|
772
|
+
}
|
|
773
|
+
if (idsToRemove.length > 0)
|
|
774
|
+
deleteChunksByIds(this.db, idsToRemove);
|
|
775
|
+
const changesMessage = `Changes: +${addedCount} -${idsToRemove.length} =${unchanged}`;
|
|
776
|
+
updateIndexingJob(this.db, kb.id, {
|
|
777
|
+
phase: "reconciling",
|
|
778
|
+
message: changesMessage,
|
|
779
|
+
processed_files: scannedFiles,
|
|
780
|
+
processed_chunks: scannedChunks,
|
|
781
|
+
added_chunks: addedCount,
|
|
782
|
+
removed_chunks: idsToRemove.length,
|
|
783
|
+
unchanged_chunks: unchanged,
|
|
784
|
+
});
|
|
785
|
+
onProgress?.(changesMessage);
|
|
786
|
+
replacementVectorPath = tempVectorPath(vectorPath);
|
|
787
|
+
const vectorWriter = openVectorWriter(replacementVectorPath);
|
|
788
|
+
const oldVectorReader = openVectorReader(vectorPath);
|
|
789
|
+
const newVectorReader = openVectorReader(addedVectorPath);
|
|
790
|
+
let finalChunkCount = 0;
|
|
791
|
+
const takeVectorIndex = (indexesByHash, hash) => {
|
|
792
|
+
const indexes = indexesByHash.get(hash);
|
|
793
|
+
return indexes?.shift();
|
|
794
|
+
};
|
|
795
|
+
try {
|
|
796
|
+
for (const chunk of iterateChunksByKB(this.db, kb.id)) {
|
|
797
|
+
const oldVectorIndex = takeVectorIndex(oldVectorIndexByHash, chunk.content_hash);
|
|
798
|
+
const newVectorIndex = takeVectorIndex(newVectorIndexByHash, chunk.content_hash);
|
|
799
|
+
const vector = oldVectorReader && oldVectorIndex !== undefined
|
|
800
|
+
? oldVectorReader.read(oldVectorIndex)
|
|
801
|
+
: newVectorReader && newVectorIndex !== undefined
|
|
802
|
+
? newVectorReader.read(newVectorIndex)
|
|
803
|
+
: undefined;
|
|
804
|
+
if (!vector)
|
|
805
|
+
throw new Error(`Missing vector while rebuilding knowledge base: ${chunk.id}`);
|
|
806
|
+
vectorWriter.append([vector]);
|
|
807
|
+
finalChunkCount++;
|
|
808
|
+
if (finalChunkCount % 1_000 === 0) {
|
|
809
|
+
const message = `Rebuilding vector file: ${finalChunkCount} chunks written`;
|
|
810
|
+
updateIndexingJob(this.db, kb.id, {
|
|
811
|
+
phase: "rebuilding_vectors",
|
|
812
|
+
message,
|
|
813
|
+
processed_files: scannedFiles,
|
|
814
|
+
processed_chunks: finalChunkCount,
|
|
815
|
+
added_chunks: addedCount,
|
|
816
|
+
removed_chunks: idsToRemove.length,
|
|
817
|
+
unchanged_chunks: unchanged,
|
|
818
|
+
});
|
|
819
|
+
onProgress?.(message);
|
|
820
|
+
}
|
|
821
|
+
}
|
|
822
|
+
}
|
|
823
|
+
finally {
|
|
824
|
+
oldVectorReader?.close();
|
|
825
|
+
newVectorReader?.close();
|
|
826
|
+
vectorWriter.close();
|
|
827
|
+
}
|
|
828
|
+
renameSync(replacementVectorPath, vectorPath);
|
|
829
|
+
replacementVectorPath = undefined;
|
|
830
|
+
if (addedVectorPath)
|
|
831
|
+
rmSync(addedVectorPath, { force: true });
|
|
832
|
+
addedVectorPath = undefined;
|
|
833
|
+
updateKBCounts(this.db, kb.id, finalChunkCount, getFileCount(this.db, kb.id));
|
|
834
|
+
updateKBStatus(this.db, kb.id, "ready");
|
|
835
|
+
const readyMessage = `Ready: +${addedCount} -${idsToRemove.length} =${unchanged}`;
|
|
836
|
+
updateIndexingJob(this.db, kb.id, {
|
|
837
|
+
phase: "ready",
|
|
838
|
+
message: readyMessage,
|
|
839
|
+
processed_files: scannedFiles,
|
|
840
|
+
processed_chunks: finalChunkCount,
|
|
841
|
+
added_chunks: addedCount,
|
|
842
|
+
removed_chunks: idsToRemove.length,
|
|
843
|
+
unchanged_chunks: unchanged,
|
|
844
|
+
});
|
|
845
|
+
finishIndexingJob(this.db, kb.id, "succeeded", readyMessage);
|
|
846
|
+
onProgress?.(readyMessage);
|
|
847
|
+
return { added: addedCount, removed: idsToRemove.length, unchanged };
|
|
848
|
+
}
|
|
849
|
+
catch (e) {
|
|
850
|
+
addedVectorWriter?.close();
|
|
851
|
+
if (replacementVectorPath)
|
|
852
|
+
rmSync(replacementVectorPath, { force: true });
|
|
853
|
+
if (addedVectorPath)
|
|
854
|
+
rmSync(addedVectorPath, { force: true });
|
|
855
|
+
updateKBStatus(this.db, kb.id, "error");
|
|
856
|
+
finishIndexingJob(this.db, kb.id, isCancellationError(e) ? "cancelled" : "failed", isCancellationError(e) ? "Update cancelled." : "Update failed.", e instanceof Error ? e.message : String(e));
|
|
857
|
+
throw e;
|
|
858
|
+
}
|
|
859
|
+
}
|
|
860
|
+
async search(query, options = {}) {
|
|
861
|
+
if (!this.db)
|
|
862
|
+
throw new Error("Engine not initialized");
|
|
863
|
+
const requestedMode = options.mode ?? "hybrid";
|
|
864
|
+
if (requestedMode === "auto") {
|
|
865
|
+
const primaryMode = chooseAutoMode(query);
|
|
866
|
+
const attempts = [primaryMode, ...fallbackModesFor(primaryMode)];
|
|
867
|
+
const tried = [];
|
|
868
|
+
const warnings = [];
|
|
869
|
+
for (const mode of attempts) {
|
|
870
|
+
if (tried.includes(mode))
|
|
871
|
+
continue;
|
|
872
|
+
tried.push(mode);
|
|
873
|
+
const response = await this.search(query, { ...options, mode });
|
|
874
|
+
if (response.warnings)
|
|
875
|
+
warnings.push(...response.warnings);
|
|
876
|
+
if (!isWeakAutoResponse(query, response, primaryMode, mode)) {
|
|
877
|
+
return {
|
|
878
|
+
...response,
|
|
879
|
+
warnings: warnings.length > 0 ? [...new Set(warnings)] : response.warnings,
|
|
880
|
+
mode_used: mode,
|
|
881
|
+
retry_modes: tried.slice(0, -1),
|
|
882
|
+
suggestions: response.suggestions,
|
|
883
|
+
};
|
|
884
|
+
}
|
|
885
|
+
if (tried.length >= 3)
|
|
886
|
+
break;
|
|
887
|
+
}
|
|
888
|
+
return {
|
|
889
|
+
results: [],
|
|
890
|
+
total_count: 0,
|
|
891
|
+
has_more: false,
|
|
892
|
+
warnings: warnings.length > 0 ? [...new Set(warnings)] : undefined,
|
|
893
|
+
mode_used: tried.at(-1),
|
|
894
|
+
retry_modes: tried.slice(0, -1),
|
|
895
|
+
suggestions: [
|
|
896
|
+
"No results after auto mode fallback. Check knowledge_status, try a more exact term, or rebuild the KB if indexing rules changed.",
|
|
897
|
+
],
|
|
898
|
+
};
|
|
899
|
+
}
|
|
900
|
+
const db = this.db;
|
|
901
|
+
const { mode = "hybrid", limit = 10, offset = 0, kb_id, filters, diversity = "balanced" } = options;
|
|
902
|
+
const retrievalMode = mode === "adaptive" ? "hybrid" : mode;
|
|
903
|
+
const candidateLimit = Math.max(50, offset + limit * 12);
|
|
904
|
+
const normalizedQuery = normalizedQueryText(query);
|
|
905
|
+
const queryTokens = tokenizeForSimilarity(query);
|
|
906
|
+
const normalizedFileType = normalizeFileTypeFilter(filters?.file_type);
|
|
907
|
+
const warnings = [];
|
|
908
|
+
const selectedKB = kb_id ? (getKB(db, kb_id) ?? getKBByName(db, kb_id)) : undefined;
|
|
909
|
+
const availableKBs = kb_id ? [selectedKB].filter(Boolean) : listKBs(db);
|
|
910
|
+
const kbs = availableKBs.filter((kb) => kb.status === "ready" || kb.status === "stale");
|
|
911
|
+
for (const kb of availableKBs) {
|
|
912
|
+
if (kb.status !== "ready" && kb.status !== "stale") {
|
|
913
|
+
warnings.push(`"${kb.name}" is ${kb.status}; search skipped until indexing is ready`);
|
|
914
|
+
}
|
|
915
|
+
}
|
|
916
|
+
if (kbs.length === 0) {
|
|
917
|
+
return {
|
|
918
|
+
results: [],
|
|
919
|
+
total_count: 0,
|
|
920
|
+
has_more: false,
|
|
921
|
+
warnings: warnings.length > 0 ? warnings : undefined,
|
|
922
|
+
mode_used: mode,
|
|
923
|
+
};
|
|
924
|
+
}
|
|
925
|
+
const allResults = [];
|
|
926
|
+
const vectorsByChunkId = new Map();
|
|
927
|
+
for (const kb of kbs) {
|
|
928
|
+
if (kb.embedding_model !== CURRENT_EMBEDDING_MODEL) {
|
|
929
|
+
warnings.push(`"${kb.name}" was indexed with ${kb.embedding_model} (current: ${CURRENT_EMBEDDING_MODEL}) — run knowledge_update for best results`);
|
|
930
|
+
}
|
|
931
|
+
if (kb.chunk_count === 0)
|
|
932
|
+
continue;
|
|
933
|
+
const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
|
|
934
|
+
if (retrievalMode === "fast") {
|
|
935
|
+
allResults.push(...searchBM25(db, normalizedQuery || query, candidateLimit, kb.id, { allowOrFallback: false }));
|
|
936
|
+
}
|
|
937
|
+
else if (retrievalMode === "semantic") {
|
|
938
|
+
const queryVec = await embedQuery(query);
|
|
939
|
+
const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), candidateLimit);
|
|
940
|
+
allResults.push(...vectorResults.results);
|
|
941
|
+
for (const [chunkId, vector] of vectorResults.vectorsByChunkId)
|
|
942
|
+
vectorsByChunkId.set(chunkId, vector);
|
|
943
|
+
}
|
|
944
|
+
else {
|
|
945
|
+
// hybrid: BM25 + vector weighted fusion (both scoped to this KB)
|
|
946
|
+
const bm25Results = searchBM25(db, normalizedQuery || query, candidateLimit, kb.id);
|
|
947
|
+
if (bm25Results.length === 0)
|
|
948
|
+
continue;
|
|
949
|
+
let vecResults = [];
|
|
950
|
+
const queryVec = await embedQuery(query);
|
|
951
|
+
const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), candidateLimit);
|
|
952
|
+
vecResults = vectorResults.results;
|
|
953
|
+
for (const [chunkId, vector] of vectorResults.vectorsByChunkId)
|
|
954
|
+
vectorsByChunkId.set(chunkId, vector);
|
|
955
|
+
const fused = weightedScoreFusion(bm25Results, vecResults);
|
|
956
|
+
allResults.push(...fused);
|
|
957
|
+
}
|
|
958
|
+
}
|
|
959
|
+
// Deduplicate and sort
|
|
960
|
+
const seen = new Set();
|
|
961
|
+
const unique = allResults.filter((r) => {
|
|
962
|
+
if (seen.has(r.chunkId))
|
|
963
|
+
return false;
|
|
964
|
+
seen.add(r.chunkId);
|
|
965
|
+
return true;
|
|
966
|
+
});
|
|
967
|
+
const rankingByChunkId = new Map();
|
|
968
|
+
for (const result of unique) {
|
|
969
|
+
const chunk = getChunkById(db, result.chunkId);
|
|
970
|
+
if (chunk) {
|
|
971
|
+
const ranking = scoreChunkForQuery(result.score, chunk, queryTokens);
|
|
972
|
+
rankingByChunkId.set(result.chunkId, ranking);
|
|
973
|
+
result.score = ranking.adjusted_score;
|
|
974
|
+
}
|
|
975
|
+
}
|
|
976
|
+
let scored = unique;
|
|
977
|
+
if (retrievalMode === "fast") {
|
|
978
|
+
scored = unique.filter((result) => {
|
|
979
|
+
const chunk = getChunkById(db, result.chunkId);
|
|
980
|
+
if (!chunk)
|
|
981
|
+
return false;
|
|
982
|
+
return hasAnyLexicalEvidence(buildChunkEmbeddingText(chunk), queryTokens);
|
|
983
|
+
});
|
|
984
|
+
}
|
|
985
|
+
else if (retrievalMode !== "semantic") {
|
|
986
|
+
scored = unique.filter((result) => {
|
|
987
|
+
const chunk = getChunkById(db, result.chunkId);
|
|
988
|
+
if (!chunk)
|
|
989
|
+
return false;
|
|
990
|
+
return result.score >= MIN_HYBRID_SCORE && hasEnoughLexicalEvidence(chunk, queryTokens);
|
|
991
|
+
});
|
|
992
|
+
}
|
|
993
|
+
scored.sort((a, b) => b.score - a.score);
|
|
994
|
+
// Apply metadata filters post-retrieval
|
|
995
|
+
let filtered = scored;
|
|
996
|
+
if (normalizedFileType || filters?.path_pattern) {
|
|
997
|
+
filtered = scored.filter((r) => {
|
|
998
|
+
const chunk = getChunkById(db, r.chunkId);
|
|
999
|
+
if (!chunk)
|
|
1000
|
+
return false;
|
|
1001
|
+
if (normalizedFileType && chunk.file_type !== normalizedFileType)
|
|
1002
|
+
return false;
|
|
1003
|
+
if (filters?.path_pattern && !chunk.file_path.includes(filters.path_pattern))
|
|
1004
|
+
return false;
|
|
1005
|
+
return true;
|
|
1006
|
+
});
|
|
1007
|
+
}
|
|
1008
|
+
if (mode === "deep" && filtered.length > 0) {
|
|
1009
|
+
const candidates = filtered
|
|
1010
|
+
.slice(0, 30)
|
|
1011
|
+
.map((r) => {
|
|
1012
|
+
const chunk = getChunkById(db, r.chunkId);
|
|
1013
|
+
return chunk ? { chunkId: r.chunkId, content: chunk.content } : null;
|
|
1014
|
+
})
|
|
1015
|
+
.filter(Boolean);
|
|
1016
|
+
const reranked = await rerank(query, candidates, Math.max(limit * 3, limit));
|
|
1017
|
+
const ranked = [];
|
|
1018
|
+
for (const r of reranked) {
|
|
1019
|
+
const chunk = getChunkById(db, r.chunkId);
|
|
1020
|
+
if (!chunk)
|
|
1021
|
+
continue;
|
|
1022
|
+
const kbObj = getKB(db, chunk.kb_id);
|
|
1023
|
+
ranked.push({
|
|
1024
|
+
chunk,
|
|
1025
|
+
kbName: kbObj?.name ?? "unknown",
|
|
1026
|
+
ranking: scoreChunkForQuery(r.score, chunk, queryTokens),
|
|
1027
|
+
score: r.score,
|
|
1028
|
+
content: chunk.content,
|
|
1029
|
+
snippet: buildQuerySnippet(chunk.content, query),
|
|
1030
|
+
startLine: chunk.start_line,
|
|
1031
|
+
endLine: chunk.end_line,
|
|
1032
|
+
sourceChunkIds: [chunk.id],
|
|
1033
|
+
});
|
|
1034
|
+
}
|
|
1035
|
+
const diversified = interleaveByFile(diversifyRankedChunks(ranked, diversity, vectorsByChunkId), diversity);
|
|
1036
|
+
const page = diversified.slice(offset, offset + limit);
|
|
1037
|
+
const results = page.map((r) => ({
|
|
1038
|
+
content: r.content,
|
|
1039
|
+
file_path: r.chunk.file_path,
|
|
1040
|
+
file_type: r.chunk.file_type,
|
|
1041
|
+
kb_name: r.kbName,
|
|
1042
|
+
score: r.score,
|
|
1043
|
+
ranking: r.ranking,
|
|
1044
|
+
snippet: r.snippet,
|
|
1045
|
+
start_line: r.startLine,
|
|
1046
|
+
end_line: r.endLine,
|
|
1047
|
+
}));
|
|
1048
|
+
return {
|
|
1049
|
+
results,
|
|
1050
|
+
total_count: diversified.length,
|
|
1051
|
+
has_more: offset + limit < diversified.length,
|
|
1052
|
+
warnings: warnings.length > 0 ? warnings : undefined,
|
|
1053
|
+
mode_used: mode,
|
|
1054
|
+
};
|
|
1055
|
+
}
|
|
1056
|
+
const ranked = [];
|
|
1057
|
+
for (const r of filtered) {
|
|
1058
|
+
const chunk = getChunkById(db, r.chunkId);
|
|
1059
|
+
if (!chunk)
|
|
1060
|
+
continue;
|
|
1061
|
+
const kb = getKB(db, chunk.kb_id);
|
|
1062
|
+
if (mode === "adaptive") {
|
|
1063
|
+
const contextChunks = getChunksByFile(db, chunk.kb_id, chunk.file_path, Math.max(1, chunk.start_line - ADAPTIVE_CONTEXT_LINES), chunk.end_line + ADAPTIVE_CONTEXT_LINES);
|
|
1064
|
+
const context = buildAdaptiveContext(chunk, contextChunks.length > 0 ? contextChunks : [chunk], queryTokens);
|
|
1065
|
+
pushAdaptiveCandidate(ranked, {
|
|
1066
|
+
chunk,
|
|
1067
|
+
kbName: kb?.name ?? "unknown",
|
|
1068
|
+
ranking: rankingByChunkId.get(r.chunkId),
|
|
1069
|
+
score: r.score,
|
|
1070
|
+
content: context.content,
|
|
1071
|
+
snippet: buildQuerySnippet(context.content, query),
|
|
1072
|
+
startLine: context.startLine,
|
|
1073
|
+
endLine: context.endLine,
|
|
1074
|
+
sourceChunkIds: context.sourceChunkIds,
|
|
1075
|
+
});
|
|
1076
|
+
}
|
|
1077
|
+
else {
|
|
1078
|
+
ranked.push({
|
|
1079
|
+
chunk,
|
|
1080
|
+
kbName: kb?.name ?? "unknown",
|
|
1081
|
+
ranking: rankingByChunkId.get(r.chunkId),
|
|
1082
|
+
score: r.score,
|
|
1083
|
+
content: chunk.content,
|
|
1084
|
+
snippet: buildQuerySnippet(chunk.content, query),
|
|
1085
|
+
startLine: chunk.start_line,
|
|
1086
|
+
endLine: chunk.end_line,
|
|
1087
|
+
sourceChunkIds: [chunk.id],
|
|
1088
|
+
});
|
|
1089
|
+
}
|
|
1090
|
+
}
|
|
1091
|
+
const diversified = interleaveByFile(diversifyRankedChunks(ranked, diversity, vectorsByChunkId), diversity);
|
|
1092
|
+
const total = diversified.length;
|
|
1093
|
+
const page = diversified.slice(offset, offset + limit);
|
|
1094
|
+
const results = page.map((r) => ({
|
|
1095
|
+
content: r.content,
|
|
1096
|
+
file_path: r.chunk.file_path,
|
|
1097
|
+
file_type: r.chunk.file_type,
|
|
1098
|
+
kb_name: r.kbName,
|
|
1099
|
+
score: r.score,
|
|
1100
|
+
ranking: r.ranking,
|
|
1101
|
+
snippet: r.snippet,
|
|
1102
|
+
start_line: r.startLine,
|
|
1103
|
+
end_line: r.endLine,
|
|
1104
|
+
}));
|
|
1105
|
+
return {
|
|
1106
|
+
results,
|
|
1107
|
+
total_count: total,
|
|
1108
|
+
has_more: offset + limit < total,
|
|
1109
|
+
warnings: warnings.length > 0 ? warnings : undefined,
|
|
1110
|
+
mode_used: mode,
|
|
1111
|
+
suggestions: results.length === 0
|
|
1112
|
+
? [
|
|
1113
|
+
"Try mode 'fast' for exact symbols or mode 'semantic' for conceptual wording.",
|
|
1114
|
+
"Run knowledge_status if the KB should contain this answer.",
|
|
1115
|
+
]
|
|
1116
|
+
: undefined,
|
|
1117
|
+
};
|
|
1118
|
+
}
|
|
1119
|
+
remove(nameOrId) {
|
|
1120
|
+
if (!this.db)
|
|
1121
|
+
return false;
|
|
1122
|
+
const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
|
|
1123
|
+
if (!kb)
|
|
1124
|
+
return false;
|
|
1125
|
+
deleteKB(this.db, kb.id);
|
|
1126
|
+
return true;
|
|
1127
|
+
}
|
|
1128
|
+
list() {
|
|
1129
|
+
if (!this.db)
|
|
1130
|
+
return [];
|
|
1131
|
+
return listKBs(this.db);
|
|
1132
|
+
}
|
|
1133
|
+
clear() {
|
|
1134
|
+
if (!this.db)
|
|
1135
|
+
return;
|
|
1136
|
+
for (const kb of listKBs(this.db))
|
|
1137
|
+
deleteKB(this.db, kb.id);
|
|
1138
|
+
}
|
|
1139
|
+
diagnose() {
|
|
1140
|
+
if (!this.db)
|
|
1141
|
+
return [];
|
|
1142
|
+
const db = this.db;
|
|
1143
|
+
return listKBs(db).map((kb) => diagnoseKB(db, kb));
|
|
1144
|
+
}
|
|
1145
|
+
doctor() {
|
|
1146
|
+
const diagnostics = this.diagnose();
|
|
1147
|
+
const issues = [];
|
|
1148
|
+
const kbs = this.list();
|
|
1149
|
+
if (kbs.length === 0) {
|
|
1150
|
+
issues.push({
|
|
1151
|
+
severity: "blocking",
|
|
1152
|
+
message: "No knowledge bases are indexed.",
|
|
1153
|
+
action: "Run knowledge_add for the project root or relevant source/docs directory.",
|
|
1154
|
+
});
|
|
1155
|
+
}
|
|
1156
|
+
for (const diagnostic of diagnostics) {
|
|
1157
|
+
if (diagnostic.stuck_indexing) {
|
|
1158
|
+
const phase = diagnostic.job ? ` during ${diagnostic.job.phase}` : "";
|
|
1159
|
+
issues.push({
|
|
1160
|
+
severity: "blocking",
|
|
1161
|
+
kb_name: diagnostic.kb_name,
|
|
1162
|
+
message: `Indexing appears stuck${phase} for ${formatDuration(diagnostic.last_progress_age_ms)}.`,
|
|
1163
|
+
action: "Check knowledge_status for the last progress message. If no Pi process is actively indexing it, remove and rebuild this KB.",
|
|
1164
|
+
});
|
|
1165
|
+
}
|
|
1166
|
+
if (diagnostic.status === "error") {
|
|
1167
|
+
issues.push({
|
|
1168
|
+
severity: "blocking",
|
|
1169
|
+
kb_name: diagnostic.kb_name,
|
|
1170
|
+
message: "Knowledge base is in error state and is skipped by search.",
|
|
1171
|
+
action: "Run knowledge_remove and knowledge_add to rebuild it from the source.",
|
|
1172
|
+
});
|
|
1173
|
+
}
|
|
1174
|
+
if (diagnostic.coverage_percent < 80) {
|
|
1175
|
+
issues.push({
|
|
1176
|
+
severity: "warning",
|
|
1177
|
+
kb_name: diagnostic.kb_name,
|
|
1178
|
+
message: `Coverage is ${diagnostic.coverage_percent}% (${diagnostic.indexed_files}/${diagnostic.total_source_files} files).`,
|
|
1179
|
+
action: "Review skipped files and source path. Rebuild if index-time rules changed.",
|
|
1180
|
+
});
|
|
1181
|
+
}
|
|
1182
|
+
if (diagnostic.stale_files.length > 0) {
|
|
1183
|
+
issues.push({
|
|
1184
|
+
severity: "warning",
|
|
1185
|
+
kb_name: diagnostic.kb_name,
|
|
1186
|
+
message: `${diagnostic.stale_files.length} files changed after indexing.`,
|
|
1187
|
+
action: "Run knowledge_update for this KB.",
|
|
1188
|
+
});
|
|
1189
|
+
}
|
|
1190
|
+
if (diagnostic.orphan_files.length > 0) {
|
|
1191
|
+
issues.push({
|
|
1192
|
+
severity: "warning",
|
|
1193
|
+
kb_name: diagnostic.kb_name,
|
|
1194
|
+
message: `${diagnostic.orphan_files.length} indexed files no longer exist in the source.`,
|
|
1195
|
+
action: "Run knowledge_update or rebuild the KB.",
|
|
1196
|
+
});
|
|
1197
|
+
}
|
|
1198
|
+
if (diagnostic.skipped_files.total > 0) {
|
|
1199
|
+
issues.push({
|
|
1200
|
+
severity: "info",
|
|
1201
|
+
kb_name: diagnostic.kb_name,
|
|
1202
|
+
message: `${diagnostic.skipped_files.total} files were skipped while scanning (${Object.entries(diagnostic.skipped_files.by_reason)
|
|
1203
|
+
.filter(([, count]) => count > 0)
|
|
1204
|
+
.map(([reason, count]) => `${reason}: ${count}`)
|
|
1205
|
+
.join(", ")}).`,
|
|
1206
|
+
action: "Use skipped samples to confirm exclusions are expected. Adjust source path or ignore rules if needed.",
|
|
1207
|
+
});
|
|
1208
|
+
}
|
|
1209
|
+
}
|
|
1210
|
+
const penalty = issues.reduce((score, issue) => {
|
|
1211
|
+
if (issue.severity === "blocking")
|
|
1212
|
+
return score + 35;
|
|
1213
|
+
if (issue.severity === "warning")
|
|
1214
|
+
return score + 12;
|
|
1215
|
+
return score + 2;
|
|
1216
|
+
}, 0);
|
|
1217
|
+
const healthScore = Math.max(0, Math.min(100, 100 - penalty));
|
|
1218
|
+
const blocking = issues.filter((issue) => issue.severity === "blocking").length;
|
|
1219
|
+
const warnings = issues.filter((issue) => issue.severity === "warning").length;
|
|
1220
|
+
const summary = issues.length === 0
|
|
1221
|
+
? "Knowledge system is healthy."
|
|
1222
|
+
: `${blocking} blocking, ${warnings} warning, ${issues.length - blocking - warnings} info issues.`;
|
|
1223
|
+
return {
|
|
1224
|
+
health_score: healthScore,
|
|
1225
|
+
summary,
|
|
1226
|
+
issues,
|
|
1227
|
+
diagnostics,
|
|
1228
|
+
};
|
|
1229
|
+
}
|
|
1230
|
+
async exportKB(nameOrId, outputPath) {
|
|
1231
|
+
if (!this.db)
|
|
1232
|
+
throw new Error("Engine not initialized");
|
|
1233
|
+
const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
|
|
1234
|
+
if (!kb)
|
|
1235
|
+
throw new Error(`Knowledge base not found: ${nameOrId}`);
|
|
1236
|
+
const chunks = getChunksByKB(this.db, kb.id);
|
|
1237
|
+
const { writeFileSync } = await import("node:fs");
|
|
1238
|
+
const header = JSON.stringify({
|
|
1239
|
+
name: kb.name,
|
|
1240
|
+
description: kb.description,
|
|
1241
|
+
source_type: "text",
|
|
1242
|
+
chunk_count: chunks.length,
|
|
1243
|
+
});
|
|
1244
|
+
const lines = [
|
|
1245
|
+
header,
|
|
1246
|
+
...chunks.map((c) => JSON.stringify({
|
|
1247
|
+
content: c.content,
|
|
1248
|
+
file_path: c.file_path,
|
|
1249
|
+
file_type: c.file_type,
|
|
1250
|
+
start_line: c.start_line,
|
|
1251
|
+
end_line: c.end_line,
|
|
1252
|
+
metadata_json: c.metadata_json,
|
|
1253
|
+
})),
|
|
1254
|
+
];
|
|
1255
|
+
writeFileSync(outputPath, `${lines.join("\n")}\n`);
|
|
1256
|
+
return chunks.length;
|
|
1257
|
+
}
|
|
1258
|
+
async importKB(inputPath, onProgress, signal) {
|
|
1259
|
+
if (!this.db)
|
|
1260
|
+
throw new Error("Engine not initialized");
|
|
1261
|
+
const { readFileSync } = await import("node:fs");
|
|
1262
|
+
const lines = readFileSync(inputPath, "utf-8").trim().split("\n");
|
|
1263
|
+
if (lines.length < 1)
|
|
1264
|
+
throw new Error("Empty import file");
|
|
1265
|
+
const header = JSON.parse(lines[0]);
|
|
1266
|
+
const kb = createKB(this.db, { name: header.name, description: header.description, source_type: "text" });
|
|
1267
|
+
updateKBStatus(this.db, kb.id, "indexing");
|
|
1268
|
+
startIndexingJob(this.db, kb.id, "import", `Starting import for "${header.name}"`);
|
|
1269
|
+
let vectorWriter;
|
|
1270
|
+
let tempVectorFile;
|
|
1271
|
+
try {
|
|
1272
|
+
const chunkData = lines.slice(1).map((l) => JSON.parse(l));
|
|
1273
|
+
const importMessage = `Importing ${chunkData.length} chunks...`;
|
|
1274
|
+
updateIndexingJob(this.db, kb.id, {
|
|
1275
|
+
phase: "importing",
|
|
1276
|
+
message: importMessage,
|
|
1277
|
+
total_files: chunkData.length,
|
|
1278
|
+
});
|
|
1279
|
+
onProgress?.(importMessage);
|
|
1280
|
+
const chunks = chunkData.map((c) => ({
|
|
1281
|
+
content_hash: chunkIdentityHash({
|
|
1282
|
+
content: c.content,
|
|
1283
|
+
filePath: c.file_path,
|
|
1284
|
+
fileType: c.file_type,
|
|
1285
|
+
startLine: c.start_line,
|
|
1286
|
+
endLine: c.end_line,
|
|
1287
|
+
metadataJson: c.metadata_json || "{}",
|
|
1288
|
+
}),
|
|
1289
|
+
content: c.content,
|
|
1290
|
+
content_tokenized: "",
|
|
1291
|
+
file_path: c.file_path,
|
|
1292
|
+
file_type: c.file_type,
|
|
1293
|
+
start_line: c.start_line,
|
|
1294
|
+
end_line: c.end_line,
|
|
1295
|
+
metadata_json: c.metadata_json || "{}",
|
|
1296
|
+
}));
|
|
1297
|
+
const indexedChunks = chunks.map((chunk) => ({
|
|
1298
|
+
...chunk,
|
|
1299
|
+
content_tokenized: preTokenizeForFTS(buildChunkEmbeddingText(chunk)),
|
|
1300
|
+
}));
|
|
1301
|
+
const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
|
|
1302
|
+
tempVectorFile = tempVectorPath(vectorPath);
|
|
1303
|
+
vectorWriter = openVectorWriter(tempVectorFile);
|
|
1304
|
+
let inserted = 0;
|
|
1305
|
+
for (let offset = 0; offset < indexedChunks.length; offset += INDEX_EMBED_BATCH_SIZE) {
|
|
1306
|
+
if (signal?.aborted)
|
|
1307
|
+
throw new Error("Cancelled");
|
|
1308
|
+
const batch = indexedChunks.slice(offset, offset + INDEX_EMBED_BATCH_SIZE);
|
|
1309
|
+
const message = `Embedding import batch ${Math.floor(offset / INDEX_EMBED_BATCH_SIZE) + 1}/${Math.ceil(indexedChunks.length / INDEX_EMBED_BATCH_SIZE)}: ${offset}/${indexedChunks.length} chunks`;
|
|
1310
|
+
updateIndexingJob(this.db, kb.id, {
|
|
1311
|
+
phase: "embedding",
|
|
1312
|
+
message,
|
|
1313
|
+
processed_chunks: offset,
|
|
1314
|
+
total_files: indexedChunks.length,
|
|
1315
|
+
added_chunks: inserted,
|
|
1316
|
+
});
|
|
1317
|
+
onProgress?.(message);
|
|
1318
|
+
const vectors = await embedDocuments(batch.map((c) => buildChunkEmbeddingText(c)), signal);
|
|
1319
|
+
if (signal?.aborted)
|
|
1320
|
+
throw new Error("Cancelled");
|
|
1321
|
+
insertChunks(this.db, kb.id, batch);
|
|
1322
|
+
vectorWriter.append(vectors);
|
|
1323
|
+
inserted += batch.length;
|
|
1324
|
+
updateKBCounts(this.db, kb.id, inserted, new Set(indexedChunks.slice(0, inserted).map((c) => c.file_path)).size);
|
|
1325
|
+
updateIndexingJob(this.db, kb.id, {
|
|
1326
|
+
phase: "storing",
|
|
1327
|
+
message: `Stored import batch: ${inserted}/${indexedChunks.length} chunks`,
|
|
1328
|
+
processed_chunks: inserted,
|
|
1329
|
+
total_files: indexedChunks.length,
|
|
1330
|
+
added_chunks: inserted,
|
|
1331
|
+
});
|
|
1332
|
+
}
|
|
1333
|
+
vectorWriter.close();
|
|
1334
|
+
vectorWriter = undefined;
|
|
1335
|
+
renameSync(tempVectorFile, vectorPath);
|
|
1336
|
+
tempVectorFile = undefined;
|
|
1337
|
+
updateKBCounts(this.db, kb.id, indexedChunks.length, new Set(indexedChunks.map((c) => c.file_path)).size);
|
|
1338
|
+
updateKBStatus(this.db, kb.id, "ready");
|
|
1339
|
+
const savedKB = getKB(this.db, kb.id);
|
|
1340
|
+
if (!savedKB)
|
|
1341
|
+
throw new Error(`Knowledge base disappeared after import: ${kb.id}`);
|
|
1342
|
+
finishIndexingJob(this.db, kb.id, "succeeded", `Ready: imported ${indexedChunks.length} chunks`);
|
|
1343
|
+
return { kb: savedKB, chunkCount: chunks.length };
|
|
1344
|
+
}
|
|
1345
|
+
catch (e) {
|
|
1346
|
+
vectorWriter?.close();
|
|
1347
|
+
if (tempVectorFile)
|
|
1348
|
+
rmSync(tempVectorFile, { force: true });
|
|
1349
|
+
if (this.db) {
|
|
1350
|
+
finishIndexingJob(this.db, kb.id, isCancellationError(e) ? "cancelled" : "failed", isCancellationError(e) ? "Import cancelled." : "Import failed.", e instanceof Error ? e.message : String(e));
|
|
1351
|
+
}
|
|
1352
|
+
deleteKB(this.db, kb.id);
|
|
1353
|
+
throw e;
|
|
1354
|
+
}
|
|
1355
|
+
}
|
|
1356
|
+
async dispose(options = {}) {
|
|
1357
|
+
const disposeModels = options.disposeModels ?? true;
|
|
1358
|
+
if (disposeModels) {
|
|
1359
|
+
await disposeEmbedding();
|
|
1360
|
+
await disposeReranker();
|
|
1361
|
+
}
|
|
1362
|
+
else {
|
|
1363
|
+
await prepareEmbeddingForShutdown();
|
|
1364
|
+
await prepareRerankerForShutdown();
|
|
1365
|
+
}
|
|
1366
|
+
shutdownModelWorker();
|
|
1367
|
+
this.db?.close();
|
|
1368
|
+
this.db = null;
|
|
1369
|
+
}
|
|
1370
|
+
}
|