pi-knowledge 0.3.4 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.pi/skills/search-docs.md +24 -4
- package/CHANGELOG.md +34 -0
- package/README.md +55 -3
- package/index.ts +263 -20
- package/package.json +1 -1
- package/src/diagnostics/health.ts +84 -14
- package/src/embedding/provider.ts +50 -12
- package/src/embedding/vectors.ts +95 -12
- package/src/engine.ts +1252 -177
- package/src/indexer/chunker.ts +331 -100
- package/src/indexer/chunkers/code-ast.ts +14 -11
- package/src/search/bm25.ts +37 -22
- package/src/search/fusion.ts +33 -0
- package/src/search/query.ts +59 -0
- package/src/search/ranking.ts +201 -0
- package/src/search/reranker.ts +49 -11
- package/src/search/vector.ts +60 -0
- package/src/storage/sqlite.ts +240 -9
package/src/engine.ts
CHANGED
|
@@ -1,39 +1,72 @@
|
|
|
1
|
-
import { existsSync, statSync } from "node:fs";
|
|
1
|
+
import { existsSync, renameSync, rmSync, statSync } from "node:fs";
|
|
2
2
|
import { join, resolve } from "node:path";
|
|
3
3
|
import type Database from "better-sqlite3";
|
|
4
4
|
import { type DiagnosticResult, diagnoseKB } from "./diagnostics/health.ts";
|
|
5
|
-
import {
|
|
6
|
-
|
|
7
|
-
|
|
5
|
+
import {
|
|
6
|
+
dispose as disposeEmbedding,
|
|
7
|
+
embedDocuments,
|
|
8
|
+
embedQuery,
|
|
9
|
+
prepareForShutdown as prepareEmbeddingForShutdown,
|
|
10
|
+
} from "./embedding/provider.ts";
|
|
11
|
+
import { openVectorReader, openVectorWriter } from "./embedding/vectors.ts";
|
|
12
|
+
import {
|
|
13
|
+
buildChunkEmbeddingText,
|
|
14
|
+
chunkFile,
|
|
15
|
+
chunkIdentityHash,
|
|
16
|
+
createSkippedScanStats,
|
|
17
|
+
isReadableTextFile,
|
|
18
|
+
iterateScannableFiles,
|
|
19
|
+
iterateScannedFiles,
|
|
20
|
+
preTokenizeForFTS,
|
|
21
|
+
type ScanOptions,
|
|
22
|
+
summarizeSkippedScan,
|
|
23
|
+
} from "./indexer/chunker.ts";
|
|
8
24
|
import { searchBM25 } from "./search/bm25.ts";
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import { searchVector } from "./search/vector.ts";
|
|
25
|
+
import { weightedScoreFusion } from "./search/fusion.ts";
|
|
26
|
+
import { normalizedQueryText, tokenizeForSearch } from "./search/query.ts";
|
|
12
27
|
import {
|
|
28
|
+
hasAnyLexicalEvidence,
|
|
29
|
+
hasEnoughLexicalEvidence,
|
|
30
|
+
MIN_HYBRID_SCORE,
|
|
31
|
+
normalizeFileTypeFilter,
|
|
32
|
+
queryCoverage,
|
|
33
|
+
type RankingDiagnostics,
|
|
34
|
+
scoreChunkForQuery,
|
|
35
|
+
} from "./search/ranking.ts";
|
|
36
|
+
import { disposeReranker, prepareRerankerForShutdown, rerank } from "./search/reranker.ts";
|
|
37
|
+
import { searchVectorFile } from "./search/vector.ts";
|
|
38
|
+
import {
|
|
39
|
+
type Chunk,
|
|
13
40
|
createKB,
|
|
14
41
|
deleteChunksByIds,
|
|
15
42
|
deleteKB,
|
|
43
|
+
finishIndexingJob,
|
|
16
44
|
getChunkById,
|
|
17
|
-
|
|
18
|
-
|
|
45
|
+
getChunkCount,
|
|
46
|
+
getChunksByFile,
|
|
19
47
|
getChunksByKB,
|
|
20
48
|
getFileCount,
|
|
21
49
|
getKB,
|
|
22
50
|
getKBByName,
|
|
23
51
|
insertChunks,
|
|
52
|
+
iterateChunkIdsByKB,
|
|
53
|
+
iterateChunksByKB,
|
|
24
54
|
type KnowledgeBase,
|
|
25
55
|
listKBs,
|
|
26
56
|
openDatabase,
|
|
57
|
+
startIndexingJob,
|
|
58
|
+
updateIndexingJob,
|
|
27
59
|
updateKBCounts,
|
|
28
60
|
updateKBStatus,
|
|
29
61
|
} from "./storage/sqlite.ts";
|
|
30
62
|
|
|
31
63
|
export interface SearchOptions {
|
|
32
|
-
mode?: "fast" | "semantic" | "hybrid" | "deep";
|
|
64
|
+
mode?: "auto" | "fast" | "semantic" | "hybrid" | "deep" | "adaptive";
|
|
33
65
|
limit?: number;
|
|
34
66
|
offset?: number;
|
|
35
67
|
kb_id?: string;
|
|
36
68
|
filters?: { file_type?: string; path_pattern?: string };
|
|
69
|
+
diversity?: "off" | "balanced" | "strong";
|
|
37
70
|
}
|
|
38
71
|
|
|
39
72
|
export interface SearchResult {
|
|
@@ -45,6 +78,7 @@ export interface SearchResult {
|
|
|
45
78
|
snippet: string;
|
|
46
79
|
start_line: number;
|
|
47
80
|
end_line: number;
|
|
81
|
+
ranking?: RankingDiagnostics;
|
|
48
82
|
}
|
|
49
83
|
|
|
50
84
|
export const CURRENT_EMBEDDING_MODEL = "multilingual-e5-small";
|
|
@@ -54,10 +88,19 @@ export interface SearchResponse {
|
|
|
54
88
|
total_count: number;
|
|
55
89
|
has_more: boolean;
|
|
56
90
|
warnings?: string[];
|
|
91
|
+
mode_used?: NonNullable<SearchOptions["mode"]>;
|
|
92
|
+
retry_modes?: NonNullable<SearchOptions["mode"]>[];
|
|
93
|
+
suggestions?: string[];
|
|
57
94
|
}
|
|
58
95
|
|
|
59
96
|
export type ProgressCallback = (msg: string) => void;
|
|
60
97
|
|
|
98
|
+
export interface AddOptions {
|
|
99
|
+
include_suggested_text?: boolean;
|
|
100
|
+
include_paths?: string[];
|
|
101
|
+
exclude_paths?: string[];
|
|
102
|
+
}
|
|
103
|
+
|
|
61
104
|
interface ImportedChunk {
|
|
62
105
|
content: string;
|
|
63
106
|
file_path: string;
|
|
@@ -67,6 +110,387 @@ interface ImportedChunk {
|
|
|
67
110
|
metadata_json?: string;
|
|
68
111
|
}
|
|
69
112
|
|
|
113
|
+
interface RankedChunk {
|
|
114
|
+
chunk: Chunk;
|
|
115
|
+
kbName: string;
|
|
116
|
+
ranking?: RankingDiagnostics;
|
|
117
|
+
score: number;
|
|
118
|
+
content: string;
|
|
119
|
+
snippet: string;
|
|
120
|
+
startLine: number;
|
|
121
|
+
endLine: number;
|
|
122
|
+
sourceChunkIds: string[];
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
export interface DoctorIssue {
|
|
126
|
+
severity: "blocking" | "warning" | "info";
|
|
127
|
+
kb_name?: string;
|
|
128
|
+
message: string;
|
|
129
|
+
action: string;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
export interface DoctorReport {
|
|
133
|
+
health_score: number;
|
|
134
|
+
summary: string;
|
|
135
|
+
issues: DoctorIssue[];
|
|
136
|
+
diagnostics: DiagnosticResult[];
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
const ADAPTIVE_CONTEXT_LINES = 80;
|
|
140
|
+
const ADAPTIVE_MAX_CONTEXT_CHARS = 6_000;
|
|
141
|
+
const ADAPTIVE_NEIGHBOR_TARGET = 5;
|
|
142
|
+
const INDEX_EMBED_BATCH_SIZE = 64;
|
|
143
|
+
const VECTOR_REDUNDANCY_WEIGHT = 0.35;
|
|
144
|
+
|
|
145
|
+
interface DirectoryScanPlan {
|
|
146
|
+
files: number;
|
|
147
|
+
bytes: number;
|
|
148
|
+
skippedTotal: number;
|
|
149
|
+
skippedSummary: string;
|
|
150
|
+
skipped: ReturnType<typeof createSkippedScanStats>;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
export interface IndexPlan {
|
|
154
|
+
source_type: "file" | "directory" | "text" | "url";
|
|
155
|
+
scannable_files: number;
|
|
156
|
+
scannable_bytes: number;
|
|
157
|
+
skipped: ReturnType<typeof createSkippedScanStats>;
|
|
158
|
+
summary: string;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function isCancellationError(error: unknown): boolean {
|
|
162
|
+
return error instanceof Error && error.message === "Cancelled";
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
function tempVectorPath(vectorPath: string): string {
|
|
166
|
+
return `${vectorPath}.${Date.now()}.${Math.random().toString(16).slice(2)}.tmp`;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
function toScanOptions(options: AddOptions = {}): ScanOptions {
|
|
170
|
+
return {
|
|
171
|
+
includeSuggestedText: options.include_suggested_text === true,
|
|
172
|
+
includePaths: options.include_paths,
|
|
173
|
+
excludePaths: options.exclude_paths,
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
function serializeAddOptions(options: AddOptions = {}): string | undefined {
|
|
178
|
+
const normalized: AddOptions = {};
|
|
179
|
+
if (options.include_suggested_text === true) normalized.include_suggested_text = true;
|
|
180
|
+
if (options.include_paths && options.include_paths.length > 0) normalized.include_paths = options.include_paths;
|
|
181
|
+
if (options.exclude_paths && options.exclude_paths.length > 0) normalized.exclude_paths = options.exclude_paths;
|
|
182
|
+
return Object.keys(normalized).length > 0 ? JSON.stringify(normalized) : undefined;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
function parseAddOptions(raw: string | null): AddOptions {
|
|
186
|
+
if (!raw) return {};
|
|
187
|
+
try {
|
|
188
|
+
const parsed = JSON.parse(raw) as AddOptions;
|
|
189
|
+
return {
|
|
190
|
+
include_suggested_text: parsed.include_suggested_text === true,
|
|
191
|
+
include_paths: Array.isArray(parsed.include_paths)
|
|
192
|
+
? parsed.include_paths.filter((item) => typeof item === "string")
|
|
193
|
+
: undefined,
|
|
194
|
+
exclude_paths: Array.isArray(parsed.exclude_paths)
|
|
195
|
+
? parsed.exclude_paths.filter((item) => typeof item === "string")
|
|
196
|
+
: undefined,
|
|
197
|
+
};
|
|
198
|
+
} catch {
|
|
199
|
+
return {};
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
function planDirectoryScan(dirPath: string, options: ScanOptions = {}): DirectoryScanPlan {
|
|
204
|
+
const skipped = createSkippedScanStats();
|
|
205
|
+
let files = 0;
|
|
206
|
+
let bytes = 0;
|
|
207
|
+
for (const file of iterateScannableFiles(dirPath, skipped, options)) {
|
|
208
|
+
files++;
|
|
209
|
+
bytes += file.size;
|
|
210
|
+
}
|
|
211
|
+
return {
|
|
212
|
+
files,
|
|
213
|
+
bytes,
|
|
214
|
+
skippedTotal: skipped.total,
|
|
215
|
+
skippedSummary: summarizeSkippedScan(skipped),
|
|
216
|
+
skipped,
|
|
217
|
+
};
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
function formatBytes(bytes: number): string {
|
|
221
|
+
if (bytes < 1024) return `${bytes} B`;
|
|
222
|
+
const kb = bytes / 1024;
|
|
223
|
+
if (kb < 1024) return `${kb.toFixed(1)} KB`;
|
|
224
|
+
const mb = kb / 1024;
|
|
225
|
+
if (mb < 1024) return `${mb.toFixed(1)} MB`;
|
|
226
|
+
return `${(mb / 1024).toFixed(2)} GB`;
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
function cosineSimilarity(a: Float32Array | undefined, b: Float32Array | undefined): number {
|
|
230
|
+
if (!a || !b || a.length !== b.length) return 0;
|
|
231
|
+
let dot = 0;
|
|
232
|
+
for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
|
|
233
|
+
return dot;
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
function tokenizeForSimilarity(text: string): Set<string> {
|
|
237
|
+
return tokenizeForSearch(text);
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
function jaccardSimilarity(a: Set<string>, b: Set<string>): number {
|
|
241
|
+
if (a.size === 0 || b.size === 0) return 0;
|
|
242
|
+
let intersection = 0;
|
|
243
|
+
for (const token of a) {
|
|
244
|
+
if (b.has(token)) intersection++;
|
|
245
|
+
}
|
|
246
|
+
return intersection / (a.size + b.size - intersection);
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
function lineProximity(a: RankedChunk, b: RankedChunk): number {
|
|
250
|
+
if (a.chunk.file_path !== b.chunk.file_path || a.chunk.kb_id !== b.chunk.kb_id) return 0;
|
|
251
|
+
if (a.sourceChunkIds.some((id) => b.sourceChunkIds.includes(id))) return 1;
|
|
252
|
+
const overlap = Math.max(0, Math.min(a.endLine, b.endLine) - Math.max(a.startLine, b.startLine) + 1);
|
|
253
|
+
if (overlap > 0) return 1;
|
|
254
|
+
const gap = Math.max(a.startLine, b.startLine) - Math.min(a.endLine, b.endLine);
|
|
255
|
+
if (gap <= 20) return 0.8;
|
|
256
|
+
if (gap <= 80) return 0.45;
|
|
257
|
+
return 0.2;
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
function normalizeScores(candidates: RankedChunk[]): Map<string, number> {
|
|
261
|
+
const scores = candidates.map((candidate) => candidate.score);
|
|
262
|
+
const min = Math.min(...scores);
|
|
263
|
+
const max = Math.max(...scores);
|
|
264
|
+
const normalized = new Map<string, number>();
|
|
265
|
+
for (const candidate of candidates) {
|
|
266
|
+
const value = max === min ? 1 : (candidate.score - min) / (max - min);
|
|
267
|
+
normalized.set(candidate.chunk.id, value);
|
|
268
|
+
}
|
|
269
|
+
return normalized;
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
function diversifyRankedChunks(
|
|
273
|
+
candidates: RankedChunk[],
|
|
274
|
+
diversity: SearchOptions["diversity"],
|
|
275
|
+
vectorsByChunkId: Map<string, Float32Array> = new Map(),
|
|
276
|
+
): RankedChunk[] {
|
|
277
|
+
if (diversity === "off" || candidates.length <= 2) return candidates;
|
|
278
|
+
const lambda = diversity === "strong" ? 0.62 : 0.76;
|
|
279
|
+
const normalized = normalizeScores(candidates);
|
|
280
|
+
const tokenSets = new Map(
|
|
281
|
+
candidates.map((candidate) => [candidate.chunk.id, tokenizeForSimilarity(candidate.content)]),
|
|
282
|
+
);
|
|
283
|
+
const selected: RankedChunk[] = [];
|
|
284
|
+
const remaining = [...candidates];
|
|
285
|
+
|
|
286
|
+
while (remaining.length > 0) {
|
|
287
|
+
let bestIndex = 0;
|
|
288
|
+
let bestScore = Number.NEGATIVE_INFINITY;
|
|
289
|
+
for (let i = 0; i < remaining.length; i++) {
|
|
290
|
+
const candidate = remaining[i];
|
|
291
|
+
let redundancy = 0;
|
|
292
|
+
for (const chosen of selected) {
|
|
293
|
+
const lexical = jaccardSimilarity(
|
|
294
|
+
tokenSets.get(candidate.chunk.id) ?? new Set<string>(),
|
|
295
|
+
tokenSets.get(chosen.chunk.id) ?? new Set<string>(),
|
|
296
|
+
);
|
|
297
|
+
const vector = Math.max(
|
|
298
|
+
0,
|
|
299
|
+
cosineSimilarity(vectorsByChunkId.get(candidate.chunk.id), vectorsByChunkId.get(chosen.chunk.id)),
|
|
300
|
+
);
|
|
301
|
+
redundancy = Math.max(
|
|
302
|
+
redundancy,
|
|
303
|
+
Math.max(lexical, lineProximity(candidate, chosen), vector * VECTOR_REDUNDANCY_WEIGHT),
|
|
304
|
+
);
|
|
305
|
+
}
|
|
306
|
+
const relevance = normalized.get(candidate.chunk.id) ?? 0;
|
|
307
|
+
const mmrScore = lambda * relevance - (1 - lambda) * redundancy;
|
|
308
|
+
if (mmrScore > bestScore) {
|
|
309
|
+
bestScore = mmrScore;
|
|
310
|
+
bestIndex = i;
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
selected.push(remaining.splice(bestIndex, 1)[0]);
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
return selected;
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
function interleaveByFile(candidates: RankedChunk[], diversity: SearchOptions["diversity"]): RankedChunk[] {
|
|
320
|
+
if (diversity === "off" || candidates.length <= 2) return candidates;
|
|
321
|
+
const buckets = new Map<string, RankedChunk[]>();
|
|
322
|
+
const fileOrder: string[] = [];
|
|
323
|
+
for (const candidate of candidates) {
|
|
324
|
+
const key = `${candidate.chunk.kb_id}:${candidate.chunk.file_path}`;
|
|
325
|
+
if (!buckets.has(key)) {
|
|
326
|
+
buckets.set(key, []);
|
|
327
|
+
fileOrder.push(key);
|
|
328
|
+
}
|
|
329
|
+
buckets.get(key)?.push(candidate);
|
|
330
|
+
}
|
|
331
|
+
if (fileOrder.length <= 1) return candidates;
|
|
332
|
+
|
|
333
|
+
const result: RankedChunk[] = [];
|
|
334
|
+
let round = 0;
|
|
335
|
+
while (result.length < candidates.length) {
|
|
336
|
+
let added = false;
|
|
337
|
+
for (const key of fileOrder) {
|
|
338
|
+
const bucket = buckets.get(key);
|
|
339
|
+
const item = bucket?.[round];
|
|
340
|
+
if (!item) continue;
|
|
341
|
+
result.push(item);
|
|
342
|
+
added = true;
|
|
343
|
+
}
|
|
344
|
+
if (!added) break;
|
|
345
|
+
round++;
|
|
346
|
+
}
|
|
347
|
+
return result;
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
function buildQuerySnippet(content: string, query: string, maxLength = 240): string {
|
|
351
|
+
const terms = [...tokenizeForSimilarity(query)].sort((a, b) => b.length - a.length);
|
|
352
|
+
const lower = content.toLowerCase();
|
|
353
|
+
let matchIndex = -1;
|
|
354
|
+
for (const term of terms) {
|
|
355
|
+
matchIndex = lower.indexOf(term.toLowerCase());
|
|
356
|
+
if (matchIndex >= 0) break;
|
|
357
|
+
}
|
|
358
|
+
if (matchIndex < 0) return content.slice(0, maxLength);
|
|
359
|
+
const half = Math.floor(maxLength / 2);
|
|
360
|
+
const start = Math.max(0, matchIndex - half);
|
|
361
|
+
const end = Math.min(content.length, start + maxLength);
|
|
362
|
+
const prefix = start > 0 ? "..." : "";
|
|
363
|
+
const suffix = end < content.length ? "..." : "";
|
|
364
|
+
return `${prefix}${content.slice(start, end)}${suffix}`;
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
function buildAdaptiveContext(
|
|
368
|
+
seed: Chunk,
|
|
369
|
+
chunks: Chunk[],
|
|
370
|
+
queryTokens: Set<string>,
|
|
371
|
+
): {
|
|
372
|
+
content: string;
|
|
373
|
+
startLine: number;
|
|
374
|
+
endLine: number;
|
|
375
|
+
sourceChunkIds: string[];
|
|
376
|
+
} {
|
|
377
|
+
const scored = chunks
|
|
378
|
+
.map((chunk) => {
|
|
379
|
+
const distance =
|
|
380
|
+
chunk.id === seed.id
|
|
381
|
+
? 0
|
|
382
|
+
: Math.min(Math.abs(chunk.start_line - seed.start_line), Math.abs(chunk.end_line - seed.end_line));
|
|
383
|
+
const proximity = 1 / (1 + distance / 20);
|
|
384
|
+
const coverage = queryCoverage(chunk.content, queryTokens);
|
|
385
|
+
const seedBoost = chunk.id === seed.id ? 2 : 0;
|
|
386
|
+
return { chunk, score: seedBoost + proximity + coverage };
|
|
387
|
+
})
|
|
388
|
+
.sort((a, b) => b.score - a.score)
|
|
389
|
+
.slice(0, ADAPTIVE_NEIGHBOR_TARGET);
|
|
390
|
+
if (!scored.some((item) => item.chunk.id === seed.id)) scored.push({ chunk: seed, score: Number.MAX_SAFE_INTEGER });
|
|
391
|
+
|
|
392
|
+
const selectedIds = new Set(scored.map((item) => item.chunk.id));
|
|
393
|
+
const ordered = chunks
|
|
394
|
+
.filter((chunk) => selectedIds.has(chunk.id))
|
|
395
|
+
.sort((a, b) => a.start_line - b.start_line || a.end_line - b.end_line);
|
|
396
|
+
const parts: string[] = [];
|
|
397
|
+
const sourceChunkIds: string[] = [];
|
|
398
|
+
let totalLength = 0;
|
|
399
|
+
for (const chunk of ordered) {
|
|
400
|
+
if (totalLength >= ADAPTIVE_MAX_CONTEXT_CHARS) break;
|
|
401
|
+
const remaining = ADAPTIVE_MAX_CONTEXT_CHARS - totalLength;
|
|
402
|
+
const text = chunk.content.slice(0, remaining);
|
|
403
|
+
parts.push(text);
|
|
404
|
+
sourceChunkIds.push(chunk.id);
|
|
405
|
+
totalLength += text.length + 2;
|
|
406
|
+
}
|
|
407
|
+
return {
|
|
408
|
+
content: parts.join("\n\n"),
|
|
409
|
+
startLine: Math.min(...ordered.map((chunk) => chunk.start_line)),
|
|
410
|
+
endLine: Math.max(...ordered.map((chunk) => chunk.end_line)),
|
|
411
|
+
sourceChunkIds,
|
|
412
|
+
};
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
function pushAdaptiveCandidate(candidates: RankedChunk[], candidate: RankedChunk): void {
|
|
416
|
+
const overlappingIndex = candidates.findIndex(
|
|
417
|
+
(existing) =>
|
|
418
|
+
existing.chunk.kb_id === candidate.chunk.kb_id &&
|
|
419
|
+
existing.chunk.file_path === candidate.chunk.file_path &&
|
|
420
|
+
existing.sourceChunkIds.some((id) => candidate.sourceChunkIds.includes(id)),
|
|
421
|
+
);
|
|
422
|
+
if (overlappingIndex < 0) {
|
|
423
|
+
candidates.push(candidate);
|
|
424
|
+
return;
|
|
425
|
+
}
|
|
426
|
+
const existing = candidates[overlappingIndex];
|
|
427
|
+
if (candidate.score > existing.score || candidate.sourceChunkIds.length > existing.sourceChunkIds.length) {
|
|
428
|
+
candidates[overlappingIndex] = candidate;
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
function formatDuration(ms: number): string {
|
|
433
|
+
if (!Number.isFinite(ms) || ms <= 0) return "0s";
|
|
434
|
+
const seconds = Math.ceil(ms / 1000);
|
|
435
|
+
if (seconds < 60) return `${seconds}s`;
|
|
436
|
+
const minutes = Math.floor(seconds / 60);
|
|
437
|
+
const rest = seconds % 60;
|
|
438
|
+
return rest === 0 ? `${minutes}m` : `${minutes}m ${rest}s`;
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
function isExactLookupQuery(query: string): boolean {
|
|
442
|
+
const trimmed = query.trim();
|
|
443
|
+
if (/["'`]/.test(trimmed)) return true;
|
|
444
|
+
if (/[./\\][\w.-]+/.test(trimmed)) return true;
|
|
445
|
+
if (/\b[A-Z][A-Za-z0-9_]*[A-Z][A-Za-z0-9_]*\b/.test(trimmed)) return true;
|
|
446
|
+
if (/\b[A-Z]{2,}[_-]?[A-Z0-9]*\b/.test(trimmed)) return true;
|
|
447
|
+
if (/\b[a-zA-Z_][\w-]*\.(ts|tsx|js|jsx|go|rs|py|java|md|json|ya?ml|toml)\b/.test(trimmed)) return true;
|
|
448
|
+
if (/\b[A-Z]+-\d+\b/.test(trimmed)) return true;
|
|
449
|
+
return false;
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
function chooseAutoMode(query: string): NonNullable<SearchOptions["mode"]> {
|
|
453
|
+
const normalized = query.trim();
|
|
454
|
+
if (isExactLookupQuery(normalized)) return "fast";
|
|
455
|
+
const wordCount = normalized.split(/\s+/).filter(Boolean).length;
|
|
456
|
+
if (wordCount >= 10 || /how|why|explain|concept|architecture|design|流程|架構|概念|為什麼|如何/i.test(normalized)) {
|
|
457
|
+
return "semantic";
|
|
458
|
+
}
|
|
459
|
+
return "hybrid";
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
function fallbackModesFor(mode: NonNullable<SearchOptions["mode"]>): NonNullable<SearchOptions["mode"]>[] {
|
|
463
|
+
if (mode === "fast") return ["hybrid", "semantic"];
|
|
464
|
+
if (mode === "semantic") return ["hybrid", "adaptive"];
|
|
465
|
+
if (mode === "adaptive") return ["hybrid", "deep"];
|
|
466
|
+
if (mode === "deep") return ["hybrid", "adaptive"];
|
|
467
|
+
return ["fast", "semantic", "adaptive"];
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
function isWeakAutoResponse(
|
|
471
|
+
query: string,
|
|
472
|
+
response: SearchResponse,
|
|
473
|
+
primaryMode: NonNullable<SearchOptions["mode"]>,
|
|
474
|
+
attemptMode: NonNullable<SearchOptions["mode"]>,
|
|
475
|
+
): boolean {
|
|
476
|
+
if (response.results.length === 0) return true;
|
|
477
|
+
if (primaryMode === "fast") {
|
|
478
|
+
const exactNeedle = query
|
|
479
|
+
.trim()
|
|
480
|
+
.replace(/^["'`]|["'`]$/g, "")
|
|
481
|
+
.toLowerCase();
|
|
482
|
+
const hasExactHit = response.results.some(
|
|
483
|
+
(result) =>
|
|
484
|
+
result.file_path.toLowerCase().includes(exactNeedle) || result.content.toLowerCase().includes(exactNeedle),
|
|
485
|
+
);
|
|
486
|
+
if (!hasExactHit && response.results.every((result) => (result.ranking?.coverage ?? 0) < 0.67)) return true;
|
|
487
|
+
}
|
|
488
|
+
if (primaryMode !== "semantic" && attemptMode === "semantic") {
|
|
489
|
+
return response.results.every((result) => (result.ranking?.coverage ?? 0) === 0);
|
|
490
|
+
}
|
|
491
|
+
return false;
|
|
492
|
+
}
|
|
493
|
+
|
|
70
494
|
async function chunkUrl(source: string, signal?: AbortSignal): Promise<Awaited<ReturnType<typeof chunkFile>>> {
|
|
71
495
|
const res = await fetch(source, { signal });
|
|
72
496
|
if (!res.ok) throw new Error(`Fetch failed: ${res.status}`);
|
|
@@ -88,25 +512,61 @@ function normalizeExtractedText(text: string | string[]): string {
|
|
|
88
512
|
export class KnowledgeEngine {
|
|
89
513
|
private db: Database.Database | null = null;
|
|
90
514
|
private knowledgeDir: string = "";
|
|
91
|
-
private vectorCache: Map<string, Float32Array[]> = new Map();
|
|
92
515
|
|
|
93
516
|
async initialize(knowledgeDir: string): Promise<void> {
|
|
94
517
|
this.knowledgeDir = knowledgeDir;
|
|
95
518
|
this.db = openDatabase(knowledgeDir);
|
|
96
|
-
this.vectorCache.clear();
|
|
97
519
|
}
|
|
98
520
|
|
|
99
|
-
|
|
100
|
-
const
|
|
101
|
-
|
|
102
|
-
const
|
|
103
|
-
const
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
521
|
+
plan(source: string, options: AddOptions = {}): IndexPlan {
|
|
522
|
+
const resolvedSource = resolve(source);
|
|
523
|
+
const isUrl = source.startsWith("http://") || source.startsWith("https://");
|
|
524
|
+
const isDir = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isDirectory();
|
|
525
|
+
const isFile = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isFile();
|
|
526
|
+
const sourceType = isUrl ? "url" : isDir ? "directory" : isFile ? "file" : "text";
|
|
527
|
+
if (isDir) {
|
|
528
|
+
const plan = planDirectoryScan(resolvedSource, toScanOptions(options));
|
|
529
|
+
return {
|
|
530
|
+
source_type: "directory",
|
|
531
|
+
scannable_files: plan.files,
|
|
532
|
+
scannable_bytes: plan.bytes,
|
|
533
|
+
skipped: plan.skipped,
|
|
534
|
+
summary: `Directory plan: ${plan.files} scannable files, ${formatBytes(plan.bytes)} scannable text, skipped ${plan.skippedTotal} (${plan.skippedSummary})`,
|
|
535
|
+
};
|
|
536
|
+
}
|
|
537
|
+
if (isFile) {
|
|
538
|
+
const skipped = createSkippedScanStats();
|
|
539
|
+
const supportedDocument =
|
|
540
|
+
resolvedSource.endsWith(".pdf") || resolvedSource.endsWith(".docx") || resolvedSource.endsWith(".doc");
|
|
541
|
+
if (!supportedDocument && !isReadableTextFile(resolvedSource)) {
|
|
542
|
+
skipped.total = 1;
|
|
543
|
+
skipped.by_reason.binary = 1;
|
|
544
|
+
skipped.samples.push({ path: resolvedSource, reason: "binary" });
|
|
545
|
+
return {
|
|
546
|
+
source_type: "file",
|
|
547
|
+
scannable_files: 0,
|
|
548
|
+
scannable_bytes: 0,
|
|
549
|
+
skipped,
|
|
550
|
+
summary: "File plan: 0 scannable files; file is unsupported binary/non-text",
|
|
551
|
+
};
|
|
552
|
+
}
|
|
553
|
+
const size = statSync(resolvedSource).size;
|
|
554
|
+
return {
|
|
555
|
+
source_type: "file",
|
|
556
|
+
scannable_files: 1,
|
|
557
|
+
scannable_bytes: size,
|
|
558
|
+
skipped,
|
|
559
|
+
summary: `File plan: 1 scannable file, ${formatBytes(size)} source size`,
|
|
560
|
+
};
|
|
561
|
+
}
|
|
562
|
+
const skipped = createSkippedScanStats();
|
|
563
|
+
return {
|
|
564
|
+
source_type: sourceType,
|
|
565
|
+
scannable_files: 1,
|
|
566
|
+
scannable_bytes: Buffer.byteLength(source),
|
|
567
|
+
skipped,
|
|
568
|
+
summary: `${sourceType === "url" ? "URL" : "Inline text"} plan: 1 source`,
|
|
569
|
+
};
|
|
110
570
|
}
|
|
111
571
|
|
|
112
572
|
async add(
|
|
@@ -114,6 +574,7 @@ export class KnowledgeEngine {
|
|
|
114
574
|
name: string,
|
|
115
575
|
onProgress?: ProgressCallback,
|
|
116
576
|
signal?: AbortSignal,
|
|
577
|
+
options: AddOptions = {},
|
|
117
578
|
): Promise<{ kb: KnowledgeBase; chunkCount: number }> {
|
|
118
579
|
if (!this.db) throw new Error("Engine not initialized");
|
|
119
580
|
const resolvedSource = resolve(source);
|
|
@@ -121,6 +582,7 @@ export class KnowledgeEngine {
|
|
|
121
582
|
const isDir = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isDirectory();
|
|
122
583
|
const isFile = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isFile();
|
|
123
584
|
const sourceType = isUrl ? "url" : isDir ? "directory" : isFile ? "file" : "text";
|
|
585
|
+
const scanOptions = toScanOptions(options);
|
|
124
586
|
|
|
125
587
|
const existingKB = getKBByName(this.db, name);
|
|
126
588
|
if (existingKB) {
|
|
@@ -133,60 +595,195 @@ export class KnowledgeEngine {
|
|
|
133
595
|
name,
|
|
134
596
|
source_path: isDir || isFile ? resolvedSource : isUrl ? source : undefined,
|
|
135
597
|
source_type: sourceType,
|
|
598
|
+
source_options: serializeAddOptions(options),
|
|
136
599
|
});
|
|
137
600
|
updateKBStatus(this.db, kb.id, "indexing");
|
|
601
|
+
startIndexingJob(this.db, kb.id, "add", `Starting indexing for "${name}"`);
|
|
138
602
|
|
|
603
|
+
let vectorWriter: ReturnType<typeof openVectorWriter> | undefined;
|
|
604
|
+
let tempVectorFile: string | undefined;
|
|
139
605
|
try {
|
|
140
|
-
|
|
606
|
+
const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
|
|
607
|
+
tempVectorFile = tempVectorPath(vectorPath);
|
|
608
|
+
vectorWriter = openVectorWriter(tempVectorFile);
|
|
609
|
+
let chunkCount = 0;
|
|
610
|
+
let fileCount = 0;
|
|
611
|
+
const pendingChunks: Awaited<ReturnType<typeof chunkFile>> = [];
|
|
612
|
+
const startedAt = Date.now();
|
|
613
|
+
let latestSkippedTotal = 0;
|
|
614
|
+
|
|
615
|
+
const reportProgress = (phase: string, processedFiles?: number, totalFiles?: number, skippedTotal = 0): void => {
|
|
616
|
+
latestSkippedTotal = skippedTotal;
|
|
617
|
+
const elapsed = Date.now() - startedAt;
|
|
618
|
+
const chunkRate = chunkCount / Math.max(1, elapsed / 1000);
|
|
619
|
+
let suffix = `${chunkCount} chunks, ${chunkRate.toFixed(1)} chunks/s, elapsed ${formatDuration(elapsed)}`;
|
|
620
|
+
if (processedFiles !== undefined && totalFiles !== undefined && totalFiles > 0) {
|
|
621
|
+
const rate = processedFiles / Math.max(1, elapsed / 1000);
|
|
622
|
+
const remainingFiles = Math.max(0, totalFiles - processedFiles);
|
|
623
|
+
const etaMs = rate > 0 ? (remainingFiles / rate) * 1000 : 0;
|
|
624
|
+
suffix = `${processedFiles}/${totalFiles} files, ${suffix}, file ETA ${formatDuration(etaMs)}`;
|
|
625
|
+
} else if (processedFiles !== undefined) {
|
|
626
|
+
suffix = `${processedFiles} files scanned, ${suffix}`;
|
|
627
|
+
}
|
|
628
|
+
if (skippedTotal > 0) suffix = `${suffix}, skipped ${skippedTotal}`;
|
|
629
|
+
const message = `${phase}: ${suffix}`;
|
|
630
|
+
updateIndexingJob(this.db, kb.id, {
|
|
631
|
+
phase,
|
|
632
|
+
message,
|
|
633
|
+
processed_files: processedFiles,
|
|
634
|
+
processed_chunks: chunkCount,
|
|
635
|
+
total_files: totalFiles,
|
|
636
|
+
skipped_total: skippedTotal,
|
|
637
|
+
added_chunks: chunkCount,
|
|
638
|
+
});
|
|
639
|
+
onProgress?.(message);
|
|
640
|
+
};
|
|
641
|
+
|
|
642
|
+
const flushPending = async (processedFiles?: number, totalFiles?: number): Promise<void> => {
|
|
643
|
+
if (!this.db || pendingChunks.length === 0) return;
|
|
644
|
+
if (signal?.aborted) throw new Error("Cancelled");
|
|
645
|
+
const batch = pendingChunks.splice(0, INDEX_EMBED_BATCH_SIZE);
|
|
646
|
+
reportProgress(`Embedding batch of ${batch.length}`, processedFiles, totalFiles);
|
|
647
|
+
const vectors = await embedDocuments(
|
|
648
|
+
batch.map((chunk) => buildChunkEmbeddingText(chunk)),
|
|
649
|
+
signal,
|
|
650
|
+
);
|
|
651
|
+
if (signal?.aborted) throw new Error("Cancelled");
|
|
652
|
+
insertChunks(this.db, kb.id, batch);
|
|
653
|
+
vectorWriter.append(vectors);
|
|
654
|
+
chunkCount += batch.length;
|
|
655
|
+
updateKBCounts(this.db, kb.id, chunkCount, fileCount);
|
|
656
|
+
reportProgress("Stored batch", processedFiles, totalFiles);
|
|
657
|
+
};
|
|
658
|
+
|
|
659
|
+
const addChunks = async (
|
|
660
|
+
chunks: Awaited<ReturnType<typeof chunkFile>>,
|
|
661
|
+
processedFiles?: number,
|
|
662
|
+
totalFiles?: number,
|
|
663
|
+
): Promise<void> => {
|
|
664
|
+
pendingChunks.push(...chunks);
|
|
665
|
+
while (pendingChunks.length >= INDEX_EMBED_BATCH_SIZE) {
|
|
666
|
+
await flushPending(processedFiles, totalFiles);
|
|
667
|
+
}
|
|
668
|
+
};
|
|
141
669
|
|
|
142
670
|
if (isUrl) {
|
|
143
|
-
|
|
144
|
-
|
|
671
|
+
const message = `Fetching ${source}...`;
|
|
672
|
+
updateIndexingJob(this.db, kb.id, { phase: "fetching", message });
|
|
673
|
+
onProgress?.(message);
|
|
674
|
+
const chunks = await chunkUrl(source, signal);
|
|
675
|
+
fileCount = 1;
|
|
676
|
+
await addChunks(chunks);
|
|
145
677
|
} else if (isFile && resolvedSource.endsWith(".pdf")) {
|
|
678
|
+
updateIndexingJob(this.db, kb.id, { phase: "extracting", message: "Extracting text from PDF..." });
|
|
146
679
|
onProgress?.("Extracting text from PDF...");
|
|
147
680
|
const { extractText } = await import("unpdf");
|
|
148
681
|
const buf = (await import("node:fs")).readFileSync(resolvedSource);
|
|
149
682
|
const { text } = await extractText(new Uint8Array(buf));
|
|
150
|
-
|
|
683
|
+
fileCount = 1;
|
|
684
|
+
await addChunks(await chunkFile(normalizeExtractedText(text), resolvedSource));
|
|
151
685
|
} else if (isFile && (resolvedSource.endsWith(".docx") || resolvedSource.endsWith(".doc"))) {
|
|
686
|
+
updateIndexingJob(this.db, kb.id, { phase: "extracting", message: "Extracting text from DOCX..." });
|
|
152
687
|
onProgress?.("Extracting text from DOCX...");
|
|
153
688
|
const mammoth = await import("mammoth");
|
|
154
689
|
const result = await mammoth.extractRawText({ path: resolvedSource });
|
|
155
|
-
|
|
690
|
+
fileCount = 1;
|
|
691
|
+
await addChunks(await chunkFile(result.value, resolvedSource));
|
|
156
692
|
} else if (isDir) {
|
|
157
|
-
|
|
158
|
-
const
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
693
|
+
const plan = planDirectoryScan(resolvedSource, scanOptions);
|
|
694
|
+
const planningMessage = `Planned directory scan: ${plan.files} files, ${formatBytes(
|
|
695
|
+
plan.bytes,
|
|
696
|
+
)} scannable text, skipped ${plan.skippedTotal} (${plan.skippedSummary})`;
|
|
697
|
+
updateIndexingJob(this.db, kb.id, {
|
|
698
|
+
phase: "planning",
|
|
699
|
+
message: planningMessage,
|
|
700
|
+
total_files: plan.files,
|
|
701
|
+
skipped_total: plan.skippedTotal,
|
|
702
|
+
});
|
|
703
|
+
onProgress?.(planningMessage);
|
|
704
|
+
const scanningMessage = `Scanning ${resolvedSource}...`;
|
|
705
|
+
updateIndexingJob(this.db, kb.id, {
|
|
706
|
+
phase: "scanning",
|
|
707
|
+
message: scanningMessage,
|
|
708
|
+
total_files: plan.files,
|
|
709
|
+
skipped_total: plan.skippedTotal,
|
|
710
|
+
});
|
|
711
|
+
onProgress?.(scanningMessage);
|
|
712
|
+
const skipped = createSkippedScanStats();
|
|
713
|
+
let processedFiles = 0;
|
|
714
|
+
for (const file of iterateScannedFiles(resolvedSource, skipped, scanOptions)) {
|
|
715
|
+
if (signal?.aborted) throw new Error("Cancelled");
|
|
716
|
+
const chunks = await chunkFile(file.content, file.relPath);
|
|
717
|
+
processedFiles++;
|
|
718
|
+
latestSkippedTotal = skipped.total;
|
|
719
|
+
if (chunks.length > 0) fileCount++;
|
|
720
|
+
await addChunks(chunks, processedFiles, plan.files);
|
|
721
|
+
if (processedFiles % 25 === 0) reportProgress("Chunking", processedFiles, plan.files, skipped.total);
|
|
162
722
|
}
|
|
723
|
+
const finalizingMessage = `Scanned ${processedFiles} files, skipped ${skipped.total} (${summarizeSkippedScan(
|
|
724
|
+
skipped,
|
|
725
|
+
)}), finalizing...`;
|
|
726
|
+
updateIndexingJob(this.db, kb.id, {
|
|
727
|
+
phase: "finalizing",
|
|
728
|
+
message: finalizingMessage,
|
|
729
|
+
processed_files: processedFiles,
|
|
730
|
+
processed_chunks: chunkCount,
|
|
731
|
+
total_files: plan.files,
|
|
732
|
+
skipped_total: skipped.total,
|
|
733
|
+
added_chunks: chunkCount,
|
|
734
|
+
});
|
|
735
|
+
onProgress?.(finalizingMessage);
|
|
163
736
|
} else if (isFile) {
|
|
737
|
+
if (!isReadableTextFile(resolvedSource)) {
|
|
738
|
+
throw new Error(`File is not readable text and has no supported extractor: ${resolvedSource}`);
|
|
739
|
+
}
|
|
164
740
|
const { readFileSync } = await import("node:fs");
|
|
165
741
|
const content = readFileSync(resolvedSource, "utf-8");
|
|
166
|
-
|
|
742
|
+
fileCount = 1;
|
|
743
|
+
await addChunks(await chunkFile(content, resolvedSource));
|
|
167
744
|
} else {
|
|
168
|
-
|
|
745
|
+
fileCount = 1;
|
|
746
|
+
await addChunks(await chunkFile(source, "inline-text"));
|
|
169
747
|
}
|
|
170
748
|
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
insertChunks(this.db, kb.id, allChunks);
|
|
177
|
-
|
|
178
|
-
const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
|
|
179
|
-
saveVectors(vectorPath, vectors);
|
|
180
|
-
this.vectorCache.set(kb.id, vectors);
|
|
749
|
+
await flushPending();
|
|
750
|
+
vectorWriter.close();
|
|
751
|
+
vectorWriter = undefined;
|
|
752
|
+
renameSync(tempVectorFile, vectorPath);
|
|
753
|
+
tempVectorFile = undefined;
|
|
181
754
|
|
|
182
|
-
const
|
|
183
|
-
updateKBCounts(this.db, kb.id,
|
|
755
|
+
const savedFileCount = isDir ? getFileCount(this.db, kb.id) : fileCount;
|
|
756
|
+
updateKBCounts(this.db, kb.id, chunkCount, savedFileCount);
|
|
184
757
|
updateKBStatus(this.db, kb.id, "ready");
|
|
185
758
|
|
|
186
759
|
const savedKB = getKB(this.db, kb.id);
|
|
187
760
|
if (!savedKB) throw new Error(`Knowledge base disappeared after add: ${kb.id}`);
|
|
188
|
-
|
|
761
|
+
const readyMessage = `Ready: ${chunkCount} chunks from ${savedFileCount} files in ${formatDuration(
|
|
762
|
+
Date.now() - startedAt,
|
|
763
|
+
)}`;
|
|
764
|
+
updateIndexingJob(this.db, kb.id, {
|
|
765
|
+
phase: "ready",
|
|
766
|
+
message: readyMessage,
|
|
767
|
+
processed_files: savedFileCount,
|
|
768
|
+
processed_chunks: chunkCount,
|
|
769
|
+
skipped_total: latestSkippedTotal,
|
|
770
|
+
added_chunks: chunkCount,
|
|
771
|
+
});
|
|
772
|
+
finishIndexingJob(this.db, kb.id, "succeeded", readyMessage);
|
|
773
|
+
onProgress?.(readyMessage);
|
|
774
|
+
return { kb: savedKB, chunkCount };
|
|
189
775
|
} catch (e) {
|
|
776
|
+
vectorWriter?.close();
|
|
777
|
+
if (tempVectorFile) rmSync(tempVectorFile, { force: true });
|
|
778
|
+
if (this.db) {
|
|
779
|
+
finishIndexingJob(
|
|
780
|
+
this.db,
|
|
781
|
+
kb.id,
|
|
782
|
+
isCancellationError(e) ? "cancelled" : "failed",
|
|
783
|
+
isCancellationError(e) ? "Indexing cancelled." : "Indexing failed.",
|
|
784
|
+
e instanceof Error ? e.message : String(e),
|
|
785
|
+
);
|
|
786
|
+
}
|
|
190
787
|
// Clean up partial state on failure
|
|
191
788
|
deleteKB(this.db, kb.id);
|
|
192
789
|
throw e;
|
|
@@ -206,92 +803,325 @@ export class KnowledgeEngine {
|
|
|
206
803
|
}
|
|
207
804
|
|
|
208
805
|
updateKBStatus(this.db, kb.id, "indexing");
|
|
806
|
+
startIndexingJob(this.db, kb.id, "update", `Starting update for "${kb.name}"`);
|
|
807
|
+
const scanOptions = toScanOptions(parseAddOptions(kb.source_options));
|
|
808
|
+
let replacementVectorPath: string | undefined;
|
|
809
|
+
let addedVectorPath: string | undefined;
|
|
810
|
+
let addedVectorWriter: ReturnType<typeof openVectorWriter> | undefined;
|
|
209
811
|
|
|
210
812
|
try {
|
|
211
|
-
// 1. Scan and chunk current source
|
|
212
|
-
onProgress?.("Scanning source...");
|
|
213
|
-
let newChunks: Awaited<ReturnType<typeof chunkFile>> = [];
|
|
214
|
-
if (kb.source_type === "url") {
|
|
215
|
-
onProgress?.(`Fetching ${kb.source_path}...`);
|
|
216
|
-
newChunks = await chunkUrl(kb.source_path, signal);
|
|
217
|
-
} else if (statSync(kb.source_path).isDirectory()) {
|
|
218
|
-
const files = walkDir(kb.source_path);
|
|
219
|
-
for (const file of files) newChunks.push(...(await chunkFile(file.content, file.relPath)));
|
|
220
|
-
} else {
|
|
221
|
-
const { readFileSync } = await import("node:fs");
|
|
222
|
-
newChunks = await chunkFile(readFileSync(kb.source_path, "utf-8"), kb.source_path);
|
|
223
|
-
}
|
|
224
|
-
|
|
225
|
-
// 2. Load existing state: chunks (hash→id) + vectors (ordered)
|
|
226
|
-
const existingHashes = getChunkHashesByKB(this.db, kb.id);
|
|
227
813
|
const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
const
|
|
233
|
-
|
|
234
|
-
|
|
814
|
+
addedVectorPath = tempVectorPath(`${vectorPath}.added`);
|
|
815
|
+
addedVectorWriter = openVectorWriter(addedVectorPath);
|
|
816
|
+
const existingHashes = new Map<string, Array<{ id: string; vectorIndex: number }>>();
|
|
817
|
+
let existingIndex = 0;
|
|
818
|
+
for (const chunk of iterateChunksByKB(this.db, kb.id)) {
|
|
819
|
+
const entries = existingHashes.get(chunk.content_hash) ?? [];
|
|
820
|
+
entries.push({ id: chunk.id, vectorIndex: existingIndex });
|
|
821
|
+
existingHashes.set(chunk.content_hash, entries);
|
|
822
|
+
existingIndex++;
|
|
235
823
|
}
|
|
236
824
|
|
|
237
|
-
|
|
238
|
-
const
|
|
239
|
-
const
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
825
|
+
const oldVectorIndexByHash = new Map<string, number[]>();
|
|
826
|
+
const newVectorIndexByHash = new Map<string, number[]>();
|
|
827
|
+
const pendingChunks: Awaited<ReturnType<typeof chunkFile>> = [];
|
|
828
|
+
let addedVectorCount = 0;
|
|
829
|
+
let addedCount = 0;
|
|
830
|
+
let unchanged = 0;
|
|
831
|
+
let scannedFiles = 0;
|
|
832
|
+
let scannedChunks = 0;
|
|
833
|
+
let plannedTotalFiles: number | undefined;
|
|
834
|
+
const startedAt = Date.now();
|
|
244
835
|
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
836
|
+
const flushPending = async (): Promise<void> => {
|
|
837
|
+
if (!this.db || !addedVectorWriter || pendingChunks.length === 0) return;
|
|
838
|
+
if (signal?.aborted) throw new Error("Cancelled");
|
|
839
|
+
const batch = pendingChunks.splice(0, INDEX_EMBED_BATCH_SIZE);
|
|
840
|
+
const elapsed = Date.now() - startedAt;
|
|
841
|
+
const message = `Embedding update batch: ${addedCount} new chunks stored, ${scannedFiles} files scanned, elapsed ${formatDuration(
|
|
842
|
+
elapsed,
|
|
843
|
+
)}`;
|
|
844
|
+
updateIndexingJob(this.db, kb.id, {
|
|
845
|
+
phase: "embedding",
|
|
846
|
+
message,
|
|
847
|
+
processed_files: scannedFiles,
|
|
848
|
+
processed_chunks: scannedChunks,
|
|
849
|
+
total_files: plannedTotalFiles,
|
|
850
|
+
added_chunks: addedCount,
|
|
851
|
+
unchanged_chunks: unchanged,
|
|
852
|
+
});
|
|
853
|
+
onProgress?.(message);
|
|
854
|
+
const newVectors = await embedDocuments(
|
|
855
|
+
batch.map((c) => buildChunkEmbeddingText(c)),
|
|
254
856
|
signal,
|
|
255
857
|
);
|
|
256
|
-
|
|
858
|
+
if (signal?.aborted) throw new Error("Cancelled");
|
|
859
|
+
addedVectorWriter.append(newVectors);
|
|
860
|
+
for (let i = 0; i < batch.length; i++) {
|
|
861
|
+
const indexes = newVectorIndexByHash.get(batch[i].content_hash) ?? [];
|
|
862
|
+
indexes.push(addedVectorCount + i);
|
|
863
|
+
newVectorIndexByHash.set(batch[i].content_hash, indexes);
|
|
864
|
+
}
|
|
865
|
+
addedVectorCount += newVectors.length;
|
|
866
|
+
insertChunks(this.db, kb.id, batch);
|
|
867
|
+
addedCount += batch.length;
|
|
868
|
+
updateKBCounts(this.db, kb.id, getChunkCount(this.db, kb.id), getFileCount(this.db, kb.id));
|
|
869
|
+
const storedMessage = `Stored update batch: +${addedCount} chunks, =${unchanged} unchanged`;
|
|
870
|
+
updateIndexingJob(this.db, kb.id, {
|
|
871
|
+
phase: "storing",
|
|
872
|
+
message: storedMessage,
|
|
873
|
+
processed_files: scannedFiles,
|
|
874
|
+
processed_chunks: scannedChunks,
|
|
875
|
+
total_files: plannedTotalFiles,
|
|
876
|
+
added_chunks: addedCount,
|
|
877
|
+
unchanged_chunks: unchanged,
|
|
878
|
+
});
|
|
879
|
+
onProgress?.(storedMessage);
|
|
880
|
+
};
|
|
881
|
+
|
|
882
|
+
const processChunks = async (chunks: Awaited<ReturnType<typeof chunkFile>>): Promise<void> => {
|
|
883
|
+
for (const chunk of chunks) {
|
|
884
|
+
scannedChunks++;
|
|
885
|
+
const existing = existingHashes.get(chunk.content_hash);
|
|
886
|
+
if (existing && existing.length > 0) {
|
|
887
|
+
const retained = existing.shift();
|
|
888
|
+
if (retained) {
|
|
889
|
+
const indexes = oldVectorIndexByHash.get(chunk.content_hash) ?? [];
|
|
890
|
+
indexes.push(retained.vectorIndex);
|
|
891
|
+
oldVectorIndexByHash.set(chunk.content_hash, indexes);
|
|
892
|
+
}
|
|
893
|
+
unchanged++;
|
|
894
|
+
continue;
|
|
895
|
+
}
|
|
896
|
+
pendingChunks.push(chunk);
|
|
897
|
+
if (pendingChunks.length >= INDEX_EMBED_BATCH_SIZE) await flushPending();
|
|
898
|
+
}
|
|
899
|
+
};
|
|
900
|
+
|
|
901
|
+
updateIndexingJob(this.db, kb.id, { phase: "scanning", message: "Scanning source..." });
|
|
902
|
+
onProgress?.("Scanning source...");
|
|
903
|
+
if (kb.source_type === "url") {
|
|
904
|
+
const message = `Fetching ${kb.source_path}...`;
|
|
905
|
+
updateIndexingJob(this.db, kb.id, { phase: "fetching", message });
|
|
906
|
+
onProgress?.(message);
|
|
907
|
+
scannedFiles = 1;
|
|
908
|
+
await processChunks(await chunkUrl(kb.source_path, signal));
|
|
909
|
+
} else if (statSync(kb.source_path).isDirectory()) {
|
|
910
|
+
const plan = planDirectoryScan(kb.source_path, scanOptions);
|
|
911
|
+
plannedTotalFiles = plan.files;
|
|
912
|
+
const planningMessage = `Planned directory scan: ${plan.files} files, ${formatBytes(
|
|
913
|
+
plan.bytes,
|
|
914
|
+
)} scannable text, skipped ${plan.skippedTotal} (${plan.skippedSummary})`;
|
|
915
|
+
updateIndexingJob(this.db, kb.id, {
|
|
916
|
+
phase: "planning",
|
|
917
|
+
message: planningMessage,
|
|
918
|
+
total_files: plan.files,
|
|
919
|
+
skipped_total: plan.skippedTotal,
|
|
920
|
+
});
|
|
921
|
+
onProgress?.(planningMessage);
|
|
922
|
+
const skipped = createSkippedScanStats();
|
|
923
|
+
for (const file of iterateScannedFiles(kb.source_path, skipped, scanOptions)) {
|
|
924
|
+
if (signal?.aborted) throw new Error("Cancelled");
|
|
925
|
+
scannedFiles++;
|
|
926
|
+
await processChunks(await chunkFile(file.content, file.relPath));
|
|
927
|
+
if (scannedFiles % 25 === 0) {
|
|
928
|
+
const message = `Scanned ${scannedFiles} files, ${scannedChunks} chunks, skipped ${skipped.total}, +${addedCount} =${unchanged}`;
|
|
929
|
+
updateIndexingJob(this.db, kb.id, {
|
|
930
|
+
phase: "scanning",
|
|
931
|
+
message,
|
|
932
|
+
processed_files: scannedFiles,
|
|
933
|
+
processed_chunks: scannedChunks,
|
|
934
|
+
total_files: plan.files,
|
|
935
|
+
skipped_total: skipped.total,
|
|
936
|
+
added_chunks: addedCount,
|
|
937
|
+
unchanged_chunks: unchanged,
|
|
938
|
+
});
|
|
939
|
+
onProgress?.(message);
|
|
940
|
+
}
|
|
941
|
+
}
|
|
942
|
+
const message = `Scanned ${scannedFiles} files, skipped ${skipped.total} (${summarizeSkippedScan(
|
|
943
|
+
skipped,
|
|
944
|
+
)}), reconciling deletes...`;
|
|
945
|
+
updateIndexingJob(this.db, kb.id, {
|
|
946
|
+
phase: "reconciling",
|
|
947
|
+
message,
|
|
948
|
+
processed_files: scannedFiles,
|
|
949
|
+
processed_chunks: scannedChunks,
|
|
950
|
+
total_files: plan.files,
|
|
951
|
+
skipped_total: skipped.total,
|
|
952
|
+
added_chunks: addedCount,
|
|
953
|
+
unchanged_chunks: unchanged,
|
|
954
|
+
});
|
|
955
|
+
onProgress?.(message);
|
|
956
|
+
} else {
|
|
957
|
+
const { readFileSync } = await import("node:fs");
|
|
958
|
+
scannedFiles = 1;
|
|
959
|
+
await processChunks(await chunkFile(readFileSync(kb.source_path, "utf-8"), kb.source_path));
|
|
257
960
|
}
|
|
961
|
+
await flushPending();
|
|
962
|
+
addedVectorWriter.close();
|
|
963
|
+
addedVectorWriter = undefined;
|
|
258
964
|
|
|
259
|
-
|
|
260
|
-
for (
|
|
261
|
-
|
|
965
|
+
const idsToRemove: string[] = [];
|
|
966
|
+
for (const entries of existingHashes.values()) {
|
|
967
|
+
idsToRemove.push(...entries.map((entry) => entry.id));
|
|
262
968
|
}
|
|
969
|
+
if (idsToRemove.length > 0) deleteChunksByIds(this.db, idsToRemove);
|
|
970
|
+
const changesMessage = `Changes: +${addedCount} -${idsToRemove.length} =${unchanged}`;
|
|
971
|
+
updateIndexingJob(this.db, kb.id, {
|
|
972
|
+
phase: "reconciling",
|
|
973
|
+
message: changesMessage,
|
|
974
|
+
processed_files: scannedFiles,
|
|
975
|
+
processed_chunks: scannedChunks,
|
|
976
|
+
added_chunks: addedCount,
|
|
977
|
+
removed_chunks: idsToRemove.length,
|
|
978
|
+
unchanged_chunks: unchanged,
|
|
979
|
+
});
|
|
980
|
+
onProgress?.(changesMessage);
|
|
263
981
|
|
|
264
|
-
|
|
265
|
-
const
|
|
266
|
-
const
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
982
|
+
replacementVectorPath = tempVectorPath(vectorPath);
|
|
983
|
+
const vectorWriter = openVectorWriter(replacementVectorPath);
|
|
984
|
+
const oldVectorReader = openVectorReader(vectorPath);
|
|
985
|
+
const newVectorReader = openVectorReader(addedVectorPath);
|
|
986
|
+
let finalChunkCount = 0;
|
|
987
|
+
const takeVectorIndex = (indexesByHash: Map<string, number[]>, hash: string): number | undefined => {
|
|
988
|
+
const indexes = indexesByHash.get(hash);
|
|
989
|
+
return indexes?.shift();
|
|
990
|
+
};
|
|
991
|
+
try {
|
|
992
|
+
for (const chunk of iterateChunksByKB(this.db, kb.id)) {
|
|
993
|
+
const oldVectorIndex = takeVectorIndex(oldVectorIndexByHash, chunk.content_hash);
|
|
994
|
+
const newVectorIndex = takeVectorIndex(newVectorIndexByHash, chunk.content_hash);
|
|
995
|
+
const vector =
|
|
996
|
+
oldVectorReader && oldVectorIndex !== undefined
|
|
997
|
+
? oldVectorReader.read(oldVectorIndex)
|
|
998
|
+
: newVectorReader && newVectorIndex !== undefined
|
|
999
|
+
? newVectorReader.read(newVectorIndex)
|
|
1000
|
+
: undefined;
|
|
1001
|
+
if (!vector) throw new Error(`Missing vector while rebuilding knowledge base: ${chunk.id}`);
|
|
1002
|
+
vectorWriter.append([vector]);
|
|
1003
|
+
finalChunkCount++;
|
|
1004
|
+
if (finalChunkCount % 1_000 === 0) {
|
|
1005
|
+
const message = `Rebuilding vector file: ${finalChunkCount} chunks written`;
|
|
1006
|
+
updateIndexingJob(this.db, kb.id, {
|
|
1007
|
+
phase: "rebuilding_vectors",
|
|
1008
|
+
message,
|
|
1009
|
+
processed_files: scannedFiles,
|
|
1010
|
+
processed_chunks: finalChunkCount,
|
|
1011
|
+
added_chunks: addedCount,
|
|
1012
|
+
removed_chunks: idsToRemove.length,
|
|
1013
|
+
unchanged_chunks: unchanged,
|
|
1014
|
+
});
|
|
1015
|
+
onProgress?.(message);
|
|
1016
|
+
}
|
|
1017
|
+
}
|
|
1018
|
+
} finally {
|
|
1019
|
+
oldVectorReader?.close();
|
|
1020
|
+
newVectorReader?.close();
|
|
1021
|
+
vectorWriter.close();
|
|
270
1022
|
}
|
|
271
|
-
|
|
272
|
-
|
|
1023
|
+
renameSync(replacementVectorPath, vectorPath);
|
|
1024
|
+
replacementVectorPath = undefined;
|
|
1025
|
+
if (addedVectorPath) rmSync(addedVectorPath, { force: true });
|
|
1026
|
+
addedVectorPath = undefined;
|
|
273
1027
|
|
|
274
|
-
|
|
275
|
-
updateKBCounts(this.db, kb.id, finalChunks.length, getFileCount(this.db, kb.id));
|
|
1028
|
+
updateKBCounts(this.db, kb.id, finalChunkCount, getFileCount(this.db, kb.id));
|
|
276
1029
|
updateKBStatus(this.db, kb.id, "ready");
|
|
1030
|
+
const readyMessage = `Ready: +${addedCount} -${idsToRemove.length} =${unchanged}`;
|
|
1031
|
+
updateIndexingJob(this.db, kb.id, {
|
|
1032
|
+
phase: "ready",
|
|
1033
|
+
message: readyMessage,
|
|
1034
|
+
processed_files: scannedFiles,
|
|
1035
|
+
processed_chunks: finalChunkCount,
|
|
1036
|
+
added_chunks: addedCount,
|
|
1037
|
+
removed_chunks: idsToRemove.length,
|
|
1038
|
+
unchanged_chunks: unchanged,
|
|
1039
|
+
});
|
|
1040
|
+
finishIndexingJob(this.db, kb.id, "succeeded", readyMessage);
|
|
1041
|
+
onProgress?.(readyMessage);
|
|
277
1042
|
|
|
278
|
-
return { added:
|
|
1043
|
+
return { added: addedCount, removed: idsToRemove.length, unchanged };
|
|
279
1044
|
} catch (e) {
|
|
1045
|
+
addedVectorWriter?.close();
|
|
1046
|
+
if (replacementVectorPath) rmSync(replacementVectorPath, { force: true });
|
|
1047
|
+
if (addedVectorPath) rmSync(addedVectorPath, { force: true });
|
|
280
1048
|
updateKBStatus(this.db, kb.id, "error");
|
|
1049
|
+
finishIndexingJob(
|
|
1050
|
+
this.db,
|
|
1051
|
+
kb.id,
|
|
1052
|
+
isCancellationError(e) ? "cancelled" : "failed",
|
|
1053
|
+
isCancellationError(e) ? "Update cancelled." : "Update failed.",
|
|
1054
|
+
e instanceof Error ? e.message : String(e),
|
|
1055
|
+
);
|
|
281
1056
|
throw e;
|
|
282
1057
|
}
|
|
283
1058
|
}
|
|
284
1059
|
|
|
285
1060
|
async search(query: string, options: SearchOptions = {}): Promise<SearchResponse> {
|
|
286
1061
|
if (!this.db) throw new Error("Engine not initialized");
|
|
1062
|
+
const requestedMode = options.mode ?? "hybrid";
|
|
1063
|
+
if (requestedMode === "auto") {
|
|
1064
|
+
const primaryMode = chooseAutoMode(query);
|
|
1065
|
+
const attempts = [primaryMode, ...fallbackModesFor(primaryMode)];
|
|
1066
|
+
const tried: NonNullable<SearchOptions["mode"]>[] = [];
|
|
1067
|
+
const warnings: string[] = [];
|
|
1068
|
+
for (const mode of attempts) {
|
|
1069
|
+
if (tried.includes(mode)) continue;
|
|
1070
|
+
tried.push(mode);
|
|
1071
|
+
const response = await this.search(query, { ...options, mode });
|
|
1072
|
+
if (response.warnings) warnings.push(...response.warnings);
|
|
1073
|
+
if (!isWeakAutoResponse(query, response, primaryMode, mode)) {
|
|
1074
|
+
return {
|
|
1075
|
+
...response,
|
|
1076
|
+
warnings: warnings.length > 0 ? [...new Set(warnings)] : response.warnings,
|
|
1077
|
+
mode_used: mode,
|
|
1078
|
+
retry_modes: tried.slice(0, -1),
|
|
1079
|
+
suggestions: response.suggestions,
|
|
1080
|
+
};
|
|
1081
|
+
}
|
|
1082
|
+
if (tried.length >= 3) break;
|
|
1083
|
+
}
|
|
1084
|
+
return {
|
|
1085
|
+
results: [],
|
|
1086
|
+
total_count: 0,
|
|
1087
|
+
has_more: false,
|
|
1088
|
+
warnings: warnings.length > 0 ? [...new Set(warnings)] : undefined,
|
|
1089
|
+
mode_used: tried.at(-1),
|
|
1090
|
+
retry_modes: tried.slice(0, -1),
|
|
1091
|
+
suggestions: [
|
|
1092
|
+
"No results after auto mode fallback. Check knowledge_status, try a more exact term, or rebuild the KB if indexing rules changed.",
|
|
1093
|
+
],
|
|
1094
|
+
};
|
|
1095
|
+
}
|
|
287
1096
|
const db = this.db;
|
|
288
|
-
const { mode = "hybrid", limit = 10, offset = 0, kb_id, filters } = options;
|
|
289
|
-
|
|
290
|
-
const
|
|
291
|
-
|
|
1097
|
+
const { mode = "hybrid", limit = 10, offset = 0, kb_id, filters, diversity = "balanced" } = options;
|
|
1098
|
+
const retrievalMode = mode === "adaptive" ? "hybrid" : mode;
|
|
1099
|
+
const candidateLimit = Math.max(50, offset + limit * 12);
|
|
1100
|
+
const normalizedQuery = normalizedQueryText(query);
|
|
1101
|
+
const queryTokens = tokenizeForSimilarity(query);
|
|
1102
|
+
const normalizedFileType = normalizeFileTypeFilter(filters?.file_type);
|
|
292
1103
|
|
|
293
1104
|
const warnings: string[] = [];
|
|
1105
|
+
const selectedKB = kb_id ? (getKB(db, kb_id) ?? getKBByName(db, kb_id)) : undefined;
|
|
1106
|
+
const availableKBs = kb_id ? ([selectedKB].filter(Boolean) as KnowledgeBase[]) : listKBs(db);
|
|
1107
|
+
const kbs = availableKBs.filter((kb) => kb.status === "ready" || kb.status === "stale");
|
|
1108
|
+
for (const kb of availableKBs) {
|
|
1109
|
+
if (kb.status !== "ready" && kb.status !== "stale") {
|
|
1110
|
+
warnings.push(`"${kb.name}" is ${kb.status}; search skipped until indexing is ready`);
|
|
1111
|
+
}
|
|
1112
|
+
}
|
|
1113
|
+
if (kbs.length === 0) {
|
|
1114
|
+
return {
|
|
1115
|
+
results: [],
|
|
1116
|
+
total_count: 0,
|
|
1117
|
+
has_more: false,
|
|
1118
|
+
warnings: warnings.length > 0 ? warnings : undefined,
|
|
1119
|
+
mode_used: mode,
|
|
1120
|
+
};
|
|
1121
|
+
}
|
|
1122
|
+
|
|
294
1123
|
const allResults: { chunkId: string; score: number }[] = [];
|
|
1124
|
+
const vectorsByChunkId = new Map<string, Float32Array>();
|
|
295
1125
|
|
|
296
1126
|
for (const kb of kbs) {
|
|
297
1127
|
if (kb.embedding_model !== CURRENT_EMBEDDING_MODEL) {
|
|
@@ -299,28 +1129,27 @@ export class KnowledgeEngine {
|
|
|
299
1129
|
`"${kb.name}" was indexed with ${kb.embedding_model} (current: ${CURRENT_EMBEDDING_MODEL}) — run knowledge_update for best results`,
|
|
300
1130
|
);
|
|
301
1131
|
}
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
if (
|
|
306
|
-
allResults.push(...searchBM25(db, query,
|
|
307
|
-
} else if (
|
|
308
|
-
const
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
}
|
|
1132
|
+
if (kb.chunk_count === 0) continue;
|
|
1133
|
+
const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
|
|
1134
|
+
|
|
1135
|
+
if (retrievalMode === "fast") {
|
|
1136
|
+
allResults.push(...searchBM25(db, normalizedQuery || query, candidateLimit, kb.id, { allowOrFallback: false }));
|
|
1137
|
+
} else if (retrievalMode === "semantic") {
|
|
1138
|
+
const queryVec = await embedQuery(query);
|
|
1139
|
+
const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), candidateLimit);
|
|
1140
|
+
allResults.push(...vectorResults.results);
|
|
1141
|
+
for (const [chunkId, vector] of vectorResults.vectorsByChunkId) vectorsByChunkId.set(chunkId, vector);
|
|
313
1142
|
} else {
|
|
314
|
-
// hybrid: BM25 + vector
|
|
315
|
-
const bm25Results = searchBM25(db, query,
|
|
1143
|
+
// hybrid: BM25 + vector weighted fusion (both scoped to this KB)
|
|
1144
|
+
const bm25Results = searchBM25(db, normalizedQuery || query, candidateLimit, kb.id);
|
|
1145
|
+
if (bm25Results.length === 0) continue;
|
|
316
1146
|
|
|
317
|
-
const vectors = this.getVectors(kb.id);
|
|
318
1147
|
let vecResults: { chunkId: string; score: number }[] = [];
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
const fused =
|
|
1148
|
+
const queryVec = await embedQuery(query);
|
|
1149
|
+
const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), candidateLimit);
|
|
1150
|
+
vecResults = vectorResults.results;
|
|
1151
|
+
for (const [chunkId, vector] of vectorResults.vectorsByChunkId) vectorsByChunkId.set(chunkId, vector);
|
|
1152
|
+
const fused = weightedScoreFusion(bm25Results, vecResults);
|
|
324
1153
|
allResults.push(...fused);
|
|
325
1154
|
}
|
|
326
1155
|
}
|
|
@@ -332,80 +1161,160 @@ export class KnowledgeEngine {
|
|
|
332
1161
|
seen.add(r.chunkId);
|
|
333
1162
|
return true;
|
|
334
1163
|
});
|
|
335
|
-
|
|
1164
|
+
const rankingByChunkId = new Map<string, RankingDiagnostics>();
|
|
1165
|
+
for (const result of unique) {
|
|
1166
|
+
const chunk = getChunkById(db, result.chunkId);
|
|
1167
|
+
if (chunk) {
|
|
1168
|
+
const ranking = scoreChunkForQuery(result.score, chunk, queryTokens);
|
|
1169
|
+
rankingByChunkId.set(result.chunkId, ranking);
|
|
1170
|
+
result.score = ranking.adjusted_score;
|
|
1171
|
+
}
|
|
1172
|
+
}
|
|
1173
|
+
let scored = unique;
|
|
1174
|
+
if (retrievalMode === "fast") {
|
|
1175
|
+
scored = unique.filter((result) => {
|
|
1176
|
+
const chunk = getChunkById(db, result.chunkId);
|
|
1177
|
+
if (!chunk) return false;
|
|
1178
|
+
return hasAnyLexicalEvidence(buildChunkEmbeddingText(chunk), queryTokens);
|
|
1179
|
+
});
|
|
1180
|
+
} else if (retrievalMode !== "semantic") {
|
|
1181
|
+
scored = unique.filter((result) => {
|
|
1182
|
+
const chunk = getChunkById(db, result.chunkId);
|
|
1183
|
+
if (!chunk) return false;
|
|
1184
|
+
return result.score >= MIN_HYBRID_SCORE && hasEnoughLexicalEvidence(chunk, queryTokens);
|
|
1185
|
+
});
|
|
1186
|
+
}
|
|
1187
|
+
scored.sort((a, b) => b.score - a.score);
|
|
336
1188
|
|
|
337
1189
|
// Apply metadata filters post-retrieval
|
|
338
|
-
let filtered =
|
|
339
|
-
if (
|
|
340
|
-
filtered =
|
|
1190
|
+
let filtered = scored;
|
|
1191
|
+
if (normalizedFileType || filters?.path_pattern) {
|
|
1192
|
+
filtered = scored.filter((r) => {
|
|
341
1193
|
const chunk = getChunkById(db, r.chunkId);
|
|
342
1194
|
if (!chunk) return false;
|
|
343
|
-
if (
|
|
344
|
-
if (filters
|
|
1195
|
+
if (normalizedFileType && chunk.file_type !== normalizedFileType) return false;
|
|
1196
|
+
if (filters?.path_pattern && !chunk.file_path.includes(filters.path_pattern)) return false;
|
|
345
1197
|
return true;
|
|
346
1198
|
});
|
|
347
1199
|
}
|
|
348
1200
|
|
|
349
|
-
// Deep mode: rerank top-20 with cross-encoder
|
|
350
1201
|
if (mode === "deep" && filtered.length > 0) {
|
|
351
1202
|
const candidates = filtered
|
|
352
|
-
.slice(0,
|
|
1203
|
+
.slice(0, 30)
|
|
353
1204
|
.map((r) => {
|
|
354
1205
|
const chunk = getChunkById(db, r.chunkId);
|
|
355
1206
|
return chunk ? { chunkId: r.chunkId, content: chunk.content } : null;
|
|
356
1207
|
})
|
|
357
1208
|
.filter(Boolean) as Array<{ chunkId: string; content: string }>;
|
|
358
|
-
const reranked = await rerank(query, candidates, limit);
|
|
359
|
-
const
|
|
1209
|
+
const reranked = await rerank(query, candidates, Math.max(limit * 3, limit));
|
|
1210
|
+
const ranked: RankedChunk[] = [];
|
|
360
1211
|
for (const r of reranked) {
|
|
361
1212
|
const chunk = getChunkById(db, r.chunkId);
|
|
362
1213
|
if (!chunk) continue;
|
|
363
1214
|
const kbObj = getKB(db, chunk.kb_id);
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
kb_name: kbObj?.name ?? "unknown",
|
|
1215
|
+
ranked.push({
|
|
1216
|
+
chunk,
|
|
1217
|
+
kbName: kbObj?.name ?? "unknown",
|
|
1218
|
+
ranking: scoreChunkForQuery(r.score, chunk, queryTokens),
|
|
369
1219
|
score: r.score,
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
1220
|
+
content: chunk.content,
|
|
1221
|
+
snippet: buildQuerySnippet(chunk.content, query),
|
|
1222
|
+
startLine: chunk.start_line,
|
|
1223
|
+
endLine: chunk.end_line,
|
|
1224
|
+
sourceChunkIds: [chunk.id],
|
|
373
1225
|
});
|
|
374
1226
|
}
|
|
1227
|
+
const diversified = interleaveByFile(diversifyRankedChunks(ranked, diversity, vectorsByChunkId), diversity);
|
|
1228
|
+
const page = diversified.slice(offset, offset + limit);
|
|
1229
|
+
const results = page.map((r) => ({
|
|
1230
|
+
content: r.content,
|
|
1231
|
+
file_path: r.chunk.file_path,
|
|
1232
|
+
file_type: r.chunk.file_type,
|
|
1233
|
+
kb_name: r.kbName,
|
|
1234
|
+
score: r.score,
|
|
1235
|
+
ranking: r.ranking,
|
|
1236
|
+
snippet: r.snippet,
|
|
1237
|
+
start_line: r.startLine,
|
|
1238
|
+
end_line: r.endLine,
|
|
1239
|
+
}));
|
|
375
1240
|
return {
|
|
376
1241
|
results,
|
|
377
|
-
total_count:
|
|
378
|
-
has_more:
|
|
1242
|
+
total_count: diversified.length,
|
|
1243
|
+
has_more: offset + limit < diversified.length,
|
|
379
1244
|
warnings: warnings.length > 0 ? warnings : undefined,
|
|
1245
|
+
mode_used: mode,
|
|
380
1246
|
};
|
|
381
1247
|
}
|
|
382
1248
|
|
|
383
|
-
const
|
|
384
|
-
const
|
|
1249
|
+
const ranked: RankedChunk[] = [];
|
|
1250
|
+
for (const r of filtered) {
|
|
1251
|
+
const chunk = getChunkById(db, r.chunkId);
|
|
1252
|
+
if (!chunk) continue;
|
|
1253
|
+
const kb = getKB(db, chunk.kb_id);
|
|
385
1254
|
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
1255
|
+
if (mode === "adaptive") {
|
|
1256
|
+
const contextChunks = getChunksByFile(
|
|
1257
|
+
db,
|
|
1258
|
+
chunk.kb_id,
|
|
1259
|
+
chunk.file_path,
|
|
1260
|
+
Math.max(1, chunk.start_line - ADAPTIVE_CONTEXT_LINES),
|
|
1261
|
+
chunk.end_line + ADAPTIVE_CONTEXT_LINES,
|
|
1262
|
+
);
|
|
1263
|
+
const context = buildAdaptiveContext(chunk, contextChunks.length > 0 ? contextChunks : [chunk], queryTokens);
|
|
1264
|
+
pushAdaptiveCandidate(ranked, {
|
|
1265
|
+
chunk,
|
|
1266
|
+
kbName: kb?.name ?? "unknown",
|
|
1267
|
+
ranking: rankingByChunkId.get(r.chunkId),
|
|
396
1268
|
score: r.score,
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
1269
|
+
content: context.content,
|
|
1270
|
+
snippet: buildQuerySnippet(context.content, query),
|
|
1271
|
+
startLine: context.startLine,
|
|
1272
|
+
endLine: context.endLine,
|
|
1273
|
+
sourceChunkIds: context.sourceChunkIds,
|
|
1274
|
+
});
|
|
1275
|
+
} else {
|
|
1276
|
+
ranked.push({
|
|
1277
|
+
chunk,
|
|
1278
|
+
kbName: kb?.name ?? "unknown",
|
|
1279
|
+
ranking: rankingByChunkId.get(r.chunkId),
|
|
1280
|
+
score: r.score,
|
|
1281
|
+
content: chunk.content,
|
|
1282
|
+
snippet: buildQuerySnippet(chunk.content, query),
|
|
1283
|
+
startLine: chunk.start_line,
|
|
1284
|
+
endLine: chunk.end_line,
|
|
1285
|
+
sourceChunkIds: [chunk.id],
|
|
1286
|
+
});
|
|
1287
|
+
}
|
|
1288
|
+
}
|
|
1289
|
+
|
|
1290
|
+
const diversified = interleaveByFile(diversifyRankedChunks(ranked, diversity, vectorsByChunkId), diversity);
|
|
1291
|
+
const total = diversified.length;
|
|
1292
|
+
const page = diversified.slice(offset, offset + limit);
|
|
1293
|
+
const results: SearchResult[] = page.map((r) => ({
|
|
1294
|
+
content: r.content,
|
|
1295
|
+
file_path: r.chunk.file_path,
|
|
1296
|
+
file_type: r.chunk.file_type,
|
|
1297
|
+
kb_name: r.kbName,
|
|
1298
|
+
score: r.score,
|
|
1299
|
+
ranking: r.ranking,
|
|
1300
|
+
snippet: r.snippet,
|
|
1301
|
+
start_line: r.startLine,
|
|
1302
|
+
end_line: r.endLine,
|
|
1303
|
+
}));
|
|
403
1304
|
|
|
404
1305
|
return {
|
|
405
1306
|
results,
|
|
406
1307
|
total_count: total,
|
|
407
1308
|
has_more: offset + limit < total,
|
|
408
1309
|
warnings: warnings.length > 0 ? warnings : undefined,
|
|
1310
|
+
mode_used: mode,
|
|
1311
|
+
suggestions:
|
|
1312
|
+
results.length === 0
|
|
1313
|
+
? [
|
|
1314
|
+
"Try mode 'fast' for exact symbols or mode 'semantic' for conceptual wording.",
|
|
1315
|
+
"Run knowledge_status if the KB should contain this answer.",
|
|
1316
|
+
]
|
|
1317
|
+
: undefined,
|
|
409
1318
|
};
|
|
410
1319
|
}
|
|
411
1320
|
|
|
@@ -413,7 +1322,6 @@ export class KnowledgeEngine {
|
|
|
413
1322
|
if (!this.db) return false;
|
|
414
1323
|
const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
|
|
415
1324
|
if (!kb) return false;
|
|
416
|
-
this.invalidateVectorCache(kb.id);
|
|
417
1325
|
deleteKB(this.db, kb.id);
|
|
418
1326
|
return true;
|
|
419
1327
|
}
|
|
@@ -426,7 +1334,6 @@ export class KnowledgeEngine {
|
|
|
426
1334
|
clear(): void {
|
|
427
1335
|
if (!this.db) return;
|
|
428
1336
|
for (const kb of listKBs(this.db)) deleteKB(this.db, kb.id);
|
|
429
|
-
this.vectorCache.clear();
|
|
430
1337
|
}
|
|
431
1338
|
|
|
432
1339
|
diagnose(): DiagnosticResult[] {
|
|
@@ -435,6 +1342,98 @@ export class KnowledgeEngine {
|
|
|
435
1342
|
return listKBs(db).map((kb) => diagnoseKB(db, kb));
|
|
436
1343
|
}
|
|
437
1344
|
|
|
1345
|
+
doctor(): DoctorReport {
|
|
1346
|
+
const diagnostics = this.diagnose();
|
|
1347
|
+
const issues: DoctorIssue[] = [];
|
|
1348
|
+
const kbs = this.list();
|
|
1349
|
+
if (kbs.length === 0) {
|
|
1350
|
+
issues.push({
|
|
1351
|
+
severity: "blocking",
|
|
1352
|
+
message: "No knowledge bases are indexed.",
|
|
1353
|
+
action: "Run knowledge_add for the project root or relevant source/docs directory.",
|
|
1354
|
+
});
|
|
1355
|
+
}
|
|
1356
|
+
|
|
1357
|
+
for (const diagnostic of diagnostics) {
|
|
1358
|
+
if (diagnostic.stuck_indexing) {
|
|
1359
|
+
const phase = diagnostic.job ? ` during ${diagnostic.job.phase}` : "";
|
|
1360
|
+
issues.push({
|
|
1361
|
+
severity: "blocking",
|
|
1362
|
+
kb_name: diagnostic.kb_name,
|
|
1363
|
+
message: `Indexing appears stuck${phase} for ${formatDuration(diagnostic.last_progress_age_ms)}.`,
|
|
1364
|
+
action:
|
|
1365
|
+
"Check knowledge_status for the last progress message. If no Pi process is actively indexing it, remove and rebuild this KB.",
|
|
1366
|
+
});
|
|
1367
|
+
}
|
|
1368
|
+
if (diagnostic.status === "error") {
|
|
1369
|
+
issues.push({
|
|
1370
|
+
severity: "blocking",
|
|
1371
|
+
kb_name: diagnostic.kb_name,
|
|
1372
|
+
message: "Knowledge base is in error state and is skipped by search.",
|
|
1373
|
+
action: "Run knowledge_remove and knowledge_add to rebuild it from the source.",
|
|
1374
|
+
});
|
|
1375
|
+
}
|
|
1376
|
+
if (diagnostic.coverage_percent < 80) {
|
|
1377
|
+
issues.push({
|
|
1378
|
+
severity: "warning",
|
|
1379
|
+
kb_name: diagnostic.kb_name,
|
|
1380
|
+
message: `Coverage is ${diagnostic.coverage_percent}% (${diagnostic.indexed_files}/${diagnostic.total_source_files} files).`,
|
|
1381
|
+
action: "Review skipped files and source path. Rebuild if index-time rules changed.",
|
|
1382
|
+
});
|
|
1383
|
+
}
|
|
1384
|
+
if (diagnostic.stale_files.length > 0) {
|
|
1385
|
+
issues.push({
|
|
1386
|
+
severity: "warning",
|
|
1387
|
+
kb_name: diagnostic.kb_name,
|
|
1388
|
+
message: `${diagnostic.stale_files.length} files changed after indexing.`,
|
|
1389
|
+
action: "Run knowledge_update for this KB.",
|
|
1390
|
+
});
|
|
1391
|
+
}
|
|
1392
|
+
if (diagnostic.orphan_files.length > 0) {
|
|
1393
|
+
issues.push({
|
|
1394
|
+
severity: "warning",
|
|
1395
|
+
kb_name: diagnostic.kb_name,
|
|
1396
|
+
message: `${diagnostic.orphan_files.length} indexed files no longer exist in the source.`,
|
|
1397
|
+
action: "Run knowledge_update or rebuild the KB.",
|
|
1398
|
+
});
|
|
1399
|
+
}
|
|
1400
|
+
if (diagnostic.skipped_files.total > 0) {
|
|
1401
|
+
issues.push({
|
|
1402
|
+
severity: "info",
|
|
1403
|
+
kb_name: diagnostic.kb_name,
|
|
1404
|
+
message: `${diagnostic.skipped_files.total} files were skipped while scanning (${Object.entries(
|
|
1405
|
+
diagnostic.skipped_files.by_reason,
|
|
1406
|
+
)
|
|
1407
|
+
.filter(([, count]) => count > 0)
|
|
1408
|
+
.map(([reason, count]) => `${reason}: ${count}`)
|
|
1409
|
+
.join(", ")}).`,
|
|
1410
|
+
action:
|
|
1411
|
+
"Use skipped samples to confirm exclusions are expected. Adjust source path or ignore rules if needed.",
|
|
1412
|
+
});
|
|
1413
|
+
}
|
|
1414
|
+
}
|
|
1415
|
+
|
|
1416
|
+
const penalty = issues.reduce((score, issue) => {
|
|
1417
|
+
if (issue.severity === "blocking") return score + 35;
|
|
1418
|
+
if (issue.severity === "warning") return score + 12;
|
|
1419
|
+
return score + 2;
|
|
1420
|
+
}, 0);
|
|
1421
|
+
const healthScore = Math.max(0, Math.min(100, 100 - penalty));
|
|
1422
|
+
const blocking = issues.filter((issue) => issue.severity === "blocking").length;
|
|
1423
|
+
const warnings = issues.filter((issue) => issue.severity === "warning").length;
|
|
1424
|
+
const summary =
|
|
1425
|
+
issues.length === 0
|
|
1426
|
+
? "Knowledge system is healthy."
|
|
1427
|
+
: `${blocking} blocking, ${warnings} warning, ${issues.length - blocking - warnings} info issues.`;
|
|
1428
|
+
|
|
1429
|
+
return {
|
|
1430
|
+
health_score: healthScore,
|
|
1431
|
+
summary,
|
|
1432
|
+
issues,
|
|
1433
|
+
diagnostics,
|
|
1434
|
+
};
|
|
1435
|
+
}
|
|
1436
|
+
|
|
438
1437
|
async exportKB(nameOrId: string, outputPath: string): Promise<number> {
|
|
439
1438
|
if (!this.db) throw new Error("Engine not initialized");
|
|
440
1439
|
const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
|
|
@@ -476,39 +1475,115 @@ export class KnowledgeEngine {
|
|
|
476
1475
|
const header = JSON.parse(lines[0]) as { name: string; description?: string };
|
|
477
1476
|
const kb = createKB(this.db, { name: header.name, description: header.description, source_type: "text" });
|
|
478
1477
|
updateKBStatus(this.db, kb.id, "indexing");
|
|
1478
|
+
startIndexingJob(this.db, kb.id, "import", `Starting import for "${header.name}"`);
|
|
1479
|
+
let vectorWriter: ReturnType<typeof openVectorWriter> | undefined;
|
|
1480
|
+
let tempVectorFile: string | undefined;
|
|
479
1481
|
try {
|
|
480
1482
|
const chunkData = lines.slice(1).map((l) => JSON.parse(l) as ImportedChunk);
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
1483
|
+
const importMessage = `Importing ${chunkData.length} chunks...`;
|
|
1484
|
+
updateIndexingJob(this.db, kb.id, {
|
|
1485
|
+
phase: "importing",
|
|
1486
|
+
message: importMessage,
|
|
1487
|
+
total_files: chunkData.length,
|
|
1488
|
+
});
|
|
1489
|
+
onProgress?.(importMessage);
|
|
484
1490
|
const chunks = chunkData.map((c) => ({
|
|
485
|
-
content_hash:
|
|
1491
|
+
content_hash: chunkIdentityHash({
|
|
1492
|
+
content: c.content,
|
|
1493
|
+
filePath: c.file_path,
|
|
1494
|
+
fileType: c.file_type,
|
|
1495
|
+
startLine: c.start_line,
|
|
1496
|
+
endLine: c.end_line,
|
|
1497
|
+
metadataJson: c.metadata_json || "{}",
|
|
1498
|
+
}),
|
|
486
1499
|
content: c.content,
|
|
487
|
-
content_tokenized:
|
|
1500
|
+
content_tokenized: "",
|
|
488
1501
|
file_path: c.file_path,
|
|
489
1502
|
file_type: c.file_type,
|
|
490
1503
|
start_line: c.start_line,
|
|
491
1504
|
end_line: c.end_line,
|
|
492
1505
|
metadata_json: c.metadata_json || "{}",
|
|
493
1506
|
}));
|
|
494
|
-
|
|
1507
|
+
const indexedChunks = chunks.map((chunk) => ({
|
|
1508
|
+
...chunk,
|
|
1509
|
+
content_tokenized: preTokenizeForFTS(buildChunkEmbeddingText(chunk)),
|
|
1510
|
+
}));
|
|
495
1511
|
const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
1512
|
+
tempVectorFile = tempVectorPath(vectorPath);
|
|
1513
|
+
vectorWriter = openVectorWriter(tempVectorFile);
|
|
1514
|
+
let inserted = 0;
|
|
1515
|
+
for (let offset = 0; offset < indexedChunks.length; offset += INDEX_EMBED_BATCH_SIZE) {
|
|
1516
|
+
if (signal?.aborted) throw new Error("Cancelled");
|
|
1517
|
+
const batch = indexedChunks.slice(offset, offset + INDEX_EMBED_BATCH_SIZE);
|
|
1518
|
+
const message = `Embedding import batch ${Math.floor(offset / INDEX_EMBED_BATCH_SIZE) + 1}/${Math.ceil(
|
|
1519
|
+
indexedChunks.length / INDEX_EMBED_BATCH_SIZE,
|
|
1520
|
+
)}: ${offset}/${indexedChunks.length} chunks`;
|
|
1521
|
+
updateIndexingJob(this.db, kb.id, {
|
|
1522
|
+
phase: "embedding",
|
|
1523
|
+
message,
|
|
1524
|
+
processed_chunks: offset,
|
|
1525
|
+
total_files: indexedChunks.length,
|
|
1526
|
+
added_chunks: inserted,
|
|
1527
|
+
});
|
|
1528
|
+
onProgress?.(message);
|
|
1529
|
+
const vectors = await embedDocuments(
|
|
1530
|
+
batch.map((c) => buildChunkEmbeddingText(c)),
|
|
1531
|
+
signal,
|
|
1532
|
+
);
|
|
1533
|
+
if (signal?.aborted) throw new Error("Cancelled");
|
|
1534
|
+
insertChunks(this.db, kb.id, batch);
|
|
1535
|
+
vectorWriter.append(vectors);
|
|
1536
|
+
inserted += batch.length;
|
|
1537
|
+
updateKBCounts(
|
|
1538
|
+
this.db,
|
|
1539
|
+
kb.id,
|
|
1540
|
+
inserted,
|
|
1541
|
+
new Set(indexedChunks.slice(0, inserted).map((c) => c.file_path)).size,
|
|
1542
|
+
);
|
|
1543
|
+
updateIndexingJob(this.db, kb.id, {
|
|
1544
|
+
phase: "storing",
|
|
1545
|
+
message: `Stored import batch: ${inserted}/${indexedChunks.length} chunks`,
|
|
1546
|
+
processed_chunks: inserted,
|
|
1547
|
+
total_files: indexedChunks.length,
|
|
1548
|
+
added_chunks: inserted,
|
|
1549
|
+
});
|
|
1550
|
+
}
|
|
1551
|
+
vectorWriter.close();
|
|
1552
|
+
vectorWriter = undefined;
|
|
1553
|
+
renameSync(tempVectorFile, vectorPath);
|
|
1554
|
+
tempVectorFile = undefined;
|
|
1555
|
+
updateKBCounts(this.db, kb.id, indexedChunks.length, new Set(indexedChunks.map((c) => c.file_path)).size);
|
|
499
1556
|
updateKBStatus(this.db, kb.id, "ready");
|
|
500
1557
|
const savedKB = getKB(this.db, kb.id);
|
|
501
1558
|
if (!savedKB) throw new Error(`Knowledge base disappeared after import: ${kb.id}`);
|
|
1559
|
+
finishIndexingJob(this.db, kb.id, "succeeded", `Ready: imported ${indexedChunks.length} chunks`);
|
|
502
1560
|
return { kb: savedKB, chunkCount: chunks.length };
|
|
503
1561
|
} catch (e) {
|
|
1562
|
+
vectorWriter?.close();
|
|
1563
|
+
if (tempVectorFile) rmSync(tempVectorFile, { force: true });
|
|
1564
|
+
if (this.db) {
|
|
1565
|
+
finishIndexingJob(
|
|
1566
|
+
this.db,
|
|
1567
|
+
kb.id,
|
|
1568
|
+
isCancellationError(e) ? "cancelled" : "failed",
|
|
1569
|
+
isCancellationError(e) ? "Import cancelled." : "Import failed.",
|
|
1570
|
+
e instanceof Error ? e.message : String(e),
|
|
1571
|
+
);
|
|
1572
|
+
}
|
|
504
1573
|
deleteKB(this.db, kb.id);
|
|
505
1574
|
throw e;
|
|
506
1575
|
}
|
|
507
1576
|
}
|
|
508
1577
|
|
|
509
|
-
async dispose(): Promise<void> {
|
|
510
|
-
|
|
511
|
-
|
|
1578
|
+
async dispose(options: { disposeModels?: boolean } = {}): Promise<void> {
|
|
1579
|
+
const disposeModels = options.disposeModels ?? true;
|
|
1580
|
+
if (disposeModels) {
|
|
1581
|
+
await disposeEmbedding();
|
|
1582
|
+
await disposeReranker();
|
|
1583
|
+
} else {
|
|
1584
|
+
await prepareEmbeddingForShutdown();
|
|
1585
|
+
await prepareRerankerForShutdown();
|
|
1586
|
+
}
|
|
512
1587
|
this.db?.close();
|
|
513
1588
|
this.db = null;
|
|
514
1589
|
}
|