pi-knowledge 0.3.6 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.pi/skills/search-docs.md +24 -4
- package/CHANGELOG.md +19 -2
- package/README.md +42 -3
- package/index.ts +201 -124
- package/package.json +1 -1
- package/src/diagnostics/health.ts +84 -14
- package/src/embedding/provider.ts +5 -39
- package/src/embedding/vectors.ts +95 -12
- package/src/engine.ts +882 -140
- package/src/indexer/chunker.ts +232 -24
- package/src/model-worker-client.ts +146 -0
- package/src/model-worker.ts +88 -0
- package/src/search/bm25.ts +15 -4
- package/src/search/ranking.ts +4 -0
- package/src/search/reranker.ts +5 -35
- package/src/search/vector.ts +60 -0
- package/src/storage/sqlite.ts +224 -9
package/src/engine.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { existsSync, statSync } from "node:fs";
|
|
1
|
+
import { existsSync, renameSync, rmSync, statSync } from "node:fs";
|
|
2
2
|
import { join, resolve } from "node:path";
|
|
3
3
|
import type Database from "better-sqlite3";
|
|
4
4
|
import { type DiagnosticResult, diagnoseKB } from "./diagnostics/health.ts";
|
|
@@ -8,12 +8,25 @@ import {
|
|
|
8
8
|
embedQuery,
|
|
9
9
|
prepareForShutdown as prepareEmbeddingForShutdown,
|
|
10
10
|
} from "./embedding/provider.ts";
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
11
|
+
import { openVectorReader, openVectorWriter } from "./embedding/vectors.ts";
|
|
12
|
+
import {
|
|
13
|
+
buildChunkEmbeddingText,
|
|
14
|
+
chunkFile,
|
|
15
|
+
chunkIdentityHash,
|
|
16
|
+
createSkippedScanStats,
|
|
17
|
+
isReadableTextFile,
|
|
18
|
+
iterateScannableFiles,
|
|
19
|
+
iterateScannedFiles,
|
|
20
|
+
preTokenizeForFTS,
|
|
21
|
+
type ScanOptions,
|
|
22
|
+
summarizeSkippedScan,
|
|
23
|
+
} from "./indexer/chunker.ts";
|
|
24
|
+
import { shutdownModelWorker } from "./model-worker-client.ts";
|
|
13
25
|
import { searchBM25 } from "./search/bm25.ts";
|
|
14
26
|
import { weightedScoreFusion } from "./search/fusion.ts";
|
|
15
27
|
import { normalizedQueryText, tokenizeForSearch } from "./search/query.ts";
|
|
16
28
|
import {
|
|
29
|
+
hasAnyLexicalEvidence,
|
|
17
30
|
hasEnoughLexicalEvidence,
|
|
18
31
|
MIN_HYBRID_SCORE,
|
|
19
32
|
normalizeFileTypeFilter,
|
|
@@ -22,30 +35,34 @@ import {
|
|
|
22
35
|
scoreChunkForQuery,
|
|
23
36
|
} from "./search/ranking.ts";
|
|
24
37
|
import { disposeReranker, prepareRerankerForShutdown, rerank } from "./search/reranker.ts";
|
|
25
|
-
import {
|
|
38
|
+
import { searchVectorFile } from "./search/vector.ts";
|
|
26
39
|
import {
|
|
27
40
|
type Chunk,
|
|
28
41
|
createKB,
|
|
29
42
|
deleteChunksByIds,
|
|
30
43
|
deleteKB,
|
|
44
|
+
finishIndexingJob,
|
|
31
45
|
getChunkById,
|
|
32
|
-
|
|
33
|
-
getChunkIdsByKB,
|
|
46
|
+
getChunkCount,
|
|
34
47
|
getChunksByFile,
|
|
35
48
|
getChunksByKB,
|
|
36
49
|
getFileCount,
|
|
37
50
|
getKB,
|
|
38
51
|
getKBByName,
|
|
39
52
|
insertChunks,
|
|
53
|
+
iterateChunkIdsByKB,
|
|
54
|
+
iterateChunksByKB,
|
|
40
55
|
type KnowledgeBase,
|
|
41
56
|
listKBs,
|
|
42
57
|
openDatabase,
|
|
58
|
+
startIndexingJob,
|
|
59
|
+
updateIndexingJob,
|
|
43
60
|
updateKBCounts,
|
|
44
61
|
updateKBStatus,
|
|
45
62
|
} from "./storage/sqlite.ts";
|
|
46
63
|
|
|
47
64
|
export interface SearchOptions {
|
|
48
|
-
mode?: "fast" | "semantic" | "hybrid" | "deep" | "adaptive";
|
|
65
|
+
mode?: "auto" | "fast" | "semantic" | "hybrid" | "deep" | "adaptive";
|
|
49
66
|
limit?: number;
|
|
50
67
|
offset?: number;
|
|
51
68
|
kb_id?: string;
|
|
@@ -72,10 +89,19 @@ export interface SearchResponse {
|
|
|
72
89
|
total_count: number;
|
|
73
90
|
has_more: boolean;
|
|
74
91
|
warnings?: string[];
|
|
92
|
+
mode_used?: NonNullable<SearchOptions["mode"]>;
|
|
93
|
+
retry_modes?: NonNullable<SearchOptions["mode"]>[];
|
|
94
|
+
suggestions?: string[];
|
|
75
95
|
}
|
|
76
96
|
|
|
77
97
|
export type ProgressCallback = (msg: string) => void;
|
|
78
98
|
|
|
99
|
+
export interface AddOptions {
|
|
100
|
+
include_suggested_text?: boolean;
|
|
101
|
+
include_paths?: string[];
|
|
102
|
+
exclude_paths?: string[];
|
|
103
|
+
}
|
|
104
|
+
|
|
79
105
|
interface ImportedChunk {
|
|
80
106
|
content: string;
|
|
81
107
|
file_path: string;
|
|
@@ -97,11 +123,110 @@ interface RankedChunk {
|
|
|
97
123
|
sourceChunkIds: string[];
|
|
98
124
|
}
|
|
99
125
|
|
|
126
|
+
export interface DoctorIssue {
|
|
127
|
+
severity: "blocking" | "warning" | "info";
|
|
128
|
+
kb_name?: string;
|
|
129
|
+
message: string;
|
|
130
|
+
action: string;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
export interface DoctorReport {
|
|
134
|
+
health_score: number;
|
|
135
|
+
summary: string;
|
|
136
|
+
issues: DoctorIssue[];
|
|
137
|
+
diagnostics: DiagnosticResult[];
|
|
138
|
+
}
|
|
139
|
+
|
|
100
140
|
const ADAPTIVE_CONTEXT_LINES = 80;
|
|
101
141
|
const ADAPTIVE_MAX_CONTEXT_CHARS = 6_000;
|
|
102
142
|
const ADAPTIVE_NEIGHBOR_TARGET = 5;
|
|
143
|
+
const INDEX_EMBED_BATCH_SIZE = 64;
|
|
103
144
|
const VECTOR_REDUNDANCY_WEIGHT = 0.35;
|
|
104
145
|
|
|
146
|
+
interface DirectoryScanPlan {
|
|
147
|
+
files: number;
|
|
148
|
+
bytes: number;
|
|
149
|
+
skippedTotal: number;
|
|
150
|
+
skippedSummary: string;
|
|
151
|
+
skipped: ReturnType<typeof createSkippedScanStats>;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
export interface IndexPlan {
|
|
155
|
+
source_type: "file" | "directory" | "text" | "url";
|
|
156
|
+
scannable_files: number;
|
|
157
|
+
scannable_bytes: number;
|
|
158
|
+
skipped: ReturnType<typeof createSkippedScanStats>;
|
|
159
|
+
summary: string;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
function isCancellationError(error: unknown): boolean {
|
|
163
|
+
return error instanceof Error && error.message === "Cancelled";
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
function tempVectorPath(vectorPath: string): string {
|
|
167
|
+
return `${vectorPath}.${Date.now()}.${Math.random().toString(16).slice(2)}.tmp`;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
function toScanOptions(options: AddOptions = {}): ScanOptions {
|
|
171
|
+
return {
|
|
172
|
+
includeSuggestedText: options.include_suggested_text === true,
|
|
173
|
+
includePaths: options.include_paths,
|
|
174
|
+
excludePaths: options.exclude_paths,
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
function serializeAddOptions(options: AddOptions = {}): string | undefined {
|
|
179
|
+
const normalized: AddOptions = {};
|
|
180
|
+
if (options.include_suggested_text === true) normalized.include_suggested_text = true;
|
|
181
|
+
if (options.include_paths && options.include_paths.length > 0) normalized.include_paths = options.include_paths;
|
|
182
|
+
if (options.exclude_paths && options.exclude_paths.length > 0) normalized.exclude_paths = options.exclude_paths;
|
|
183
|
+
return Object.keys(normalized).length > 0 ? JSON.stringify(normalized) : undefined;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
function parseAddOptions(raw: string | null): AddOptions {
|
|
187
|
+
if (!raw) return {};
|
|
188
|
+
try {
|
|
189
|
+
const parsed = JSON.parse(raw) as AddOptions;
|
|
190
|
+
return {
|
|
191
|
+
include_suggested_text: parsed.include_suggested_text === true,
|
|
192
|
+
include_paths: Array.isArray(parsed.include_paths)
|
|
193
|
+
? parsed.include_paths.filter((item) => typeof item === "string")
|
|
194
|
+
: undefined,
|
|
195
|
+
exclude_paths: Array.isArray(parsed.exclude_paths)
|
|
196
|
+
? parsed.exclude_paths.filter((item) => typeof item === "string")
|
|
197
|
+
: undefined,
|
|
198
|
+
};
|
|
199
|
+
} catch {
|
|
200
|
+
return {};
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
function planDirectoryScan(dirPath: string, options: ScanOptions = {}): DirectoryScanPlan {
|
|
205
|
+
const skipped = createSkippedScanStats();
|
|
206
|
+
let files = 0;
|
|
207
|
+
let bytes = 0;
|
|
208
|
+
for (const file of iterateScannableFiles(dirPath, skipped, options)) {
|
|
209
|
+
files++;
|
|
210
|
+
bytes += file.size;
|
|
211
|
+
}
|
|
212
|
+
return {
|
|
213
|
+
files,
|
|
214
|
+
bytes,
|
|
215
|
+
skippedTotal: skipped.total,
|
|
216
|
+
skippedSummary: summarizeSkippedScan(skipped),
|
|
217
|
+
skipped,
|
|
218
|
+
};
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
function formatBytes(bytes: number): string {
|
|
222
|
+
if (bytes < 1024) return `${bytes} B`;
|
|
223
|
+
const kb = bytes / 1024;
|
|
224
|
+
if (kb < 1024) return `${kb.toFixed(1)} KB`;
|
|
225
|
+
const mb = kb / 1024;
|
|
226
|
+
if (mb < 1024) return `${mb.toFixed(1)} MB`;
|
|
227
|
+
return `${(mb / 1024).toFixed(2)} GB`;
|
|
228
|
+
}
|
|
229
|
+
|
|
105
230
|
function cosineSimilarity(a: Float32Array | undefined, b: Float32Array | undefined): number {
|
|
106
231
|
if (!a || !b || a.length !== b.length) return 0;
|
|
107
232
|
let dot = 0;
|
|
@@ -305,6 +430,68 @@ function pushAdaptiveCandidate(candidates: RankedChunk[], candidate: RankedChunk
|
|
|
305
430
|
}
|
|
306
431
|
}
|
|
307
432
|
|
|
433
|
+
function formatDuration(ms: number): string {
|
|
434
|
+
if (!Number.isFinite(ms) || ms <= 0) return "0s";
|
|
435
|
+
const seconds = Math.ceil(ms / 1000);
|
|
436
|
+
if (seconds < 60) return `${seconds}s`;
|
|
437
|
+
const minutes = Math.floor(seconds / 60);
|
|
438
|
+
const rest = seconds % 60;
|
|
439
|
+
return rest === 0 ? `${minutes}m` : `${minutes}m ${rest}s`;
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
function isExactLookupQuery(query: string): boolean {
|
|
443
|
+
const trimmed = query.trim();
|
|
444
|
+
if (/["'`]/.test(trimmed)) return true;
|
|
445
|
+
if (/[./\\][\w.-]+/.test(trimmed)) return true;
|
|
446
|
+
if (/\b[A-Z][A-Za-z0-9_]*[A-Z][A-Za-z0-9_]*\b/.test(trimmed)) return true;
|
|
447
|
+
if (/\b[A-Z]{2,}[_-]?[A-Z0-9]*\b/.test(trimmed)) return true;
|
|
448
|
+
if (/\b[a-zA-Z_][\w-]*\.(ts|tsx|js|jsx|go|rs|py|java|md|json|ya?ml|toml)\b/.test(trimmed)) return true;
|
|
449
|
+
if (/\b[A-Z]+-\d+\b/.test(trimmed)) return true;
|
|
450
|
+
return false;
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
function chooseAutoMode(query: string): NonNullable<SearchOptions["mode"]> {
|
|
454
|
+
const normalized = query.trim();
|
|
455
|
+
if (isExactLookupQuery(normalized)) return "fast";
|
|
456
|
+
const wordCount = normalized.split(/\s+/).filter(Boolean).length;
|
|
457
|
+
if (wordCount >= 10 || /how|why|explain|concept|architecture|design|流程|架構|概念|為什麼|如何/i.test(normalized)) {
|
|
458
|
+
return "semantic";
|
|
459
|
+
}
|
|
460
|
+
return "hybrid";
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
function fallbackModesFor(mode: NonNullable<SearchOptions["mode"]>): NonNullable<SearchOptions["mode"]>[] {
|
|
464
|
+
if (mode === "fast") return ["hybrid", "semantic"];
|
|
465
|
+
if (mode === "semantic") return ["hybrid", "adaptive"];
|
|
466
|
+
if (mode === "adaptive") return ["hybrid", "deep"];
|
|
467
|
+
if (mode === "deep") return ["hybrid", "adaptive"];
|
|
468
|
+
return ["fast", "semantic", "adaptive"];
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
function isWeakAutoResponse(
|
|
472
|
+
query: string,
|
|
473
|
+
response: SearchResponse,
|
|
474
|
+
primaryMode: NonNullable<SearchOptions["mode"]>,
|
|
475
|
+
attemptMode: NonNullable<SearchOptions["mode"]>,
|
|
476
|
+
): boolean {
|
|
477
|
+
if (response.results.length === 0) return true;
|
|
478
|
+
if (primaryMode === "fast") {
|
|
479
|
+
const exactNeedle = query
|
|
480
|
+
.trim()
|
|
481
|
+
.replace(/^["'`]|["'`]$/g, "")
|
|
482
|
+
.toLowerCase();
|
|
483
|
+
const hasExactHit = response.results.some(
|
|
484
|
+
(result) =>
|
|
485
|
+
result.file_path.toLowerCase().includes(exactNeedle) || result.content.toLowerCase().includes(exactNeedle),
|
|
486
|
+
);
|
|
487
|
+
if (!hasExactHit && response.results.every((result) => (result.ranking?.coverage ?? 0) < 0.67)) return true;
|
|
488
|
+
}
|
|
489
|
+
if (primaryMode !== "semantic" && attemptMode === "semantic") {
|
|
490
|
+
return response.results.every((result) => (result.ranking?.coverage ?? 0) === 0);
|
|
491
|
+
}
|
|
492
|
+
return false;
|
|
493
|
+
}
|
|
494
|
+
|
|
308
495
|
async function chunkUrl(source: string, signal?: AbortSignal): Promise<Awaited<ReturnType<typeof chunkFile>>> {
|
|
309
496
|
const res = await fetch(source, { signal });
|
|
310
497
|
if (!res.ok) throw new Error(`Fetch failed: ${res.status}`);
|
|
@@ -326,34 +513,61 @@ function normalizeExtractedText(text: string | string[]): string {
|
|
|
326
513
|
export class KnowledgeEngine {
|
|
327
514
|
private db: Database.Database | null = null;
|
|
328
515
|
private knowledgeDir: string = "";
|
|
329
|
-
private vectorCache: Map<string, Float32Array[]> = new Map();
|
|
330
516
|
|
|
331
517
|
async initialize(knowledgeDir: string): Promise<void> {
|
|
332
518
|
this.knowledgeDir = knowledgeDir;
|
|
333
519
|
this.db = openDatabase(knowledgeDir);
|
|
334
|
-
this.vectorCache.clear();
|
|
335
|
-
}
|
|
336
|
-
|
|
337
|
-
private getVectors(kbId: string): Float32Array[] {
|
|
338
|
-
const cached = this.vectorCache.get(kbId);
|
|
339
|
-
if (cached) return cached;
|
|
340
|
-
const vectorPath = join(this.knowledgeDir, "vectors", `${kbId}.bin`);
|
|
341
|
-
const vectors = loadVectors(vectorPath);
|
|
342
|
-
this.vectorCache.set(kbId, vectors);
|
|
343
|
-
return vectors;
|
|
344
|
-
}
|
|
345
|
-
|
|
346
|
-
private invalidateVectorCache(kbId: string): void {
|
|
347
|
-
this.vectorCache.delete(kbId);
|
|
348
520
|
}
|
|
349
521
|
|
|
350
|
-
|
|
351
|
-
const
|
|
352
|
-
const
|
|
353
|
-
|
|
354
|
-
|
|
522
|
+
plan(source: string, options: AddOptions = {}): IndexPlan {
|
|
523
|
+
const resolvedSource = resolve(source);
|
|
524
|
+
const isUrl = source.startsWith("http://") || source.startsWith("https://");
|
|
525
|
+
const isDir = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isDirectory();
|
|
526
|
+
const isFile = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isFile();
|
|
527
|
+
const sourceType = isUrl ? "url" : isDir ? "directory" : isFile ? "file" : "text";
|
|
528
|
+
if (isDir) {
|
|
529
|
+
const plan = planDirectoryScan(resolvedSource, toScanOptions(options));
|
|
530
|
+
return {
|
|
531
|
+
source_type: "directory",
|
|
532
|
+
scannable_files: plan.files,
|
|
533
|
+
scannable_bytes: plan.bytes,
|
|
534
|
+
skipped: plan.skipped,
|
|
535
|
+
summary: `Directory plan: ${plan.files} scannable files, ${formatBytes(plan.bytes)} scannable text, skipped ${plan.skippedTotal} (${plan.skippedSummary})`,
|
|
536
|
+
};
|
|
537
|
+
}
|
|
538
|
+
if (isFile) {
|
|
539
|
+
const skipped = createSkippedScanStats();
|
|
540
|
+
const supportedDocument =
|
|
541
|
+
resolvedSource.endsWith(".pdf") || resolvedSource.endsWith(".docx") || resolvedSource.endsWith(".doc");
|
|
542
|
+
if (!supportedDocument && !isReadableTextFile(resolvedSource)) {
|
|
543
|
+
skipped.total = 1;
|
|
544
|
+
skipped.by_reason.binary = 1;
|
|
545
|
+
skipped.samples.push({ path: resolvedSource, reason: "binary" });
|
|
546
|
+
return {
|
|
547
|
+
source_type: "file",
|
|
548
|
+
scannable_files: 0,
|
|
549
|
+
scannable_bytes: 0,
|
|
550
|
+
skipped,
|
|
551
|
+
summary: "File plan: 0 scannable files; file is unsupported binary/non-text",
|
|
552
|
+
};
|
|
553
|
+
}
|
|
554
|
+
const size = statSync(resolvedSource).size;
|
|
555
|
+
return {
|
|
556
|
+
source_type: "file",
|
|
557
|
+
scannable_files: 1,
|
|
558
|
+
scannable_bytes: size,
|
|
559
|
+
skipped,
|
|
560
|
+
summary: `File plan: 1 scannable file, ${formatBytes(size)} source size`,
|
|
561
|
+
};
|
|
355
562
|
}
|
|
356
|
-
|
|
563
|
+
const skipped = createSkippedScanStats();
|
|
564
|
+
return {
|
|
565
|
+
source_type: sourceType,
|
|
566
|
+
scannable_files: 1,
|
|
567
|
+
scannable_bytes: Buffer.byteLength(source),
|
|
568
|
+
skipped,
|
|
569
|
+
summary: `${sourceType === "url" ? "URL" : "Inline text"} plan: 1 source`,
|
|
570
|
+
};
|
|
357
571
|
}
|
|
358
572
|
|
|
359
573
|
async add(
|
|
@@ -361,6 +575,7 @@ export class KnowledgeEngine {
|
|
|
361
575
|
name: string,
|
|
362
576
|
onProgress?: ProgressCallback,
|
|
363
577
|
signal?: AbortSignal,
|
|
578
|
+
options: AddOptions = {},
|
|
364
579
|
): Promise<{ kb: KnowledgeBase; chunkCount: number }> {
|
|
365
580
|
if (!this.db) throw new Error("Engine not initialized");
|
|
366
581
|
const resolvedSource = resolve(source);
|
|
@@ -368,6 +583,7 @@ export class KnowledgeEngine {
|
|
|
368
583
|
const isDir = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isDirectory();
|
|
369
584
|
const isFile = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isFile();
|
|
370
585
|
const sourceType = isUrl ? "url" : isDir ? "directory" : isFile ? "file" : "text";
|
|
586
|
+
const scanOptions = toScanOptions(options);
|
|
371
587
|
|
|
372
588
|
const existingKB = getKBByName(this.db, name);
|
|
373
589
|
if (existingKB) {
|
|
@@ -380,60 +596,195 @@ export class KnowledgeEngine {
|
|
|
380
596
|
name,
|
|
381
597
|
source_path: isDir || isFile ? resolvedSource : isUrl ? source : undefined,
|
|
382
598
|
source_type: sourceType,
|
|
599
|
+
source_options: serializeAddOptions(options),
|
|
383
600
|
});
|
|
384
601
|
updateKBStatus(this.db, kb.id, "indexing");
|
|
602
|
+
startIndexingJob(this.db, kb.id, "add", `Starting indexing for "${name}"`);
|
|
385
603
|
|
|
604
|
+
let vectorWriter: ReturnType<typeof openVectorWriter> | undefined;
|
|
605
|
+
let tempVectorFile: string | undefined;
|
|
386
606
|
try {
|
|
387
|
-
|
|
607
|
+
const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
|
|
608
|
+
tempVectorFile = tempVectorPath(vectorPath);
|
|
609
|
+
vectorWriter = openVectorWriter(tempVectorFile);
|
|
610
|
+
let chunkCount = 0;
|
|
611
|
+
let fileCount = 0;
|
|
612
|
+
const pendingChunks: Awaited<ReturnType<typeof chunkFile>> = [];
|
|
613
|
+
const startedAt = Date.now();
|
|
614
|
+
let latestSkippedTotal = 0;
|
|
615
|
+
|
|
616
|
+
const reportProgress = (phase: string, processedFiles?: number, totalFiles?: number, skippedTotal = 0): void => {
|
|
617
|
+
latestSkippedTotal = skippedTotal;
|
|
618
|
+
const elapsed = Date.now() - startedAt;
|
|
619
|
+
const chunkRate = chunkCount / Math.max(1, elapsed / 1000);
|
|
620
|
+
let suffix = `${chunkCount} chunks, ${chunkRate.toFixed(1)} chunks/s, elapsed ${formatDuration(elapsed)}`;
|
|
621
|
+
if (processedFiles !== undefined && totalFiles !== undefined && totalFiles > 0) {
|
|
622
|
+
const rate = processedFiles / Math.max(1, elapsed / 1000);
|
|
623
|
+
const remainingFiles = Math.max(0, totalFiles - processedFiles);
|
|
624
|
+
const etaMs = rate > 0 ? (remainingFiles / rate) * 1000 : 0;
|
|
625
|
+
suffix = `${processedFiles}/${totalFiles} files, ${suffix}, file ETA ${formatDuration(etaMs)}`;
|
|
626
|
+
} else if (processedFiles !== undefined) {
|
|
627
|
+
suffix = `${processedFiles} files scanned, ${suffix}`;
|
|
628
|
+
}
|
|
629
|
+
if (skippedTotal > 0) suffix = `${suffix}, skipped ${skippedTotal}`;
|
|
630
|
+
const message = `${phase}: ${suffix}`;
|
|
631
|
+
updateIndexingJob(this.db, kb.id, {
|
|
632
|
+
phase,
|
|
633
|
+
message,
|
|
634
|
+
processed_files: processedFiles,
|
|
635
|
+
processed_chunks: chunkCount,
|
|
636
|
+
total_files: totalFiles,
|
|
637
|
+
skipped_total: skippedTotal,
|
|
638
|
+
added_chunks: chunkCount,
|
|
639
|
+
});
|
|
640
|
+
onProgress?.(message);
|
|
641
|
+
};
|
|
642
|
+
|
|
643
|
+
const flushPending = async (processedFiles?: number, totalFiles?: number): Promise<void> => {
|
|
644
|
+
if (!this.db || pendingChunks.length === 0) return;
|
|
645
|
+
if (signal?.aborted) throw new Error("Cancelled");
|
|
646
|
+
const batch = pendingChunks.splice(0, INDEX_EMBED_BATCH_SIZE);
|
|
647
|
+
reportProgress(`Embedding batch of ${batch.length}`, processedFiles, totalFiles);
|
|
648
|
+
const vectors = await embedDocuments(
|
|
649
|
+
batch.map((chunk) => buildChunkEmbeddingText(chunk)),
|
|
650
|
+
signal,
|
|
651
|
+
);
|
|
652
|
+
if (signal?.aborted) throw new Error("Cancelled");
|
|
653
|
+
insertChunks(this.db, kb.id, batch);
|
|
654
|
+
vectorWriter.append(vectors);
|
|
655
|
+
chunkCount += batch.length;
|
|
656
|
+
updateKBCounts(this.db, kb.id, chunkCount, fileCount);
|
|
657
|
+
reportProgress("Stored batch", processedFiles, totalFiles);
|
|
658
|
+
};
|
|
659
|
+
|
|
660
|
+
const addChunks = async (
|
|
661
|
+
chunks: Awaited<ReturnType<typeof chunkFile>>,
|
|
662
|
+
processedFiles?: number,
|
|
663
|
+
totalFiles?: number,
|
|
664
|
+
): Promise<void> => {
|
|
665
|
+
pendingChunks.push(...chunks);
|
|
666
|
+
while (pendingChunks.length >= INDEX_EMBED_BATCH_SIZE) {
|
|
667
|
+
await flushPending(processedFiles, totalFiles);
|
|
668
|
+
}
|
|
669
|
+
};
|
|
388
670
|
|
|
389
671
|
if (isUrl) {
|
|
390
|
-
|
|
391
|
-
|
|
672
|
+
const message = `Fetching ${source}...`;
|
|
673
|
+
updateIndexingJob(this.db, kb.id, { phase: "fetching", message });
|
|
674
|
+
onProgress?.(message);
|
|
675
|
+
const chunks = await chunkUrl(source, signal);
|
|
676
|
+
fileCount = 1;
|
|
677
|
+
await addChunks(chunks);
|
|
392
678
|
} else if (isFile && resolvedSource.endsWith(".pdf")) {
|
|
679
|
+
updateIndexingJob(this.db, kb.id, { phase: "extracting", message: "Extracting text from PDF..." });
|
|
393
680
|
onProgress?.("Extracting text from PDF...");
|
|
394
681
|
const { extractText } = await import("unpdf");
|
|
395
682
|
const buf = (await import("node:fs")).readFileSync(resolvedSource);
|
|
396
683
|
const { text } = await extractText(new Uint8Array(buf));
|
|
397
|
-
|
|
684
|
+
fileCount = 1;
|
|
685
|
+
await addChunks(await chunkFile(normalizeExtractedText(text), resolvedSource));
|
|
398
686
|
} else if (isFile && (resolvedSource.endsWith(".docx") || resolvedSource.endsWith(".doc"))) {
|
|
687
|
+
updateIndexingJob(this.db, kb.id, { phase: "extracting", message: "Extracting text from DOCX..." });
|
|
399
688
|
onProgress?.("Extracting text from DOCX...");
|
|
400
689
|
const mammoth = await import("mammoth");
|
|
401
690
|
const result = await mammoth.extractRawText({ path: resolvedSource });
|
|
402
|
-
|
|
691
|
+
fileCount = 1;
|
|
692
|
+
await addChunks(await chunkFile(result.value, resolvedSource));
|
|
403
693
|
} else if (isDir) {
|
|
404
|
-
|
|
405
|
-
const
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
694
|
+
const plan = planDirectoryScan(resolvedSource, scanOptions);
|
|
695
|
+
const planningMessage = `Planned directory scan: ${plan.files} files, ${formatBytes(
|
|
696
|
+
plan.bytes,
|
|
697
|
+
)} scannable text, skipped ${plan.skippedTotal} (${plan.skippedSummary})`;
|
|
698
|
+
updateIndexingJob(this.db, kb.id, {
|
|
699
|
+
phase: "planning",
|
|
700
|
+
message: planningMessage,
|
|
701
|
+
total_files: plan.files,
|
|
702
|
+
skipped_total: plan.skippedTotal,
|
|
703
|
+
});
|
|
704
|
+
onProgress?.(planningMessage);
|
|
705
|
+
const scanningMessage = `Scanning ${resolvedSource}...`;
|
|
706
|
+
updateIndexingJob(this.db, kb.id, {
|
|
707
|
+
phase: "scanning",
|
|
708
|
+
message: scanningMessage,
|
|
709
|
+
total_files: plan.files,
|
|
710
|
+
skipped_total: plan.skippedTotal,
|
|
711
|
+
});
|
|
712
|
+
onProgress?.(scanningMessage);
|
|
713
|
+
const skipped = createSkippedScanStats();
|
|
714
|
+
let processedFiles = 0;
|
|
715
|
+
for (const file of iterateScannedFiles(resolvedSource, skipped, scanOptions)) {
|
|
716
|
+
if (signal?.aborted) throw new Error("Cancelled");
|
|
717
|
+
const chunks = await chunkFile(file.content, file.relPath);
|
|
718
|
+
processedFiles++;
|
|
719
|
+
latestSkippedTotal = skipped.total;
|
|
720
|
+
if (chunks.length > 0) fileCount++;
|
|
721
|
+
await addChunks(chunks, processedFiles, plan.files);
|
|
722
|
+
if (processedFiles % 25 === 0) reportProgress("Chunking", processedFiles, plan.files, skipped.total);
|
|
409
723
|
}
|
|
724
|
+
const finalizingMessage = `Scanned ${processedFiles} files, skipped ${skipped.total} (${summarizeSkippedScan(
|
|
725
|
+
skipped,
|
|
726
|
+
)}), finalizing...`;
|
|
727
|
+
updateIndexingJob(this.db, kb.id, {
|
|
728
|
+
phase: "finalizing",
|
|
729
|
+
message: finalizingMessage,
|
|
730
|
+
processed_files: processedFiles,
|
|
731
|
+
processed_chunks: chunkCount,
|
|
732
|
+
total_files: plan.files,
|
|
733
|
+
skipped_total: skipped.total,
|
|
734
|
+
added_chunks: chunkCount,
|
|
735
|
+
});
|
|
736
|
+
onProgress?.(finalizingMessage);
|
|
410
737
|
} else if (isFile) {
|
|
738
|
+
if (!isReadableTextFile(resolvedSource)) {
|
|
739
|
+
throw new Error(`File is not readable text and has no supported extractor: ${resolvedSource}`);
|
|
740
|
+
}
|
|
411
741
|
const { readFileSync } = await import("node:fs");
|
|
412
742
|
const content = readFileSync(resolvedSource, "utf-8");
|
|
413
|
-
|
|
743
|
+
fileCount = 1;
|
|
744
|
+
await addChunks(await chunkFile(content, resolvedSource));
|
|
414
745
|
} else {
|
|
415
|
-
|
|
746
|
+
fileCount = 1;
|
|
747
|
+
await addChunks(await chunkFile(source, "inline-text"));
|
|
416
748
|
}
|
|
417
749
|
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
insertChunks(this.db, kb.id, allChunks);
|
|
750
|
+
await flushPending();
|
|
751
|
+
vectorWriter.close();
|
|
752
|
+
vectorWriter = undefined;
|
|
753
|
+
renameSync(tempVectorFile, vectorPath);
|
|
754
|
+
tempVectorFile = undefined;
|
|
424
755
|
|
|
425
|
-
const
|
|
426
|
-
|
|
427
|
-
this.vectorCache.set(kb.id, vectors);
|
|
428
|
-
|
|
429
|
-
const fileCount = isDir ? getFileCount(this.db, kb.id) : 1;
|
|
430
|
-
updateKBCounts(this.db, kb.id, allChunks.length, fileCount);
|
|
756
|
+
const savedFileCount = isDir ? getFileCount(this.db, kb.id) : fileCount;
|
|
757
|
+
updateKBCounts(this.db, kb.id, chunkCount, savedFileCount);
|
|
431
758
|
updateKBStatus(this.db, kb.id, "ready");
|
|
432
759
|
|
|
433
760
|
const savedKB = getKB(this.db, kb.id);
|
|
434
761
|
if (!savedKB) throw new Error(`Knowledge base disappeared after add: ${kb.id}`);
|
|
435
|
-
|
|
762
|
+
const readyMessage = `Ready: ${chunkCount} chunks from ${savedFileCount} files in ${formatDuration(
|
|
763
|
+
Date.now() - startedAt,
|
|
764
|
+
)}`;
|
|
765
|
+
updateIndexingJob(this.db, kb.id, {
|
|
766
|
+
phase: "ready",
|
|
767
|
+
message: readyMessage,
|
|
768
|
+
processed_files: savedFileCount,
|
|
769
|
+
processed_chunks: chunkCount,
|
|
770
|
+
skipped_total: latestSkippedTotal,
|
|
771
|
+
added_chunks: chunkCount,
|
|
772
|
+
});
|
|
773
|
+
finishIndexingJob(this.db, kb.id, "succeeded", readyMessage);
|
|
774
|
+
onProgress?.(readyMessage);
|
|
775
|
+
return { kb: savedKB, chunkCount };
|
|
436
776
|
} catch (e) {
|
|
777
|
+
vectorWriter?.close();
|
|
778
|
+
if (tempVectorFile) rmSync(tempVectorFile, { force: true });
|
|
779
|
+
if (this.db) {
|
|
780
|
+
finishIndexingJob(
|
|
781
|
+
this.db,
|
|
782
|
+
kb.id,
|
|
783
|
+
isCancellationError(e) ? "cancelled" : "failed",
|
|
784
|
+
isCancellationError(e) ? "Indexing cancelled." : "Indexing failed.",
|
|
785
|
+
e instanceof Error ? e.message : String(e),
|
|
786
|
+
);
|
|
787
|
+
}
|
|
437
788
|
// Clean up partial state on failure
|
|
438
789
|
deleteKB(this.db, kb.id);
|
|
439
790
|
throw e;
|
|
@@ -453,84 +804,296 @@ export class KnowledgeEngine {
|
|
|
453
804
|
}
|
|
454
805
|
|
|
455
806
|
updateKBStatus(this.db, kb.id, "indexing");
|
|
807
|
+
startIndexingJob(this.db, kb.id, "update", `Starting update for "${kb.name}"`);
|
|
808
|
+
const scanOptions = toScanOptions(parseAddOptions(kb.source_options));
|
|
809
|
+
let replacementVectorPath: string | undefined;
|
|
810
|
+
let addedVectorPath: string | undefined;
|
|
811
|
+
let addedVectorWriter: ReturnType<typeof openVectorWriter> | undefined;
|
|
456
812
|
|
|
457
813
|
try {
|
|
458
|
-
|
|
814
|
+
const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
|
|
815
|
+
addedVectorPath = tempVectorPath(`${vectorPath}.added`);
|
|
816
|
+
addedVectorWriter = openVectorWriter(addedVectorPath);
|
|
817
|
+
const existingHashes = new Map<string, Array<{ id: string; vectorIndex: number }>>();
|
|
818
|
+
let existingIndex = 0;
|
|
819
|
+
for (const chunk of iterateChunksByKB(this.db, kb.id)) {
|
|
820
|
+
const entries = existingHashes.get(chunk.content_hash) ?? [];
|
|
821
|
+
entries.push({ id: chunk.id, vectorIndex: existingIndex });
|
|
822
|
+
existingHashes.set(chunk.content_hash, entries);
|
|
823
|
+
existingIndex++;
|
|
824
|
+
}
|
|
825
|
+
|
|
826
|
+
const oldVectorIndexByHash = new Map<string, number[]>();
|
|
827
|
+
const newVectorIndexByHash = new Map<string, number[]>();
|
|
828
|
+
const pendingChunks: Awaited<ReturnType<typeof chunkFile>> = [];
|
|
829
|
+
let addedVectorCount = 0;
|
|
830
|
+
let addedCount = 0;
|
|
831
|
+
let unchanged = 0;
|
|
832
|
+
let scannedFiles = 0;
|
|
833
|
+
let scannedChunks = 0;
|
|
834
|
+
let plannedTotalFiles: number | undefined;
|
|
835
|
+
const startedAt = Date.now();
|
|
836
|
+
|
|
837
|
+
const flushPending = async (): Promise<void> => {
|
|
838
|
+
if (!this.db || !addedVectorWriter || pendingChunks.length === 0) return;
|
|
839
|
+
if (signal?.aborted) throw new Error("Cancelled");
|
|
840
|
+
const batch = pendingChunks.splice(0, INDEX_EMBED_BATCH_SIZE);
|
|
841
|
+
const elapsed = Date.now() - startedAt;
|
|
842
|
+
const message = `Embedding update batch: ${addedCount} new chunks stored, ${scannedFiles} files scanned, elapsed ${formatDuration(
|
|
843
|
+
elapsed,
|
|
844
|
+
)}`;
|
|
845
|
+
updateIndexingJob(this.db, kb.id, {
|
|
846
|
+
phase: "embedding",
|
|
847
|
+
message,
|
|
848
|
+
processed_files: scannedFiles,
|
|
849
|
+
processed_chunks: scannedChunks,
|
|
850
|
+
total_files: plannedTotalFiles,
|
|
851
|
+
added_chunks: addedCount,
|
|
852
|
+
unchanged_chunks: unchanged,
|
|
853
|
+
});
|
|
854
|
+
onProgress?.(message);
|
|
855
|
+
const newVectors = await embedDocuments(
|
|
856
|
+
batch.map((c) => buildChunkEmbeddingText(c)),
|
|
857
|
+
signal,
|
|
858
|
+
);
|
|
859
|
+
if (signal?.aborted) throw new Error("Cancelled");
|
|
860
|
+
addedVectorWriter.append(newVectors);
|
|
861
|
+
for (let i = 0; i < batch.length; i++) {
|
|
862
|
+
const indexes = newVectorIndexByHash.get(batch[i].content_hash) ?? [];
|
|
863
|
+
indexes.push(addedVectorCount + i);
|
|
864
|
+
newVectorIndexByHash.set(batch[i].content_hash, indexes);
|
|
865
|
+
}
|
|
866
|
+
addedVectorCount += newVectors.length;
|
|
867
|
+
insertChunks(this.db, kb.id, batch);
|
|
868
|
+
addedCount += batch.length;
|
|
869
|
+
updateKBCounts(this.db, kb.id, getChunkCount(this.db, kb.id), getFileCount(this.db, kb.id));
|
|
870
|
+
const storedMessage = `Stored update batch: +${addedCount} chunks, =${unchanged} unchanged`;
|
|
871
|
+
updateIndexingJob(this.db, kb.id, {
|
|
872
|
+
phase: "storing",
|
|
873
|
+
message: storedMessage,
|
|
874
|
+
processed_files: scannedFiles,
|
|
875
|
+
processed_chunks: scannedChunks,
|
|
876
|
+
total_files: plannedTotalFiles,
|
|
877
|
+
added_chunks: addedCount,
|
|
878
|
+
unchanged_chunks: unchanged,
|
|
879
|
+
});
|
|
880
|
+
onProgress?.(storedMessage);
|
|
881
|
+
};
|
|
882
|
+
|
|
883
|
+
const processChunks = async (chunks: Awaited<ReturnType<typeof chunkFile>>): Promise<void> => {
|
|
884
|
+
for (const chunk of chunks) {
|
|
885
|
+
scannedChunks++;
|
|
886
|
+
const existing = existingHashes.get(chunk.content_hash);
|
|
887
|
+
if (existing && existing.length > 0) {
|
|
888
|
+
const retained = existing.shift();
|
|
889
|
+
if (retained) {
|
|
890
|
+
const indexes = oldVectorIndexByHash.get(chunk.content_hash) ?? [];
|
|
891
|
+
indexes.push(retained.vectorIndex);
|
|
892
|
+
oldVectorIndexByHash.set(chunk.content_hash, indexes);
|
|
893
|
+
}
|
|
894
|
+
unchanged++;
|
|
895
|
+
continue;
|
|
896
|
+
}
|
|
897
|
+
pendingChunks.push(chunk);
|
|
898
|
+
if (pendingChunks.length >= INDEX_EMBED_BATCH_SIZE) await flushPending();
|
|
899
|
+
}
|
|
900
|
+
};
|
|
901
|
+
|
|
902
|
+
updateIndexingJob(this.db, kb.id, { phase: "scanning", message: "Scanning source..." });
|
|
459
903
|
onProgress?.("Scanning source...");
|
|
460
|
-
let newChunks: Awaited<ReturnType<typeof chunkFile>> = [];
|
|
461
904
|
if (kb.source_type === "url") {
|
|
462
|
-
|
|
463
|
-
|
|
905
|
+
const message = `Fetching ${kb.source_path}...`;
|
|
906
|
+
updateIndexingJob(this.db, kb.id, { phase: "fetching", message });
|
|
907
|
+
onProgress?.(message);
|
|
908
|
+
scannedFiles = 1;
|
|
909
|
+
await processChunks(await chunkUrl(kb.source_path, signal));
|
|
464
910
|
} else if (statSync(kb.source_path).isDirectory()) {
|
|
465
|
-
const
|
|
466
|
-
|
|
911
|
+
const plan = planDirectoryScan(kb.source_path, scanOptions);
|
|
912
|
+
plannedTotalFiles = plan.files;
|
|
913
|
+
const planningMessage = `Planned directory scan: ${plan.files} files, ${formatBytes(
|
|
914
|
+
plan.bytes,
|
|
915
|
+
)} scannable text, skipped ${plan.skippedTotal} (${plan.skippedSummary})`;
|
|
916
|
+
updateIndexingJob(this.db, kb.id, {
|
|
917
|
+
phase: "planning",
|
|
918
|
+
message: planningMessage,
|
|
919
|
+
total_files: plan.files,
|
|
920
|
+
skipped_total: plan.skippedTotal,
|
|
921
|
+
});
|
|
922
|
+
onProgress?.(planningMessage);
|
|
923
|
+
const skipped = createSkippedScanStats();
|
|
924
|
+
for (const file of iterateScannedFiles(kb.source_path, skipped, scanOptions)) {
|
|
925
|
+
if (signal?.aborted) throw new Error("Cancelled");
|
|
926
|
+
scannedFiles++;
|
|
927
|
+
await processChunks(await chunkFile(file.content, file.relPath));
|
|
928
|
+
if (scannedFiles % 25 === 0) {
|
|
929
|
+
const message = `Scanned ${scannedFiles} files, ${scannedChunks} chunks, skipped ${skipped.total}, +${addedCount} =${unchanged}`;
|
|
930
|
+
updateIndexingJob(this.db, kb.id, {
|
|
931
|
+
phase: "scanning",
|
|
932
|
+
message,
|
|
933
|
+
processed_files: scannedFiles,
|
|
934
|
+
processed_chunks: scannedChunks,
|
|
935
|
+
total_files: plan.files,
|
|
936
|
+
skipped_total: skipped.total,
|
|
937
|
+
added_chunks: addedCount,
|
|
938
|
+
unchanged_chunks: unchanged,
|
|
939
|
+
});
|
|
940
|
+
onProgress?.(message);
|
|
941
|
+
}
|
|
942
|
+
}
|
|
943
|
+
const message = `Scanned ${scannedFiles} files, skipped ${skipped.total} (${summarizeSkippedScan(
|
|
944
|
+
skipped,
|
|
945
|
+
)}), reconciling deletes...`;
|
|
946
|
+
updateIndexingJob(this.db, kb.id, {
|
|
947
|
+
phase: "reconciling",
|
|
948
|
+
message,
|
|
949
|
+
processed_files: scannedFiles,
|
|
950
|
+
processed_chunks: scannedChunks,
|
|
951
|
+
total_files: plan.files,
|
|
952
|
+
skipped_total: skipped.total,
|
|
953
|
+
added_chunks: addedCount,
|
|
954
|
+
unchanged_chunks: unchanged,
|
|
955
|
+
});
|
|
956
|
+
onProgress?.(message);
|
|
467
957
|
} else {
|
|
468
958
|
const { readFileSync } = await import("node:fs");
|
|
469
|
-
|
|
959
|
+
scannedFiles = 1;
|
|
960
|
+
await processChunks(await chunkFile(readFileSync(kb.source_path, "utf-8"), kb.source_path));
|
|
470
961
|
}
|
|
962
|
+
await flushPending();
|
|
963
|
+
addedVectorWriter.close();
|
|
964
|
+
addedVectorWriter = undefined;
|
|
471
965
|
|
|
472
|
-
|
|
473
|
-
const
|
|
474
|
-
|
|
475
|
-
const existingVectors = loadVectors(vectorPath);
|
|
476
|
-
|
|
477
|
-
// Build hash→vector cache from existing data
|
|
478
|
-
const vectorCache = new Map<string, Float32Array>();
|
|
479
|
-
const existingChunks = getChunksByKB(this.db, kb.id);
|
|
480
|
-
for (let i = 0; i < existingChunks.length; i++) {
|
|
481
|
-
if (existingVectors[i]) vectorCache.set(existingChunks[i].content_hash, existingVectors[i]);
|
|
966
|
+
const idsToRemove: string[] = [];
|
|
967
|
+
for (const entries of existingHashes.values()) {
|
|
968
|
+
idsToRemove.push(...entries.map((entry) => entry.id));
|
|
482
969
|
}
|
|
483
|
-
|
|
484
|
-
// 3. Identify changes
|
|
485
|
-
const newHashSet = new Set(newChunks.map((c) => c.content_hash));
|
|
486
|
-
const chunksToAdd = newChunks.filter((c) => !existingHashes.has(c.content_hash));
|
|
487
|
-
const idsToRemove = [...existingHashes.entries()].filter(([hash]) => !newHashSet.has(hash)).map(([, id]) => id);
|
|
488
|
-
const unchanged = newChunks.length - chunksToAdd.length;
|
|
489
|
-
|
|
490
|
-
onProgress?.(`Changes: +${chunksToAdd.length} -${idsToRemove.length} =${unchanged}`);
|
|
491
|
-
|
|
492
|
-
// 4. Remove deleted chunks from DB
|
|
493
970
|
if (idsToRemove.length > 0) deleteChunksByIds(this.db, idsToRemove);
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
971
|
+
const changesMessage = `Changes: +${addedCount} -${idsToRemove.length} =${unchanged}`;
|
|
972
|
+
updateIndexingJob(this.db, kb.id, {
|
|
973
|
+
phase: "reconciling",
|
|
974
|
+
message: changesMessage,
|
|
975
|
+
processed_files: scannedFiles,
|
|
976
|
+
processed_chunks: scannedChunks,
|
|
977
|
+
added_chunks: addedCount,
|
|
978
|
+
removed_chunks: idsToRemove.length,
|
|
979
|
+
unchanged_chunks: unchanged,
|
|
980
|
+
});
|
|
981
|
+
onProgress?.(changesMessage);
|
|
982
|
+
|
|
983
|
+
replacementVectorPath = tempVectorPath(vectorPath);
|
|
984
|
+
const vectorWriter = openVectorWriter(replacementVectorPath);
|
|
985
|
+
const oldVectorReader = openVectorReader(vectorPath);
|
|
986
|
+
const newVectorReader = openVectorReader(addedVectorPath);
|
|
987
|
+
let finalChunkCount = 0;
|
|
988
|
+
const takeVectorIndex = (indexesByHash: Map<string, number[]>, hash: string): number | undefined => {
|
|
989
|
+
const indexes = indexesByHash.get(hash);
|
|
990
|
+
return indexes?.shift();
|
|
991
|
+
};
|
|
992
|
+
try {
|
|
993
|
+
for (const chunk of iterateChunksByKB(this.db, kb.id)) {
|
|
994
|
+
const oldVectorIndex = takeVectorIndex(oldVectorIndexByHash, chunk.content_hash);
|
|
995
|
+
const newVectorIndex = takeVectorIndex(newVectorIndexByHash, chunk.content_hash);
|
|
996
|
+
const vector =
|
|
997
|
+
oldVectorReader && oldVectorIndex !== undefined
|
|
998
|
+
? oldVectorReader.read(oldVectorIndex)
|
|
999
|
+
: newVectorReader && newVectorIndex !== undefined
|
|
1000
|
+
? newVectorReader.read(newVectorIndex)
|
|
1001
|
+
: undefined;
|
|
1002
|
+
if (!vector) throw new Error(`Missing vector while rebuilding knowledge base: ${chunk.id}`);
|
|
1003
|
+
vectorWriter.append([vector]);
|
|
1004
|
+
finalChunkCount++;
|
|
1005
|
+
if (finalChunkCount % 1_000 === 0) {
|
|
1006
|
+
const message = `Rebuilding vector file: ${finalChunkCount} chunks written`;
|
|
1007
|
+
updateIndexingJob(this.db, kb.id, {
|
|
1008
|
+
phase: "rebuilding_vectors",
|
|
1009
|
+
message,
|
|
1010
|
+
processed_files: scannedFiles,
|
|
1011
|
+
processed_chunks: finalChunkCount,
|
|
1012
|
+
added_chunks: addedCount,
|
|
1013
|
+
removed_chunks: idsToRemove.length,
|
|
1014
|
+
unchanged_chunks: unchanged,
|
|
1015
|
+
});
|
|
1016
|
+
onProgress?.(message);
|
|
1017
|
+
}
|
|
1018
|
+
}
|
|
1019
|
+
} finally {
|
|
1020
|
+
oldVectorReader?.close();
|
|
1021
|
+
newVectorReader?.close();
|
|
1022
|
+
vectorWriter.close();
|
|
517
1023
|
}
|
|
518
|
-
|
|
519
|
-
|
|
1024
|
+
renameSync(replacementVectorPath, vectorPath);
|
|
1025
|
+
replacementVectorPath = undefined;
|
|
1026
|
+
if (addedVectorPath) rmSync(addedVectorPath, { force: true });
|
|
1027
|
+
addedVectorPath = undefined;
|
|
520
1028
|
|
|
521
|
-
|
|
522
|
-
updateKBCounts(this.db, kb.id, finalChunks.length, getFileCount(this.db, kb.id));
|
|
1029
|
+
updateKBCounts(this.db, kb.id, finalChunkCount, getFileCount(this.db, kb.id));
|
|
523
1030
|
updateKBStatus(this.db, kb.id, "ready");
|
|
1031
|
+
const readyMessage = `Ready: +${addedCount} -${idsToRemove.length} =${unchanged}`;
|
|
1032
|
+
updateIndexingJob(this.db, kb.id, {
|
|
1033
|
+
phase: "ready",
|
|
1034
|
+
message: readyMessage,
|
|
1035
|
+
processed_files: scannedFiles,
|
|
1036
|
+
processed_chunks: finalChunkCount,
|
|
1037
|
+
added_chunks: addedCount,
|
|
1038
|
+
removed_chunks: idsToRemove.length,
|
|
1039
|
+
unchanged_chunks: unchanged,
|
|
1040
|
+
});
|
|
1041
|
+
finishIndexingJob(this.db, kb.id, "succeeded", readyMessage);
|
|
1042
|
+
onProgress?.(readyMessage);
|
|
524
1043
|
|
|
525
|
-
return { added:
|
|
1044
|
+
return { added: addedCount, removed: idsToRemove.length, unchanged };
|
|
526
1045
|
} catch (e) {
|
|
1046
|
+
addedVectorWriter?.close();
|
|
1047
|
+
if (replacementVectorPath) rmSync(replacementVectorPath, { force: true });
|
|
1048
|
+
if (addedVectorPath) rmSync(addedVectorPath, { force: true });
|
|
527
1049
|
updateKBStatus(this.db, kb.id, "error");
|
|
1050
|
+
finishIndexingJob(
|
|
1051
|
+
this.db,
|
|
1052
|
+
kb.id,
|
|
1053
|
+
isCancellationError(e) ? "cancelled" : "failed",
|
|
1054
|
+
isCancellationError(e) ? "Update cancelled." : "Update failed.",
|
|
1055
|
+
e instanceof Error ? e.message : String(e),
|
|
1056
|
+
);
|
|
528
1057
|
throw e;
|
|
529
1058
|
}
|
|
530
1059
|
}
|
|
531
1060
|
|
|
532
1061
|
async search(query: string, options: SearchOptions = {}): Promise<SearchResponse> {
|
|
533
1062
|
if (!this.db) throw new Error("Engine not initialized");
|
|
1063
|
+
const requestedMode = options.mode ?? "hybrid";
|
|
1064
|
+
if (requestedMode === "auto") {
|
|
1065
|
+
const primaryMode = chooseAutoMode(query);
|
|
1066
|
+
const attempts = [primaryMode, ...fallbackModesFor(primaryMode)];
|
|
1067
|
+
const tried: NonNullable<SearchOptions["mode"]>[] = [];
|
|
1068
|
+
const warnings: string[] = [];
|
|
1069
|
+
for (const mode of attempts) {
|
|
1070
|
+
if (tried.includes(mode)) continue;
|
|
1071
|
+
tried.push(mode);
|
|
1072
|
+
const response = await this.search(query, { ...options, mode });
|
|
1073
|
+
if (response.warnings) warnings.push(...response.warnings);
|
|
1074
|
+
if (!isWeakAutoResponse(query, response, primaryMode, mode)) {
|
|
1075
|
+
return {
|
|
1076
|
+
...response,
|
|
1077
|
+
warnings: warnings.length > 0 ? [...new Set(warnings)] : response.warnings,
|
|
1078
|
+
mode_used: mode,
|
|
1079
|
+
retry_modes: tried.slice(0, -1),
|
|
1080
|
+
suggestions: response.suggestions,
|
|
1081
|
+
};
|
|
1082
|
+
}
|
|
1083
|
+
if (tried.length >= 3) break;
|
|
1084
|
+
}
|
|
1085
|
+
return {
|
|
1086
|
+
results: [],
|
|
1087
|
+
total_count: 0,
|
|
1088
|
+
has_more: false,
|
|
1089
|
+
warnings: warnings.length > 0 ? [...new Set(warnings)] : undefined,
|
|
1090
|
+
mode_used: tried.at(-1),
|
|
1091
|
+
retry_modes: tried.slice(0, -1),
|
|
1092
|
+
suggestions: [
|
|
1093
|
+
"No results after auto mode fallback. Check knowledge_status, try a more exact term, or rebuild the KB if indexing rules changed.",
|
|
1094
|
+
],
|
|
1095
|
+
};
|
|
1096
|
+
}
|
|
534
1097
|
const db = this.db;
|
|
535
1098
|
const { mode = "hybrid", limit = 10, offset = 0, kb_id, filters, diversity = "balanced" } = options;
|
|
536
1099
|
const retrievalMode = mode === "adaptive" ? "hybrid" : mode;
|
|
@@ -539,11 +1102,25 @@ export class KnowledgeEngine {
|
|
|
539
1102
|
const queryTokens = tokenizeForSimilarity(query);
|
|
540
1103
|
const normalizedFileType = normalizeFileTypeFilter(filters?.file_type);
|
|
541
1104
|
|
|
1105
|
+
const warnings: string[] = [];
|
|
542
1106
|
const selectedKB = kb_id ? (getKB(db, kb_id) ?? getKBByName(db, kb_id)) : undefined;
|
|
543
|
-
const
|
|
544
|
-
|
|
1107
|
+
const availableKBs = kb_id ? ([selectedKB].filter(Boolean) as KnowledgeBase[]) : listKBs(db);
|
|
1108
|
+
const kbs = availableKBs.filter((kb) => kb.status === "ready" || kb.status === "stale");
|
|
1109
|
+
for (const kb of availableKBs) {
|
|
1110
|
+
if (kb.status !== "ready" && kb.status !== "stale") {
|
|
1111
|
+
warnings.push(`"${kb.name}" is ${kb.status}; search skipped until indexing is ready`);
|
|
1112
|
+
}
|
|
1113
|
+
}
|
|
1114
|
+
if (kbs.length === 0) {
|
|
1115
|
+
return {
|
|
1116
|
+
results: [],
|
|
1117
|
+
total_count: 0,
|
|
1118
|
+
has_more: false,
|
|
1119
|
+
warnings: warnings.length > 0 ? warnings : undefined,
|
|
1120
|
+
mode_used: mode,
|
|
1121
|
+
};
|
|
1122
|
+
}
|
|
545
1123
|
|
|
546
|
-
const warnings: string[] = [];
|
|
547
1124
|
const allResults: { chunkId: string; score: number }[] = [];
|
|
548
1125
|
const vectorsByChunkId = new Map<string, Float32Array>();
|
|
549
1126
|
|
|
@@ -553,31 +1130,26 @@ export class KnowledgeEngine {
|
|
|
553
1130
|
`"${kb.name}" was indexed with ${kb.embedding_model} (current: ${CURRENT_EMBEDDING_MODEL}) — run knowledge_update for best results`,
|
|
554
1131
|
);
|
|
555
1132
|
}
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
for (const [chunkId, vector] of this.getVectorsByChunkId(kb.id, chunkIds)) {
|
|
559
|
-
vectorsByChunkId.set(chunkId, vector);
|
|
560
|
-
}
|
|
1133
|
+
if (kb.chunk_count === 0) continue;
|
|
1134
|
+
const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
|
|
561
1135
|
|
|
562
1136
|
if (retrievalMode === "fast") {
|
|
563
|
-
allResults.push(...searchBM25(db, normalizedQuery || query, candidateLimit, kb.id));
|
|
1137
|
+
allResults.push(...searchBM25(db, normalizedQuery || query, candidateLimit, kb.id, { allowOrFallback: false }));
|
|
564
1138
|
} else if (retrievalMode === "semantic") {
|
|
565
|
-
const
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
}
|
|
1139
|
+
const queryVec = await embedQuery(query);
|
|
1140
|
+
const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), candidateLimit);
|
|
1141
|
+
allResults.push(...vectorResults.results);
|
|
1142
|
+
for (const [chunkId, vector] of vectorResults.vectorsByChunkId) vectorsByChunkId.set(chunkId, vector);
|
|
570
1143
|
} else {
|
|
571
1144
|
// hybrid: BM25 + vector weighted fusion (both scoped to this KB)
|
|
572
1145
|
const bm25Results = searchBM25(db, normalizedQuery || query, candidateLimit, kb.id);
|
|
1146
|
+
if (bm25Results.length === 0) continue;
|
|
573
1147
|
|
|
574
|
-
const vectors = [...this.getVectorsByChunkId(kb.id, chunkIds).values()];
|
|
575
1148
|
let vecResults: { chunkId: string; score: number }[] = [];
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
if (bm25Results.length === 0) continue;
|
|
1149
|
+
const queryVec = await embedQuery(query);
|
|
1150
|
+
const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), candidateLimit);
|
|
1151
|
+
vecResults = vectorResults.results;
|
|
1152
|
+
for (const [chunkId, vector] of vectorResults.vectorsByChunkId) vectorsByChunkId.set(chunkId, vector);
|
|
581
1153
|
const fused = weightedScoreFusion(bm25Results, vecResults);
|
|
582
1154
|
allResults.push(...fused);
|
|
583
1155
|
}
|
|
@@ -600,7 +1172,13 @@ export class KnowledgeEngine {
|
|
|
600
1172
|
}
|
|
601
1173
|
}
|
|
602
1174
|
let scored = unique;
|
|
603
|
-
if (retrievalMode
|
|
1175
|
+
if (retrievalMode === "fast") {
|
|
1176
|
+
scored = unique.filter((result) => {
|
|
1177
|
+
const chunk = getChunkById(db, result.chunkId);
|
|
1178
|
+
if (!chunk) return false;
|
|
1179
|
+
return hasAnyLexicalEvidence(buildChunkEmbeddingText(chunk), queryTokens);
|
|
1180
|
+
});
|
|
1181
|
+
} else if (retrievalMode !== "semantic") {
|
|
604
1182
|
scored = unique.filter((result) => {
|
|
605
1183
|
const chunk = getChunkById(db, result.chunkId);
|
|
606
1184
|
if (!chunk) return false;
|
|
@@ -665,6 +1243,7 @@ export class KnowledgeEngine {
|
|
|
665
1243
|
total_count: diversified.length,
|
|
666
1244
|
has_more: offset + limit < diversified.length,
|
|
667
1245
|
warnings: warnings.length > 0 ? warnings : undefined,
|
|
1246
|
+
mode_used: mode,
|
|
668
1247
|
};
|
|
669
1248
|
}
|
|
670
1249
|
|
|
@@ -729,6 +1308,14 @@ export class KnowledgeEngine {
|
|
|
729
1308
|
total_count: total,
|
|
730
1309
|
has_more: offset + limit < total,
|
|
731
1310
|
warnings: warnings.length > 0 ? warnings : undefined,
|
|
1311
|
+
mode_used: mode,
|
|
1312
|
+
suggestions:
|
|
1313
|
+
results.length === 0
|
|
1314
|
+
? [
|
|
1315
|
+
"Try mode 'fast' for exact symbols or mode 'semantic' for conceptual wording.",
|
|
1316
|
+
"Run knowledge_status if the KB should contain this answer.",
|
|
1317
|
+
]
|
|
1318
|
+
: undefined,
|
|
732
1319
|
};
|
|
733
1320
|
}
|
|
734
1321
|
|
|
@@ -736,7 +1323,6 @@ export class KnowledgeEngine {
|
|
|
736
1323
|
if (!this.db) return false;
|
|
737
1324
|
const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
|
|
738
1325
|
if (!kb) return false;
|
|
739
|
-
this.invalidateVectorCache(kb.id);
|
|
740
1326
|
deleteKB(this.db, kb.id);
|
|
741
1327
|
return true;
|
|
742
1328
|
}
|
|
@@ -749,7 +1335,6 @@ export class KnowledgeEngine {
|
|
|
749
1335
|
clear(): void {
|
|
750
1336
|
if (!this.db) return;
|
|
751
1337
|
for (const kb of listKBs(this.db)) deleteKB(this.db, kb.id);
|
|
752
|
-
this.vectorCache.clear();
|
|
753
1338
|
}
|
|
754
1339
|
|
|
755
1340
|
diagnose(): DiagnosticResult[] {
|
|
@@ -758,6 +1343,98 @@ export class KnowledgeEngine {
|
|
|
758
1343
|
return listKBs(db).map((kb) => diagnoseKB(db, kb));
|
|
759
1344
|
}
|
|
760
1345
|
|
|
1346
|
+
doctor(): DoctorReport {
|
|
1347
|
+
const diagnostics = this.diagnose();
|
|
1348
|
+
const issues: DoctorIssue[] = [];
|
|
1349
|
+
const kbs = this.list();
|
|
1350
|
+
if (kbs.length === 0) {
|
|
1351
|
+
issues.push({
|
|
1352
|
+
severity: "blocking",
|
|
1353
|
+
message: "No knowledge bases are indexed.",
|
|
1354
|
+
action: "Run knowledge_add for the project root or relevant source/docs directory.",
|
|
1355
|
+
});
|
|
1356
|
+
}
|
|
1357
|
+
|
|
1358
|
+
for (const diagnostic of diagnostics) {
|
|
1359
|
+
if (diagnostic.stuck_indexing) {
|
|
1360
|
+
const phase = diagnostic.job ? ` during ${diagnostic.job.phase}` : "";
|
|
1361
|
+
issues.push({
|
|
1362
|
+
severity: "blocking",
|
|
1363
|
+
kb_name: diagnostic.kb_name,
|
|
1364
|
+
message: `Indexing appears stuck${phase} for ${formatDuration(diagnostic.last_progress_age_ms)}.`,
|
|
1365
|
+
action:
|
|
1366
|
+
"Check knowledge_status for the last progress message. If no Pi process is actively indexing it, remove and rebuild this KB.",
|
|
1367
|
+
});
|
|
1368
|
+
}
|
|
1369
|
+
if (diagnostic.status === "error") {
|
|
1370
|
+
issues.push({
|
|
1371
|
+
severity: "blocking",
|
|
1372
|
+
kb_name: diagnostic.kb_name,
|
|
1373
|
+
message: "Knowledge base is in error state and is skipped by search.",
|
|
1374
|
+
action: "Run knowledge_remove and knowledge_add to rebuild it from the source.",
|
|
1375
|
+
});
|
|
1376
|
+
}
|
|
1377
|
+
if (diagnostic.coverage_percent < 80) {
|
|
1378
|
+
issues.push({
|
|
1379
|
+
severity: "warning",
|
|
1380
|
+
kb_name: diagnostic.kb_name,
|
|
1381
|
+
message: `Coverage is ${diagnostic.coverage_percent}% (${diagnostic.indexed_files}/${diagnostic.total_source_files} files).`,
|
|
1382
|
+
action: "Review skipped files and source path. Rebuild if index-time rules changed.",
|
|
1383
|
+
});
|
|
1384
|
+
}
|
|
1385
|
+
if (diagnostic.stale_files.length > 0) {
|
|
1386
|
+
issues.push({
|
|
1387
|
+
severity: "warning",
|
|
1388
|
+
kb_name: diagnostic.kb_name,
|
|
1389
|
+
message: `${diagnostic.stale_files.length} files changed after indexing.`,
|
|
1390
|
+
action: "Run knowledge_update for this KB.",
|
|
1391
|
+
});
|
|
1392
|
+
}
|
|
1393
|
+
if (diagnostic.orphan_files.length > 0) {
|
|
1394
|
+
issues.push({
|
|
1395
|
+
severity: "warning",
|
|
1396
|
+
kb_name: diagnostic.kb_name,
|
|
1397
|
+
message: `${diagnostic.orphan_files.length} indexed files no longer exist in the source.`,
|
|
1398
|
+
action: "Run knowledge_update or rebuild the KB.",
|
|
1399
|
+
});
|
|
1400
|
+
}
|
|
1401
|
+
if (diagnostic.skipped_files.total > 0) {
|
|
1402
|
+
issues.push({
|
|
1403
|
+
severity: "info",
|
|
1404
|
+
kb_name: diagnostic.kb_name,
|
|
1405
|
+
message: `${diagnostic.skipped_files.total} files were skipped while scanning (${Object.entries(
|
|
1406
|
+
diagnostic.skipped_files.by_reason,
|
|
1407
|
+
)
|
|
1408
|
+
.filter(([, count]) => count > 0)
|
|
1409
|
+
.map(([reason, count]) => `${reason}: ${count}`)
|
|
1410
|
+
.join(", ")}).`,
|
|
1411
|
+
action:
|
|
1412
|
+
"Use skipped samples to confirm exclusions are expected. Adjust source path or ignore rules if needed.",
|
|
1413
|
+
});
|
|
1414
|
+
}
|
|
1415
|
+
}
|
|
1416
|
+
|
|
1417
|
+
const penalty = issues.reduce((score, issue) => {
|
|
1418
|
+
if (issue.severity === "blocking") return score + 35;
|
|
1419
|
+
if (issue.severity === "warning") return score + 12;
|
|
1420
|
+
return score + 2;
|
|
1421
|
+
}, 0);
|
|
1422
|
+
const healthScore = Math.max(0, Math.min(100, 100 - penalty));
|
|
1423
|
+
const blocking = issues.filter((issue) => issue.severity === "blocking").length;
|
|
1424
|
+
const warnings = issues.filter((issue) => issue.severity === "warning").length;
|
|
1425
|
+
const summary =
|
|
1426
|
+
issues.length === 0
|
|
1427
|
+
? "Knowledge system is healthy."
|
|
1428
|
+
: `${blocking} blocking, ${warnings} warning, ${issues.length - blocking - warnings} info issues.`;
|
|
1429
|
+
|
|
1430
|
+
return {
|
|
1431
|
+
health_score: healthScore,
|
|
1432
|
+
summary,
|
|
1433
|
+
issues,
|
|
1434
|
+
diagnostics,
|
|
1435
|
+
};
|
|
1436
|
+
}
|
|
1437
|
+
|
|
761
1438
|
async exportKB(nameOrId: string, outputPath: string): Promise<number> {
|
|
762
1439
|
if (!this.db) throw new Error("Engine not initialized");
|
|
763
1440
|
const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
|
|
@@ -799,11 +1476,27 @@ export class KnowledgeEngine {
|
|
|
799
1476
|
const header = JSON.parse(lines[0]) as { name: string; description?: string };
|
|
800
1477
|
const kb = createKB(this.db, { name: header.name, description: header.description, source_type: "text" });
|
|
801
1478
|
updateKBStatus(this.db, kb.id, "indexing");
|
|
1479
|
+
startIndexingJob(this.db, kb.id, "import", `Starting import for "${header.name}"`);
|
|
1480
|
+
let vectorWriter: ReturnType<typeof openVectorWriter> | undefined;
|
|
1481
|
+
let tempVectorFile: string | undefined;
|
|
802
1482
|
try {
|
|
803
1483
|
const chunkData = lines.slice(1).map((l) => JSON.parse(l) as ImportedChunk);
|
|
804
|
-
|
|
1484
|
+
const importMessage = `Importing ${chunkData.length} chunks...`;
|
|
1485
|
+
updateIndexingJob(this.db, kb.id, {
|
|
1486
|
+
phase: "importing",
|
|
1487
|
+
message: importMessage,
|
|
1488
|
+
total_files: chunkData.length,
|
|
1489
|
+
});
|
|
1490
|
+
onProgress?.(importMessage);
|
|
805
1491
|
const chunks = chunkData.map((c) => ({
|
|
806
|
-
content_hash:
|
|
1492
|
+
content_hash: chunkIdentityHash({
|
|
1493
|
+
content: c.content,
|
|
1494
|
+
filePath: c.file_path,
|
|
1495
|
+
fileType: c.file_type,
|
|
1496
|
+
startLine: c.start_line,
|
|
1497
|
+
endLine: c.end_line,
|
|
1498
|
+
metadataJson: c.metadata_json || "{}",
|
|
1499
|
+
}),
|
|
807
1500
|
content: c.content,
|
|
808
1501
|
content_tokenized: "",
|
|
809
1502
|
file_path: c.file_path,
|
|
@@ -816,20 +1509,68 @@ export class KnowledgeEngine {
|
|
|
816
1509
|
...chunk,
|
|
817
1510
|
content_tokenized: preTokenizeForFTS(buildChunkEmbeddingText(chunk)),
|
|
818
1511
|
}));
|
|
819
|
-
const vectors = await embedDocuments(
|
|
820
|
-
indexedChunks.map((c) => buildChunkEmbeddingText(c)),
|
|
821
|
-
signal,
|
|
822
|
-
);
|
|
823
|
-
insertChunks(this.db, kb.id, indexedChunks);
|
|
824
1512
|
const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
|
|
825
|
-
|
|
826
|
-
|
|
1513
|
+
tempVectorFile = tempVectorPath(vectorPath);
|
|
1514
|
+
vectorWriter = openVectorWriter(tempVectorFile);
|
|
1515
|
+
let inserted = 0;
|
|
1516
|
+
for (let offset = 0; offset < indexedChunks.length; offset += INDEX_EMBED_BATCH_SIZE) {
|
|
1517
|
+
if (signal?.aborted) throw new Error("Cancelled");
|
|
1518
|
+
const batch = indexedChunks.slice(offset, offset + INDEX_EMBED_BATCH_SIZE);
|
|
1519
|
+
const message = `Embedding import batch ${Math.floor(offset / INDEX_EMBED_BATCH_SIZE) + 1}/${Math.ceil(
|
|
1520
|
+
indexedChunks.length / INDEX_EMBED_BATCH_SIZE,
|
|
1521
|
+
)}: ${offset}/${indexedChunks.length} chunks`;
|
|
1522
|
+
updateIndexingJob(this.db, kb.id, {
|
|
1523
|
+
phase: "embedding",
|
|
1524
|
+
message,
|
|
1525
|
+
processed_chunks: offset,
|
|
1526
|
+
total_files: indexedChunks.length,
|
|
1527
|
+
added_chunks: inserted,
|
|
1528
|
+
});
|
|
1529
|
+
onProgress?.(message);
|
|
1530
|
+
const vectors = await embedDocuments(
|
|
1531
|
+
batch.map((c) => buildChunkEmbeddingText(c)),
|
|
1532
|
+
signal,
|
|
1533
|
+
);
|
|
1534
|
+
if (signal?.aborted) throw new Error("Cancelled");
|
|
1535
|
+
insertChunks(this.db, kb.id, batch);
|
|
1536
|
+
vectorWriter.append(vectors);
|
|
1537
|
+
inserted += batch.length;
|
|
1538
|
+
updateKBCounts(
|
|
1539
|
+
this.db,
|
|
1540
|
+
kb.id,
|
|
1541
|
+
inserted,
|
|
1542
|
+
new Set(indexedChunks.slice(0, inserted).map((c) => c.file_path)).size,
|
|
1543
|
+
);
|
|
1544
|
+
updateIndexingJob(this.db, kb.id, {
|
|
1545
|
+
phase: "storing",
|
|
1546
|
+
message: `Stored import batch: ${inserted}/${indexedChunks.length} chunks`,
|
|
1547
|
+
processed_chunks: inserted,
|
|
1548
|
+
total_files: indexedChunks.length,
|
|
1549
|
+
added_chunks: inserted,
|
|
1550
|
+
});
|
|
1551
|
+
}
|
|
1552
|
+
vectorWriter.close();
|
|
1553
|
+
vectorWriter = undefined;
|
|
1554
|
+
renameSync(tempVectorFile, vectorPath);
|
|
1555
|
+
tempVectorFile = undefined;
|
|
827
1556
|
updateKBCounts(this.db, kb.id, indexedChunks.length, new Set(indexedChunks.map((c) => c.file_path)).size);
|
|
828
1557
|
updateKBStatus(this.db, kb.id, "ready");
|
|
829
1558
|
const savedKB = getKB(this.db, kb.id);
|
|
830
1559
|
if (!savedKB) throw new Error(`Knowledge base disappeared after import: ${kb.id}`);
|
|
1560
|
+
finishIndexingJob(this.db, kb.id, "succeeded", `Ready: imported ${indexedChunks.length} chunks`);
|
|
831
1561
|
return { kb: savedKB, chunkCount: chunks.length };
|
|
832
1562
|
} catch (e) {
|
|
1563
|
+
vectorWriter?.close();
|
|
1564
|
+
if (tempVectorFile) rmSync(tempVectorFile, { force: true });
|
|
1565
|
+
if (this.db) {
|
|
1566
|
+
finishIndexingJob(
|
|
1567
|
+
this.db,
|
|
1568
|
+
kb.id,
|
|
1569
|
+
isCancellationError(e) ? "cancelled" : "failed",
|
|
1570
|
+
isCancellationError(e) ? "Import cancelled." : "Import failed.",
|
|
1571
|
+
e instanceof Error ? e.message : String(e),
|
|
1572
|
+
);
|
|
1573
|
+
}
|
|
833
1574
|
deleteKB(this.db, kb.id);
|
|
834
1575
|
throw e;
|
|
835
1576
|
}
|
|
@@ -844,6 +1585,7 @@ export class KnowledgeEngine {
|
|
|
844
1585
|
await prepareEmbeddingForShutdown();
|
|
845
1586
|
await prepareRerankerForShutdown();
|
|
846
1587
|
}
|
|
1588
|
+
shutdownModelWorker();
|
|
847
1589
|
this.db?.close();
|
|
848
1590
|
this.db = null;
|
|
849
1591
|
}
|