pi-knowledge 0.3.6 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/engine.ts CHANGED
@@ -1,4 +1,4 @@
1
- import { existsSync, statSync } from "node:fs";
1
+ import { existsSync, renameSync, rmSync, statSync } from "node:fs";
2
2
  import { join, resolve } from "node:path";
3
3
  import type Database from "better-sqlite3";
4
4
  import { type DiagnosticResult, diagnoseKB } from "./diagnostics/health.ts";
@@ -8,12 +8,24 @@ import {
8
8
  embedQuery,
9
9
  prepareForShutdown as prepareEmbeddingForShutdown,
10
10
  } from "./embedding/provider.ts";
11
- import { loadVectors, saveVectors } from "./embedding/vectors.ts";
12
- import { buildChunkEmbeddingText, chunkFile, contentHash, preTokenizeForFTS, walkDir } from "./indexer/chunker.ts";
11
+ import { openVectorReader, openVectorWriter } from "./embedding/vectors.ts";
12
+ import {
13
+ buildChunkEmbeddingText,
14
+ chunkFile,
15
+ chunkIdentityHash,
16
+ createSkippedScanStats,
17
+ isReadableTextFile,
18
+ iterateScannableFiles,
19
+ iterateScannedFiles,
20
+ preTokenizeForFTS,
21
+ type ScanOptions,
22
+ summarizeSkippedScan,
23
+ } from "./indexer/chunker.ts";
13
24
  import { searchBM25 } from "./search/bm25.ts";
14
25
  import { weightedScoreFusion } from "./search/fusion.ts";
15
26
  import { normalizedQueryText, tokenizeForSearch } from "./search/query.ts";
16
27
  import {
28
+ hasAnyLexicalEvidence,
17
29
  hasEnoughLexicalEvidence,
18
30
  MIN_HYBRID_SCORE,
19
31
  normalizeFileTypeFilter,
@@ -22,30 +34,34 @@ import {
22
34
  scoreChunkForQuery,
23
35
  } from "./search/ranking.ts";
24
36
  import { disposeReranker, prepareRerankerForShutdown, rerank } from "./search/reranker.ts";
25
- import { searchVector } from "./search/vector.ts";
37
+ import { searchVectorFile } from "./search/vector.ts";
26
38
  import {
27
39
  type Chunk,
28
40
  createKB,
29
41
  deleteChunksByIds,
30
42
  deleteKB,
43
+ finishIndexingJob,
31
44
  getChunkById,
32
- getChunkHashesByKB,
33
- getChunkIdsByKB,
45
+ getChunkCount,
34
46
  getChunksByFile,
35
47
  getChunksByKB,
36
48
  getFileCount,
37
49
  getKB,
38
50
  getKBByName,
39
51
  insertChunks,
52
+ iterateChunkIdsByKB,
53
+ iterateChunksByKB,
40
54
  type KnowledgeBase,
41
55
  listKBs,
42
56
  openDatabase,
57
+ startIndexingJob,
58
+ updateIndexingJob,
43
59
  updateKBCounts,
44
60
  updateKBStatus,
45
61
  } from "./storage/sqlite.ts";
46
62
 
47
63
  export interface SearchOptions {
48
- mode?: "fast" | "semantic" | "hybrid" | "deep" | "adaptive";
64
+ mode?: "auto" | "fast" | "semantic" | "hybrid" | "deep" | "adaptive";
49
65
  limit?: number;
50
66
  offset?: number;
51
67
  kb_id?: string;
@@ -72,10 +88,19 @@ export interface SearchResponse {
72
88
  total_count: number;
73
89
  has_more: boolean;
74
90
  warnings?: string[];
91
+ mode_used?: NonNullable<SearchOptions["mode"]>;
92
+ retry_modes?: NonNullable<SearchOptions["mode"]>[];
93
+ suggestions?: string[];
75
94
  }
76
95
 
77
96
  export type ProgressCallback = (msg: string) => void;
78
97
 
98
+ export interface AddOptions {
99
+ include_suggested_text?: boolean;
100
+ include_paths?: string[];
101
+ exclude_paths?: string[];
102
+ }
103
+
79
104
  interface ImportedChunk {
80
105
  content: string;
81
106
  file_path: string;
@@ -97,11 +122,110 @@ interface RankedChunk {
97
122
  sourceChunkIds: string[];
98
123
  }
99
124
 
125
+ export interface DoctorIssue {
126
+ severity: "blocking" | "warning" | "info";
127
+ kb_name?: string;
128
+ message: string;
129
+ action: string;
130
+ }
131
+
132
+ export interface DoctorReport {
133
+ health_score: number;
134
+ summary: string;
135
+ issues: DoctorIssue[];
136
+ diagnostics: DiagnosticResult[];
137
+ }
138
+
100
139
  const ADAPTIVE_CONTEXT_LINES = 80;
101
140
  const ADAPTIVE_MAX_CONTEXT_CHARS = 6_000;
102
141
  const ADAPTIVE_NEIGHBOR_TARGET = 5;
142
+ const INDEX_EMBED_BATCH_SIZE = 64;
103
143
  const VECTOR_REDUNDANCY_WEIGHT = 0.35;
104
144
 
145
+ interface DirectoryScanPlan {
146
+ files: number;
147
+ bytes: number;
148
+ skippedTotal: number;
149
+ skippedSummary: string;
150
+ skipped: ReturnType<typeof createSkippedScanStats>;
151
+ }
152
+
153
+ export interface IndexPlan {
154
+ source_type: "file" | "directory" | "text" | "url";
155
+ scannable_files: number;
156
+ scannable_bytes: number;
157
+ skipped: ReturnType<typeof createSkippedScanStats>;
158
+ summary: string;
159
+ }
160
+
161
+ function isCancellationError(error: unknown): boolean {
162
+ return error instanceof Error && error.message === "Cancelled";
163
+ }
164
+
165
+ function tempVectorPath(vectorPath: string): string {
166
+ return `${vectorPath}.${Date.now()}.${Math.random().toString(16).slice(2)}.tmp`;
167
+ }
168
+
169
+ function toScanOptions(options: AddOptions = {}): ScanOptions {
170
+ return {
171
+ includeSuggestedText: options.include_suggested_text === true,
172
+ includePaths: options.include_paths,
173
+ excludePaths: options.exclude_paths,
174
+ };
175
+ }
176
+
177
+ function serializeAddOptions(options: AddOptions = {}): string | undefined {
178
+ const normalized: AddOptions = {};
179
+ if (options.include_suggested_text === true) normalized.include_suggested_text = true;
180
+ if (options.include_paths && options.include_paths.length > 0) normalized.include_paths = options.include_paths;
181
+ if (options.exclude_paths && options.exclude_paths.length > 0) normalized.exclude_paths = options.exclude_paths;
182
+ return Object.keys(normalized).length > 0 ? JSON.stringify(normalized) : undefined;
183
+ }
184
+
185
+ function parseAddOptions(raw: string | null): AddOptions {
186
+ if (!raw) return {};
187
+ try {
188
+ const parsed = JSON.parse(raw) as AddOptions;
189
+ return {
190
+ include_suggested_text: parsed.include_suggested_text === true,
191
+ include_paths: Array.isArray(parsed.include_paths)
192
+ ? parsed.include_paths.filter((item) => typeof item === "string")
193
+ : undefined,
194
+ exclude_paths: Array.isArray(parsed.exclude_paths)
195
+ ? parsed.exclude_paths.filter((item) => typeof item === "string")
196
+ : undefined,
197
+ };
198
+ } catch {
199
+ return {};
200
+ }
201
+ }
202
+
203
+ function planDirectoryScan(dirPath: string, options: ScanOptions = {}): DirectoryScanPlan {
204
+ const skipped = createSkippedScanStats();
205
+ let files = 0;
206
+ let bytes = 0;
207
+ for (const file of iterateScannableFiles(dirPath, skipped, options)) {
208
+ files++;
209
+ bytes += file.size;
210
+ }
211
+ return {
212
+ files,
213
+ bytes,
214
+ skippedTotal: skipped.total,
215
+ skippedSummary: summarizeSkippedScan(skipped),
216
+ skipped,
217
+ };
218
+ }
219
+
220
+ function formatBytes(bytes: number): string {
221
+ if (bytes < 1024) return `${bytes} B`;
222
+ const kb = bytes / 1024;
223
+ if (kb < 1024) return `${kb.toFixed(1)} KB`;
224
+ const mb = kb / 1024;
225
+ if (mb < 1024) return `${mb.toFixed(1)} MB`;
226
+ return `${(mb / 1024).toFixed(2)} GB`;
227
+ }
228
+
105
229
  function cosineSimilarity(a: Float32Array | undefined, b: Float32Array | undefined): number {
106
230
  if (!a || !b || a.length !== b.length) return 0;
107
231
  let dot = 0;
@@ -305,6 +429,68 @@ function pushAdaptiveCandidate(candidates: RankedChunk[], candidate: RankedChunk
305
429
  }
306
430
  }
307
431
 
432
+ function formatDuration(ms: number): string {
433
+ if (!Number.isFinite(ms) || ms <= 0) return "0s";
434
+ const seconds = Math.ceil(ms / 1000);
435
+ if (seconds < 60) return `${seconds}s`;
436
+ const minutes = Math.floor(seconds / 60);
437
+ const rest = seconds % 60;
438
+ return rest === 0 ? `${minutes}m` : `${minutes}m ${rest}s`;
439
+ }
440
+
441
+ function isExactLookupQuery(query: string): boolean {
442
+ const trimmed = query.trim();
443
+ if (/["'`]/.test(trimmed)) return true;
444
+ if (/[./\\][\w.-]+/.test(trimmed)) return true;
445
+ if (/\b[A-Z][A-Za-z0-9_]*[A-Z][A-Za-z0-9_]*\b/.test(trimmed)) return true;
446
+ if (/\b[A-Z]{2,}[_-]?[A-Z0-9]*\b/.test(trimmed)) return true;
447
+ if (/\b[a-zA-Z_][\w-]*\.(ts|tsx|js|jsx|go|rs|py|java|md|json|ya?ml|toml)\b/.test(trimmed)) return true;
448
+ if (/\b[A-Z]+-\d+\b/.test(trimmed)) return true;
449
+ return false;
450
+ }
451
+
452
+ function chooseAutoMode(query: string): NonNullable<SearchOptions["mode"]> {
453
+ const normalized = query.trim();
454
+ if (isExactLookupQuery(normalized)) return "fast";
455
+ const wordCount = normalized.split(/\s+/).filter(Boolean).length;
456
+ if (wordCount >= 10 || /how|why|explain|concept|architecture|design|流程|架構|概念|為什麼|如何/i.test(normalized)) {
457
+ return "semantic";
458
+ }
459
+ return "hybrid";
460
+ }
461
+
462
+ function fallbackModesFor(mode: NonNullable<SearchOptions["mode"]>): NonNullable<SearchOptions["mode"]>[] {
463
+ if (mode === "fast") return ["hybrid", "semantic"];
464
+ if (mode === "semantic") return ["hybrid", "adaptive"];
465
+ if (mode === "adaptive") return ["hybrid", "deep"];
466
+ if (mode === "deep") return ["hybrid", "adaptive"];
467
+ return ["fast", "semantic", "adaptive"];
468
+ }
469
+
470
+ function isWeakAutoResponse(
471
+ query: string,
472
+ response: SearchResponse,
473
+ primaryMode: NonNullable<SearchOptions["mode"]>,
474
+ attemptMode: NonNullable<SearchOptions["mode"]>,
475
+ ): boolean {
476
+ if (response.results.length === 0) return true;
477
+ if (primaryMode === "fast") {
478
+ const exactNeedle = query
479
+ .trim()
480
+ .replace(/^["'`]|["'`]$/g, "")
481
+ .toLowerCase();
482
+ const hasExactHit = response.results.some(
483
+ (result) =>
484
+ result.file_path.toLowerCase().includes(exactNeedle) || result.content.toLowerCase().includes(exactNeedle),
485
+ );
486
+ if (!hasExactHit && response.results.every((result) => (result.ranking?.coverage ?? 0) < 0.67)) return true;
487
+ }
488
+ if (primaryMode !== "semantic" && attemptMode === "semantic") {
489
+ return response.results.every((result) => (result.ranking?.coverage ?? 0) === 0);
490
+ }
491
+ return false;
492
+ }
493
+
308
494
  async function chunkUrl(source: string, signal?: AbortSignal): Promise<Awaited<ReturnType<typeof chunkFile>>> {
309
495
  const res = await fetch(source, { signal });
310
496
  if (!res.ok) throw new Error(`Fetch failed: ${res.status}`);
@@ -326,34 +512,61 @@ function normalizeExtractedText(text: string | string[]): string {
326
512
  export class KnowledgeEngine {
327
513
  private db: Database.Database | null = null;
328
514
  private knowledgeDir: string = "";
329
- private vectorCache: Map<string, Float32Array[]> = new Map();
330
515
 
331
516
  async initialize(knowledgeDir: string): Promise<void> {
332
517
  this.knowledgeDir = knowledgeDir;
333
518
  this.db = openDatabase(knowledgeDir);
334
- this.vectorCache.clear();
335
- }
336
-
337
- private getVectors(kbId: string): Float32Array[] {
338
- const cached = this.vectorCache.get(kbId);
339
- if (cached) return cached;
340
- const vectorPath = join(this.knowledgeDir, "vectors", `${kbId}.bin`);
341
- const vectors = loadVectors(vectorPath);
342
- this.vectorCache.set(kbId, vectors);
343
- return vectors;
344
- }
345
-
346
- private invalidateVectorCache(kbId: string): void {
347
- this.vectorCache.delete(kbId);
348
519
  }
349
520
 
350
- private getVectorsByChunkId(kbId: string, chunkIds: string[]): Map<string, Float32Array> {
351
- const vectors = this.getVectors(kbId);
352
- const mapped = new Map<string, Float32Array>();
353
- for (let i = 0; i < chunkIds.length; i++) {
354
- if (vectors[i]) mapped.set(chunkIds[i], vectors[i]);
521
+ plan(source: string, options: AddOptions = {}): IndexPlan {
522
+ const resolvedSource = resolve(source);
523
+ const isUrl = source.startsWith("http://") || source.startsWith("https://");
524
+ const isDir = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isDirectory();
525
+ const isFile = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isFile();
526
+ const sourceType = isUrl ? "url" : isDir ? "directory" : isFile ? "file" : "text";
527
+ if (isDir) {
528
+ const plan = planDirectoryScan(resolvedSource, toScanOptions(options));
529
+ return {
530
+ source_type: "directory",
531
+ scannable_files: plan.files,
532
+ scannable_bytes: plan.bytes,
533
+ skipped: plan.skipped,
534
+ summary: `Directory plan: ${plan.files} scannable files, ${formatBytes(plan.bytes)} scannable text, skipped ${plan.skippedTotal} (${plan.skippedSummary})`,
535
+ };
536
+ }
537
+ if (isFile) {
538
+ const skipped = createSkippedScanStats();
539
+ const supportedDocument =
540
+ resolvedSource.endsWith(".pdf") || resolvedSource.endsWith(".docx") || resolvedSource.endsWith(".doc");
541
+ if (!supportedDocument && !isReadableTextFile(resolvedSource)) {
542
+ skipped.total = 1;
543
+ skipped.by_reason.binary = 1;
544
+ skipped.samples.push({ path: resolvedSource, reason: "binary" });
545
+ return {
546
+ source_type: "file",
547
+ scannable_files: 0,
548
+ scannable_bytes: 0,
549
+ skipped,
550
+ summary: "File plan: 0 scannable files; file is unsupported binary/non-text",
551
+ };
552
+ }
553
+ const size = statSync(resolvedSource).size;
554
+ return {
555
+ source_type: "file",
556
+ scannable_files: 1,
557
+ scannable_bytes: size,
558
+ skipped,
559
+ summary: `File plan: 1 scannable file, ${formatBytes(size)} source size`,
560
+ };
355
561
  }
356
- return mapped;
562
+ const skipped = createSkippedScanStats();
563
+ return {
564
+ source_type: sourceType,
565
+ scannable_files: 1,
566
+ scannable_bytes: Buffer.byteLength(source),
567
+ skipped,
568
+ summary: `${sourceType === "url" ? "URL" : "Inline text"} plan: 1 source`,
569
+ };
357
570
  }
358
571
 
359
572
  async add(
@@ -361,6 +574,7 @@ export class KnowledgeEngine {
361
574
  name: string,
362
575
  onProgress?: ProgressCallback,
363
576
  signal?: AbortSignal,
577
+ options: AddOptions = {},
364
578
  ): Promise<{ kb: KnowledgeBase; chunkCount: number }> {
365
579
  if (!this.db) throw new Error("Engine not initialized");
366
580
  const resolvedSource = resolve(source);
@@ -368,6 +582,7 @@ export class KnowledgeEngine {
368
582
  const isDir = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isDirectory();
369
583
  const isFile = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isFile();
370
584
  const sourceType = isUrl ? "url" : isDir ? "directory" : isFile ? "file" : "text";
585
+ const scanOptions = toScanOptions(options);
371
586
 
372
587
  const existingKB = getKBByName(this.db, name);
373
588
  if (existingKB) {
@@ -380,60 +595,195 @@ export class KnowledgeEngine {
380
595
  name,
381
596
  source_path: isDir || isFile ? resolvedSource : isUrl ? source : undefined,
382
597
  source_type: sourceType,
598
+ source_options: serializeAddOptions(options),
383
599
  });
384
600
  updateKBStatus(this.db, kb.id, "indexing");
601
+ startIndexingJob(this.db, kb.id, "add", `Starting indexing for "${name}"`);
385
602
 
603
+ let vectorWriter: ReturnType<typeof openVectorWriter> | undefined;
604
+ let tempVectorFile: string | undefined;
386
605
  try {
387
- let allChunks: Awaited<ReturnType<typeof chunkFile>> = [];
606
+ const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
607
+ tempVectorFile = tempVectorPath(vectorPath);
608
+ vectorWriter = openVectorWriter(tempVectorFile);
609
+ let chunkCount = 0;
610
+ let fileCount = 0;
611
+ const pendingChunks: Awaited<ReturnType<typeof chunkFile>> = [];
612
+ const startedAt = Date.now();
613
+ let latestSkippedTotal = 0;
614
+
615
+ const reportProgress = (phase: string, processedFiles?: number, totalFiles?: number, skippedTotal = 0): void => {
616
+ latestSkippedTotal = skippedTotal;
617
+ const elapsed = Date.now() - startedAt;
618
+ const chunkRate = chunkCount / Math.max(1, elapsed / 1000);
619
+ let suffix = `${chunkCount} chunks, ${chunkRate.toFixed(1)} chunks/s, elapsed ${formatDuration(elapsed)}`;
620
+ if (processedFiles !== undefined && totalFiles !== undefined && totalFiles > 0) {
621
+ const rate = processedFiles / Math.max(1, elapsed / 1000);
622
+ const remainingFiles = Math.max(0, totalFiles - processedFiles);
623
+ const etaMs = rate > 0 ? (remainingFiles / rate) * 1000 : 0;
624
+ suffix = `${processedFiles}/${totalFiles} files, ${suffix}, file ETA ${formatDuration(etaMs)}`;
625
+ } else if (processedFiles !== undefined) {
626
+ suffix = `${processedFiles} files scanned, ${suffix}`;
627
+ }
628
+ if (skippedTotal > 0) suffix = `${suffix}, skipped ${skippedTotal}`;
629
+ const message = `${phase}: ${suffix}`;
630
+ updateIndexingJob(this.db, kb.id, {
631
+ phase,
632
+ message,
633
+ processed_files: processedFiles,
634
+ processed_chunks: chunkCount,
635
+ total_files: totalFiles,
636
+ skipped_total: skippedTotal,
637
+ added_chunks: chunkCount,
638
+ });
639
+ onProgress?.(message);
640
+ };
641
+
642
+ const flushPending = async (processedFiles?: number, totalFiles?: number): Promise<void> => {
643
+ if (!this.db || pendingChunks.length === 0) return;
644
+ if (signal?.aborted) throw new Error("Cancelled");
645
+ const batch = pendingChunks.splice(0, INDEX_EMBED_BATCH_SIZE);
646
+ reportProgress(`Embedding batch of ${batch.length}`, processedFiles, totalFiles);
647
+ const vectors = await embedDocuments(
648
+ batch.map((chunk) => buildChunkEmbeddingText(chunk)),
649
+ signal,
650
+ );
651
+ if (signal?.aborted) throw new Error("Cancelled");
652
+ insertChunks(this.db, kb.id, batch);
653
+ vectorWriter.append(vectors);
654
+ chunkCount += batch.length;
655
+ updateKBCounts(this.db, kb.id, chunkCount, fileCount);
656
+ reportProgress("Stored batch", processedFiles, totalFiles);
657
+ };
658
+
659
+ const addChunks = async (
660
+ chunks: Awaited<ReturnType<typeof chunkFile>>,
661
+ processedFiles?: number,
662
+ totalFiles?: number,
663
+ ): Promise<void> => {
664
+ pendingChunks.push(...chunks);
665
+ while (pendingChunks.length >= INDEX_EMBED_BATCH_SIZE) {
666
+ await flushPending(processedFiles, totalFiles);
667
+ }
668
+ };
388
669
 
389
670
  if (isUrl) {
390
- onProgress?.(`Fetching ${source}...`);
391
- allChunks = await chunkUrl(source, signal);
671
+ const message = `Fetching ${source}...`;
672
+ updateIndexingJob(this.db, kb.id, { phase: "fetching", message });
673
+ onProgress?.(message);
674
+ const chunks = await chunkUrl(source, signal);
675
+ fileCount = 1;
676
+ await addChunks(chunks);
392
677
  } else if (isFile && resolvedSource.endsWith(".pdf")) {
678
+ updateIndexingJob(this.db, kb.id, { phase: "extracting", message: "Extracting text from PDF..." });
393
679
  onProgress?.("Extracting text from PDF...");
394
680
  const { extractText } = await import("unpdf");
395
681
  const buf = (await import("node:fs")).readFileSync(resolvedSource);
396
682
  const { text } = await extractText(new Uint8Array(buf));
397
- allChunks = await chunkFile(normalizeExtractedText(text), resolvedSource);
683
+ fileCount = 1;
684
+ await addChunks(await chunkFile(normalizeExtractedText(text), resolvedSource));
398
685
  } else if (isFile && (resolvedSource.endsWith(".docx") || resolvedSource.endsWith(".doc"))) {
686
+ updateIndexingJob(this.db, kb.id, { phase: "extracting", message: "Extracting text from DOCX..." });
399
687
  onProgress?.("Extracting text from DOCX...");
400
688
  const mammoth = await import("mammoth");
401
689
  const result = await mammoth.extractRawText({ path: resolvedSource });
402
- allChunks = await chunkFile(result.value, resolvedSource);
690
+ fileCount = 1;
691
+ await addChunks(await chunkFile(result.value, resolvedSource));
403
692
  } else if (isDir) {
404
- onProgress?.(`Scanning ${resolvedSource}...`);
405
- const files = walkDir(resolvedSource);
406
- onProgress?.(`Found ${files.length} files, chunking...`);
407
- for (const file of files) {
408
- allChunks.push(...(await chunkFile(file.content, file.relPath)));
693
+ const plan = planDirectoryScan(resolvedSource, scanOptions);
694
+ const planningMessage = `Planned directory scan: ${plan.files} files, ${formatBytes(
695
+ plan.bytes,
696
+ )} scannable text, skipped ${plan.skippedTotal} (${plan.skippedSummary})`;
697
+ updateIndexingJob(this.db, kb.id, {
698
+ phase: "planning",
699
+ message: planningMessage,
700
+ total_files: plan.files,
701
+ skipped_total: plan.skippedTotal,
702
+ });
703
+ onProgress?.(planningMessage);
704
+ const scanningMessage = `Scanning ${resolvedSource}...`;
705
+ updateIndexingJob(this.db, kb.id, {
706
+ phase: "scanning",
707
+ message: scanningMessage,
708
+ total_files: plan.files,
709
+ skipped_total: plan.skippedTotal,
710
+ });
711
+ onProgress?.(scanningMessage);
712
+ const skipped = createSkippedScanStats();
713
+ let processedFiles = 0;
714
+ for (const file of iterateScannedFiles(resolvedSource, skipped, scanOptions)) {
715
+ if (signal?.aborted) throw new Error("Cancelled");
716
+ const chunks = await chunkFile(file.content, file.relPath);
717
+ processedFiles++;
718
+ latestSkippedTotal = skipped.total;
719
+ if (chunks.length > 0) fileCount++;
720
+ await addChunks(chunks, processedFiles, plan.files);
721
+ if (processedFiles % 25 === 0) reportProgress("Chunking", processedFiles, plan.files, skipped.total);
409
722
  }
723
+ const finalizingMessage = `Scanned ${processedFiles} files, skipped ${skipped.total} (${summarizeSkippedScan(
724
+ skipped,
725
+ )}), finalizing...`;
726
+ updateIndexingJob(this.db, kb.id, {
727
+ phase: "finalizing",
728
+ message: finalizingMessage,
729
+ processed_files: processedFiles,
730
+ processed_chunks: chunkCount,
731
+ total_files: plan.files,
732
+ skipped_total: skipped.total,
733
+ added_chunks: chunkCount,
734
+ });
735
+ onProgress?.(finalizingMessage);
410
736
  } else if (isFile) {
737
+ if (!isReadableTextFile(resolvedSource)) {
738
+ throw new Error(`File is not readable text and has no supported extractor: ${resolvedSource}`);
739
+ }
411
740
  const { readFileSync } = await import("node:fs");
412
741
  const content = readFileSync(resolvedSource, "utf-8");
413
- allChunks = await chunkFile(content, resolvedSource);
742
+ fileCount = 1;
743
+ await addChunks(await chunkFile(content, resolvedSource));
414
744
  } else {
415
- allChunks = await chunkFile(source, "inline-text");
745
+ fileCount = 1;
746
+ await addChunks(await chunkFile(source, "inline-text"));
416
747
  }
417
748
 
418
- onProgress?.(`${allChunks.length} chunks, embedding...`);
419
- const texts = allChunks.map((c) => buildChunkEmbeddingText(c));
420
- const vectors = await embedDocuments(texts, signal);
421
-
422
- onProgress?.(`Storing...`);
423
- insertChunks(this.db, kb.id, allChunks);
749
+ await flushPending();
750
+ vectorWriter.close();
751
+ vectorWriter = undefined;
752
+ renameSync(tempVectorFile, vectorPath);
753
+ tempVectorFile = undefined;
424
754
 
425
- const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
426
- saveVectors(vectorPath, vectors);
427
- this.vectorCache.set(kb.id, vectors);
428
-
429
- const fileCount = isDir ? getFileCount(this.db, kb.id) : 1;
430
- updateKBCounts(this.db, kb.id, allChunks.length, fileCount);
755
+ const savedFileCount = isDir ? getFileCount(this.db, kb.id) : fileCount;
756
+ updateKBCounts(this.db, kb.id, chunkCount, savedFileCount);
431
757
  updateKBStatus(this.db, kb.id, "ready");
432
758
 
433
759
  const savedKB = getKB(this.db, kb.id);
434
760
  if (!savedKB) throw new Error(`Knowledge base disappeared after add: ${kb.id}`);
435
- return { kb: savedKB, chunkCount: allChunks.length };
761
+ const readyMessage = `Ready: ${chunkCount} chunks from ${savedFileCount} files in ${formatDuration(
762
+ Date.now() - startedAt,
763
+ )}`;
764
+ updateIndexingJob(this.db, kb.id, {
765
+ phase: "ready",
766
+ message: readyMessage,
767
+ processed_files: savedFileCount,
768
+ processed_chunks: chunkCount,
769
+ skipped_total: latestSkippedTotal,
770
+ added_chunks: chunkCount,
771
+ });
772
+ finishIndexingJob(this.db, kb.id, "succeeded", readyMessage);
773
+ onProgress?.(readyMessage);
774
+ return { kb: savedKB, chunkCount };
436
775
  } catch (e) {
776
+ vectorWriter?.close();
777
+ if (tempVectorFile) rmSync(tempVectorFile, { force: true });
778
+ if (this.db) {
779
+ finishIndexingJob(
780
+ this.db,
781
+ kb.id,
782
+ isCancellationError(e) ? "cancelled" : "failed",
783
+ isCancellationError(e) ? "Indexing cancelled." : "Indexing failed.",
784
+ e instanceof Error ? e.message : String(e),
785
+ );
786
+ }
437
787
  // Clean up partial state on failure
438
788
  deleteKB(this.db, kb.id);
439
789
  throw e;
@@ -453,84 +803,296 @@ export class KnowledgeEngine {
453
803
  }
454
804
 
455
805
  updateKBStatus(this.db, kb.id, "indexing");
806
+ startIndexingJob(this.db, kb.id, "update", `Starting update for "${kb.name}"`);
807
+ const scanOptions = toScanOptions(parseAddOptions(kb.source_options));
808
+ let replacementVectorPath: string | undefined;
809
+ let addedVectorPath: string | undefined;
810
+ let addedVectorWriter: ReturnType<typeof openVectorWriter> | undefined;
456
811
 
457
812
  try {
458
- // 1. Scan and chunk current source
813
+ const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
814
+ addedVectorPath = tempVectorPath(`${vectorPath}.added`);
815
+ addedVectorWriter = openVectorWriter(addedVectorPath);
816
+ const existingHashes = new Map<string, Array<{ id: string; vectorIndex: number }>>();
817
+ let existingIndex = 0;
818
+ for (const chunk of iterateChunksByKB(this.db, kb.id)) {
819
+ const entries = existingHashes.get(chunk.content_hash) ?? [];
820
+ entries.push({ id: chunk.id, vectorIndex: existingIndex });
821
+ existingHashes.set(chunk.content_hash, entries);
822
+ existingIndex++;
823
+ }
824
+
825
+ const oldVectorIndexByHash = new Map<string, number[]>();
826
+ const newVectorIndexByHash = new Map<string, number[]>();
827
+ const pendingChunks: Awaited<ReturnType<typeof chunkFile>> = [];
828
+ let addedVectorCount = 0;
829
+ let addedCount = 0;
830
+ let unchanged = 0;
831
+ let scannedFiles = 0;
832
+ let scannedChunks = 0;
833
+ let plannedTotalFiles: number | undefined;
834
+ const startedAt = Date.now();
835
+
836
+ const flushPending = async (): Promise<void> => {
837
+ if (!this.db || !addedVectorWriter || pendingChunks.length === 0) return;
838
+ if (signal?.aborted) throw new Error("Cancelled");
839
+ const batch = pendingChunks.splice(0, INDEX_EMBED_BATCH_SIZE);
840
+ const elapsed = Date.now() - startedAt;
841
+ const message = `Embedding update batch: ${addedCount} new chunks stored, ${scannedFiles} files scanned, elapsed ${formatDuration(
842
+ elapsed,
843
+ )}`;
844
+ updateIndexingJob(this.db, kb.id, {
845
+ phase: "embedding",
846
+ message,
847
+ processed_files: scannedFiles,
848
+ processed_chunks: scannedChunks,
849
+ total_files: plannedTotalFiles,
850
+ added_chunks: addedCount,
851
+ unchanged_chunks: unchanged,
852
+ });
853
+ onProgress?.(message);
854
+ const newVectors = await embedDocuments(
855
+ batch.map((c) => buildChunkEmbeddingText(c)),
856
+ signal,
857
+ );
858
+ if (signal?.aborted) throw new Error("Cancelled");
859
+ addedVectorWriter.append(newVectors);
860
+ for (let i = 0; i < batch.length; i++) {
861
+ const indexes = newVectorIndexByHash.get(batch[i].content_hash) ?? [];
862
+ indexes.push(addedVectorCount + i);
863
+ newVectorIndexByHash.set(batch[i].content_hash, indexes);
864
+ }
865
+ addedVectorCount += newVectors.length;
866
+ insertChunks(this.db, kb.id, batch);
867
+ addedCount += batch.length;
868
+ updateKBCounts(this.db, kb.id, getChunkCount(this.db, kb.id), getFileCount(this.db, kb.id));
869
+ const storedMessage = `Stored update batch: +${addedCount} chunks, =${unchanged} unchanged`;
870
+ updateIndexingJob(this.db, kb.id, {
871
+ phase: "storing",
872
+ message: storedMessage,
873
+ processed_files: scannedFiles,
874
+ processed_chunks: scannedChunks,
875
+ total_files: plannedTotalFiles,
876
+ added_chunks: addedCount,
877
+ unchanged_chunks: unchanged,
878
+ });
879
+ onProgress?.(storedMessage);
880
+ };
881
+
882
+ const processChunks = async (chunks: Awaited<ReturnType<typeof chunkFile>>): Promise<void> => {
883
+ for (const chunk of chunks) {
884
+ scannedChunks++;
885
+ const existing = existingHashes.get(chunk.content_hash);
886
+ if (existing && existing.length > 0) {
887
+ const retained = existing.shift();
888
+ if (retained) {
889
+ const indexes = oldVectorIndexByHash.get(chunk.content_hash) ?? [];
890
+ indexes.push(retained.vectorIndex);
891
+ oldVectorIndexByHash.set(chunk.content_hash, indexes);
892
+ }
893
+ unchanged++;
894
+ continue;
895
+ }
896
+ pendingChunks.push(chunk);
897
+ if (pendingChunks.length >= INDEX_EMBED_BATCH_SIZE) await flushPending();
898
+ }
899
+ };
900
+
901
+ updateIndexingJob(this.db, kb.id, { phase: "scanning", message: "Scanning source..." });
459
902
  onProgress?.("Scanning source...");
460
- let newChunks: Awaited<ReturnType<typeof chunkFile>> = [];
461
903
  if (kb.source_type === "url") {
462
- onProgress?.(`Fetching ${kb.source_path}...`);
463
- newChunks = await chunkUrl(kb.source_path, signal);
904
+ const message = `Fetching ${kb.source_path}...`;
905
+ updateIndexingJob(this.db, kb.id, { phase: "fetching", message });
906
+ onProgress?.(message);
907
+ scannedFiles = 1;
908
+ await processChunks(await chunkUrl(kb.source_path, signal));
464
909
  } else if (statSync(kb.source_path).isDirectory()) {
465
- const files = walkDir(kb.source_path);
466
- for (const file of files) newChunks.push(...(await chunkFile(file.content, file.relPath)));
910
+ const plan = planDirectoryScan(kb.source_path, scanOptions);
911
+ plannedTotalFiles = plan.files;
912
+ const planningMessage = `Planned directory scan: ${plan.files} files, ${formatBytes(
913
+ plan.bytes,
914
+ )} scannable text, skipped ${plan.skippedTotal} (${plan.skippedSummary})`;
915
+ updateIndexingJob(this.db, kb.id, {
916
+ phase: "planning",
917
+ message: planningMessage,
918
+ total_files: plan.files,
919
+ skipped_total: plan.skippedTotal,
920
+ });
921
+ onProgress?.(planningMessage);
922
+ const skipped = createSkippedScanStats();
923
+ for (const file of iterateScannedFiles(kb.source_path, skipped, scanOptions)) {
924
+ if (signal?.aborted) throw new Error("Cancelled");
925
+ scannedFiles++;
926
+ await processChunks(await chunkFile(file.content, file.relPath));
927
+ if (scannedFiles % 25 === 0) {
928
+ const message = `Scanned ${scannedFiles} files, ${scannedChunks} chunks, skipped ${skipped.total}, +${addedCount} =${unchanged}`;
929
+ updateIndexingJob(this.db, kb.id, {
930
+ phase: "scanning",
931
+ message,
932
+ processed_files: scannedFiles,
933
+ processed_chunks: scannedChunks,
934
+ total_files: plan.files,
935
+ skipped_total: skipped.total,
936
+ added_chunks: addedCount,
937
+ unchanged_chunks: unchanged,
938
+ });
939
+ onProgress?.(message);
940
+ }
941
+ }
942
+ const message = `Scanned ${scannedFiles} files, skipped ${skipped.total} (${summarizeSkippedScan(
943
+ skipped,
944
+ )}), reconciling deletes...`;
945
+ updateIndexingJob(this.db, kb.id, {
946
+ phase: "reconciling",
947
+ message,
948
+ processed_files: scannedFiles,
949
+ processed_chunks: scannedChunks,
950
+ total_files: plan.files,
951
+ skipped_total: skipped.total,
952
+ added_chunks: addedCount,
953
+ unchanged_chunks: unchanged,
954
+ });
955
+ onProgress?.(message);
467
956
  } else {
468
957
  const { readFileSync } = await import("node:fs");
469
- newChunks = await chunkFile(readFileSync(kb.source_path, "utf-8"), kb.source_path);
958
+ scannedFiles = 1;
959
+ await processChunks(await chunkFile(readFileSync(kb.source_path, "utf-8"), kb.source_path));
470
960
  }
961
+ await flushPending();
962
+ addedVectorWriter.close();
963
+ addedVectorWriter = undefined;
471
964
 
472
- // 2. Load existing state: chunks (hash→id) + vectors (ordered)
473
- const existingHashes = getChunkHashesByKB(this.db, kb.id);
474
- const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
475
- const existingVectors = loadVectors(vectorPath);
476
-
477
- // Build hash→vector cache from existing data
478
- const vectorCache = new Map<string, Float32Array>();
479
- const existingChunks = getChunksByKB(this.db, kb.id);
480
- for (let i = 0; i < existingChunks.length; i++) {
481
- if (existingVectors[i]) vectorCache.set(existingChunks[i].content_hash, existingVectors[i]);
965
+ const idsToRemove: string[] = [];
966
+ for (const entries of existingHashes.values()) {
967
+ idsToRemove.push(...entries.map((entry) => entry.id));
482
968
  }
483
-
484
- // 3. Identify changes
485
- const newHashSet = new Set(newChunks.map((c) => c.content_hash));
486
- const chunksToAdd = newChunks.filter((c) => !existingHashes.has(c.content_hash));
487
- const idsToRemove = [...existingHashes.entries()].filter(([hash]) => !newHashSet.has(hash)).map(([, id]) => id);
488
- const unchanged = newChunks.length - chunksToAdd.length;
489
-
490
- onProgress?.(`Changes: +${chunksToAdd.length} -${idsToRemove.length} =${unchanged}`);
491
-
492
- // 4. Remove deleted chunks from DB
493
969
  if (idsToRemove.length > 0) deleteChunksByIds(this.db, idsToRemove);
494
-
495
- // 5. Embed ONLY new chunks
496
- let newVectors: Float32Array[] = [];
497
- if (chunksToAdd.length > 0) {
498
- onProgress?.(`Embedding ${chunksToAdd.length} new chunks...`);
499
- newVectors = await embedDocuments(
500
- chunksToAdd.map((c) => buildChunkEmbeddingText(c)),
501
- signal,
502
- );
503
- insertChunks(this.db, kb.id, chunksToAdd);
504
- }
505
-
506
- // Add new vectors to cache
507
- for (let i = 0; i < chunksToAdd.length; i++) {
508
- vectorCache.set(chunksToAdd[i].content_hash, newVectors[i]);
509
- }
510
-
511
- // 6. Rebuild vector file in DB chunk order (reusing cached vectors)
512
- const finalChunks = getChunksByKB(this.db, kb.id);
513
- const finalVectors: Float32Array[] = [];
514
- for (const chunk of finalChunks) {
515
- const vector = vectorCache.get(chunk.content_hash);
516
- if (vector) finalVectors.push(vector);
970
+ const changesMessage = `Changes: +${addedCount} -${idsToRemove.length} =${unchanged}`;
971
+ updateIndexingJob(this.db, kb.id, {
972
+ phase: "reconciling",
973
+ message: changesMessage,
974
+ processed_files: scannedFiles,
975
+ processed_chunks: scannedChunks,
976
+ added_chunks: addedCount,
977
+ removed_chunks: idsToRemove.length,
978
+ unchanged_chunks: unchanged,
979
+ });
980
+ onProgress?.(changesMessage);
981
+
982
+ replacementVectorPath = tempVectorPath(vectorPath);
983
+ const vectorWriter = openVectorWriter(replacementVectorPath);
984
+ const oldVectorReader = openVectorReader(vectorPath);
985
+ const newVectorReader = openVectorReader(addedVectorPath);
986
+ let finalChunkCount = 0;
987
+ const takeVectorIndex = (indexesByHash: Map<string, number[]>, hash: string): number | undefined => {
988
+ const indexes = indexesByHash.get(hash);
989
+ return indexes?.shift();
990
+ };
991
+ try {
992
+ for (const chunk of iterateChunksByKB(this.db, kb.id)) {
993
+ const oldVectorIndex = takeVectorIndex(oldVectorIndexByHash, chunk.content_hash);
994
+ const newVectorIndex = takeVectorIndex(newVectorIndexByHash, chunk.content_hash);
995
+ const vector =
996
+ oldVectorReader && oldVectorIndex !== undefined
997
+ ? oldVectorReader.read(oldVectorIndex)
998
+ : newVectorReader && newVectorIndex !== undefined
999
+ ? newVectorReader.read(newVectorIndex)
1000
+ : undefined;
1001
+ if (!vector) throw new Error(`Missing vector while rebuilding knowledge base: ${chunk.id}`);
1002
+ vectorWriter.append([vector]);
1003
+ finalChunkCount++;
1004
+ if (finalChunkCount % 1_000 === 0) {
1005
+ const message = `Rebuilding vector file: ${finalChunkCount} chunks written`;
1006
+ updateIndexingJob(this.db, kb.id, {
1007
+ phase: "rebuilding_vectors",
1008
+ message,
1009
+ processed_files: scannedFiles,
1010
+ processed_chunks: finalChunkCount,
1011
+ added_chunks: addedCount,
1012
+ removed_chunks: idsToRemove.length,
1013
+ unchanged_chunks: unchanged,
1014
+ });
1015
+ onProgress?.(message);
1016
+ }
1017
+ }
1018
+ } finally {
1019
+ oldVectorReader?.close();
1020
+ newVectorReader?.close();
1021
+ vectorWriter.close();
517
1022
  }
518
- saveVectors(vectorPath, finalVectors);
519
- this.vectorCache.set(kb.id, finalVectors);
1023
+ renameSync(replacementVectorPath, vectorPath);
1024
+ replacementVectorPath = undefined;
1025
+ if (addedVectorPath) rmSync(addedVectorPath, { force: true });
1026
+ addedVectorPath = undefined;
520
1027
 
521
- // 7. Update counts
522
- updateKBCounts(this.db, kb.id, finalChunks.length, getFileCount(this.db, kb.id));
1028
+ updateKBCounts(this.db, kb.id, finalChunkCount, getFileCount(this.db, kb.id));
523
1029
  updateKBStatus(this.db, kb.id, "ready");
1030
+ const readyMessage = `Ready: +${addedCount} -${idsToRemove.length} =${unchanged}`;
1031
+ updateIndexingJob(this.db, kb.id, {
1032
+ phase: "ready",
1033
+ message: readyMessage,
1034
+ processed_files: scannedFiles,
1035
+ processed_chunks: finalChunkCount,
1036
+ added_chunks: addedCount,
1037
+ removed_chunks: idsToRemove.length,
1038
+ unchanged_chunks: unchanged,
1039
+ });
1040
+ finishIndexingJob(this.db, kb.id, "succeeded", readyMessage);
1041
+ onProgress?.(readyMessage);
524
1042
 
525
- return { added: chunksToAdd.length, removed: idsToRemove.length, unchanged };
1043
+ return { added: addedCount, removed: idsToRemove.length, unchanged };
526
1044
  } catch (e) {
1045
+ addedVectorWriter?.close();
1046
+ if (replacementVectorPath) rmSync(replacementVectorPath, { force: true });
1047
+ if (addedVectorPath) rmSync(addedVectorPath, { force: true });
527
1048
  updateKBStatus(this.db, kb.id, "error");
1049
+ finishIndexingJob(
1050
+ this.db,
1051
+ kb.id,
1052
+ isCancellationError(e) ? "cancelled" : "failed",
1053
+ isCancellationError(e) ? "Update cancelled." : "Update failed.",
1054
+ e instanceof Error ? e.message : String(e),
1055
+ );
528
1056
  throw e;
529
1057
  }
530
1058
  }
531
1059
 
532
1060
  async search(query: string, options: SearchOptions = {}): Promise<SearchResponse> {
533
1061
  if (!this.db) throw new Error("Engine not initialized");
1062
+ const requestedMode = options.mode ?? "hybrid";
1063
+ if (requestedMode === "auto") {
1064
+ const primaryMode = chooseAutoMode(query);
1065
+ const attempts = [primaryMode, ...fallbackModesFor(primaryMode)];
1066
+ const tried: NonNullable<SearchOptions["mode"]>[] = [];
1067
+ const warnings: string[] = [];
1068
+ for (const mode of attempts) {
1069
+ if (tried.includes(mode)) continue;
1070
+ tried.push(mode);
1071
+ const response = await this.search(query, { ...options, mode });
1072
+ if (response.warnings) warnings.push(...response.warnings);
1073
+ if (!isWeakAutoResponse(query, response, primaryMode, mode)) {
1074
+ return {
1075
+ ...response,
1076
+ warnings: warnings.length > 0 ? [...new Set(warnings)] : response.warnings,
1077
+ mode_used: mode,
1078
+ retry_modes: tried.slice(0, -1),
1079
+ suggestions: response.suggestions,
1080
+ };
1081
+ }
1082
+ if (tried.length >= 3) break;
1083
+ }
1084
+ return {
1085
+ results: [],
1086
+ total_count: 0,
1087
+ has_more: false,
1088
+ warnings: warnings.length > 0 ? [...new Set(warnings)] : undefined,
1089
+ mode_used: tried.at(-1),
1090
+ retry_modes: tried.slice(0, -1),
1091
+ suggestions: [
1092
+ "No results after auto mode fallback. Check knowledge_status, try a more exact term, or rebuild the KB if indexing rules changed.",
1093
+ ],
1094
+ };
1095
+ }
534
1096
  const db = this.db;
535
1097
  const { mode = "hybrid", limit = 10, offset = 0, kb_id, filters, diversity = "balanced" } = options;
536
1098
  const retrievalMode = mode === "adaptive" ? "hybrid" : mode;
@@ -539,11 +1101,25 @@ export class KnowledgeEngine {
539
1101
  const queryTokens = tokenizeForSimilarity(query);
540
1102
  const normalizedFileType = normalizeFileTypeFilter(filters?.file_type);
541
1103
 
1104
+ const warnings: string[] = [];
542
1105
  const selectedKB = kb_id ? (getKB(db, kb_id) ?? getKBByName(db, kb_id)) : undefined;
543
- const kbs = kb_id ? ([selectedKB].filter(Boolean) as KnowledgeBase[]) : listKBs(db);
544
- if (kbs.length === 0) return { results: [], total_count: 0, has_more: false };
1106
+ const availableKBs = kb_id ? ([selectedKB].filter(Boolean) as KnowledgeBase[]) : listKBs(db);
1107
+ const kbs = availableKBs.filter((kb) => kb.status === "ready" || kb.status === "stale");
1108
+ for (const kb of availableKBs) {
1109
+ if (kb.status !== "ready" && kb.status !== "stale") {
1110
+ warnings.push(`"${kb.name}" is ${kb.status}; search skipped until indexing is ready`);
1111
+ }
1112
+ }
1113
+ if (kbs.length === 0) {
1114
+ return {
1115
+ results: [],
1116
+ total_count: 0,
1117
+ has_more: false,
1118
+ warnings: warnings.length > 0 ? warnings : undefined,
1119
+ mode_used: mode,
1120
+ };
1121
+ }
545
1122
 
546
- const warnings: string[] = [];
547
1123
  const allResults: { chunkId: string; score: number }[] = [];
548
1124
  const vectorsByChunkId = new Map<string, Float32Array>();
549
1125
 
@@ -553,31 +1129,26 @@ export class KnowledgeEngine {
553
1129
  `"${kb.name}" was indexed with ${kb.embedding_model} (current: ${CURRENT_EMBEDDING_MODEL}) — run knowledge_update for best results`,
554
1130
  );
555
1131
  }
556
- const chunkIds = getChunkIdsByKB(db, kb.id);
557
- if (chunkIds.length === 0) continue;
558
- for (const [chunkId, vector] of this.getVectorsByChunkId(kb.id, chunkIds)) {
559
- vectorsByChunkId.set(chunkId, vector);
560
- }
1132
+ if (kb.chunk_count === 0) continue;
1133
+ const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
561
1134
 
562
1135
  if (retrievalMode === "fast") {
563
- allResults.push(...searchBM25(db, normalizedQuery || query, candidateLimit, kb.id));
1136
+ allResults.push(...searchBM25(db, normalizedQuery || query, candidateLimit, kb.id, { allowOrFallback: false }));
564
1137
  } else if (retrievalMode === "semantic") {
565
- const vectors = [...this.getVectorsByChunkId(kb.id, chunkIds).values()];
566
- if (vectors.length > 0) {
567
- const queryVec = await embedQuery(query);
568
- allResults.push(...searchVector(queryVec, vectors, chunkIds, candidateLimit));
569
- }
1138
+ const queryVec = await embedQuery(query);
1139
+ const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), candidateLimit);
1140
+ allResults.push(...vectorResults.results);
1141
+ for (const [chunkId, vector] of vectorResults.vectorsByChunkId) vectorsByChunkId.set(chunkId, vector);
570
1142
  } else {
571
1143
  // hybrid: BM25 + vector weighted fusion (both scoped to this KB)
572
1144
  const bm25Results = searchBM25(db, normalizedQuery || query, candidateLimit, kb.id);
1145
+ if (bm25Results.length === 0) continue;
573
1146
 
574
- const vectors = [...this.getVectorsByChunkId(kb.id, chunkIds).values()];
575
1147
  let vecResults: { chunkId: string; score: number }[] = [];
576
- if (vectors.length > 0) {
577
- const queryVec = await embedQuery(query);
578
- vecResults = searchVector(queryVec, vectors, chunkIds, candidateLimit);
579
- }
580
- if (bm25Results.length === 0) continue;
1148
+ const queryVec = await embedQuery(query);
1149
+ const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), candidateLimit);
1150
+ vecResults = vectorResults.results;
1151
+ for (const [chunkId, vector] of vectorResults.vectorsByChunkId) vectorsByChunkId.set(chunkId, vector);
581
1152
  const fused = weightedScoreFusion(bm25Results, vecResults);
582
1153
  allResults.push(...fused);
583
1154
  }
@@ -600,7 +1171,13 @@ export class KnowledgeEngine {
600
1171
  }
601
1172
  }
602
1173
  let scored = unique;
603
- if (retrievalMode !== "fast" && retrievalMode !== "semantic") {
1174
+ if (retrievalMode === "fast") {
1175
+ scored = unique.filter((result) => {
1176
+ const chunk = getChunkById(db, result.chunkId);
1177
+ if (!chunk) return false;
1178
+ return hasAnyLexicalEvidence(buildChunkEmbeddingText(chunk), queryTokens);
1179
+ });
1180
+ } else if (retrievalMode !== "semantic") {
604
1181
  scored = unique.filter((result) => {
605
1182
  const chunk = getChunkById(db, result.chunkId);
606
1183
  if (!chunk) return false;
@@ -665,6 +1242,7 @@ export class KnowledgeEngine {
665
1242
  total_count: diversified.length,
666
1243
  has_more: offset + limit < diversified.length,
667
1244
  warnings: warnings.length > 0 ? warnings : undefined,
1245
+ mode_used: mode,
668
1246
  };
669
1247
  }
670
1248
 
@@ -729,6 +1307,14 @@ export class KnowledgeEngine {
729
1307
  total_count: total,
730
1308
  has_more: offset + limit < total,
731
1309
  warnings: warnings.length > 0 ? warnings : undefined,
1310
+ mode_used: mode,
1311
+ suggestions:
1312
+ results.length === 0
1313
+ ? [
1314
+ "Try mode 'fast' for exact symbols or mode 'semantic' for conceptual wording.",
1315
+ "Run knowledge_status if the KB should contain this answer.",
1316
+ ]
1317
+ : undefined,
732
1318
  };
733
1319
  }
734
1320
 
@@ -736,7 +1322,6 @@ export class KnowledgeEngine {
736
1322
  if (!this.db) return false;
737
1323
  const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
738
1324
  if (!kb) return false;
739
- this.invalidateVectorCache(kb.id);
740
1325
  deleteKB(this.db, kb.id);
741
1326
  return true;
742
1327
  }
@@ -749,7 +1334,6 @@ export class KnowledgeEngine {
749
1334
  clear(): void {
750
1335
  if (!this.db) return;
751
1336
  for (const kb of listKBs(this.db)) deleteKB(this.db, kb.id);
752
- this.vectorCache.clear();
753
1337
  }
754
1338
 
755
1339
  diagnose(): DiagnosticResult[] {
@@ -758,6 +1342,98 @@ export class KnowledgeEngine {
758
1342
  return listKBs(db).map((kb) => diagnoseKB(db, kb));
759
1343
  }
760
1344
 
1345
+ doctor(): DoctorReport {
1346
+ const diagnostics = this.diagnose();
1347
+ const issues: DoctorIssue[] = [];
1348
+ const kbs = this.list();
1349
+ if (kbs.length === 0) {
1350
+ issues.push({
1351
+ severity: "blocking",
1352
+ message: "No knowledge bases are indexed.",
1353
+ action: "Run knowledge_add for the project root or relevant source/docs directory.",
1354
+ });
1355
+ }
1356
+
1357
+ for (const diagnostic of diagnostics) {
1358
+ if (diagnostic.stuck_indexing) {
1359
+ const phase = diagnostic.job ? ` during ${diagnostic.job.phase}` : "";
1360
+ issues.push({
1361
+ severity: "blocking",
1362
+ kb_name: diagnostic.kb_name,
1363
+ message: `Indexing appears stuck${phase} for ${formatDuration(diagnostic.last_progress_age_ms)}.`,
1364
+ action:
1365
+ "Check knowledge_status for the last progress message. If no Pi process is actively indexing it, remove and rebuild this KB.",
1366
+ });
1367
+ }
1368
+ if (diagnostic.status === "error") {
1369
+ issues.push({
1370
+ severity: "blocking",
1371
+ kb_name: diagnostic.kb_name,
1372
+ message: "Knowledge base is in error state and is skipped by search.",
1373
+ action: "Run knowledge_remove and knowledge_add to rebuild it from the source.",
1374
+ });
1375
+ }
1376
+ if (diagnostic.coverage_percent < 80) {
1377
+ issues.push({
1378
+ severity: "warning",
1379
+ kb_name: diagnostic.kb_name,
1380
+ message: `Coverage is ${diagnostic.coverage_percent}% (${diagnostic.indexed_files}/${diagnostic.total_source_files} files).`,
1381
+ action: "Review skipped files and source path. Rebuild if index-time rules changed.",
1382
+ });
1383
+ }
1384
+ if (diagnostic.stale_files.length > 0) {
1385
+ issues.push({
1386
+ severity: "warning",
1387
+ kb_name: diagnostic.kb_name,
1388
+ message: `${diagnostic.stale_files.length} files changed after indexing.`,
1389
+ action: "Run knowledge_update for this KB.",
1390
+ });
1391
+ }
1392
+ if (diagnostic.orphan_files.length > 0) {
1393
+ issues.push({
1394
+ severity: "warning",
1395
+ kb_name: diagnostic.kb_name,
1396
+ message: `${diagnostic.orphan_files.length} indexed files no longer exist in the source.`,
1397
+ action: "Run knowledge_update or rebuild the KB.",
1398
+ });
1399
+ }
1400
+ if (diagnostic.skipped_files.total > 0) {
1401
+ issues.push({
1402
+ severity: "info",
1403
+ kb_name: diagnostic.kb_name,
1404
+ message: `${diagnostic.skipped_files.total} files were skipped while scanning (${Object.entries(
1405
+ diagnostic.skipped_files.by_reason,
1406
+ )
1407
+ .filter(([, count]) => count > 0)
1408
+ .map(([reason, count]) => `${reason}: ${count}`)
1409
+ .join(", ")}).`,
1410
+ action:
1411
+ "Use skipped samples to confirm exclusions are expected. Adjust source path or ignore rules if needed.",
1412
+ });
1413
+ }
1414
+ }
1415
+
1416
+ const penalty = issues.reduce((score, issue) => {
1417
+ if (issue.severity === "blocking") return score + 35;
1418
+ if (issue.severity === "warning") return score + 12;
1419
+ return score + 2;
1420
+ }, 0);
1421
+ const healthScore = Math.max(0, Math.min(100, 100 - penalty));
1422
+ const blocking = issues.filter((issue) => issue.severity === "blocking").length;
1423
+ const warnings = issues.filter((issue) => issue.severity === "warning").length;
1424
+ const summary =
1425
+ issues.length === 0
1426
+ ? "Knowledge system is healthy."
1427
+ : `${blocking} blocking, ${warnings} warning, ${issues.length - blocking - warnings} info issues.`;
1428
+
1429
+ return {
1430
+ health_score: healthScore,
1431
+ summary,
1432
+ issues,
1433
+ diagnostics,
1434
+ };
1435
+ }
1436
+
761
1437
  async exportKB(nameOrId: string, outputPath: string): Promise<number> {
762
1438
  if (!this.db) throw new Error("Engine not initialized");
763
1439
  const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
@@ -799,11 +1475,27 @@ export class KnowledgeEngine {
799
1475
  const header = JSON.parse(lines[0]) as { name: string; description?: string };
800
1476
  const kb = createKB(this.db, { name: header.name, description: header.description, source_type: "text" });
801
1477
  updateKBStatus(this.db, kb.id, "indexing");
1478
+ startIndexingJob(this.db, kb.id, "import", `Starting import for "${header.name}"`);
1479
+ let vectorWriter: ReturnType<typeof openVectorWriter> | undefined;
1480
+ let tempVectorFile: string | undefined;
802
1481
  try {
803
1482
  const chunkData = lines.slice(1).map((l) => JSON.parse(l) as ImportedChunk);
804
- onProgress?.(`Importing ${chunkData.length} chunks, embedding...`);
1483
+ const importMessage = `Importing ${chunkData.length} chunks...`;
1484
+ updateIndexingJob(this.db, kb.id, {
1485
+ phase: "importing",
1486
+ message: importMessage,
1487
+ total_files: chunkData.length,
1488
+ });
1489
+ onProgress?.(importMessage);
805
1490
  const chunks = chunkData.map((c) => ({
806
- content_hash: contentHash(c.content),
1491
+ content_hash: chunkIdentityHash({
1492
+ content: c.content,
1493
+ filePath: c.file_path,
1494
+ fileType: c.file_type,
1495
+ startLine: c.start_line,
1496
+ endLine: c.end_line,
1497
+ metadataJson: c.metadata_json || "{}",
1498
+ }),
807
1499
  content: c.content,
808
1500
  content_tokenized: "",
809
1501
  file_path: c.file_path,
@@ -816,20 +1508,68 @@ export class KnowledgeEngine {
816
1508
  ...chunk,
817
1509
  content_tokenized: preTokenizeForFTS(buildChunkEmbeddingText(chunk)),
818
1510
  }));
819
- const vectors = await embedDocuments(
820
- indexedChunks.map((c) => buildChunkEmbeddingText(c)),
821
- signal,
822
- );
823
- insertChunks(this.db, kb.id, indexedChunks);
824
1511
  const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
825
- saveVectors(vectorPath, vectors);
826
- this.vectorCache.set(kb.id, vectors);
1512
+ tempVectorFile = tempVectorPath(vectorPath);
1513
+ vectorWriter = openVectorWriter(tempVectorFile);
1514
+ let inserted = 0;
1515
+ for (let offset = 0; offset < indexedChunks.length; offset += INDEX_EMBED_BATCH_SIZE) {
1516
+ if (signal?.aborted) throw new Error("Cancelled");
1517
+ const batch = indexedChunks.slice(offset, offset + INDEX_EMBED_BATCH_SIZE);
1518
+ const message = `Embedding import batch ${Math.floor(offset / INDEX_EMBED_BATCH_SIZE) + 1}/${Math.ceil(
1519
+ indexedChunks.length / INDEX_EMBED_BATCH_SIZE,
1520
+ )}: ${offset}/${indexedChunks.length} chunks`;
1521
+ updateIndexingJob(this.db, kb.id, {
1522
+ phase: "embedding",
1523
+ message,
1524
+ processed_chunks: offset,
1525
+ total_files: indexedChunks.length,
1526
+ added_chunks: inserted,
1527
+ });
1528
+ onProgress?.(message);
1529
+ const vectors = await embedDocuments(
1530
+ batch.map((c) => buildChunkEmbeddingText(c)),
1531
+ signal,
1532
+ );
1533
+ if (signal?.aborted) throw new Error("Cancelled");
1534
+ insertChunks(this.db, kb.id, batch);
1535
+ vectorWriter.append(vectors);
1536
+ inserted += batch.length;
1537
+ updateKBCounts(
1538
+ this.db,
1539
+ kb.id,
1540
+ inserted,
1541
+ new Set(indexedChunks.slice(0, inserted).map((c) => c.file_path)).size,
1542
+ );
1543
+ updateIndexingJob(this.db, kb.id, {
1544
+ phase: "storing",
1545
+ message: `Stored import batch: ${inserted}/${indexedChunks.length} chunks`,
1546
+ processed_chunks: inserted,
1547
+ total_files: indexedChunks.length,
1548
+ added_chunks: inserted,
1549
+ });
1550
+ }
1551
+ vectorWriter.close();
1552
+ vectorWriter = undefined;
1553
+ renameSync(tempVectorFile, vectorPath);
1554
+ tempVectorFile = undefined;
827
1555
  updateKBCounts(this.db, kb.id, indexedChunks.length, new Set(indexedChunks.map((c) => c.file_path)).size);
828
1556
  updateKBStatus(this.db, kb.id, "ready");
829
1557
  const savedKB = getKB(this.db, kb.id);
830
1558
  if (!savedKB) throw new Error(`Knowledge base disappeared after import: ${kb.id}`);
1559
+ finishIndexingJob(this.db, kb.id, "succeeded", `Ready: imported ${indexedChunks.length} chunks`);
831
1560
  return { kb: savedKB, chunkCount: chunks.length };
832
1561
  } catch (e) {
1562
+ vectorWriter?.close();
1563
+ if (tempVectorFile) rmSync(tempVectorFile, { force: true });
1564
+ if (this.db) {
1565
+ finishIndexingJob(
1566
+ this.db,
1567
+ kb.id,
1568
+ isCancellationError(e) ? "cancelled" : "failed",
1569
+ isCancellationError(e) ? "Import cancelled." : "Import failed.",
1570
+ e instanceof Error ? e.message : String(e),
1571
+ );
1572
+ }
833
1573
  deleteKB(this.db, kb.id);
834
1574
  throw e;
835
1575
  }