pi-knowledge 0.4.7 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,9 +1,10 @@
1
- import { existsSync, renameSync, rmSync, statSync } from "node:fs";
1
+ import { existsSync, readFileSync, renameSync, rmSync, statSync } from "node:fs";
2
2
  import { join, resolve } from "node:path";
3
3
  import { diagnoseKB } from "./diagnostics/health.js";
4
4
  import { dispose as disposeEmbedding, embedDocuments, embedQuery, prepareForShutdown as prepareEmbeddingForShutdown, } from "./embedding/provider.js";
5
5
  import { openVectorReader, openVectorWriter } from "./embedding/vectors.js";
6
- import { buildChunkEmbeddingText, chunkFile, chunkIdentityHash, createSkippedScanStats, isReadableTextFile, iterateScannableFiles, iterateScannedFiles, preTokenizeForFTS, summarizeSkippedScan, } from "./indexer/chunker.js";
6
+ import { addSkippedScanEntry, buildChunkEmbeddingText, chunkFile, chunkIdentityHash, createSkippedScanStats, isReadableTextFile, isSupportedDocumentFile, iterateScannableFiles, preTokenizeForFTS, summarizeSkippedScan, } from "./indexer/chunker.js";
7
+ import { extractSymbols } from "./indexer/symbols.js";
7
8
  import { shutdownModelWorker } from "./model-worker-client.js";
8
9
  import { searchBM25 } from "./search/bm25.js";
9
10
  import { weightedScoreFusion } from "./search/fusion.js";
@@ -11,7 +12,7 @@ import { normalizedQueryText, tokenizeForSearch } from "./search/query.js";
11
12
  import { hasAnyLexicalEvidence, hasEnoughLexicalEvidence, MIN_HYBRID_SCORE, normalizeFileTypeFilter, queryCoverage, scoreChunkForQuery, } from "./search/ranking.js";
12
13
  import { disposeReranker, prepareRerankerForShutdown, rerank } from "./search/reranker.js";
13
14
  import { searchVectorFile } from "./search/vector.js";
14
- import { createKB, deleteChunksByIds, deleteKB, finishIndexingJob, getChunkById, getChunkCount, getChunksByFile, getChunksByKB, getFileCount, getKB, getKBByName, insertChunks, iterateChunkIdsByKB, iterateChunksByKB, listKBs, openDatabase, startIndexingJob, updateIndexingJob, updateKBCounts, updateKBStatus, } from "./storage/sqlite.js";
15
+ import { countSymbols, createKB, deleteChunksByIds, deleteKB, deleteSymbolsByKB, finishIndexingJob, getChunkById, getChunkCount, getChunksByFile, getFileCount, getKB, getKBByName, getSymbolCount, insertChunks, insertSymbols, iterateChunkIdsByKB, iterateChunksByKB, listKBs, openDatabase, searchSymbols, startIndexingJob, updateIndexingJob, updateKBCounts, updateKBStatus, } from "./storage/sqlite.js";
15
16
  export const CURRENT_EMBEDDING_MODEL = "multilingual-e5-small";
16
17
  const ADAPTIVE_CONTEXT_LINES = 80;
17
18
  const ADAPTIVE_MAX_CONTEXT_CHARS = 6_000;
@@ -24,6 +25,26 @@ function isCancellationError(error) {
24
25
  function tempVectorPath(vectorPath) {
25
26
  return `${vectorPath}.${Date.now()}.${Math.random().toString(16).slice(2)}.tmp`;
26
27
  }
28
+ function retrievalModeFor(mode) {
29
+ if (mode === "auto")
30
+ return "hybrid";
31
+ if (mode === "code" || mode === "config" || mode === "errors")
32
+ return "fast";
33
+ if (mode === "docs" || mode === "decision")
34
+ return "hybrid";
35
+ return mode;
36
+ }
37
+ function matchReasonFor(mode) {
38
+ if (mode === "deep")
39
+ return "rerank";
40
+ if (mode === "adaptive")
41
+ return "adaptive";
42
+ if (mode === "semantic")
43
+ return "vector";
44
+ if (mode === "fast" || mode === "code" || mode === "config" || mode === "errors")
45
+ return "bm25";
46
+ return "hybrid";
47
+ }
27
48
  function toScanOptions(options = {}) {
28
49
  return {
29
50
  includeSuggestedText: options.include_suggested_text === true,
@@ -60,11 +81,12 @@ function parseAddOptions(raw) {
60
81
  return {};
61
82
  }
62
83
  }
63
- function planDirectoryScan(dirPath, options = {}) {
84
+ function planDirectoryScan(dirPath, options = {}, signal) {
64
85
  const skipped = createSkippedScanStats();
65
86
  let files = 0;
66
87
  let bytes = 0;
67
88
  for (const file of iterateScannableFiles(dirPath, skipped, options)) {
89
+ throwIfAborted(signal);
68
90
  files++;
69
91
  bytes += file.size;
70
92
  }
@@ -76,6 +98,25 @@ function planDirectoryScan(dirPath, options = {}) {
76
98
  skipped,
77
99
  };
78
100
  }
101
+ function sourceMtimeFor(kb, filePath) {
102
+ if (!kb?.source_path)
103
+ return undefined;
104
+ const absPath = kb.source_type === "directory" ? join(kb.source_path, filePath) : kb.source_path;
105
+ try {
106
+ return statSync(absPath).mtimeMs;
107
+ }
108
+ catch {
109
+ return undefined;
110
+ }
111
+ }
112
+ function kbTrustMultiplier(kb) {
113
+ let multiplier = kb.status === "stale" ? 0.92 : 1;
114
+ if (kb.source_type === "directory" || kb.source_type === "file")
115
+ multiplier *= 1.04;
116
+ if (kb.source_type === "text" || kb.source_type === "url")
117
+ multiplier *= 0.98;
118
+ return multiplier;
119
+ }
79
120
  function formatBytes(bytes) {
80
121
  if (bytes < 1024)
81
122
  return `${bytes} B`;
@@ -347,22 +388,181 @@ async function chunkUrl(source, signal) {
347
388
  function normalizeExtractedText(text) {
348
389
  return Array.isArray(text) ? text.join("\n\n") : text;
349
390
  }
391
+ async function extractSourceFileContent(filePath, signal) {
392
+ const lowerPath = filePath.toLowerCase();
393
+ if (lowerPath.endsWith(".pdf")) {
394
+ throwIfAborted(signal);
395
+ // PDF extraction is an optional heavy runtime path; keep parser loading out of extension startup.
396
+ const { extractText } = await import("unpdf");
397
+ throwIfAborted(signal);
398
+ const buf = readFileSync(filePath);
399
+ throwIfAborted(signal);
400
+ const { text } = await extractText(new Uint8Array(buf));
401
+ throwIfAborted(signal);
402
+ return { content: normalizeExtractedText(text), fileType: "pdf" };
403
+ }
404
+ if (lowerPath.endsWith(".docx") || lowerPath.endsWith(".doc")) {
405
+ throwIfAborted(signal);
406
+ // DOCX extraction is an optional heavy runtime path; keep parser loading out of extension startup.
407
+ const mammoth = await import("mammoth");
408
+ throwIfAborted(signal);
409
+ const result = await mammoth.extractRawText({ path: filePath });
410
+ throwIfAborted(signal);
411
+ return { content: result.value, fileType: "docx" };
412
+ }
413
+ if (!isReadableTextFile(filePath)) {
414
+ throw new Error(`File is not readable text and has no supported extractor: ${filePath}`);
415
+ }
416
+ throwIfAborted(signal);
417
+ return { content: readFileSync(filePath, "utf-8"), fileType: "text" };
418
+ }
419
+ function looksLikeLocalPath(source) {
420
+ const trimmed = source.trim();
421
+ if (!trimmed || trimmed.includes("\n") || trimmed.includes("\r"))
422
+ return false;
423
+ if (/^(?:https?:)?\/\//.test(trimmed))
424
+ return false;
425
+ if (trimmed.startsWith(".") || trimmed.startsWith("/") || trimmed.startsWith("~"))
426
+ return true;
427
+ if (trimmed.includes("/") || trimmed.includes("\\"))
428
+ return true;
429
+ return /\.[A-Za-z0-9][A-Za-z0-9_-]{0,15}$/.test(trimmed);
430
+ }
431
+ function classifySource(source) {
432
+ const resolvedSource = resolve(source);
433
+ const isUrl = source.startsWith("http://") || source.startsWith("https://");
434
+ const isDir = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isDirectory();
435
+ const isFile = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isFile();
436
+ if (!isUrl && !isDir && !isFile && looksLikeLocalPath(source)) {
437
+ throw new Error(`Local path does not exist: ${source}. Pass inline text only when the source itself is text.`);
438
+ }
439
+ return {
440
+ resolvedSource,
441
+ isUrl,
442
+ isDir,
443
+ isFile,
444
+ sourceType: isUrl ? "url" : isDir ? "directory" : isFile ? "file" : "text",
445
+ };
446
+ }
447
+ async function extractScannableFileContent(file, signal) {
448
+ return extractSourceFileContent(file.path, signal);
449
+ }
450
+ async function extractScannableFileContentOrSkip(file, skipped, signal) {
451
+ try {
452
+ return await extractScannableFileContent(file, signal);
453
+ }
454
+ catch (error) {
455
+ if (signal?.aborted || isCancellationError(error))
456
+ throw error;
457
+ addSkippedScanEntry(skipped, { path: file.relPath, reason: "extraction_failed", size: file.size });
458
+ return undefined;
459
+ }
460
+ }
461
+ function throwIfAborted(signal) {
462
+ if (signal?.aborted)
463
+ throw new Error("Cancelled");
464
+ }
465
+ function importedChunkToInsert(chunk) {
466
+ const metadataJson = chunk.metadata_json || "{}";
467
+ return {
468
+ content_hash: chunkIdentityHash({
469
+ content: chunk.content,
470
+ filePath: chunk.file_path,
471
+ fileType: chunk.file_type,
472
+ startLine: chunk.start_line,
473
+ endLine: chunk.end_line,
474
+ metadataJson,
475
+ }),
476
+ content: chunk.content,
477
+ content_tokenized: "",
478
+ file_path: chunk.file_path,
479
+ file_type: chunk.file_type,
480
+ start_line: chunk.start_line,
481
+ end_line: chunk.end_line,
482
+ metadata_json: metadataJson,
483
+ };
484
+ }
485
+ async function writeLine(stream, line) {
486
+ if (stream.write(`${line}\n`))
487
+ return;
488
+ await new Promise((resolve, reject) => {
489
+ const cleanup = () => {
490
+ stream.off("drain", onDrain);
491
+ stream.off("error", onError);
492
+ };
493
+ const onDrain = () => {
494
+ cleanup();
495
+ resolve();
496
+ };
497
+ const onError = (error) => {
498
+ cleanup();
499
+ reject(error);
500
+ };
501
+ stream.once("drain", onDrain);
502
+ stream.once("error", onError);
503
+ });
504
+ }
505
+ async function finishWriteStream(stream) {
506
+ await new Promise((resolve, reject) => {
507
+ const cleanup = () => {
508
+ stream.off("finish", onFinish);
509
+ stream.off("error", onError);
510
+ };
511
+ const onFinish = () => {
512
+ cleanup();
513
+ resolve();
514
+ };
515
+ const onError = (error) => {
516
+ cleanup();
517
+ reject(error);
518
+ };
519
+ stream.once("finish", onFinish);
520
+ stream.once("error", onError);
521
+ stream.end();
522
+ });
523
+ }
350
524
  export class KnowledgeEngine {
351
525
  db = null;
352
526
  knowledgeDir = "";
353
527
  activeUpdates = new Map();
528
+ activeMutations = new Map();
529
+ disposing = false;
530
+ runExclusive(keys, description, operation) {
531
+ if (this.disposing)
532
+ throw new Error("Knowledge engine is shutting down");
533
+ const mutationKeys = Array.isArray(keys) ? keys : [keys];
534
+ for (const key of mutationKeys) {
535
+ if (this.activeMutations.has(key))
536
+ throw new Error(`${description} is already running`);
537
+ }
538
+ const run = Promise.resolve()
539
+ .then(operation)
540
+ .finally(() => {
541
+ for (const key of mutationKeys) {
542
+ if (this.activeMutations.get(key) === run)
543
+ this.activeMutations.delete(key);
544
+ }
545
+ });
546
+ for (const key of mutationKeys)
547
+ this.activeMutations.set(key, run);
548
+ return run;
549
+ }
354
550
  async initialize(knowledgeDir) {
355
551
  this.knowledgeDir = knowledgeDir;
356
552
  this.db = openDatabase(knowledgeDir);
553
+ this.disposing = false;
554
+ }
555
+ vectorPathFor(kbId) {
556
+ return join(this.knowledgeDir, "vectors", `${kbId}.bin`);
357
557
  }
358
- plan(source, options = {}) {
359
- const resolvedSource = resolve(source);
360
- const isUrl = source.startsWith("http://") || source.startsWith("https://");
361
- const isDir = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isDirectory();
362
- const isFile = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isFile();
363
- const sourceType = isUrl ? "url" : isDir ? "directory" : isFile ? "file" : "text";
558
+ deleteVectorFile(kbId) {
559
+ rmSync(this.vectorPathFor(kbId), { force: true });
560
+ }
561
+ plan(source, options = {}, signal) {
562
+ throwIfAborted(signal);
563
+ const { resolvedSource, isDir, isFile, sourceType } = classifySource(source);
364
564
  if (isDir) {
365
- const plan = planDirectoryScan(resolvedSource, toScanOptions(options));
565
+ const plan = planDirectoryScan(resolvedSource, toScanOptions(options), signal);
366
566
  return {
367
567
  source_type: "directory",
368
568
  scannable_files: plan.files,
@@ -373,7 +573,7 @@ export class KnowledgeEngine {
373
573
  }
374
574
  if (isFile) {
375
575
  const skipped = createSkippedScanStats();
376
- const supportedDocument = resolvedSource.endsWith(".pdf") || resolvedSource.endsWith(".docx") || resolvedSource.endsWith(".doc");
576
+ const supportedDocument = isSupportedDocumentFile(resolvedSource);
377
577
  if (!supportedDocument && !isReadableTextFile(resolvedSource)) {
378
578
  skipped.total = 1;
379
579
  skipped.by_reason.binary = 1;
@@ -405,14 +605,15 @@ export class KnowledgeEngine {
405
605
  };
406
606
  }
407
607
  async add(source, name, onProgress, signal, options = {}) {
608
+ throwIfAborted(signal);
609
+ return this.runExclusive(["global:mutation", `kb-name:${name}`], `Mutation for "${name}"`, () => this.addUnlocked(source, name, onProgress, signal, options));
610
+ }
611
+ async addUnlocked(source, name, onProgress, signal, options = {}) {
408
612
  if (!this.db)
409
613
  throw new Error("Engine not initialized");
614
+ throwIfAborted(signal);
410
615
  const db = this.db;
411
- const resolvedSource = resolve(source);
412
- const isUrl = source.startsWith("http://") || source.startsWith("https://");
413
- const isDir = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isDirectory();
414
- const isFile = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isFile();
415
- const sourceType = isUrl ? "url" : isDir ? "directory" : isFile ? "file" : "text";
616
+ const { resolvedSource, isUrl, isDir, isFile, sourceType } = classifySource(source);
416
617
  const scanOptions = toScanOptions(options);
417
618
  const existingKB = getKBByName(db, name);
418
619
  if (existingKB) {
@@ -429,7 +630,7 @@ export class KnowledgeEngine {
429
630
  let vectorWriter;
430
631
  let tempVectorFile;
431
632
  try {
432
- const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
633
+ const vectorPath = this.vectorPathFor(kb.id);
433
634
  tempVectorFile = tempVectorPath(vectorPath);
434
635
  const writer = openVectorWriter(tempVectorFile);
435
636
  vectorWriter = writer;
@@ -438,7 +639,8 @@ export class KnowledgeEngine {
438
639
  const pendingChunks = [];
439
640
  const startedAt = Date.now();
440
641
  let latestSkippedTotal = 0;
441
- const reportProgress = (phase, processedFiles, totalFiles, skippedTotal = 0) => {
642
+ let latestSkippedSummary = "none";
643
+ const reportProgress = (phase, processedFiles, totalFiles, skippedTotal = latestSkippedTotal) => {
442
644
  latestSkippedTotal = skippedTotal;
443
645
  const elapsed = Date.now() - startedAt;
444
646
  const chunkRate = chunkCount / Math.max(1, elapsed / 1000);
@@ -488,33 +690,26 @@ export class KnowledgeEngine {
488
690
  await flushPending(processedFiles, totalFiles);
489
691
  }
490
692
  };
693
+ const addSymbols = (content, filePath, fileType) => {
694
+ insertSymbols(db, kb.id, extractSymbols(content, filePath, fileType));
695
+ };
491
696
  if (isUrl) {
492
697
  const message = `Fetching ${source}...`;
493
698
  updateIndexingJob(this.db, kb.id, { phase: "fetching", message });
494
699
  onProgress?.(message);
495
700
  const chunks = await chunkUrl(source, signal);
496
701
  fileCount = 1;
702
+ addSymbols(chunks.map((chunk) => chunk.content).join("\n\n"), source, "html");
497
703
  await addChunks(chunks);
498
704
  }
499
- else if (isFile && resolvedSource.endsWith(".pdf")) {
500
- updateIndexingJob(this.db, kb.id, { phase: "extracting", message: "Extracting text from PDF..." });
501
- onProgress?.("Extracting text from PDF...");
502
- const { extractText } = await import("unpdf");
503
- const buf = (await import("node:fs")).readFileSync(resolvedSource);
504
- const { text } = await extractText(new Uint8Array(buf));
505
- fileCount = 1;
506
- await addChunks(await chunkFile(normalizeExtractedText(text), resolvedSource));
507
- }
508
- else if (isFile && (resolvedSource.endsWith(".docx") || resolvedSource.endsWith(".doc"))) {
509
- updateIndexingJob(this.db, kb.id, { phase: "extracting", message: "Extracting text from DOCX..." });
510
- onProgress?.("Extracting text from DOCX...");
511
- const mammoth = await import("mammoth");
512
- const result = await mammoth.extractRawText({ path: resolvedSource });
705
+ else if (isFile) {
706
+ const extracted = await extractSourceFileContent(resolvedSource, signal);
513
707
  fileCount = 1;
514
- await addChunks(await chunkFile(result.value, resolvedSource));
708
+ addSymbols(extracted.content, resolvedSource, extracted.fileType);
709
+ await addChunks(await chunkFile(extracted.content, resolvedSource));
515
710
  }
516
711
  else if (isDir) {
517
- const plan = planDirectoryScan(resolvedSource, scanOptions);
712
+ const plan = planDirectoryScan(resolvedSource, scanOptions, signal);
518
713
  const planningMessage = `Planned directory scan: ${plan.files} files, ${formatBytes(plan.bytes)} scannable text, skipped ${plan.skippedTotal} (${plan.skippedSummary})`;
519
714
  updateIndexingJob(this.db, kb.id, {
520
715
  phase: "planning",
@@ -533,19 +728,27 @@ export class KnowledgeEngine {
533
728
  onProgress?.(scanningMessage);
534
729
  const skipped = createSkippedScanStats();
535
730
  let processedFiles = 0;
536
- for (const file of iterateScannedFiles(resolvedSource, skipped, scanOptions)) {
731
+ for (const file of iterateScannableFiles(resolvedSource, skipped, scanOptions)) {
537
732
  if (signal?.aborted)
538
733
  throw new Error("Cancelled");
539
- const chunks = await chunkFile(file.content, file.relPath);
734
+ const extracted = await extractScannableFileContentOrSkip(file, skipped, signal);
735
+ if (!extracted) {
736
+ latestSkippedTotal = skipped.total;
737
+ continue;
738
+ }
739
+ const chunks = await chunkFile(extracted.content, file.relPath);
540
740
  processedFiles++;
541
741
  latestSkippedTotal = skipped.total;
542
742
  if (chunks.length > 0)
543
743
  fileCount++;
744
+ addSymbols(extracted.content, file.relPath, extracted.fileType);
544
745
  await addChunks(chunks, processedFiles, plan.files);
545
746
  if (processedFiles % 25 === 0)
546
747
  reportProgress("Chunking", processedFiles, plan.files, skipped.total);
547
748
  }
548
- const finalizingMessage = `Scanned ${processedFiles} files, skipped ${skipped.total} (${summarizeSkippedScan(skipped)}), finalizing...`;
749
+ latestSkippedTotal = skipped.total;
750
+ latestSkippedSummary = summarizeSkippedScan(skipped);
751
+ const finalizingMessage = `Scanned ${processedFiles} files, skipped ${skipped.total} (${latestSkippedSummary}), finalizing...`;
549
752
  updateIndexingJob(this.db, kb.id, {
550
753
  phase: "finalizing",
551
754
  message: finalizingMessage,
@@ -557,17 +760,9 @@ export class KnowledgeEngine {
557
760
  });
558
761
  onProgress?.(finalizingMessage);
559
762
  }
560
- else if (isFile) {
561
- if (!isReadableTextFile(resolvedSource)) {
562
- throw new Error(`File is not readable text and has no supported extractor: ${resolvedSource}`);
563
- }
564
- const { readFileSync } = await import("node:fs");
565
- const content = readFileSync(resolvedSource, "utf-8");
566
- fileCount = 1;
567
- await addChunks(await chunkFile(content, resolvedSource));
568
- }
569
763
  else {
570
764
  fileCount = 1;
765
+ addSymbols(source, "inline-text", "text");
571
766
  await addChunks(await chunkFile(source, "inline-text"));
572
767
  }
573
768
  await flushPending();
@@ -581,7 +776,8 @@ export class KnowledgeEngine {
581
776
  const savedKB = getKB(this.db, kb.id);
582
777
  if (!savedKB)
583
778
  throw new Error(`Knowledge base disappeared after add: ${kb.id}`);
584
- const readyMessage = `Ready: ${chunkCount} chunks from ${savedFileCount} files in ${formatDuration(Date.now() - startedAt)}`;
779
+ const skippedReadySuffix = latestSkippedTotal > 0 ? `, skipped ${latestSkippedTotal} (${latestSkippedSummary})` : "";
780
+ const readyMessage = `Ready: ${chunkCount} chunks from ${savedFileCount} files in ${formatDuration(Date.now() - startedAt)}${skippedReadySuffix}`;
585
781
  updateIndexingJob(this.db, kb.id, {
586
782
  phase: "ready",
587
783
  message: readyMessage,
@@ -600,15 +796,18 @@ export class KnowledgeEngine {
600
796
  rmSync(tempVectorFile, { force: true });
601
797
  if (this.db) {
602
798
  finishIndexingJob(this.db, kb.id, isCancellationError(e) ? "cancelled" : "failed", isCancellationError(e) ? "Indexing cancelled." : "Indexing failed.", e instanceof Error ? e.message : String(e));
799
+ deleteKB(this.db, kb.id);
800
+ this.deleteVectorFile(kb.id);
603
801
  }
604
- // Clean up partial state on failure
605
- deleteKB(this.db, kb.id);
606
802
  throw e;
607
803
  }
608
804
  }
609
805
  async update(nameOrId, onProgress, signal) {
610
806
  if (!this.db)
611
807
  throw new Error("Engine not initialized");
808
+ if (this.disposing)
809
+ throw new Error("Knowledge engine is shutting down");
810
+ throwIfAborted(signal);
612
811
  const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
613
812
  if (!kb)
614
813
  throw new Error(`Knowledge base not found: ${nameOrId}`);
@@ -621,7 +820,7 @@ export class KnowledgeEngine {
621
820
  onProgress?.(`Update already running for "${updateableKB.name}"; waiting for the active update.`);
622
821
  return activeUpdate;
623
822
  }
624
- const updateRun = this.runUpdate(updateableKB, onProgress, signal).finally(() => {
823
+ const updateRun = this.runExclusive(["global:mutation", `kb:${updateableKB.id}`], `Mutation for "${updateableKB.name}"`, () => this.runUpdate(updateableKB, onProgress, signal)).finally(() => {
625
824
  if (this.activeUpdates.get(updateableKB.id) === updateRun)
626
825
  this.activeUpdates.delete(updateableKB.id);
627
826
  });
@@ -631,16 +830,21 @@ export class KnowledgeEngine {
631
830
  async runUpdate(kb, onProgress, signal) {
632
831
  if (!this.db)
633
832
  throw new Error("Engine not initialized");
833
+ throwIfAborted(signal);
634
834
  updateKBStatus(this.db, kb.id, "indexing");
635
835
  startIndexingJob(this.db, kb.id, "update", `Starting update for "${kb.name}"`);
636
836
  const scanOptions = toScanOptions(parseAddOptions(kb.source_options));
637
837
  let replacementVectorPath;
638
838
  let addedVectorPath;
639
839
  let addedVectorWriter;
840
+ const insertedChunkIds = [];
841
+ const stagedSymbols = [];
640
842
  try {
641
- const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
843
+ const vectorPath = this.vectorPathFor(kb.id);
642
844
  addedVectorPath = tempVectorPath(`${vectorPath}.added`);
643
845
  addedVectorWriter = openVectorWriter(addedVectorPath);
846
+ let latestSkippedTotal = 0;
847
+ let latestSkippedSummary = "none";
644
848
  const existingHashes = new Map();
645
849
  let existingIndex = 0;
646
850
  for (const chunk of iterateChunksByKB(this.db, kb.id)) {
@@ -687,7 +891,7 @@ export class KnowledgeEngine {
687
891
  newVectorIndexByHash.set(batch[i].content_hash, indexes);
688
892
  }
689
893
  addedVectorCount += newVectors.length;
690
- insertChunks(this.db, kb.id, batch);
894
+ insertedChunkIds.push(...insertChunks(this.db, kb.id, batch));
691
895
  addedCount += batch.length;
692
896
  updateKBCounts(this.db, kb.id, getChunkCount(this.db, kb.id), getFileCount(this.db, kb.id));
693
897
  const storedMessage = `Stored update batch: +${addedCount} chunks, =${unchanged} unchanged`;
@@ -728,10 +932,12 @@ export class KnowledgeEngine {
728
932
  updateIndexingJob(this.db, kb.id, { phase: "fetching", message });
729
933
  onProgress?.(message);
730
934
  scannedFiles = 1;
731
- await processChunks(await chunkUrl(kb.source_path, signal));
935
+ const chunks = await chunkUrl(kb.source_path, signal);
936
+ stagedSymbols.push(...extractSymbols(chunks.map((chunk) => chunk.content).join("\n\n"), kb.source_path, "html"));
937
+ await processChunks(chunks);
732
938
  }
733
939
  else if (statSync(kb.source_path).isDirectory()) {
734
- const plan = planDirectoryScan(kb.source_path, scanOptions);
940
+ const plan = planDirectoryScan(kb.source_path, scanOptions, signal);
735
941
  plannedTotalFiles = plan.files;
736
942
  const planningMessage = `Planned directory scan: ${plan.files} files, ${formatBytes(plan.bytes)} scannable text, skipped ${plan.skippedTotal} (${plan.skippedSummary})`;
737
943
  updateIndexingJob(this.db, kb.id, {
@@ -742,11 +948,15 @@ export class KnowledgeEngine {
742
948
  });
743
949
  onProgress?.(planningMessage);
744
950
  const skipped = createSkippedScanStats();
745
- for (const file of iterateScannedFiles(kb.source_path, skipped, scanOptions)) {
951
+ for (const file of iterateScannableFiles(kb.source_path, skipped, scanOptions)) {
746
952
  if (signal?.aborted)
747
953
  throw new Error("Cancelled");
954
+ const extracted = await extractScannableFileContentOrSkip(file, skipped, signal);
955
+ if (!extracted)
956
+ continue;
748
957
  scannedFiles++;
749
- await processChunks(await chunkFile(file.content, file.relPath));
958
+ stagedSymbols.push(...extractSymbols(extracted.content, file.relPath, extracted.fileType));
959
+ await processChunks(await chunkFile(extracted.content, file.relPath));
750
960
  if (scannedFiles % 25 === 0) {
751
961
  const message = `Scanned ${scannedFiles} files, ${scannedChunks} chunks, skipped ${skipped.total}, +${addedCount} =${unchanged}`;
752
962
  updateIndexingJob(this.db, kb.id, {
@@ -762,7 +972,9 @@ export class KnowledgeEngine {
762
972
  onProgress?.(message);
763
973
  }
764
974
  }
765
- const message = `Scanned ${scannedFiles} files, skipped ${skipped.total} (${summarizeSkippedScan(skipped)}), reconciling deletes...`;
975
+ latestSkippedTotal = skipped.total;
976
+ latestSkippedSummary = summarizeSkippedScan(skipped);
977
+ const message = `Scanned ${scannedFiles} files, skipped ${skipped.total} (${latestSkippedSummary}), reconciling deletes...`;
766
978
  updateIndexingJob(this.db, kb.id, {
767
979
  phase: "reconciling",
768
980
  message,
@@ -776,9 +988,10 @@ export class KnowledgeEngine {
776
988
  onProgress?.(message);
777
989
  }
778
990
  else {
779
- const { readFileSync } = await import("node:fs");
991
+ const extracted = await extractSourceFileContent(kb.source_path, signal);
780
992
  scannedFiles = 1;
781
- await processChunks(await chunkFile(readFileSync(kb.source_path, "utf-8"), kb.source_path));
993
+ stagedSymbols.push(...extractSymbols(extracted.content, kb.source_path, extracted.fileType));
994
+ await processChunks(await chunkFile(extracted.content, kb.source_path));
782
995
  }
783
996
  await flushPending();
784
997
  addedVectorWriter.close();
@@ -787,8 +1000,7 @@ export class KnowledgeEngine {
787
1000
  for (const entries of existingHashes.values()) {
788
1001
  idsToRemove.push(...entries.map((entry) => entry.id));
789
1002
  }
790
- if (idsToRemove.length > 0)
791
- deleteChunksByIds(this.db, idsToRemove);
1003
+ const idsToRemoveSet = new Set(idsToRemove);
792
1004
  const changesMessage = `Changes: +${addedCount} -${idsToRemove.length} =${unchanged}`;
793
1005
  updateIndexingJob(this.db, kb.id, {
794
1006
  phase: "reconciling",
@@ -811,6 +1023,10 @@ export class KnowledgeEngine {
811
1023
  };
812
1024
  try {
813
1025
  for (const chunk of iterateChunksByKB(this.db, kb.id)) {
1026
+ if (signal?.aborted)
1027
+ throw new Error("Cancelled");
1028
+ if (idsToRemoveSet.has(chunk.id))
1029
+ continue;
814
1030
  const oldVectorIndex = takeVectorIndex(oldVectorIndexByHash, chunk.content_hash);
815
1031
  const newVectorIndex = takeVectorIndex(newVectorIndexByHash, chunk.content_hash);
816
1032
  const vector = oldVectorReader && oldVectorIndex !== undefined
@@ -847,9 +1063,14 @@ export class KnowledgeEngine {
847
1063
  if (addedVectorPath)
848
1064
  rmSync(addedVectorPath, { force: true });
849
1065
  addedVectorPath = undefined;
1066
+ if (idsToRemove.length > 0)
1067
+ deleteChunksByIds(this.db, idsToRemove);
1068
+ deleteSymbolsByKB(this.db, kb.id);
1069
+ insertSymbols(this.db, kb.id, stagedSymbols);
850
1070
  updateKBCounts(this.db, kb.id, finalChunkCount, getFileCount(this.db, kb.id));
851
1071
  updateKBStatus(this.db, kb.id, "ready");
852
- const readyMessage = `Ready: +${addedCount} -${idsToRemove.length} =${unchanged}`;
1072
+ const skippedReadySuffix = latestSkippedTotal > 0 ? `, skipped ${latestSkippedTotal} (${latestSkippedSummary})` : "";
1073
+ const readyMessage = `Ready: +${addedCount} -${idsToRemove.length} =${unchanged}${skippedReadySuffix}`;
853
1074
  updateIndexingJob(this.db, kb.id, {
854
1075
  phase: "ready",
855
1076
  message: readyMessage,
@@ -869,14 +1090,19 @@ export class KnowledgeEngine {
869
1090
  rmSync(replacementVectorPath, { force: true });
870
1091
  if (addedVectorPath)
871
1092
  rmSync(addedVectorPath, { force: true });
872
- updateKBStatus(this.db, kb.id, "error");
1093
+ if (insertedChunkIds.length > 0) {
1094
+ deleteChunksByIds(this.db, insertedChunkIds);
1095
+ updateKBCounts(this.db, kb.id, getChunkCount(this.db, kb.id), getFileCount(this.db, kb.id));
1096
+ }
1097
+ updateKBStatus(this.db, kb.id, isCancellationError(e) ? kb.status : "error");
873
1098
  finishIndexingJob(this.db, kb.id, isCancellationError(e) ? "cancelled" : "failed", isCancellationError(e) ? "Update cancelled." : "Update failed.", e instanceof Error ? e.message : String(e));
874
1099
  throw e;
875
1100
  }
876
1101
  }
877
- async search(query, options = {}) {
1102
+ async search(query, options = {}, signal) {
878
1103
  if (!this.db)
879
1104
  throw new Error("Engine not initialized");
1105
+ throwIfAborted(signal);
880
1106
  const requestedMode = options.mode ?? "hybrid";
881
1107
  if (requestedMode === "auto") {
882
1108
  const primaryMode = chooseAutoMode(query);
@@ -887,7 +1113,8 @@ export class KnowledgeEngine {
887
1113
  if (tried.includes(mode))
888
1114
  continue;
889
1115
  tried.push(mode);
890
- const response = await this.search(query, { ...options, mode });
1116
+ throwIfAborted(signal);
1117
+ const response = await this.search(query, { ...options, mode }, signal);
891
1118
  if (response.warnings)
892
1119
  warnings.push(...response.warnings);
893
1120
  if (!isWeakAutoResponse(query, response, primaryMode, mode)) {
@@ -916,7 +1143,8 @@ export class KnowledgeEngine {
916
1143
  }
917
1144
  const db = this.db;
918
1145
  const { mode = "hybrid", limit = 10, offset = 0, kb_id, filters, diversity = "balanced" } = options;
919
- const retrievalMode = mode === "adaptive" ? "hybrid" : mode;
1146
+ const resolvedMode = retrievalModeFor(mode);
1147
+ const retrievalMode = resolvedMode === "adaptive" ? "hybrid" : resolvedMode;
920
1148
  const candidateLimit = Math.max(50, offset + limit * 12);
921
1149
  const normalizedQuery = normalizedQueryText(query);
922
1150
  const queryTokens = tokenizeForSimilarity(query);
@@ -925,6 +1153,7 @@ export class KnowledgeEngine {
925
1153
  const selectedKB = kb_id ? (getKB(db, kb_id) ?? getKBByName(db, kb_id)) : undefined;
926
1154
  const availableKBs = kb_id ? [selectedKB].filter(Boolean) : listKBs(db);
927
1155
  const kbs = availableKBs.filter((kb) => kb.status === "ready" || kb.status === "stale");
1156
+ const kbById = new Map(kbs.map((kb) => [kb.id, kb]));
928
1157
  for (const kb of availableKBs) {
929
1158
  if (kb.status !== "ready" && kb.status !== "stale") {
930
1159
  warnings.push(`"${kb.name}" is ${kb.status}; search skipped until indexing is ready`);
@@ -942,19 +1171,24 @@ export class KnowledgeEngine {
942
1171
  const allResults = [];
943
1172
  const vectorsByChunkId = new Map();
944
1173
  for (const kb of kbs) {
1174
+ throwIfAborted(signal);
945
1175
  if (kb.embedding_model !== CURRENT_EMBEDDING_MODEL) {
946
1176
  warnings.push(`"${kb.name}" was indexed with ${kb.embedding_model} (current: ${CURRENT_EMBEDDING_MODEL}) — run knowledge_update for best results`);
947
1177
  }
948
1178
  if (kb.chunk_count === 0)
949
1179
  continue;
950
- const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
1180
+ const vectorPath = this.vectorPathFor(kb.id);
951
1181
  if (retrievalMode === "fast") {
952
- allResults.push(...searchBM25(db, normalizedQuery || query, candidateLimit, kb.id, { allowOrFallback: false }));
1182
+ allResults.push(...searchBM25(db, normalizedQuery || query, candidateLimit, kb.id, { allowOrFallback: false }).map((result) => ({
1183
+ ...result,
1184
+ score: result.score * kbTrustMultiplier(kb),
1185
+ })));
953
1186
  }
954
1187
  else if (retrievalMode === "semantic") {
955
- const queryVec = await embedQuery(query);
956
- const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), candidateLimit);
957
- allResults.push(...vectorResults.results);
1188
+ const queryVec = await embedQuery(query, signal);
1189
+ throwIfAborted(signal);
1190
+ const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), candidateLimit, signal);
1191
+ allResults.push(...vectorResults.results.map((result) => ({ ...result, score: result.score * kbTrustMultiplier(kb) })));
958
1192
  for (const [chunkId, vector] of vectorResults.vectorsByChunkId)
959
1193
  vectorsByChunkId.set(chunkId, vector);
960
1194
  }
@@ -964,13 +1198,14 @@ export class KnowledgeEngine {
964
1198
  if (bm25Results.length === 0)
965
1199
  continue;
966
1200
  let vecResults = [];
967
- const queryVec = await embedQuery(query);
968
- const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), candidateLimit);
1201
+ const queryVec = await embedQuery(query, signal);
1202
+ throwIfAborted(signal);
1203
+ const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), candidateLimit, signal);
969
1204
  vecResults = vectorResults.results;
970
1205
  for (const [chunkId, vector] of vectorResults.vectorsByChunkId)
971
1206
  vectorsByChunkId.set(chunkId, vector);
972
1207
  const fused = weightedScoreFusion(bm25Results, vecResults);
973
- allResults.push(...fused);
1208
+ allResults.push(...fused.map((result) => ({ ...result, score: result.score * kbTrustMultiplier(kb) })));
974
1209
  }
975
1210
  }
976
1211
  // Deduplicate and sort
@@ -983,6 +1218,7 @@ export class KnowledgeEngine {
983
1218
  });
984
1219
  const rankingByChunkId = new Map();
985
1220
  for (const result of unique) {
1221
+ throwIfAborted(signal);
986
1222
  const chunk = getChunkById(db, result.chunkId);
987
1223
  if (chunk) {
988
1224
  const ranking = scoreChunkForQuery(result.score, chunk, queryTokens);
@@ -1030,9 +1266,11 @@ export class KnowledgeEngine {
1030
1266
  return chunk ? { chunkId: r.chunkId, content: chunk.content } : null;
1031
1267
  })
1032
1268
  .filter(Boolean);
1033
- const reranked = await rerank(query, candidates, Math.max(limit * 3, limit));
1269
+ const reranked = await rerank(query, candidates, Math.max(limit * 3, limit), signal);
1270
+ throwIfAborted(signal);
1034
1271
  const ranked = [];
1035
1272
  for (const r of reranked) {
1273
+ throwIfAborted(signal);
1036
1274
  const chunk = getChunkById(db, r.chunkId);
1037
1275
  if (!chunk)
1038
1276
  continue;
@@ -1051,17 +1289,30 @@ export class KnowledgeEngine {
1051
1289
  }
1052
1290
  const diversified = interleaveByFile(diversifyRankedChunks(ranked, diversity, vectorsByChunkId), diversity);
1053
1291
  const page = diversified.slice(offset, offset + limit);
1054
- const results = page.map((r) => ({
1055
- content: r.content,
1056
- file_path: r.chunk.file_path,
1057
- file_type: r.chunk.file_type,
1058
- kb_name: r.kbName,
1059
- score: r.score,
1060
- ranking: r.ranking,
1061
- snippet: r.snippet,
1062
- start_line: r.startLine,
1063
- end_line: r.endLine,
1064
- }));
1292
+ const results = page.map((r) => {
1293
+ const kb = kbById.get(r.chunk.kb_id);
1294
+ const sourceMtime = sourceMtimeFor(kb, r.chunk.file_path);
1295
+ return {
1296
+ content: r.content,
1297
+ file_path: r.chunk.file_path,
1298
+ file_type: r.chunk.file_type,
1299
+ kb_name: r.kbName,
1300
+ score: r.score,
1301
+ ranking: r.ranking,
1302
+ snippet: r.snippet,
1303
+ start_line: r.startLine,
1304
+ end_line: r.endLine,
1305
+ provenance: {
1306
+ chunk_id: r.chunk.id,
1307
+ chunk_hash: r.chunk.content_hash,
1308
+ indexed_at: r.chunk.indexed_at,
1309
+ source_mtime: sourceMtime,
1310
+ stale: sourceMtime !== undefined ? sourceMtime > r.chunk.indexed_at : kb?.status === "stale",
1311
+ match_reason: matchReasonFor(mode),
1312
+ source_chunk_ids: r.sourceChunkIds,
1313
+ },
1314
+ };
1315
+ });
1065
1316
  return {
1066
1317
  results,
1067
1318
  total_count: diversified.length,
@@ -1072,6 +1323,7 @@ export class KnowledgeEngine {
1072
1323
  }
1073
1324
  const ranked = [];
1074
1325
  for (const r of filtered) {
1326
+ throwIfAborted(signal);
1075
1327
  const chunk = getChunkById(db, r.chunkId);
1076
1328
  if (!chunk)
1077
1329
  continue;
@@ -1118,6 +1370,19 @@ export class KnowledgeEngine {
1118
1370
  snippet: r.snippet,
1119
1371
  start_line: r.startLine,
1120
1372
  end_line: r.endLine,
1373
+ provenance: (() => {
1374
+ const kb = kbById.get(r.chunk.kb_id);
1375
+ const sourceMtime = sourceMtimeFor(kb, r.chunk.file_path);
1376
+ return {
1377
+ chunk_id: r.chunk.id,
1378
+ chunk_hash: r.chunk.content_hash,
1379
+ indexed_at: r.chunk.indexed_at,
1380
+ source_mtime: sourceMtime,
1381
+ stale: sourceMtime !== undefined ? sourceMtime > r.chunk.indexed_at : kb?.status === "stale",
1382
+ match_reason: matchReasonFor(mode),
1383
+ source_chunk_ids: r.sourceChunkIds,
1384
+ };
1385
+ })(),
1121
1386
  }));
1122
1387
  return {
1123
1388
  results,
@@ -1136,31 +1401,94 @@ export class KnowledgeEngine {
1136
1401
  remove(nameOrId) {
1137
1402
  if (!this.db)
1138
1403
  return false;
1404
+ if (this.disposing)
1405
+ throw new Error("Knowledge engine is shutting down");
1406
+ if (this.activeMutations.size > 0)
1407
+ throw new Error("A knowledge-base mutation is already running");
1139
1408
  const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
1140
1409
  if (!kb)
1141
1410
  return false;
1142
1411
  deleteKB(this.db, kb.id);
1412
+ this.deleteVectorFile(kb.id);
1143
1413
  return true;
1144
1414
  }
1145
- list() {
1415
+ list(signal) {
1146
1416
  if (!this.db)
1147
1417
  return [];
1418
+ throwIfAborted(signal);
1148
1419
  return listKBs(this.db);
1149
1420
  }
1150
1421
  clear() {
1151
1422
  if (!this.db)
1152
1423
  return;
1153
- for (const kb of listKBs(this.db))
1424
+ if (this.disposing)
1425
+ throw new Error("Knowledge engine is shutting down");
1426
+ if (this.activeMutations.size > 0)
1427
+ throw new Error("A knowledge-base mutation is already running");
1428
+ for (const kb of listKBs(this.db)) {
1154
1429
  deleteKB(this.db, kb.id);
1430
+ this.deleteVectorFile(kb.id);
1431
+ }
1155
1432
  }
1156
- diagnose() {
1433
+ diagnose(signal) {
1157
1434
  if (!this.db)
1158
1435
  return [];
1436
+ throwIfAborted(signal);
1437
+ const db = this.db;
1438
+ return listKBs(db).map((kb) => {
1439
+ throwIfAborted(signal);
1440
+ return diagnoseKB(db, kb, signal);
1441
+ });
1442
+ }
1443
+ symbolSearch(query, options = {}, signal) {
1444
+ if (!this.db)
1445
+ throw new Error("Engine not initialized");
1446
+ throwIfAborted(signal);
1159
1447
  const db = this.db;
1160
- return listKBs(db).map((kb) => diagnoseKB(db, kb));
1448
+ const selectedKB = options.kb_id ? (getKB(db, options.kb_id) ?? getKBByName(db, options.kb_id)) : undefined;
1449
+ if (options.kb_id && !selectedKB)
1450
+ throw new Error(`Knowledge base not found: ${options.kb_id}`);
1451
+ const limit = Math.max(1, Math.min(200, options.limit ?? 20));
1452
+ const offset = Math.max(0, options.offset ?? 0);
1453
+ const symbolOptions = {
1454
+ kbId: selectedKB?.id,
1455
+ kind: options.kind,
1456
+ filePattern: options.file_pattern,
1457
+ exact: options.exact,
1458
+ };
1459
+ const symbols = searchSymbols(db, query, {
1460
+ ...symbolOptions,
1461
+ limit: limit + 1,
1462
+ offset,
1463
+ });
1464
+ throwIfAborted(signal);
1465
+ const page = symbols.slice(0, limit);
1466
+ const total = countSymbols(db, query, symbolOptions);
1467
+ throwIfAborted(signal);
1468
+ return {
1469
+ results: page.map((symbol) => {
1470
+ const kb = getKB(db, symbol.kb_id);
1471
+ return {
1472
+ name: symbol.name,
1473
+ kind: symbol.kind,
1474
+ file_path: symbol.file_path,
1475
+ file_type: symbol.file_type,
1476
+ kb_name: kb?.name ?? "unknown",
1477
+ start_line: symbol.start_line,
1478
+ end_line: symbol.end_line,
1479
+ signature: symbol.signature ?? undefined,
1480
+ container_name: symbol.container_name ?? undefined,
1481
+ text: symbol.text,
1482
+ indexed_at: symbol.indexed_at,
1483
+ };
1484
+ }),
1485
+ total_count: total,
1486
+ has_more: symbols.length > limit,
1487
+ };
1161
1488
  }
1162
- doctor() {
1163
- const diagnostics = this.diagnose();
1489
+ doctor(signal) {
1490
+ throwIfAborted(signal);
1491
+ const diagnostics = this.diagnose(signal);
1164
1492
  const issues = [];
1165
1493
  const kbs = this.list();
1166
1494
  if (kbs.length === 0) {
@@ -1168,9 +1496,11 @@ export class KnowledgeEngine {
1168
1496
  severity: "blocking",
1169
1497
  message: "No knowledge bases are indexed.",
1170
1498
  action: "Run knowledge_add for the project root or relevant source/docs directory.",
1499
+ action_code: "rebuild_kb",
1171
1500
  });
1172
1501
  }
1173
1502
  for (const diagnostic of diagnostics) {
1503
+ throwIfAborted(signal);
1174
1504
  if (diagnostic.stuck_indexing) {
1175
1505
  const phase = diagnostic.job ? ` during ${diagnostic.job.phase}` : "";
1176
1506
  issues.push({
@@ -1178,6 +1508,7 @@ export class KnowledgeEngine {
1178
1508
  kb_name: diagnostic.kb_name,
1179
1509
  message: `Indexing appears stuck${phase} for ${formatDuration(diagnostic.last_progress_age_ms)}.`,
1180
1510
  action: "Check knowledge_status for the last progress message. If no Pi process is actively indexing it, remove and rebuild this KB.",
1511
+ action_code: "wait_for_indexing",
1181
1512
  });
1182
1513
  }
1183
1514
  if (diagnostic.status === "error") {
@@ -1186,6 +1517,7 @@ export class KnowledgeEngine {
1186
1517
  kb_name: diagnostic.kb_name,
1187
1518
  message: "Knowledge base is in error state and is skipped by search.",
1188
1519
  action: "Run knowledge_remove and knowledge_add to rebuild it from the source.",
1520
+ action_code: "rebuild_kb",
1189
1521
  });
1190
1522
  }
1191
1523
  if (diagnostic.coverage_percent < 80) {
@@ -1194,6 +1526,7 @@ export class KnowledgeEngine {
1194
1526
  kb_name: diagnostic.kb_name,
1195
1527
  message: `Coverage is ${diagnostic.coverage_percent}% (${diagnostic.indexed_files}/${diagnostic.total_source_files} files).`,
1196
1528
  action: "Review skipped files and source path. Rebuild if index-time rules changed.",
1529
+ action_code: "review_skipped_scope",
1197
1530
  });
1198
1531
  }
1199
1532
  if (diagnostic.stale_files.length > 0) {
@@ -1202,6 +1535,7 @@ export class KnowledgeEngine {
1202
1535
  kb_name: diagnostic.kb_name,
1203
1536
  message: `${diagnostic.stale_files.length} files changed after indexing.`,
1204
1537
  action: "Run knowledge_update for this KB.",
1538
+ action_code: "run_update",
1205
1539
  });
1206
1540
  }
1207
1541
  if (diagnostic.orphan_files.length > 0) {
@@ -1210,6 +1544,7 @@ export class KnowledgeEngine {
1210
1544
  kb_name: diagnostic.kb_name,
1211
1545
  message: `${diagnostic.orphan_files.length} indexed files no longer exist in the source.`,
1212
1546
  action: "Run knowledge_update or rebuild the KB.",
1547
+ action_code: "run_update",
1213
1548
  });
1214
1549
  }
1215
1550
  if (diagnostic.skipped_files.total > 0) {
@@ -1221,9 +1556,24 @@ export class KnowledgeEngine {
1221
1556
  .map(([reason, count]) => `${reason}: ${count}`)
1222
1557
  .join(", ")}).`,
1223
1558
  action: "Use skipped samples to confirm exclusions are expected. Adjust source path or ignore rules if needed.",
1559
+ action_code: "review_skipped_scope",
1224
1560
  });
1225
1561
  }
1226
1562
  }
1563
+ if (this.db) {
1564
+ for (const kb of kbs) {
1565
+ throwIfAborted(signal);
1566
+ if (kb.status === "ready" && kb.chunk_count > 0 && getSymbolCount(this.db, kb.id) === 0) {
1567
+ issues.push({
1568
+ severity: "info",
1569
+ kb_name: kb.name,
1570
+ message: "No lightweight symbols are indexed for this KB.",
1571
+ action: "Run knowledge_update to rebuild symbol/config/heading lookup metadata.",
1572
+ action_code: "run_update",
1573
+ });
1574
+ }
1575
+ }
1576
+ }
1227
1577
  const penalty = issues.reduce((score, issue) => {
1228
1578
  if (issue.severity === "blocking")
1229
1579
  return score + 35;
@@ -1242,138 +1592,193 @@ export class KnowledgeEngine {
1242
1592
  summary,
1243
1593
  issues,
1244
1594
  diagnostics,
1595
+ actions: issues.map((issue) => ({
1596
+ code: issue.action_code ?? "none",
1597
+ kb_name: issue.kb_name,
1598
+ target: issue.kb_name,
1599
+ description: issue.action,
1600
+ })),
1245
1601
  };
1246
1602
  }
1247
- async exportKB(nameOrId, outputPath) {
1603
+ async exportKB(nameOrId, outputPath, signal, onProgress) {
1248
1604
  if (!this.db)
1249
1605
  throw new Error("Engine not initialized");
1606
+ throwIfAborted(signal);
1250
1607
  const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
1251
1608
  if (!kb)
1252
1609
  throw new Error(`Knowledge base not found: ${nameOrId}`);
1253
- const chunks = getChunksByKB(this.db, kb.id);
1254
- const { writeFileSync } = await import("node:fs");
1610
+ return this.runExclusive(["global:mutation", `kb:${kb.id}`], `Operation for "${kb.name}"`, () => this.exportKBUnlocked(kb, outputPath, signal, onProgress));
1611
+ }
1612
+ async exportKBUnlocked(kb, outputPath, signal, onProgress) {
1613
+ if (!this.db)
1614
+ throw new Error("Engine not initialized");
1615
+ const { createWriteStream } = await import("node:fs");
1616
+ const tempOutputPath = tempVectorPath(outputPath);
1617
+ const stream = createWriteStream(tempOutputPath, { encoding: "utf-8" });
1618
+ let count = 0;
1255
1619
  const header = JSON.stringify({
1256
1620
  name: kb.name,
1257
1621
  description: kb.description,
1258
1622
  source_type: "text",
1259
- chunk_count: chunks.length,
1623
+ chunk_count: kb.chunk_count,
1260
1624
  });
1261
- const lines = [
1262
- header,
1263
- ...chunks.map((c) => JSON.stringify({
1264
- content: c.content,
1265
- file_path: c.file_path,
1266
- file_type: c.file_type,
1267
- start_line: c.start_line,
1268
- end_line: c.end_line,
1269
- metadata_json: c.metadata_json,
1270
- })),
1271
- ];
1272
- writeFileSync(outputPath, `${lines.join("\n")}\n`);
1273
- return chunks.length;
1625
+ try {
1626
+ await writeLine(stream, header);
1627
+ for (const chunk of iterateChunksByKB(this.db, kb.id)) {
1628
+ throwIfAborted(signal);
1629
+ await writeLine(stream, JSON.stringify({
1630
+ content: chunk.content,
1631
+ file_path: chunk.file_path,
1632
+ file_type: chunk.file_type,
1633
+ start_line: chunk.start_line,
1634
+ end_line: chunk.end_line,
1635
+ metadata_json: chunk.metadata_json,
1636
+ }));
1637
+ count++;
1638
+ if (count % INDEX_EMBED_BATCH_SIZE === 0)
1639
+ onProgress?.(`Exported ${count}/${kb.chunk_count} chunks...`);
1640
+ }
1641
+ await finishWriteStream(stream);
1642
+ throwIfAborted(signal);
1643
+ renameSync(tempOutputPath, outputPath);
1644
+ onProgress?.(`Exported ${count}/${kb.chunk_count} chunks.`);
1645
+ return count;
1646
+ }
1647
+ catch (error) {
1648
+ stream.destroy();
1649
+ rmSync(tempOutputPath, { force: true });
1650
+ throw error;
1651
+ }
1274
1652
  }
1275
1653
  async importKB(inputPath, onProgress, signal) {
1276
1654
  if (!this.db)
1277
1655
  throw new Error("Engine not initialized");
1278
- const { readFileSync } = await import("node:fs");
1279
- const lines = readFileSync(inputPath, "utf-8").trim().split("\n");
1280
- if (lines.length < 1)
1281
- throw new Error("Empty import file");
1282
- const header = JSON.parse(lines[0]);
1283
- const kb = createKB(this.db, { name: header.name, description: header.description, source_type: "text" });
1284
- updateKBStatus(this.db, kb.id, "indexing");
1285
- startIndexingJob(this.db, kb.id, "import", `Starting import for "${header.name}"`);
1656
+ return this.runExclusive("global:mutation", "Knowledge-base import", () => this.importKBUnlocked(inputPath, onProgress, signal));
1657
+ }
1658
+ async importKBUnlocked(inputPath, onProgress, signal) {
1659
+ if (!this.db)
1660
+ throw new Error("Engine not initialized");
1661
+ throwIfAborted(signal);
1662
+ const { createReadStream } = await import("node:fs");
1663
+ const { createInterface } = await import("node:readline");
1664
+ const stream = createReadStream(inputPath, { encoding: "utf-8" });
1665
+ const lines = createInterface({ input: stream, crlfDelay: Infinity });
1666
+ let header;
1667
+ let kb;
1286
1668
  let vectorWriter;
1287
1669
  let tempVectorFile;
1288
- try {
1289
- const chunkData = lines.slice(1).map((l) => JSON.parse(l));
1290
- const importMessage = `Importing ${chunkData.length} chunks...`;
1670
+ let inserted = 0;
1671
+ let lineNumber = 0;
1672
+ const pendingChunks = [];
1673
+ const pendingSymbols = [];
1674
+ const flushPending = async () => {
1675
+ if (!this.db || !kb || !vectorWriter || pendingChunks.length === 0)
1676
+ return;
1677
+ throwIfAborted(signal);
1678
+ const batch = pendingChunks.splice(0, INDEX_EMBED_BATCH_SIZE);
1679
+ const symbols = pendingSymbols.splice(0);
1680
+ const total = header?.chunk_count;
1681
+ const message = `Embedding import batch: ${inserted}/${total ?? "unknown"} chunks stored`;
1291
1682
  updateIndexingJob(this.db, kb.id, {
1292
- phase: "importing",
1293
- message: importMessage,
1294
- total_files: chunkData.length,
1683
+ phase: "embedding",
1684
+ message,
1685
+ processed_chunks: inserted,
1686
+ total_files: total,
1687
+ added_chunks: inserted,
1295
1688
  });
1296
- onProgress?.(importMessage);
1297
- const chunks = chunkData.map((c) => ({
1298
- content_hash: chunkIdentityHash({
1299
- content: c.content,
1300
- filePath: c.file_path,
1301
- fileType: c.file_type,
1302
- startLine: c.start_line,
1303
- endLine: c.end_line,
1304
- metadataJson: c.metadata_json || "{}",
1305
- }),
1306
- content: c.content,
1307
- content_tokenized: "",
1308
- file_path: c.file_path,
1309
- file_type: c.file_type,
1310
- start_line: c.start_line,
1311
- end_line: c.end_line,
1312
- metadata_json: c.metadata_json || "{}",
1313
- }));
1314
- const indexedChunks = chunks.map((chunk) => ({
1315
- ...chunk,
1316
- content_tokenized: preTokenizeForFTS(buildChunkEmbeddingText(chunk)),
1317
- }));
1318
- const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
1319
- tempVectorFile = tempVectorPath(vectorPath);
1320
- vectorWriter = openVectorWriter(tempVectorFile);
1321
- let inserted = 0;
1322
- for (let offset = 0; offset < indexedChunks.length; offset += INDEX_EMBED_BATCH_SIZE) {
1323
- if (signal?.aborted)
1324
- throw new Error("Cancelled");
1325
- const batch = indexedChunks.slice(offset, offset + INDEX_EMBED_BATCH_SIZE);
1326
- const message = `Embedding import batch ${Math.floor(offset / INDEX_EMBED_BATCH_SIZE) + 1}/${Math.ceil(indexedChunks.length / INDEX_EMBED_BATCH_SIZE)}: ${offset}/${indexedChunks.length} chunks`;
1327
- updateIndexingJob(this.db, kb.id, {
1328
- phase: "embedding",
1329
- message,
1330
- processed_chunks: offset,
1331
- total_files: indexedChunks.length,
1332
- added_chunks: inserted,
1333
- });
1334
- onProgress?.(message);
1335
- const vectors = await embedDocuments(batch.map((c) => buildChunkEmbeddingText(c)), signal);
1336
- if (signal?.aborted)
1337
- throw new Error("Cancelled");
1338
- insertChunks(this.db, kb.id, batch);
1339
- vectorWriter.append(vectors);
1340
- inserted += batch.length;
1341
- updateKBCounts(this.db, kb.id, inserted, new Set(indexedChunks.slice(0, inserted).map((c) => c.file_path)).size);
1342
- updateIndexingJob(this.db, kb.id, {
1343
- phase: "storing",
1344
- message: `Stored import batch: ${inserted}/${indexedChunks.length} chunks`,
1345
- processed_chunks: inserted,
1346
- total_files: indexedChunks.length,
1347
- added_chunks: inserted,
1348
- });
1689
+ onProgress?.(message);
1690
+ const vectors = await embedDocuments(batch.map((chunk) => buildChunkEmbeddingText(chunk)), signal);
1691
+ throwIfAborted(signal);
1692
+ insertChunks(this.db, kb.id, batch);
1693
+ insertSymbols(this.db, kb.id, symbols);
1694
+ vectorWriter.append(vectors);
1695
+ inserted += batch.length;
1696
+ updateKBCounts(this.db, kb.id, inserted, getFileCount(this.db, kb.id));
1697
+ updateIndexingJob(this.db, kb.id, {
1698
+ phase: "storing",
1699
+ message: `Stored import batch: ${inserted}/${total ?? "unknown"} chunks`,
1700
+ processed_chunks: inserted,
1701
+ total_files: total,
1702
+ added_chunks: inserted,
1703
+ });
1704
+ };
1705
+ try {
1706
+ for await (const rawLine of lines) {
1707
+ throwIfAborted(signal);
1708
+ lineNumber++;
1709
+ const line = rawLine.trim();
1710
+ if (!line)
1711
+ continue;
1712
+ if (!header) {
1713
+ header = JSON.parse(line);
1714
+ if (!header.name)
1715
+ throw new Error("Import header is missing a knowledge base name");
1716
+ const existingKB = getKBByName(this.db, header.name);
1717
+ if (existingKB) {
1718
+ throw new Error(`Knowledge base "${header.name}" already exists. Use knowledge_update to refresh it, or knowledge_remove before importing a replacement.`);
1719
+ }
1720
+ kb = createKB(this.db, { name: header.name, description: header.description, source_type: "text" });
1721
+ updateKBStatus(this.db, kb.id, "indexing");
1722
+ startIndexingJob(this.db, kb.id, "import", `Starting import for "${header.name}"`);
1723
+ const importMessage = `Importing ${header.chunk_count ?? "unknown"} chunks...`;
1724
+ updateIndexingJob(this.db, kb.id, {
1725
+ phase: "importing",
1726
+ message: importMessage,
1727
+ total_files: header.chunk_count,
1728
+ });
1729
+ onProgress?.(importMessage);
1730
+ const vectorPath = this.vectorPathFor(kb.id);
1731
+ tempVectorFile = tempVectorPath(vectorPath);
1732
+ vectorWriter = openVectorWriter(tempVectorFile);
1733
+ continue;
1734
+ }
1735
+ const imported = JSON.parse(line);
1736
+ const chunk = importedChunkToInsert(imported);
1737
+ chunk.content_tokenized = preTokenizeForFTS(buildChunkEmbeddingText(chunk));
1738
+ pendingChunks.push(chunk);
1739
+ pendingSymbols.push(...extractSymbols(imported.content, imported.file_path, imported.file_type));
1740
+ if (pendingChunks.length >= INDEX_EMBED_BATCH_SIZE)
1741
+ await flushPending();
1349
1742
  }
1743
+ if (!header)
1744
+ throw new Error("Empty import file");
1745
+ if (!kb || !vectorWriter || !tempVectorFile)
1746
+ throw new Error("Import did not create a knowledge base");
1747
+ await flushPending();
1350
1748
  vectorWriter.close();
1351
1749
  vectorWriter = undefined;
1352
- renameSync(tempVectorFile, vectorPath);
1750
+ renameSync(tempVectorFile, this.vectorPathFor(kb.id));
1353
1751
  tempVectorFile = undefined;
1354
- updateKBCounts(this.db, kb.id, indexedChunks.length, new Set(indexedChunks.map((c) => c.file_path)).size);
1752
+ updateKBCounts(this.db, kb.id, inserted, getFileCount(this.db, kb.id));
1355
1753
  updateKBStatus(this.db, kb.id, "ready");
1356
1754
  const savedKB = getKB(this.db, kb.id);
1357
1755
  if (!savedKB)
1358
1756
  throw new Error(`Knowledge base disappeared after import: ${kb.id}`);
1359
- finishIndexingJob(this.db, kb.id, "succeeded", `Ready: imported ${indexedChunks.length} chunks`);
1360
- return { kb: savedKB, chunkCount: chunks.length };
1757
+ finishIndexingJob(this.db, kb.id, "succeeded", `Ready: imported ${inserted} chunks`);
1758
+ return { kb: savedKB, chunkCount: inserted };
1361
1759
  }
1362
- catch (e) {
1760
+ catch (error) {
1761
+ lines.close();
1762
+ stream.destroy();
1363
1763
  vectorWriter?.close();
1364
1764
  if (tempVectorFile)
1365
1765
  rmSync(tempVectorFile, { force: true });
1366
- if (this.db) {
1367
- finishIndexingJob(this.db, kb.id, isCancellationError(e) ? "cancelled" : "failed", isCancellationError(e) ? "Import cancelled." : "Import failed.", e instanceof Error ? e.message : String(e));
1766
+ if (this.db && kb) {
1767
+ finishIndexingJob(this.db, kb.id, isCancellationError(error) ? "cancelled" : "failed", isCancellationError(error) ? "Import cancelled." : `Import failed near line ${lineNumber}.`, error instanceof Error ? error.message : String(error));
1768
+ deleteKB(this.db, kb.id);
1769
+ this.deleteVectorFile(kb.id);
1368
1770
  }
1369
- deleteKB(this.db, kb.id);
1370
- throw e;
1771
+ throw error;
1371
1772
  }
1372
1773
  }
1373
1774
  async dispose(options = {}) {
1775
+ this.disposing = true;
1374
1776
  const activeUpdates = [...this.activeUpdates.values()];
1375
1777
  if (activeUpdates.length > 0)
1376
1778
  await Promise.allSettled(activeUpdates);
1779
+ const activeMutations = [...this.activeMutations.values()];
1780
+ if (activeMutations.length > 0)
1781
+ await Promise.allSettled(activeMutations);
1377
1782
  const disposeModels = options.disposeModels ?? true;
1378
1783
  if (disposeModels) {
1379
1784
  await disposeEmbedding();