pi-knowledge 0.4.7 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,9 +1,10 @@
1
- import { existsSync, renameSync, rmSync, statSync } from "node:fs";
1
+ import { existsSync, readFileSync, renameSync, rmSync, statSync } from "node:fs";
2
2
  import { join, resolve } from "node:path";
3
3
  import { diagnoseKB } from "./diagnostics/health.js";
4
4
  import { dispose as disposeEmbedding, embedDocuments, embedQuery, prepareForShutdown as prepareEmbeddingForShutdown, } from "./embedding/provider.js";
5
5
  import { openVectorReader, openVectorWriter } from "./embedding/vectors.js";
6
6
  import { buildChunkEmbeddingText, chunkFile, chunkIdentityHash, createSkippedScanStats, isReadableTextFile, iterateScannableFiles, iterateScannedFiles, preTokenizeForFTS, summarizeSkippedScan, } from "./indexer/chunker.js";
7
+ import { extractSymbols } from "./indexer/symbols.js";
7
8
  import { shutdownModelWorker } from "./model-worker-client.js";
8
9
  import { searchBM25 } from "./search/bm25.js";
9
10
  import { weightedScoreFusion } from "./search/fusion.js";
@@ -11,7 +12,7 @@ import { normalizedQueryText, tokenizeForSearch } from "./search/query.js";
11
12
  import { hasAnyLexicalEvidence, hasEnoughLexicalEvidence, MIN_HYBRID_SCORE, normalizeFileTypeFilter, queryCoverage, scoreChunkForQuery, } from "./search/ranking.js";
12
13
  import { disposeReranker, prepareRerankerForShutdown, rerank } from "./search/reranker.js";
13
14
  import { searchVectorFile } from "./search/vector.js";
14
- import { createKB, deleteChunksByIds, deleteKB, finishIndexingJob, getChunkById, getChunkCount, getChunksByFile, getChunksByKB, getFileCount, getKB, getKBByName, insertChunks, iterateChunkIdsByKB, iterateChunksByKB, listKBs, openDatabase, startIndexingJob, updateIndexingJob, updateKBCounts, updateKBStatus, } from "./storage/sqlite.js";
15
+ import { countSymbols, createKB, deleteChunksByIds, deleteKB, deleteSymbolsByKB, finishIndexingJob, getChunkById, getChunkCount, getChunksByFile, getFileCount, getKB, getKBByName, getSymbolCount, insertChunks, insertSymbols, iterateChunkIdsByKB, iterateChunksByKB, listKBs, openDatabase, searchSymbols, startIndexingJob, updateIndexingJob, updateKBCounts, updateKBStatus, } from "./storage/sqlite.js";
15
16
  export const CURRENT_EMBEDDING_MODEL = "multilingual-e5-small";
16
17
  const ADAPTIVE_CONTEXT_LINES = 80;
17
18
  const ADAPTIVE_MAX_CONTEXT_CHARS = 6_000;
@@ -24,6 +25,26 @@ function isCancellationError(error) {
24
25
  function tempVectorPath(vectorPath) {
25
26
  return `${vectorPath}.${Date.now()}.${Math.random().toString(16).slice(2)}.tmp`;
26
27
  }
28
+ function retrievalModeFor(mode) {
29
+ if (mode === "auto")
30
+ return "hybrid";
31
+ if (mode === "code" || mode === "config" || mode === "errors")
32
+ return "fast";
33
+ if (mode === "docs" || mode === "decision")
34
+ return "hybrid";
35
+ return mode;
36
+ }
37
+ function matchReasonFor(mode) {
38
+ if (mode === "deep")
39
+ return "rerank";
40
+ if (mode === "adaptive")
41
+ return "adaptive";
42
+ if (mode === "semantic")
43
+ return "vector";
44
+ if (mode === "fast" || mode === "code" || mode === "config" || mode === "errors")
45
+ return "bm25";
46
+ return "hybrid";
47
+ }
27
48
  function toScanOptions(options = {}) {
28
49
  return {
29
50
  includeSuggestedText: options.include_suggested_text === true,
@@ -60,11 +81,12 @@ function parseAddOptions(raw) {
60
81
  return {};
61
82
  }
62
83
  }
63
- function planDirectoryScan(dirPath, options = {}) {
84
+ function planDirectoryScan(dirPath, options = {}, signal) {
64
85
  const skipped = createSkippedScanStats();
65
86
  let files = 0;
66
87
  let bytes = 0;
67
88
  for (const file of iterateScannableFiles(dirPath, skipped, options)) {
89
+ throwIfAborted(signal);
68
90
  files++;
69
91
  bytes += file.size;
70
92
  }
@@ -76,6 +98,25 @@ function planDirectoryScan(dirPath, options = {}) {
76
98
  skipped,
77
99
  };
78
100
  }
101
+ function sourceMtimeFor(kb, filePath) {
102
+ if (!kb?.source_path)
103
+ return undefined;
104
+ const absPath = kb.source_type === "directory" ? join(kb.source_path, filePath) : kb.source_path;
105
+ try {
106
+ return statSync(absPath).mtimeMs;
107
+ }
108
+ catch {
109
+ return undefined;
110
+ }
111
+ }
112
+ function kbTrustMultiplier(kb) {
113
+ let multiplier = kb.status === "stale" ? 0.92 : 1;
114
+ if (kb.source_type === "directory" || kb.source_type === "file")
115
+ multiplier *= 1.04;
116
+ if (kb.source_type === "text" || kb.source_type === "url")
117
+ multiplier *= 0.98;
118
+ return multiplier;
119
+ }
79
120
  function formatBytes(bytes) {
80
121
  if (bytes < 1024)
81
122
  return `${bytes} B`;
@@ -347,22 +388,143 @@ async function chunkUrl(source, signal) {
347
388
  function normalizeExtractedText(text) {
348
389
  return Array.isArray(text) ? text.join("\n\n") : text;
349
390
  }
391
+ async function extractSourceFileContent(filePath, signal) {
392
+ const lowerPath = filePath.toLowerCase();
393
+ if (lowerPath.endsWith(".pdf")) {
394
+ throwIfAborted(signal);
395
+ // PDF extraction is an optional heavy runtime path; keep parser loading out of extension startup.
396
+ const { extractText } = await import("unpdf");
397
+ throwIfAborted(signal);
398
+ const buf = readFileSync(filePath);
399
+ throwIfAborted(signal);
400
+ const { text } = await extractText(new Uint8Array(buf));
401
+ throwIfAborted(signal);
402
+ return { content: normalizeExtractedText(text), fileType: "pdf" };
403
+ }
404
+ if (lowerPath.endsWith(".docx") || lowerPath.endsWith(".doc")) {
405
+ throwIfAborted(signal);
406
+ // DOCX extraction is an optional heavy runtime path; keep parser loading out of extension startup.
407
+ const mammoth = await import("mammoth");
408
+ throwIfAborted(signal);
409
+ const result = await mammoth.extractRawText({ path: filePath });
410
+ throwIfAborted(signal);
411
+ return { content: result.value, fileType: "docx" };
412
+ }
413
+ if (!isReadableTextFile(filePath)) {
414
+ throw new Error(`File is not readable text and has no supported extractor: ${filePath}`);
415
+ }
416
+ throwIfAborted(signal);
417
+ return { content: readFileSync(filePath, "utf-8"), fileType: "text" };
418
+ }
419
+ function throwIfAborted(signal) {
420
+ if (signal?.aborted)
421
+ throw new Error("Cancelled");
422
+ }
423
+ function importedChunkToInsert(chunk) {
424
+ const metadataJson = chunk.metadata_json || "{}";
425
+ return {
426
+ content_hash: chunkIdentityHash({
427
+ content: chunk.content,
428
+ filePath: chunk.file_path,
429
+ fileType: chunk.file_type,
430
+ startLine: chunk.start_line,
431
+ endLine: chunk.end_line,
432
+ metadataJson,
433
+ }),
434
+ content: chunk.content,
435
+ content_tokenized: "",
436
+ file_path: chunk.file_path,
437
+ file_type: chunk.file_type,
438
+ start_line: chunk.start_line,
439
+ end_line: chunk.end_line,
440
+ metadata_json: metadataJson,
441
+ };
442
+ }
443
+ async function writeLine(stream, line) {
444
+ if (stream.write(`${line}\n`))
445
+ return;
446
+ await new Promise((resolve, reject) => {
447
+ const cleanup = () => {
448
+ stream.off("drain", onDrain);
449
+ stream.off("error", onError);
450
+ };
451
+ const onDrain = () => {
452
+ cleanup();
453
+ resolve();
454
+ };
455
+ const onError = (error) => {
456
+ cleanup();
457
+ reject(error);
458
+ };
459
+ stream.once("drain", onDrain);
460
+ stream.once("error", onError);
461
+ });
462
+ }
463
+ async function finishWriteStream(stream) {
464
+ await new Promise((resolve, reject) => {
465
+ const cleanup = () => {
466
+ stream.off("finish", onFinish);
467
+ stream.off("error", onError);
468
+ };
469
+ const onFinish = () => {
470
+ cleanup();
471
+ resolve();
472
+ };
473
+ const onError = (error) => {
474
+ cleanup();
475
+ reject(error);
476
+ };
477
+ stream.once("finish", onFinish);
478
+ stream.once("error", onError);
479
+ stream.end();
480
+ });
481
+ }
350
482
  export class KnowledgeEngine {
351
483
  db = null;
352
484
  knowledgeDir = "";
353
485
  activeUpdates = new Map();
486
+ activeMutations = new Map();
487
+ disposing = false;
488
+ runExclusive(keys, description, operation) {
489
+ if (this.disposing)
490
+ throw new Error("Knowledge engine is shutting down");
491
+ const mutationKeys = Array.isArray(keys) ? keys : [keys];
492
+ for (const key of mutationKeys) {
493
+ if (this.activeMutations.has(key))
494
+ throw new Error(`${description} is already running`);
495
+ }
496
+ const run = Promise.resolve()
497
+ .then(operation)
498
+ .finally(() => {
499
+ for (const key of mutationKeys) {
500
+ if (this.activeMutations.get(key) === run)
501
+ this.activeMutations.delete(key);
502
+ }
503
+ });
504
+ for (const key of mutationKeys)
505
+ this.activeMutations.set(key, run);
506
+ return run;
507
+ }
354
508
  async initialize(knowledgeDir) {
355
509
  this.knowledgeDir = knowledgeDir;
356
510
  this.db = openDatabase(knowledgeDir);
511
+ this.disposing = false;
357
512
  }
358
- plan(source, options = {}) {
513
+ vectorPathFor(kbId) {
514
+ return join(this.knowledgeDir, "vectors", `${kbId}.bin`);
515
+ }
516
+ deleteVectorFile(kbId) {
517
+ rmSync(this.vectorPathFor(kbId), { force: true });
518
+ }
519
+ plan(source, options = {}, signal) {
520
+ throwIfAborted(signal);
359
521
  const resolvedSource = resolve(source);
360
522
  const isUrl = source.startsWith("http://") || source.startsWith("https://");
361
523
  const isDir = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isDirectory();
362
524
  const isFile = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isFile();
363
525
  const sourceType = isUrl ? "url" : isDir ? "directory" : isFile ? "file" : "text";
364
526
  if (isDir) {
365
- const plan = planDirectoryScan(resolvedSource, toScanOptions(options));
527
+ const plan = planDirectoryScan(resolvedSource, toScanOptions(options), signal);
366
528
  return {
367
529
  source_type: "directory",
368
530
  scannable_files: plan.files,
@@ -405,8 +567,13 @@ export class KnowledgeEngine {
405
567
  };
406
568
  }
407
569
  async add(source, name, onProgress, signal, options = {}) {
570
+ throwIfAborted(signal);
571
+ return this.runExclusive(["global:mutation", `kb-name:${name}`], `Mutation for "${name}"`, () => this.addUnlocked(source, name, onProgress, signal, options));
572
+ }
573
+ async addUnlocked(source, name, onProgress, signal, options = {}) {
408
574
  if (!this.db)
409
575
  throw new Error("Engine not initialized");
576
+ throwIfAborted(signal);
410
577
  const db = this.db;
411
578
  const resolvedSource = resolve(source);
412
579
  const isUrl = source.startsWith("http://") || source.startsWith("https://");
@@ -429,7 +596,7 @@ export class KnowledgeEngine {
429
596
  let vectorWriter;
430
597
  let tempVectorFile;
431
598
  try {
432
- const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
599
+ const vectorPath = this.vectorPathFor(kb.id);
433
600
  tempVectorFile = tempVectorPath(vectorPath);
434
601
  const writer = openVectorWriter(tempVectorFile);
435
602
  vectorWriter = writer;
@@ -488,33 +655,26 @@ export class KnowledgeEngine {
488
655
  await flushPending(processedFiles, totalFiles);
489
656
  }
490
657
  };
658
+ const addSymbols = (content, filePath, fileType) => {
659
+ insertSymbols(db, kb.id, extractSymbols(content, filePath, fileType));
660
+ };
491
661
  if (isUrl) {
492
662
  const message = `Fetching ${source}...`;
493
663
  updateIndexingJob(this.db, kb.id, { phase: "fetching", message });
494
664
  onProgress?.(message);
495
665
  const chunks = await chunkUrl(source, signal);
496
666
  fileCount = 1;
667
+ addSymbols(chunks.map((chunk) => chunk.content).join("\n\n"), source, "html");
497
668
  await addChunks(chunks);
498
669
  }
499
- else if (isFile && resolvedSource.endsWith(".pdf")) {
500
- updateIndexingJob(this.db, kb.id, { phase: "extracting", message: "Extracting text from PDF..." });
501
- onProgress?.("Extracting text from PDF...");
502
- const { extractText } = await import("unpdf");
503
- const buf = (await import("node:fs")).readFileSync(resolvedSource);
504
- const { text } = await extractText(new Uint8Array(buf));
505
- fileCount = 1;
506
- await addChunks(await chunkFile(normalizeExtractedText(text), resolvedSource));
507
- }
508
- else if (isFile && (resolvedSource.endsWith(".docx") || resolvedSource.endsWith(".doc"))) {
509
- updateIndexingJob(this.db, kb.id, { phase: "extracting", message: "Extracting text from DOCX..." });
510
- onProgress?.("Extracting text from DOCX...");
511
- const mammoth = await import("mammoth");
512
- const result = await mammoth.extractRawText({ path: resolvedSource });
670
+ else if (isFile) {
671
+ const extracted = await extractSourceFileContent(resolvedSource, signal);
513
672
  fileCount = 1;
514
- await addChunks(await chunkFile(result.value, resolvedSource));
673
+ addSymbols(extracted.content, resolvedSource, extracted.fileType);
674
+ await addChunks(await chunkFile(extracted.content, resolvedSource));
515
675
  }
516
676
  else if (isDir) {
517
- const plan = planDirectoryScan(resolvedSource, scanOptions);
677
+ const plan = planDirectoryScan(resolvedSource, scanOptions, signal);
518
678
  const planningMessage = `Planned directory scan: ${plan.files} files, ${formatBytes(plan.bytes)} scannable text, skipped ${plan.skippedTotal} (${plan.skippedSummary})`;
519
679
  updateIndexingJob(this.db, kb.id, {
520
680
  phase: "planning",
@@ -541,6 +701,7 @@ export class KnowledgeEngine {
541
701
  latestSkippedTotal = skipped.total;
542
702
  if (chunks.length > 0)
543
703
  fileCount++;
704
+ addSymbols(file.content, file.relPath, file.fileType);
544
705
  await addChunks(chunks, processedFiles, plan.files);
545
706
  if (processedFiles % 25 === 0)
546
707
  reportProgress("Chunking", processedFiles, plan.files, skipped.total);
@@ -557,17 +718,9 @@ export class KnowledgeEngine {
557
718
  });
558
719
  onProgress?.(finalizingMessage);
559
720
  }
560
- else if (isFile) {
561
- if (!isReadableTextFile(resolvedSource)) {
562
- throw new Error(`File is not readable text and has no supported extractor: ${resolvedSource}`);
563
- }
564
- const { readFileSync } = await import("node:fs");
565
- const content = readFileSync(resolvedSource, "utf-8");
566
- fileCount = 1;
567
- await addChunks(await chunkFile(content, resolvedSource));
568
- }
569
721
  else {
570
722
  fileCount = 1;
723
+ addSymbols(source, "inline-text", "text");
571
724
  await addChunks(await chunkFile(source, "inline-text"));
572
725
  }
573
726
  await flushPending();
@@ -600,15 +753,18 @@ export class KnowledgeEngine {
600
753
  rmSync(tempVectorFile, { force: true });
601
754
  if (this.db) {
602
755
  finishIndexingJob(this.db, kb.id, isCancellationError(e) ? "cancelled" : "failed", isCancellationError(e) ? "Indexing cancelled." : "Indexing failed.", e instanceof Error ? e.message : String(e));
756
+ deleteKB(this.db, kb.id);
757
+ this.deleteVectorFile(kb.id);
603
758
  }
604
- // Clean up partial state on failure
605
- deleteKB(this.db, kb.id);
606
759
  throw e;
607
760
  }
608
761
  }
609
762
  async update(nameOrId, onProgress, signal) {
610
763
  if (!this.db)
611
764
  throw new Error("Engine not initialized");
765
+ if (this.disposing)
766
+ throw new Error("Knowledge engine is shutting down");
767
+ throwIfAborted(signal);
612
768
  const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
613
769
  if (!kb)
614
770
  throw new Error(`Knowledge base not found: ${nameOrId}`);
@@ -621,7 +777,7 @@ export class KnowledgeEngine {
621
777
  onProgress?.(`Update already running for "${updateableKB.name}"; waiting for the active update.`);
622
778
  return activeUpdate;
623
779
  }
624
- const updateRun = this.runUpdate(updateableKB, onProgress, signal).finally(() => {
780
+ const updateRun = this.runExclusive(["global:mutation", `kb:${updateableKB.id}`], `Mutation for "${updateableKB.name}"`, () => this.runUpdate(updateableKB, onProgress, signal)).finally(() => {
625
781
  if (this.activeUpdates.get(updateableKB.id) === updateRun)
626
782
  this.activeUpdates.delete(updateableKB.id);
627
783
  });
@@ -631,14 +787,17 @@ export class KnowledgeEngine {
631
787
  async runUpdate(kb, onProgress, signal) {
632
788
  if (!this.db)
633
789
  throw new Error("Engine not initialized");
790
+ throwIfAborted(signal);
634
791
  updateKBStatus(this.db, kb.id, "indexing");
635
792
  startIndexingJob(this.db, kb.id, "update", `Starting update for "${kb.name}"`);
636
793
  const scanOptions = toScanOptions(parseAddOptions(kb.source_options));
637
794
  let replacementVectorPath;
638
795
  let addedVectorPath;
639
796
  let addedVectorWriter;
797
+ const insertedChunkIds = [];
798
+ const stagedSymbols = [];
640
799
  try {
641
- const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
800
+ const vectorPath = this.vectorPathFor(kb.id);
642
801
  addedVectorPath = tempVectorPath(`${vectorPath}.added`);
643
802
  addedVectorWriter = openVectorWriter(addedVectorPath);
644
803
  const existingHashes = new Map();
@@ -687,7 +846,7 @@ export class KnowledgeEngine {
687
846
  newVectorIndexByHash.set(batch[i].content_hash, indexes);
688
847
  }
689
848
  addedVectorCount += newVectors.length;
690
- insertChunks(this.db, kb.id, batch);
849
+ insertedChunkIds.push(...insertChunks(this.db, kb.id, batch));
691
850
  addedCount += batch.length;
692
851
  updateKBCounts(this.db, kb.id, getChunkCount(this.db, kb.id), getFileCount(this.db, kb.id));
693
852
  const storedMessage = `Stored update batch: +${addedCount} chunks, =${unchanged} unchanged`;
@@ -728,10 +887,12 @@ export class KnowledgeEngine {
728
887
  updateIndexingJob(this.db, kb.id, { phase: "fetching", message });
729
888
  onProgress?.(message);
730
889
  scannedFiles = 1;
731
- await processChunks(await chunkUrl(kb.source_path, signal));
890
+ const chunks = await chunkUrl(kb.source_path, signal);
891
+ stagedSymbols.push(...extractSymbols(chunks.map((chunk) => chunk.content).join("\n\n"), kb.source_path, "html"));
892
+ await processChunks(chunks);
732
893
  }
733
894
  else if (statSync(kb.source_path).isDirectory()) {
734
- const plan = planDirectoryScan(kb.source_path, scanOptions);
895
+ const plan = planDirectoryScan(kb.source_path, scanOptions, signal);
735
896
  plannedTotalFiles = plan.files;
736
897
  const planningMessage = `Planned directory scan: ${plan.files} files, ${formatBytes(plan.bytes)} scannable text, skipped ${plan.skippedTotal} (${plan.skippedSummary})`;
737
898
  updateIndexingJob(this.db, kb.id, {
@@ -746,6 +907,7 @@ export class KnowledgeEngine {
746
907
  if (signal?.aborted)
747
908
  throw new Error("Cancelled");
748
909
  scannedFiles++;
910
+ stagedSymbols.push(...extractSymbols(file.content, file.relPath, file.fileType));
749
911
  await processChunks(await chunkFile(file.content, file.relPath));
750
912
  if (scannedFiles % 25 === 0) {
751
913
  const message = `Scanned ${scannedFiles} files, ${scannedChunks} chunks, skipped ${skipped.total}, +${addedCount} =${unchanged}`;
@@ -776,9 +938,10 @@ export class KnowledgeEngine {
776
938
  onProgress?.(message);
777
939
  }
778
940
  else {
779
- const { readFileSync } = await import("node:fs");
941
+ const extracted = await extractSourceFileContent(kb.source_path, signal);
780
942
  scannedFiles = 1;
781
- await processChunks(await chunkFile(readFileSync(kb.source_path, "utf-8"), kb.source_path));
943
+ stagedSymbols.push(...extractSymbols(extracted.content, kb.source_path, extracted.fileType));
944
+ await processChunks(await chunkFile(extracted.content, kb.source_path));
782
945
  }
783
946
  await flushPending();
784
947
  addedVectorWriter.close();
@@ -787,8 +950,7 @@ export class KnowledgeEngine {
787
950
  for (const entries of existingHashes.values()) {
788
951
  idsToRemove.push(...entries.map((entry) => entry.id));
789
952
  }
790
- if (idsToRemove.length > 0)
791
- deleteChunksByIds(this.db, idsToRemove);
953
+ const idsToRemoveSet = new Set(idsToRemove);
792
954
  const changesMessage = `Changes: +${addedCount} -${idsToRemove.length} =${unchanged}`;
793
955
  updateIndexingJob(this.db, kb.id, {
794
956
  phase: "reconciling",
@@ -811,6 +973,10 @@ export class KnowledgeEngine {
811
973
  };
812
974
  try {
813
975
  for (const chunk of iterateChunksByKB(this.db, kb.id)) {
976
+ if (signal?.aborted)
977
+ throw new Error("Cancelled");
978
+ if (idsToRemoveSet.has(chunk.id))
979
+ continue;
814
980
  const oldVectorIndex = takeVectorIndex(oldVectorIndexByHash, chunk.content_hash);
815
981
  const newVectorIndex = takeVectorIndex(newVectorIndexByHash, chunk.content_hash);
816
982
  const vector = oldVectorReader && oldVectorIndex !== undefined
@@ -847,6 +1013,10 @@ export class KnowledgeEngine {
847
1013
  if (addedVectorPath)
848
1014
  rmSync(addedVectorPath, { force: true });
849
1015
  addedVectorPath = undefined;
1016
+ if (idsToRemove.length > 0)
1017
+ deleteChunksByIds(this.db, idsToRemove);
1018
+ deleteSymbolsByKB(this.db, kb.id);
1019
+ insertSymbols(this.db, kb.id, stagedSymbols);
850
1020
  updateKBCounts(this.db, kb.id, finalChunkCount, getFileCount(this.db, kb.id));
851
1021
  updateKBStatus(this.db, kb.id, "ready");
852
1022
  const readyMessage = `Ready: +${addedCount} -${idsToRemove.length} =${unchanged}`;
@@ -869,14 +1039,19 @@ export class KnowledgeEngine {
869
1039
  rmSync(replacementVectorPath, { force: true });
870
1040
  if (addedVectorPath)
871
1041
  rmSync(addedVectorPath, { force: true });
872
- updateKBStatus(this.db, kb.id, "error");
1042
+ if (insertedChunkIds.length > 0) {
1043
+ deleteChunksByIds(this.db, insertedChunkIds);
1044
+ updateKBCounts(this.db, kb.id, getChunkCount(this.db, kb.id), getFileCount(this.db, kb.id));
1045
+ }
1046
+ updateKBStatus(this.db, kb.id, isCancellationError(e) ? kb.status : "error");
873
1047
  finishIndexingJob(this.db, kb.id, isCancellationError(e) ? "cancelled" : "failed", isCancellationError(e) ? "Update cancelled." : "Update failed.", e instanceof Error ? e.message : String(e));
874
1048
  throw e;
875
1049
  }
876
1050
  }
877
- async search(query, options = {}) {
1051
+ async search(query, options = {}, signal) {
878
1052
  if (!this.db)
879
1053
  throw new Error("Engine not initialized");
1054
+ throwIfAborted(signal);
880
1055
  const requestedMode = options.mode ?? "hybrid";
881
1056
  if (requestedMode === "auto") {
882
1057
  const primaryMode = chooseAutoMode(query);
@@ -887,7 +1062,8 @@ export class KnowledgeEngine {
887
1062
  if (tried.includes(mode))
888
1063
  continue;
889
1064
  tried.push(mode);
890
- const response = await this.search(query, { ...options, mode });
1065
+ throwIfAborted(signal);
1066
+ const response = await this.search(query, { ...options, mode }, signal);
891
1067
  if (response.warnings)
892
1068
  warnings.push(...response.warnings);
893
1069
  if (!isWeakAutoResponse(query, response, primaryMode, mode)) {
@@ -916,7 +1092,8 @@ export class KnowledgeEngine {
916
1092
  }
917
1093
  const db = this.db;
918
1094
  const { mode = "hybrid", limit = 10, offset = 0, kb_id, filters, diversity = "balanced" } = options;
919
- const retrievalMode = mode === "adaptive" ? "hybrid" : mode;
1095
+ const resolvedMode = retrievalModeFor(mode);
1096
+ const retrievalMode = resolvedMode === "adaptive" ? "hybrid" : resolvedMode;
920
1097
  const candidateLimit = Math.max(50, offset + limit * 12);
921
1098
  const normalizedQuery = normalizedQueryText(query);
922
1099
  const queryTokens = tokenizeForSimilarity(query);
@@ -925,6 +1102,7 @@ export class KnowledgeEngine {
925
1102
  const selectedKB = kb_id ? (getKB(db, kb_id) ?? getKBByName(db, kb_id)) : undefined;
926
1103
  const availableKBs = kb_id ? [selectedKB].filter(Boolean) : listKBs(db);
927
1104
  const kbs = availableKBs.filter((kb) => kb.status === "ready" || kb.status === "stale");
1105
+ const kbById = new Map(kbs.map((kb) => [kb.id, kb]));
928
1106
  for (const kb of availableKBs) {
929
1107
  if (kb.status !== "ready" && kb.status !== "stale") {
930
1108
  warnings.push(`"${kb.name}" is ${kb.status}; search skipped until indexing is ready`);
@@ -942,19 +1120,24 @@ export class KnowledgeEngine {
942
1120
  const allResults = [];
943
1121
  const vectorsByChunkId = new Map();
944
1122
  for (const kb of kbs) {
1123
+ throwIfAborted(signal);
945
1124
  if (kb.embedding_model !== CURRENT_EMBEDDING_MODEL) {
946
1125
  warnings.push(`"${kb.name}" was indexed with ${kb.embedding_model} (current: ${CURRENT_EMBEDDING_MODEL}) — run knowledge_update for best results`);
947
1126
  }
948
1127
  if (kb.chunk_count === 0)
949
1128
  continue;
950
- const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
1129
+ const vectorPath = this.vectorPathFor(kb.id);
951
1130
  if (retrievalMode === "fast") {
952
- allResults.push(...searchBM25(db, normalizedQuery || query, candidateLimit, kb.id, { allowOrFallback: false }));
1131
+ allResults.push(...searchBM25(db, normalizedQuery || query, candidateLimit, kb.id, { allowOrFallback: false }).map((result) => ({
1132
+ ...result,
1133
+ score: result.score * kbTrustMultiplier(kb),
1134
+ })));
953
1135
  }
954
1136
  else if (retrievalMode === "semantic") {
955
- const queryVec = await embedQuery(query);
956
- const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), candidateLimit);
957
- allResults.push(...vectorResults.results);
1137
+ const queryVec = await embedQuery(query, signal);
1138
+ throwIfAborted(signal);
1139
+ const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), candidateLimit, signal);
1140
+ allResults.push(...vectorResults.results.map((result) => ({ ...result, score: result.score * kbTrustMultiplier(kb) })));
958
1141
  for (const [chunkId, vector] of vectorResults.vectorsByChunkId)
959
1142
  vectorsByChunkId.set(chunkId, vector);
960
1143
  }
@@ -964,13 +1147,14 @@ export class KnowledgeEngine {
964
1147
  if (bm25Results.length === 0)
965
1148
  continue;
966
1149
  let vecResults = [];
967
- const queryVec = await embedQuery(query);
968
- const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), candidateLimit);
1150
+ const queryVec = await embedQuery(query, signal);
1151
+ throwIfAborted(signal);
1152
+ const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), candidateLimit, signal);
969
1153
  vecResults = vectorResults.results;
970
1154
  for (const [chunkId, vector] of vectorResults.vectorsByChunkId)
971
1155
  vectorsByChunkId.set(chunkId, vector);
972
1156
  const fused = weightedScoreFusion(bm25Results, vecResults);
973
- allResults.push(...fused);
1157
+ allResults.push(...fused.map((result) => ({ ...result, score: result.score * kbTrustMultiplier(kb) })));
974
1158
  }
975
1159
  }
976
1160
  // Deduplicate and sort
@@ -983,6 +1167,7 @@ export class KnowledgeEngine {
983
1167
  });
984
1168
  const rankingByChunkId = new Map();
985
1169
  for (const result of unique) {
1170
+ throwIfAborted(signal);
986
1171
  const chunk = getChunkById(db, result.chunkId);
987
1172
  if (chunk) {
988
1173
  const ranking = scoreChunkForQuery(result.score, chunk, queryTokens);
@@ -1030,9 +1215,11 @@ export class KnowledgeEngine {
1030
1215
  return chunk ? { chunkId: r.chunkId, content: chunk.content } : null;
1031
1216
  })
1032
1217
  .filter(Boolean);
1033
- const reranked = await rerank(query, candidates, Math.max(limit * 3, limit));
1218
+ const reranked = await rerank(query, candidates, Math.max(limit * 3, limit), signal);
1219
+ throwIfAborted(signal);
1034
1220
  const ranked = [];
1035
1221
  for (const r of reranked) {
1222
+ throwIfAborted(signal);
1036
1223
  const chunk = getChunkById(db, r.chunkId);
1037
1224
  if (!chunk)
1038
1225
  continue;
@@ -1051,17 +1238,30 @@ export class KnowledgeEngine {
1051
1238
  }
1052
1239
  const diversified = interleaveByFile(diversifyRankedChunks(ranked, diversity, vectorsByChunkId), diversity);
1053
1240
  const page = diversified.slice(offset, offset + limit);
1054
- const results = page.map((r) => ({
1055
- content: r.content,
1056
- file_path: r.chunk.file_path,
1057
- file_type: r.chunk.file_type,
1058
- kb_name: r.kbName,
1059
- score: r.score,
1060
- ranking: r.ranking,
1061
- snippet: r.snippet,
1062
- start_line: r.startLine,
1063
- end_line: r.endLine,
1064
- }));
1241
+ const results = page.map((r) => {
1242
+ const kb = kbById.get(r.chunk.kb_id);
1243
+ const sourceMtime = sourceMtimeFor(kb, r.chunk.file_path);
1244
+ return {
1245
+ content: r.content,
1246
+ file_path: r.chunk.file_path,
1247
+ file_type: r.chunk.file_type,
1248
+ kb_name: r.kbName,
1249
+ score: r.score,
1250
+ ranking: r.ranking,
1251
+ snippet: r.snippet,
1252
+ start_line: r.startLine,
1253
+ end_line: r.endLine,
1254
+ provenance: {
1255
+ chunk_id: r.chunk.id,
1256
+ chunk_hash: r.chunk.content_hash,
1257
+ indexed_at: r.chunk.indexed_at,
1258
+ source_mtime: sourceMtime,
1259
+ stale: sourceMtime !== undefined ? sourceMtime > r.chunk.indexed_at : kb?.status === "stale",
1260
+ match_reason: matchReasonFor(mode),
1261
+ source_chunk_ids: r.sourceChunkIds,
1262
+ },
1263
+ };
1264
+ });
1065
1265
  return {
1066
1266
  results,
1067
1267
  total_count: diversified.length,
@@ -1072,6 +1272,7 @@ export class KnowledgeEngine {
1072
1272
  }
1073
1273
  const ranked = [];
1074
1274
  for (const r of filtered) {
1275
+ throwIfAborted(signal);
1075
1276
  const chunk = getChunkById(db, r.chunkId);
1076
1277
  if (!chunk)
1077
1278
  continue;
@@ -1118,6 +1319,19 @@ export class KnowledgeEngine {
1118
1319
  snippet: r.snippet,
1119
1320
  start_line: r.startLine,
1120
1321
  end_line: r.endLine,
1322
+ provenance: (() => {
1323
+ const kb = kbById.get(r.chunk.kb_id);
1324
+ const sourceMtime = sourceMtimeFor(kb, r.chunk.file_path);
1325
+ return {
1326
+ chunk_id: r.chunk.id,
1327
+ chunk_hash: r.chunk.content_hash,
1328
+ indexed_at: r.chunk.indexed_at,
1329
+ source_mtime: sourceMtime,
1330
+ stale: sourceMtime !== undefined ? sourceMtime > r.chunk.indexed_at : kb?.status === "stale",
1331
+ match_reason: matchReasonFor(mode),
1332
+ source_chunk_ids: r.sourceChunkIds,
1333
+ };
1334
+ })(),
1121
1335
  }));
1122
1336
  return {
1123
1337
  results,
@@ -1136,31 +1350,94 @@ export class KnowledgeEngine {
1136
1350
  remove(nameOrId) {
1137
1351
  if (!this.db)
1138
1352
  return false;
1353
+ if (this.disposing)
1354
+ throw new Error("Knowledge engine is shutting down");
1355
+ if (this.activeMutations.size > 0)
1356
+ throw new Error("A knowledge-base mutation is already running");
1139
1357
  const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
1140
1358
  if (!kb)
1141
1359
  return false;
1142
1360
  deleteKB(this.db, kb.id);
1361
+ this.deleteVectorFile(kb.id);
1143
1362
  return true;
1144
1363
  }
1145
- list() {
1364
+ list(signal) {
1146
1365
  if (!this.db)
1147
1366
  return [];
1367
+ throwIfAborted(signal);
1148
1368
  return listKBs(this.db);
1149
1369
  }
1150
1370
  clear() {
1151
1371
  if (!this.db)
1152
1372
  return;
1153
- for (const kb of listKBs(this.db))
1373
+ if (this.disposing)
1374
+ throw new Error("Knowledge engine is shutting down");
1375
+ if (this.activeMutations.size > 0)
1376
+ throw new Error("A knowledge-base mutation is already running");
1377
+ for (const kb of listKBs(this.db)) {
1154
1378
  deleteKB(this.db, kb.id);
1379
+ this.deleteVectorFile(kb.id);
1380
+ }
1155
1381
  }
1156
- diagnose() {
1382
+ diagnose(signal) {
1157
1383
  if (!this.db)
1158
1384
  return [];
1385
+ throwIfAborted(signal);
1159
1386
  const db = this.db;
1160
- return listKBs(db).map((kb) => diagnoseKB(db, kb));
1387
+ return listKBs(db).map((kb) => {
1388
+ throwIfAborted(signal);
1389
+ return diagnoseKB(db, kb, signal);
1390
+ });
1161
1391
  }
1162
- doctor() {
1163
- const diagnostics = this.diagnose();
1392
+ symbolSearch(query, options = {}, signal) {
1393
+ if (!this.db)
1394
+ throw new Error("Engine not initialized");
1395
+ throwIfAborted(signal);
1396
+ const db = this.db;
1397
+ const selectedKB = options.kb_id ? (getKB(db, options.kb_id) ?? getKBByName(db, options.kb_id)) : undefined;
1398
+ if (options.kb_id && !selectedKB)
1399
+ throw new Error(`Knowledge base not found: ${options.kb_id}`);
1400
+ const limit = Math.max(1, Math.min(200, options.limit ?? 20));
1401
+ const offset = Math.max(0, options.offset ?? 0);
1402
+ const symbolOptions = {
1403
+ kbId: selectedKB?.id,
1404
+ kind: options.kind,
1405
+ filePattern: options.file_pattern,
1406
+ exact: options.exact,
1407
+ };
1408
+ const symbols = searchSymbols(db, query, {
1409
+ ...symbolOptions,
1410
+ limit: limit + 1,
1411
+ offset,
1412
+ });
1413
+ throwIfAborted(signal);
1414
+ const page = symbols.slice(0, limit);
1415
+ const total = countSymbols(db, query, symbolOptions);
1416
+ throwIfAborted(signal);
1417
+ return {
1418
+ results: page.map((symbol) => {
1419
+ const kb = getKB(db, symbol.kb_id);
1420
+ return {
1421
+ name: symbol.name,
1422
+ kind: symbol.kind,
1423
+ file_path: symbol.file_path,
1424
+ file_type: symbol.file_type,
1425
+ kb_name: kb?.name ?? "unknown",
1426
+ start_line: symbol.start_line,
1427
+ end_line: symbol.end_line,
1428
+ signature: symbol.signature ?? undefined,
1429
+ container_name: symbol.container_name ?? undefined,
1430
+ text: symbol.text,
1431
+ indexed_at: symbol.indexed_at,
1432
+ };
1433
+ }),
1434
+ total_count: total,
1435
+ has_more: symbols.length > limit,
1436
+ };
1437
+ }
1438
+ doctor(signal) {
1439
+ throwIfAborted(signal);
1440
+ const diagnostics = this.diagnose(signal);
1164
1441
  const issues = [];
1165
1442
  const kbs = this.list();
1166
1443
  if (kbs.length === 0) {
@@ -1168,9 +1445,11 @@ export class KnowledgeEngine {
1168
1445
  severity: "blocking",
1169
1446
  message: "No knowledge bases are indexed.",
1170
1447
  action: "Run knowledge_add for the project root or relevant source/docs directory.",
1448
+ action_code: "rebuild_kb",
1171
1449
  });
1172
1450
  }
1173
1451
  for (const diagnostic of diagnostics) {
1452
+ throwIfAborted(signal);
1174
1453
  if (diagnostic.stuck_indexing) {
1175
1454
  const phase = diagnostic.job ? ` during ${diagnostic.job.phase}` : "";
1176
1455
  issues.push({
@@ -1178,6 +1457,7 @@ export class KnowledgeEngine {
1178
1457
  kb_name: diagnostic.kb_name,
1179
1458
  message: `Indexing appears stuck${phase} for ${formatDuration(diagnostic.last_progress_age_ms)}.`,
1180
1459
  action: "Check knowledge_status for the last progress message. If no Pi process is actively indexing it, remove and rebuild this KB.",
1460
+ action_code: "wait_for_indexing",
1181
1461
  });
1182
1462
  }
1183
1463
  if (diagnostic.status === "error") {
@@ -1186,6 +1466,7 @@ export class KnowledgeEngine {
1186
1466
  kb_name: diagnostic.kb_name,
1187
1467
  message: "Knowledge base is in error state and is skipped by search.",
1188
1468
  action: "Run knowledge_remove and knowledge_add to rebuild it from the source.",
1469
+ action_code: "rebuild_kb",
1189
1470
  });
1190
1471
  }
1191
1472
  if (diagnostic.coverage_percent < 80) {
@@ -1194,6 +1475,7 @@ export class KnowledgeEngine {
1194
1475
  kb_name: diagnostic.kb_name,
1195
1476
  message: `Coverage is ${diagnostic.coverage_percent}% (${diagnostic.indexed_files}/${diagnostic.total_source_files} files).`,
1196
1477
  action: "Review skipped files and source path. Rebuild if index-time rules changed.",
1478
+ action_code: "review_skipped_scope",
1197
1479
  });
1198
1480
  }
1199
1481
  if (diagnostic.stale_files.length > 0) {
@@ -1202,6 +1484,7 @@ export class KnowledgeEngine {
1202
1484
  kb_name: diagnostic.kb_name,
1203
1485
  message: `${diagnostic.stale_files.length} files changed after indexing.`,
1204
1486
  action: "Run knowledge_update for this KB.",
1487
+ action_code: "run_update",
1205
1488
  });
1206
1489
  }
1207
1490
  if (diagnostic.orphan_files.length > 0) {
@@ -1210,6 +1493,7 @@ export class KnowledgeEngine {
1210
1493
  kb_name: diagnostic.kb_name,
1211
1494
  message: `${diagnostic.orphan_files.length} indexed files no longer exist in the source.`,
1212
1495
  action: "Run knowledge_update or rebuild the KB.",
1496
+ action_code: "run_update",
1213
1497
  });
1214
1498
  }
1215
1499
  if (diagnostic.skipped_files.total > 0) {
@@ -1221,9 +1505,24 @@ export class KnowledgeEngine {
1221
1505
  .map(([reason, count]) => `${reason}: ${count}`)
1222
1506
  .join(", ")}).`,
1223
1507
  action: "Use skipped samples to confirm exclusions are expected. Adjust source path or ignore rules if needed.",
1508
+ action_code: "review_skipped_scope",
1224
1509
  });
1225
1510
  }
1226
1511
  }
1512
+ if (this.db) {
1513
+ for (const kb of kbs) {
1514
+ throwIfAborted(signal);
1515
+ if (kb.status === "ready" && kb.chunk_count > 0 && getSymbolCount(this.db, kb.id) === 0) {
1516
+ issues.push({
1517
+ severity: "info",
1518
+ kb_name: kb.name,
1519
+ message: "No lightweight symbols are indexed for this KB.",
1520
+ action: "Run knowledge_update to rebuild symbol/config/heading lookup metadata.",
1521
+ action_code: "run_update",
1522
+ });
1523
+ }
1524
+ }
1525
+ }
1227
1526
  const penalty = issues.reduce((score, issue) => {
1228
1527
  if (issue.severity === "blocking")
1229
1528
  return score + 35;
@@ -1242,138 +1541,193 @@ export class KnowledgeEngine {
1242
1541
  summary,
1243
1542
  issues,
1244
1543
  diagnostics,
1544
+ actions: issues.map((issue) => ({
1545
+ code: issue.action_code ?? "none",
1546
+ kb_name: issue.kb_name,
1547
+ target: issue.kb_name,
1548
+ description: issue.action,
1549
+ })),
1245
1550
  };
1246
1551
  }
1247
- async exportKB(nameOrId, outputPath) {
1552
+ async exportKB(nameOrId, outputPath, signal, onProgress) {
1248
1553
  if (!this.db)
1249
1554
  throw new Error("Engine not initialized");
1555
+ throwIfAborted(signal);
1250
1556
  const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
1251
1557
  if (!kb)
1252
1558
  throw new Error(`Knowledge base not found: ${nameOrId}`);
1253
- const chunks = getChunksByKB(this.db, kb.id);
1254
- const { writeFileSync } = await import("node:fs");
1559
+ return this.runExclusive(["global:mutation", `kb:${kb.id}`], `Operation for "${kb.name}"`, () => this.exportKBUnlocked(kb, outputPath, signal, onProgress));
1560
+ }
1561
+ async exportKBUnlocked(kb, outputPath, signal, onProgress) {
1562
+ if (!this.db)
1563
+ throw new Error("Engine not initialized");
1564
+ const { createWriteStream } = await import("node:fs");
1565
+ const tempOutputPath = tempVectorPath(outputPath);
1566
+ const stream = createWriteStream(tempOutputPath, { encoding: "utf-8" });
1567
+ let count = 0;
1255
1568
  const header = JSON.stringify({
1256
1569
  name: kb.name,
1257
1570
  description: kb.description,
1258
1571
  source_type: "text",
1259
- chunk_count: chunks.length,
1572
+ chunk_count: kb.chunk_count,
1260
1573
  });
1261
- const lines = [
1262
- header,
1263
- ...chunks.map((c) => JSON.stringify({
1264
- content: c.content,
1265
- file_path: c.file_path,
1266
- file_type: c.file_type,
1267
- start_line: c.start_line,
1268
- end_line: c.end_line,
1269
- metadata_json: c.metadata_json,
1270
- })),
1271
- ];
1272
- writeFileSync(outputPath, `${lines.join("\n")}\n`);
1273
- return chunks.length;
1574
+ try {
1575
+ await writeLine(stream, header);
1576
+ for (const chunk of iterateChunksByKB(this.db, kb.id)) {
1577
+ throwIfAborted(signal);
1578
+ await writeLine(stream, JSON.stringify({
1579
+ content: chunk.content,
1580
+ file_path: chunk.file_path,
1581
+ file_type: chunk.file_type,
1582
+ start_line: chunk.start_line,
1583
+ end_line: chunk.end_line,
1584
+ metadata_json: chunk.metadata_json,
1585
+ }));
1586
+ count++;
1587
+ if (count % INDEX_EMBED_BATCH_SIZE === 0)
1588
+ onProgress?.(`Exported ${count}/${kb.chunk_count} chunks...`);
1589
+ }
1590
+ await finishWriteStream(stream);
1591
+ throwIfAborted(signal);
1592
+ renameSync(tempOutputPath, outputPath);
1593
+ onProgress?.(`Exported ${count}/${kb.chunk_count} chunks.`);
1594
+ return count;
1595
+ }
1596
+ catch (error) {
1597
+ stream.destroy();
1598
+ rmSync(tempOutputPath, { force: true });
1599
+ throw error;
1600
+ }
1274
1601
  }
1275
1602
  async importKB(inputPath, onProgress, signal) {
1276
1603
  if (!this.db)
1277
1604
  throw new Error("Engine not initialized");
1278
- const { readFileSync } = await import("node:fs");
1279
- const lines = readFileSync(inputPath, "utf-8").trim().split("\n");
1280
- if (lines.length < 1)
1281
- throw new Error("Empty import file");
1282
- const header = JSON.parse(lines[0]);
1283
- const kb = createKB(this.db, { name: header.name, description: header.description, source_type: "text" });
1284
- updateKBStatus(this.db, kb.id, "indexing");
1285
- startIndexingJob(this.db, kb.id, "import", `Starting import for "${header.name}"`);
1605
+ return this.runExclusive("global:mutation", "Knowledge-base import", () => this.importKBUnlocked(inputPath, onProgress, signal));
1606
+ }
1607
+ async importKBUnlocked(inputPath, onProgress, signal) {
1608
+ if (!this.db)
1609
+ throw new Error("Engine not initialized");
1610
+ throwIfAborted(signal);
1611
+ const { createReadStream } = await import("node:fs");
1612
+ const { createInterface } = await import("node:readline");
1613
+ const stream = createReadStream(inputPath, { encoding: "utf-8" });
1614
+ const lines = createInterface({ input: stream, crlfDelay: Infinity });
1615
+ let header;
1616
+ let kb;
1286
1617
  let vectorWriter;
1287
1618
  let tempVectorFile;
1288
- try {
1289
- const chunkData = lines.slice(1).map((l) => JSON.parse(l));
1290
- const importMessage = `Importing ${chunkData.length} chunks...`;
1619
+ let inserted = 0;
1620
+ let lineNumber = 0;
1621
+ const pendingChunks = [];
1622
+ const pendingSymbols = [];
1623
+ const flushPending = async () => {
1624
+ if (!this.db || !kb || !vectorWriter || pendingChunks.length === 0)
1625
+ return;
1626
+ throwIfAborted(signal);
1627
+ const batch = pendingChunks.splice(0, INDEX_EMBED_BATCH_SIZE);
1628
+ const symbols = pendingSymbols.splice(0);
1629
+ const total = header?.chunk_count;
1630
+ const message = `Embedding import batch: ${inserted}/${total ?? "unknown"} chunks stored`;
1291
1631
  updateIndexingJob(this.db, kb.id, {
1292
- phase: "importing",
1293
- message: importMessage,
1294
- total_files: chunkData.length,
1632
+ phase: "embedding",
1633
+ message,
1634
+ processed_chunks: inserted,
1635
+ total_files: total,
1636
+ added_chunks: inserted,
1295
1637
  });
1296
- onProgress?.(importMessage);
1297
- const chunks = chunkData.map((c) => ({
1298
- content_hash: chunkIdentityHash({
1299
- content: c.content,
1300
- filePath: c.file_path,
1301
- fileType: c.file_type,
1302
- startLine: c.start_line,
1303
- endLine: c.end_line,
1304
- metadataJson: c.metadata_json || "{}",
1305
- }),
1306
- content: c.content,
1307
- content_tokenized: "",
1308
- file_path: c.file_path,
1309
- file_type: c.file_type,
1310
- start_line: c.start_line,
1311
- end_line: c.end_line,
1312
- metadata_json: c.metadata_json || "{}",
1313
- }));
1314
- const indexedChunks = chunks.map((chunk) => ({
1315
- ...chunk,
1316
- content_tokenized: preTokenizeForFTS(buildChunkEmbeddingText(chunk)),
1317
- }));
1318
- const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
1319
- tempVectorFile = tempVectorPath(vectorPath);
1320
- vectorWriter = openVectorWriter(tempVectorFile);
1321
- let inserted = 0;
1322
- for (let offset = 0; offset < indexedChunks.length; offset += INDEX_EMBED_BATCH_SIZE) {
1323
- if (signal?.aborted)
1324
- throw new Error("Cancelled");
1325
- const batch = indexedChunks.slice(offset, offset + INDEX_EMBED_BATCH_SIZE);
1326
- const message = `Embedding import batch ${Math.floor(offset / INDEX_EMBED_BATCH_SIZE) + 1}/${Math.ceil(indexedChunks.length / INDEX_EMBED_BATCH_SIZE)}: ${offset}/${indexedChunks.length} chunks`;
1327
- updateIndexingJob(this.db, kb.id, {
1328
- phase: "embedding",
1329
- message,
1330
- processed_chunks: offset,
1331
- total_files: indexedChunks.length,
1332
- added_chunks: inserted,
1333
- });
1334
- onProgress?.(message);
1335
- const vectors = await embedDocuments(batch.map((c) => buildChunkEmbeddingText(c)), signal);
1336
- if (signal?.aborted)
1337
- throw new Error("Cancelled");
1338
- insertChunks(this.db, kb.id, batch);
1339
- vectorWriter.append(vectors);
1340
- inserted += batch.length;
1341
- updateKBCounts(this.db, kb.id, inserted, new Set(indexedChunks.slice(0, inserted).map((c) => c.file_path)).size);
1342
- updateIndexingJob(this.db, kb.id, {
1343
- phase: "storing",
1344
- message: `Stored import batch: ${inserted}/${indexedChunks.length} chunks`,
1345
- processed_chunks: inserted,
1346
- total_files: indexedChunks.length,
1347
- added_chunks: inserted,
1348
- });
1638
+ onProgress?.(message);
1639
+ const vectors = await embedDocuments(batch.map((chunk) => buildChunkEmbeddingText(chunk)), signal);
1640
+ throwIfAborted(signal);
1641
+ insertChunks(this.db, kb.id, batch);
1642
+ insertSymbols(this.db, kb.id, symbols);
1643
+ vectorWriter.append(vectors);
1644
+ inserted += batch.length;
1645
+ updateKBCounts(this.db, kb.id, inserted, getFileCount(this.db, kb.id));
1646
+ updateIndexingJob(this.db, kb.id, {
1647
+ phase: "storing",
1648
+ message: `Stored import batch: ${inserted}/${total ?? "unknown"} chunks`,
1649
+ processed_chunks: inserted,
1650
+ total_files: total,
1651
+ added_chunks: inserted,
1652
+ });
1653
+ };
1654
+ try {
1655
+ for await (const rawLine of lines) {
1656
+ throwIfAborted(signal);
1657
+ lineNumber++;
1658
+ const line = rawLine.trim();
1659
+ if (!line)
1660
+ continue;
1661
+ if (!header) {
1662
+ header = JSON.parse(line);
1663
+ if (!header.name)
1664
+ throw new Error("Import header is missing a knowledge base name");
1665
+ const existingKB = getKBByName(this.db, header.name);
1666
+ if (existingKB) {
1667
+ throw new Error(`Knowledge base "${header.name}" already exists. Use knowledge_update to refresh it, or knowledge_remove before importing a replacement.`);
1668
+ }
1669
+ kb = createKB(this.db, { name: header.name, description: header.description, source_type: "text" });
1670
+ updateKBStatus(this.db, kb.id, "indexing");
1671
+ startIndexingJob(this.db, kb.id, "import", `Starting import for "${header.name}"`);
1672
+ const importMessage = `Importing ${header.chunk_count ?? "unknown"} chunks...`;
1673
+ updateIndexingJob(this.db, kb.id, {
1674
+ phase: "importing",
1675
+ message: importMessage,
1676
+ total_files: header.chunk_count,
1677
+ });
1678
+ onProgress?.(importMessage);
1679
+ const vectorPath = this.vectorPathFor(kb.id);
1680
+ tempVectorFile = tempVectorPath(vectorPath);
1681
+ vectorWriter = openVectorWriter(tempVectorFile);
1682
+ continue;
1683
+ }
1684
+ const imported = JSON.parse(line);
1685
+ const chunk = importedChunkToInsert(imported);
1686
+ chunk.content_tokenized = preTokenizeForFTS(buildChunkEmbeddingText(chunk));
1687
+ pendingChunks.push(chunk);
1688
+ pendingSymbols.push(...extractSymbols(imported.content, imported.file_path, imported.file_type));
1689
+ if (pendingChunks.length >= INDEX_EMBED_BATCH_SIZE)
1690
+ await flushPending();
1349
1691
  }
1692
+ if (!header)
1693
+ throw new Error("Empty import file");
1694
+ if (!kb || !vectorWriter || !tempVectorFile)
1695
+ throw new Error("Import did not create a knowledge base");
1696
+ await flushPending();
1350
1697
  vectorWriter.close();
1351
1698
  vectorWriter = undefined;
1352
- renameSync(tempVectorFile, vectorPath);
1699
+ renameSync(tempVectorFile, this.vectorPathFor(kb.id));
1353
1700
  tempVectorFile = undefined;
1354
- updateKBCounts(this.db, kb.id, indexedChunks.length, new Set(indexedChunks.map((c) => c.file_path)).size);
1701
+ updateKBCounts(this.db, kb.id, inserted, getFileCount(this.db, kb.id));
1355
1702
  updateKBStatus(this.db, kb.id, "ready");
1356
1703
  const savedKB = getKB(this.db, kb.id);
1357
1704
  if (!savedKB)
1358
1705
  throw new Error(`Knowledge base disappeared after import: ${kb.id}`);
1359
- finishIndexingJob(this.db, kb.id, "succeeded", `Ready: imported ${indexedChunks.length} chunks`);
1360
- return { kb: savedKB, chunkCount: chunks.length };
1706
+ finishIndexingJob(this.db, kb.id, "succeeded", `Ready: imported ${inserted} chunks`);
1707
+ return { kb: savedKB, chunkCount: inserted };
1361
1708
  }
1362
- catch (e) {
1709
+ catch (error) {
1710
+ lines.close();
1711
+ stream.destroy();
1363
1712
  vectorWriter?.close();
1364
1713
  if (tempVectorFile)
1365
1714
  rmSync(tempVectorFile, { force: true });
1366
- if (this.db) {
1367
- finishIndexingJob(this.db, kb.id, isCancellationError(e) ? "cancelled" : "failed", isCancellationError(e) ? "Import cancelled." : "Import failed.", e instanceof Error ? e.message : String(e));
1715
+ if (this.db && kb) {
1716
+ finishIndexingJob(this.db, kb.id, isCancellationError(error) ? "cancelled" : "failed", isCancellationError(error) ? "Import cancelled." : `Import failed near line ${lineNumber}.`, error instanceof Error ? error.message : String(error));
1717
+ deleteKB(this.db, kb.id);
1718
+ this.deleteVectorFile(kb.id);
1368
1719
  }
1369
- deleteKB(this.db, kb.id);
1370
- throw e;
1720
+ throw error;
1371
1721
  }
1372
1722
  }
1373
1723
  async dispose(options = {}) {
1724
+ this.disposing = true;
1374
1725
  const activeUpdates = [...this.activeUpdates.values()];
1375
1726
  if (activeUpdates.length > 0)
1376
1727
  await Promise.allSettled(activeUpdates);
1728
+ const activeMutations = [...this.activeMutations.values()];
1729
+ if (activeMutations.length > 0)
1730
+ await Promise.allSettled(activeMutations);
1377
1731
  const disposeModels = options.disposeModels ?? true;
1378
1732
  if (disposeModels) {
1379
1733
  await disposeEmbedding();