pi-knowledge 0.4.6 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,9 +1,10 @@
1
- import { existsSync, renameSync, rmSync, statSync } from "node:fs";
1
+ import { existsSync, readFileSync, renameSync, rmSync, statSync } from "node:fs";
2
2
  import { join, resolve } from "node:path";
3
3
  import { diagnoseKB } from "./diagnostics/health.js";
4
4
  import { dispose as disposeEmbedding, embedDocuments, embedQuery, prepareForShutdown as prepareEmbeddingForShutdown, } from "./embedding/provider.js";
5
5
  import { openVectorReader, openVectorWriter } from "./embedding/vectors.js";
6
6
  import { buildChunkEmbeddingText, chunkFile, chunkIdentityHash, createSkippedScanStats, isReadableTextFile, iterateScannableFiles, iterateScannedFiles, preTokenizeForFTS, summarizeSkippedScan, } from "./indexer/chunker.js";
7
+ import { extractSymbols } from "./indexer/symbols.js";
7
8
  import { shutdownModelWorker } from "./model-worker-client.js";
8
9
  import { searchBM25 } from "./search/bm25.js";
9
10
  import { weightedScoreFusion } from "./search/fusion.js";
@@ -11,7 +12,7 @@ import { normalizedQueryText, tokenizeForSearch } from "./search/query.js";
11
12
  import { hasAnyLexicalEvidence, hasEnoughLexicalEvidence, MIN_HYBRID_SCORE, normalizeFileTypeFilter, queryCoverage, scoreChunkForQuery, } from "./search/ranking.js";
12
13
  import { disposeReranker, prepareRerankerForShutdown, rerank } from "./search/reranker.js";
13
14
  import { searchVectorFile } from "./search/vector.js";
14
- import { createKB, deleteChunksByIds, deleteKB, finishIndexingJob, getChunkById, getChunkCount, getChunksByFile, getChunksByKB, getFileCount, getKB, getKBByName, insertChunks, iterateChunkIdsByKB, iterateChunksByKB, listKBs, openDatabase, startIndexingJob, updateIndexingJob, updateKBCounts, updateKBStatus, } from "./storage/sqlite.js";
15
+ import { countSymbols, createKB, deleteChunksByIds, deleteKB, deleteSymbolsByKB, finishIndexingJob, getChunkById, getChunkCount, getChunksByFile, getFileCount, getKB, getKBByName, getSymbolCount, insertChunks, insertSymbols, iterateChunkIdsByKB, iterateChunksByKB, listKBs, openDatabase, searchSymbols, startIndexingJob, updateIndexingJob, updateKBCounts, updateKBStatus, } from "./storage/sqlite.js";
15
16
  export const CURRENT_EMBEDDING_MODEL = "multilingual-e5-small";
16
17
  const ADAPTIVE_CONTEXT_LINES = 80;
17
18
  const ADAPTIVE_MAX_CONTEXT_CHARS = 6_000;
@@ -24,6 +25,26 @@ function isCancellationError(error) {
24
25
  function tempVectorPath(vectorPath) {
25
26
  return `${vectorPath}.${Date.now()}.${Math.random().toString(16).slice(2)}.tmp`;
26
27
  }
28
+ function retrievalModeFor(mode) {
29
+ if (mode === "auto")
30
+ return "hybrid";
31
+ if (mode === "code" || mode === "config" || mode === "errors")
32
+ return "fast";
33
+ if (mode === "docs" || mode === "decision")
34
+ return "hybrid";
35
+ return mode;
36
+ }
37
+ function matchReasonFor(mode) {
38
+ if (mode === "deep")
39
+ return "rerank";
40
+ if (mode === "adaptive")
41
+ return "adaptive";
42
+ if (mode === "semantic")
43
+ return "vector";
44
+ if (mode === "fast" || mode === "code" || mode === "config" || mode === "errors")
45
+ return "bm25";
46
+ return "hybrid";
47
+ }
27
48
  function toScanOptions(options = {}) {
28
49
  return {
29
50
  includeSuggestedText: options.include_suggested_text === true,
@@ -60,11 +81,12 @@ function parseAddOptions(raw) {
60
81
  return {};
61
82
  }
62
83
  }
63
- function planDirectoryScan(dirPath, options = {}) {
84
+ function planDirectoryScan(dirPath, options = {}, signal) {
64
85
  const skipped = createSkippedScanStats();
65
86
  let files = 0;
66
87
  let bytes = 0;
67
88
  for (const file of iterateScannableFiles(dirPath, skipped, options)) {
89
+ throwIfAborted(signal);
68
90
  files++;
69
91
  bytes += file.size;
70
92
  }
@@ -76,6 +98,25 @@ function planDirectoryScan(dirPath, options = {}) {
76
98
  skipped,
77
99
  };
78
100
  }
101
+ function sourceMtimeFor(kb, filePath) {
102
+ if (!kb?.source_path)
103
+ return undefined;
104
+ const absPath = kb.source_type === "directory" ? join(kb.source_path, filePath) : kb.source_path;
105
+ try {
106
+ return statSync(absPath).mtimeMs;
107
+ }
108
+ catch {
109
+ return undefined;
110
+ }
111
+ }
112
+ function kbTrustMultiplier(kb) {
113
+ let multiplier = kb.status === "stale" ? 0.92 : 1;
114
+ if (kb.source_type === "directory" || kb.source_type === "file")
115
+ multiplier *= 1.04;
116
+ if (kb.source_type === "text" || kb.source_type === "url")
117
+ multiplier *= 0.98;
118
+ return multiplier;
119
+ }
79
120
  function formatBytes(bytes) {
80
121
  if (bytes < 1024)
81
122
  return `${bytes} B`;
@@ -347,21 +388,143 @@ async function chunkUrl(source, signal) {
347
388
  function normalizeExtractedText(text) {
348
389
  return Array.isArray(text) ? text.join("\n\n") : text;
349
390
  }
391
+ async function extractSourceFileContent(filePath, signal) {
392
+ const lowerPath = filePath.toLowerCase();
393
+ if (lowerPath.endsWith(".pdf")) {
394
+ throwIfAborted(signal);
395
+ // PDF extraction is an optional heavy runtime path; keep parser loading out of extension startup.
396
+ const { extractText } = await import("unpdf");
397
+ throwIfAborted(signal);
398
+ const buf = readFileSync(filePath);
399
+ throwIfAborted(signal);
400
+ const { text } = await extractText(new Uint8Array(buf));
401
+ throwIfAborted(signal);
402
+ return { content: normalizeExtractedText(text), fileType: "pdf" };
403
+ }
404
+ if (lowerPath.endsWith(".docx") || lowerPath.endsWith(".doc")) {
405
+ throwIfAborted(signal);
406
+ // DOCX extraction is an optional heavy runtime path; keep parser loading out of extension startup.
407
+ const mammoth = await import("mammoth");
408
+ throwIfAborted(signal);
409
+ const result = await mammoth.extractRawText({ path: filePath });
410
+ throwIfAborted(signal);
411
+ return { content: result.value, fileType: "docx" };
412
+ }
413
+ if (!isReadableTextFile(filePath)) {
414
+ throw new Error(`File is not readable text and has no supported extractor: ${filePath}`);
415
+ }
416
+ throwIfAborted(signal);
417
+ return { content: readFileSync(filePath, "utf-8"), fileType: "text" };
418
+ }
419
+ function throwIfAborted(signal) {
420
+ if (signal?.aborted)
421
+ throw new Error("Cancelled");
422
+ }
423
+ function importedChunkToInsert(chunk) {
424
+ const metadataJson = chunk.metadata_json || "{}";
425
+ return {
426
+ content_hash: chunkIdentityHash({
427
+ content: chunk.content,
428
+ filePath: chunk.file_path,
429
+ fileType: chunk.file_type,
430
+ startLine: chunk.start_line,
431
+ endLine: chunk.end_line,
432
+ metadataJson,
433
+ }),
434
+ content: chunk.content,
435
+ content_tokenized: "",
436
+ file_path: chunk.file_path,
437
+ file_type: chunk.file_type,
438
+ start_line: chunk.start_line,
439
+ end_line: chunk.end_line,
440
+ metadata_json: metadataJson,
441
+ };
442
+ }
443
+ async function writeLine(stream, line) {
444
+ if (stream.write(`${line}\n`))
445
+ return;
446
+ await new Promise((resolve, reject) => {
447
+ const cleanup = () => {
448
+ stream.off("drain", onDrain);
449
+ stream.off("error", onError);
450
+ };
451
+ const onDrain = () => {
452
+ cleanup();
453
+ resolve();
454
+ };
455
+ const onError = (error) => {
456
+ cleanup();
457
+ reject(error);
458
+ };
459
+ stream.once("drain", onDrain);
460
+ stream.once("error", onError);
461
+ });
462
+ }
463
+ async function finishWriteStream(stream) {
464
+ await new Promise((resolve, reject) => {
465
+ const cleanup = () => {
466
+ stream.off("finish", onFinish);
467
+ stream.off("error", onError);
468
+ };
469
+ const onFinish = () => {
470
+ cleanup();
471
+ resolve();
472
+ };
473
+ const onError = (error) => {
474
+ cleanup();
475
+ reject(error);
476
+ };
477
+ stream.once("finish", onFinish);
478
+ stream.once("error", onError);
479
+ stream.end();
480
+ });
481
+ }
350
482
  export class KnowledgeEngine {
351
483
  db = null;
352
484
  knowledgeDir = "";
485
+ activeUpdates = new Map();
486
+ activeMutations = new Map();
487
+ disposing = false;
488
+ runExclusive(keys, description, operation) {
489
+ if (this.disposing)
490
+ throw new Error("Knowledge engine is shutting down");
491
+ const mutationKeys = Array.isArray(keys) ? keys : [keys];
492
+ for (const key of mutationKeys) {
493
+ if (this.activeMutations.has(key))
494
+ throw new Error(`${description} is already running`);
495
+ }
496
+ const run = Promise.resolve()
497
+ .then(operation)
498
+ .finally(() => {
499
+ for (const key of mutationKeys) {
500
+ if (this.activeMutations.get(key) === run)
501
+ this.activeMutations.delete(key);
502
+ }
503
+ });
504
+ for (const key of mutationKeys)
505
+ this.activeMutations.set(key, run);
506
+ return run;
507
+ }
353
508
  async initialize(knowledgeDir) {
354
509
  this.knowledgeDir = knowledgeDir;
355
510
  this.db = openDatabase(knowledgeDir);
511
+ this.disposing = false;
512
+ }
513
+ vectorPathFor(kbId) {
514
+ return join(this.knowledgeDir, "vectors", `${kbId}.bin`);
515
+ }
516
+ deleteVectorFile(kbId) {
517
+ rmSync(this.vectorPathFor(kbId), { force: true });
356
518
  }
357
- plan(source, options = {}) {
519
+ plan(source, options = {}, signal) {
520
+ throwIfAborted(signal);
358
521
  const resolvedSource = resolve(source);
359
522
  const isUrl = source.startsWith("http://") || source.startsWith("https://");
360
523
  const isDir = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isDirectory();
361
524
  const isFile = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isFile();
362
525
  const sourceType = isUrl ? "url" : isDir ? "directory" : isFile ? "file" : "text";
363
526
  if (isDir) {
364
- const plan = planDirectoryScan(resolvedSource, toScanOptions(options));
527
+ const plan = planDirectoryScan(resolvedSource, toScanOptions(options), signal);
365
528
  return {
366
529
  source_type: "directory",
367
530
  scannable_files: plan.files,
@@ -404,8 +567,13 @@ export class KnowledgeEngine {
404
567
  };
405
568
  }
406
569
  async add(source, name, onProgress, signal, options = {}) {
570
+ throwIfAborted(signal);
571
+ return this.runExclusive(["global:mutation", `kb-name:${name}`], `Mutation for "${name}"`, () => this.addUnlocked(source, name, onProgress, signal, options));
572
+ }
573
+ async addUnlocked(source, name, onProgress, signal, options = {}) {
407
574
  if (!this.db)
408
575
  throw new Error("Engine not initialized");
576
+ throwIfAborted(signal);
409
577
  const db = this.db;
410
578
  const resolvedSource = resolve(source);
411
579
  const isUrl = source.startsWith("http://") || source.startsWith("https://");
@@ -428,7 +596,7 @@ export class KnowledgeEngine {
428
596
  let vectorWriter;
429
597
  let tempVectorFile;
430
598
  try {
431
- const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
599
+ const vectorPath = this.vectorPathFor(kb.id);
432
600
  tempVectorFile = tempVectorPath(vectorPath);
433
601
  const writer = openVectorWriter(tempVectorFile);
434
602
  vectorWriter = writer;
@@ -487,33 +655,26 @@ export class KnowledgeEngine {
487
655
  await flushPending(processedFiles, totalFiles);
488
656
  }
489
657
  };
658
+ const addSymbols = (content, filePath, fileType) => {
659
+ insertSymbols(db, kb.id, extractSymbols(content, filePath, fileType));
660
+ };
490
661
  if (isUrl) {
491
662
  const message = `Fetching ${source}...`;
492
663
  updateIndexingJob(this.db, kb.id, { phase: "fetching", message });
493
664
  onProgress?.(message);
494
665
  const chunks = await chunkUrl(source, signal);
495
666
  fileCount = 1;
667
+ addSymbols(chunks.map((chunk) => chunk.content).join("\n\n"), source, "html");
496
668
  await addChunks(chunks);
497
669
  }
498
- else if (isFile && resolvedSource.endsWith(".pdf")) {
499
- updateIndexingJob(this.db, kb.id, { phase: "extracting", message: "Extracting text from PDF..." });
500
- onProgress?.("Extracting text from PDF...");
501
- const { extractText } = await import("unpdf");
502
- const buf = (await import("node:fs")).readFileSync(resolvedSource);
503
- const { text } = await extractText(new Uint8Array(buf));
504
- fileCount = 1;
505
- await addChunks(await chunkFile(normalizeExtractedText(text), resolvedSource));
506
- }
507
- else if (isFile && (resolvedSource.endsWith(".docx") || resolvedSource.endsWith(".doc"))) {
508
- updateIndexingJob(this.db, kb.id, { phase: "extracting", message: "Extracting text from DOCX..." });
509
- onProgress?.("Extracting text from DOCX...");
510
- const mammoth = await import("mammoth");
511
- const result = await mammoth.extractRawText({ path: resolvedSource });
670
+ else if (isFile) {
671
+ const extracted = await extractSourceFileContent(resolvedSource, signal);
512
672
  fileCount = 1;
513
- await addChunks(await chunkFile(result.value, resolvedSource));
673
+ addSymbols(extracted.content, resolvedSource, extracted.fileType);
674
+ await addChunks(await chunkFile(extracted.content, resolvedSource));
514
675
  }
515
676
  else if (isDir) {
516
- const plan = planDirectoryScan(resolvedSource, scanOptions);
677
+ const plan = planDirectoryScan(resolvedSource, scanOptions, signal);
517
678
  const planningMessage = `Planned directory scan: ${plan.files} files, ${formatBytes(plan.bytes)} scannable text, skipped ${plan.skippedTotal} (${plan.skippedSummary})`;
518
679
  updateIndexingJob(this.db, kb.id, {
519
680
  phase: "planning",
@@ -540,6 +701,7 @@ export class KnowledgeEngine {
540
701
  latestSkippedTotal = skipped.total;
541
702
  if (chunks.length > 0)
542
703
  fileCount++;
704
+ addSymbols(file.content, file.relPath, file.fileType);
543
705
  await addChunks(chunks, processedFiles, plan.files);
544
706
  if (processedFiles % 25 === 0)
545
707
  reportProgress("Chunking", processedFiles, plan.files, skipped.total);
@@ -556,17 +718,9 @@ export class KnowledgeEngine {
556
718
  });
557
719
  onProgress?.(finalizingMessage);
558
720
  }
559
- else if (isFile) {
560
- if (!isReadableTextFile(resolvedSource)) {
561
- throw new Error(`File is not readable text and has no supported extractor: ${resolvedSource}`);
562
- }
563
- const { readFileSync } = await import("node:fs");
564
- const content = readFileSync(resolvedSource, "utf-8");
565
- fileCount = 1;
566
- await addChunks(await chunkFile(content, resolvedSource));
567
- }
568
721
  else {
569
722
  fileCount = 1;
723
+ addSymbols(source, "inline-text", "text");
570
724
  await addChunks(await chunkFile(source, "inline-text"));
571
725
  }
572
726
  await flushPending();
@@ -599,29 +753,51 @@ export class KnowledgeEngine {
599
753
  rmSync(tempVectorFile, { force: true });
600
754
  if (this.db) {
601
755
  finishIndexingJob(this.db, kb.id, isCancellationError(e) ? "cancelled" : "failed", isCancellationError(e) ? "Indexing cancelled." : "Indexing failed.", e instanceof Error ? e.message : String(e));
756
+ deleteKB(this.db, kb.id);
757
+ this.deleteVectorFile(kb.id);
602
758
  }
603
- // Clean up partial state on failure
604
- deleteKB(this.db, kb.id);
605
759
  throw e;
606
760
  }
607
761
  }
608
762
  async update(nameOrId, onProgress, signal) {
609
763
  if (!this.db)
610
764
  throw new Error("Engine not initialized");
765
+ if (this.disposing)
766
+ throw new Error("Knowledge engine is shutting down");
767
+ throwIfAborted(signal);
611
768
  const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
612
769
  if (!kb)
613
770
  throw new Error(`Knowledge base not found: ${nameOrId}`);
614
771
  if (!kb.source_path || (kb.source_type !== "url" && !existsSync(kb.source_path))) {
615
772
  throw new Error(`Source path not available or missing: ${kb.source_path}`);
616
773
  }
774
+ const updateableKB = { ...kb, source_path: kb.source_path };
775
+ const activeUpdate = this.activeUpdates.get(updateableKB.id);
776
+ if (activeUpdate) {
777
+ onProgress?.(`Update already running for "${updateableKB.name}"; waiting for the active update.`);
778
+ return activeUpdate;
779
+ }
780
+ const updateRun = this.runExclusive(["global:mutation", `kb:${updateableKB.id}`], `Mutation for "${updateableKB.name}"`, () => this.runUpdate(updateableKB, onProgress, signal)).finally(() => {
781
+ if (this.activeUpdates.get(updateableKB.id) === updateRun)
782
+ this.activeUpdates.delete(updateableKB.id);
783
+ });
784
+ this.activeUpdates.set(updateableKB.id, updateRun);
785
+ return updateRun;
786
+ }
787
+ async runUpdate(kb, onProgress, signal) {
788
+ if (!this.db)
789
+ throw new Error("Engine not initialized");
790
+ throwIfAborted(signal);
617
791
  updateKBStatus(this.db, kb.id, "indexing");
618
792
  startIndexingJob(this.db, kb.id, "update", `Starting update for "${kb.name}"`);
619
793
  const scanOptions = toScanOptions(parseAddOptions(kb.source_options));
620
794
  let replacementVectorPath;
621
795
  let addedVectorPath;
622
796
  let addedVectorWriter;
797
+ const insertedChunkIds = [];
798
+ const stagedSymbols = [];
623
799
  try {
624
- const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
800
+ const vectorPath = this.vectorPathFor(kb.id);
625
801
  addedVectorPath = tempVectorPath(`${vectorPath}.added`);
626
802
  addedVectorWriter = openVectorWriter(addedVectorPath);
627
803
  const existingHashes = new Map();
@@ -670,7 +846,7 @@ export class KnowledgeEngine {
670
846
  newVectorIndexByHash.set(batch[i].content_hash, indexes);
671
847
  }
672
848
  addedVectorCount += newVectors.length;
673
- insertChunks(this.db, kb.id, batch);
849
+ insertedChunkIds.push(...insertChunks(this.db, kb.id, batch));
674
850
  addedCount += batch.length;
675
851
  updateKBCounts(this.db, kb.id, getChunkCount(this.db, kb.id), getFileCount(this.db, kb.id));
676
852
  const storedMessage = `Stored update batch: +${addedCount} chunks, =${unchanged} unchanged`;
@@ -711,10 +887,12 @@ export class KnowledgeEngine {
711
887
  updateIndexingJob(this.db, kb.id, { phase: "fetching", message });
712
888
  onProgress?.(message);
713
889
  scannedFiles = 1;
714
- await processChunks(await chunkUrl(kb.source_path, signal));
890
+ const chunks = await chunkUrl(kb.source_path, signal);
891
+ stagedSymbols.push(...extractSymbols(chunks.map((chunk) => chunk.content).join("\n\n"), kb.source_path, "html"));
892
+ await processChunks(chunks);
715
893
  }
716
894
  else if (statSync(kb.source_path).isDirectory()) {
717
- const plan = planDirectoryScan(kb.source_path, scanOptions);
895
+ const plan = planDirectoryScan(kb.source_path, scanOptions, signal);
718
896
  plannedTotalFiles = plan.files;
719
897
  const planningMessage = `Planned directory scan: ${plan.files} files, ${formatBytes(plan.bytes)} scannable text, skipped ${plan.skippedTotal} (${plan.skippedSummary})`;
720
898
  updateIndexingJob(this.db, kb.id, {
@@ -729,6 +907,7 @@ export class KnowledgeEngine {
729
907
  if (signal?.aborted)
730
908
  throw new Error("Cancelled");
731
909
  scannedFiles++;
910
+ stagedSymbols.push(...extractSymbols(file.content, file.relPath, file.fileType));
732
911
  await processChunks(await chunkFile(file.content, file.relPath));
733
912
  if (scannedFiles % 25 === 0) {
734
913
  const message = `Scanned ${scannedFiles} files, ${scannedChunks} chunks, skipped ${skipped.total}, +${addedCount} =${unchanged}`;
@@ -759,9 +938,10 @@ export class KnowledgeEngine {
759
938
  onProgress?.(message);
760
939
  }
761
940
  else {
762
- const { readFileSync } = await import("node:fs");
941
+ const extracted = await extractSourceFileContent(kb.source_path, signal);
763
942
  scannedFiles = 1;
764
- await processChunks(await chunkFile(readFileSync(kb.source_path, "utf-8"), kb.source_path));
943
+ stagedSymbols.push(...extractSymbols(extracted.content, kb.source_path, extracted.fileType));
944
+ await processChunks(await chunkFile(extracted.content, kb.source_path));
765
945
  }
766
946
  await flushPending();
767
947
  addedVectorWriter.close();
@@ -770,8 +950,7 @@ export class KnowledgeEngine {
770
950
  for (const entries of existingHashes.values()) {
771
951
  idsToRemove.push(...entries.map((entry) => entry.id));
772
952
  }
773
- if (idsToRemove.length > 0)
774
- deleteChunksByIds(this.db, idsToRemove);
953
+ const idsToRemoveSet = new Set(idsToRemove);
775
954
  const changesMessage = `Changes: +${addedCount} -${idsToRemove.length} =${unchanged}`;
776
955
  updateIndexingJob(this.db, kb.id, {
777
956
  phase: "reconciling",
@@ -794,6 +973,10 @@ export class KnowledgeEngine {
794
973
  };
795
974
  try {
796
975
  for (const chunk of iterateChunksByKB(this.db, kb.id)) {
976
+ if (signal?.aborted)
977
+ throw new Error("Cancelled");
978
+ if (idsToRemoveSet.has(chunk.id))
979
+ continue;
797
980
  const oldVectorIndex = takeVectorIndex(oldVectorIndexByHash, chunk.content_hash);
798
981
  const newVectorIndex = takeVectorIndex(newVectorIndexByHash, chunk.content_hash);
799
982
  const vector = oldVectorReader && oldVectorIndex !== undefined
@@ -830,6 +1013,10 @@ export class KnowledgeEngine {
830
1013
  if (addedVectorPath)
831
1014
  rmSync(addedVectorPath, { force: true });
832
1015
  addedVectorPath = undefined;
1016
+ if (idsToRemove.length > 0)
1017
+ deleteChunksByIds(this.db, idsToRemove);
1018
+ deleteSymbolsByKB(this.db, kb.id);
1019
+ insertSymbols(this.db, kb.id, stagedSymbols);
833
1020
  updateKBCounts(this.db, kb.id, finalChunkCount, getFileCount(this.db, kb.id));
834
1021
  updateKBStatus(this.db, kb.id, "ready");
835
1022
  const readyMessage = `Ready: +${addedCount} -${idsToRemove.length} =${unchanged}`;
@@ -852,14 +1039,19 @@ export class KnowledgeEngine {
852
1039
  rmSync(replacementVectorPath, { force: true });
853
1040
  if (addedVectorPath)
854
1041
  rmSync(addedVectorPath, { force: true });
855
- updateKBStatus(this.db, kb.id, "error");
1042
+ if (insertedChunkIds.length > 0) {
1043
+ deleteChunksByIds(this.db, insertedChunkIds);
1044
+ updateKBCounts(this.db, kb.id, getChunkCount(this.db, kb.id), getFileCount(this.db, kb.id));
1045
+ }
1046
+ updateKBStatus(this.db, kb.id, isCancellationError(e) ? kb.status : "error");
856
1047
  finishIndexingJob(this.db, kb.id, isCancellationError(e) ? "cancelled" : "failed", isCancellationError(e) ? "Update cancelled." : "Update failed.", e instanceof Error ? e.message : String(e));
857
1048
  throw e;
858
1049
  }
859
1050
  }
860
- async search(query, options = {}) {
1051
+ async search(query, options = {}, signal) {
861
1052
  if (!this.db)
862
1053
  throw new Error("Engine not initialized");
1054
+ throwIfAborted(signal);
863
1055
  const requestedMode = options.mode ?? "hybrid";
864
1056
  if (requestedMode === "auto") {
865
1057
  const primaryMode = chooseAutoMode(query);
@@ -870,7 +1062,8 @@ export class KnowledgeEngine {
870
1062
  if (tried.includes(mode))
871
1063
  continue;
872
1064
  tried.push(mode);
873
- const response = await this.search(query, { ...options, mode });
1065
+ throwIfAborted(signal);
1066
+ const response = await this.search(query, { ...options, mode }, signal);
874
1067
  if (response.warnings)
875
1068
  warnings.push(...response.warnings);
876
1069
  if (!isWeakAutoResponse(query, response, primaryMode, mode)) {
@@ -899,7 +1092,8 @@ export class KnowledgeEngine {
899
1092
  }
900
1093
  const db = this.db;
901
1094
  const { mode = "hybrid", limit = 10, offset = 0, kb_id, filters, diversity = "balanced" } = options;
902
- const retrievalMode = mode === "adaptive" ? "hybrid" : mode;
1095
+ const resolvedMode = retrievalModeFor(mode);
1096
+ const retrievalMode = resolvedMode === "adaptive" ? "hybrid" : resolvedMode;
903
1097
  const candidateLimit = Math.max(50, offset + limit * 12);
904
1098
  const normalizedQuery = normalizedQueryText(query);
905
1099
  const queryTokens = tokenizeForSimilarity(query);
@@ -908,6 +1102,7 @@ export class KnowledgeEngine {
908
1102
  const selectedKB = kb_id ? (getKB(db, kb_id) ?? getKBByName(db, kb_id)) : undefined;
909
1103
  const availableKBs = kb_id ? [selectedKB].filter(Boolean) : listKBs(db);
910
1104
  const kbs = availableKBs.filter((kb) => kb.status === "ready" || kb.status === "stale");
1105
+ const kbById = new Map(kbs.map((kb) => [kb.id, kb]));
911
1106
  for (const kb of availableKBs) {
912
1107
  if (kb.status !== "ready" && kb.status !== "stale") {
913
1108
  warnings.push(`"${kb.name}" is ${kb.status}; search skipped until indexing is ready`);
@@ -925,19 +1120,24 @@ export class KnowledgeEngine {
925
1120
  const allResults = [];
926
1121
  const vectorsByChunkId = new Map();
927
1122
  for (const kb of kbs) {
1123
+ throwIfAborted(signal);
928
1124
  if (kb.embedding_model !== CURRENT_EMBEDDING_MODEL) {
929
1125
  warnings.push(`"${kb.name}" was indexed with ${kb.embedding_model} (current: ${CURRENT_EMBEDDING_MODEL}) — run knowledge_update for best results`);
930
1126
  }
931
1127
  if (kb.chunk_count === 0)
932
1128
  continue;
933
- const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
1129
+ const vectorPath = this.vectorPathFor(kb.id);
934
1130
  if (retrievalMode === "fast") {
935
- allResults.push(...searchBM25(db, normalizedQuery || query, candidateLimit, kb.id, { allowOrFallback: false }));
1131
+ allResults.push(...searchBM25(db, normalizedQuery || query, candidateLimit, kb.id, { allowOrFallback: false }).map((result) => ({
1132
+ ...result,
1133
+ score: result.score * kbTrustMultiplier(kb),
1134
+ })));
936
1135
  }
937
1136
  else if (retrievalMode === "semantic") {
938
- const queryVec = await embedQuery(query);
939
- const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), candidateLimit);
940
- allResults.push(...vectorResults.results);
1137
+ const queryVec = await embedQuery(query, signal);
1138
+ throwIfAborted(signal);
1139
+ const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), candidateLimit, signal);
1140
+ allResults.push(...vectorResults.results.map((result) => ({ ...result, score: result.score * kbTrustMultiplier(kb) })));
941
1141
  for (const [chunkId, vector] of vectorResults.vectorsByChunkId)
942
1142
  vectorsByChunkId.set(chunkId, vector);
943
1143
  }
@@ -947,13 +1147,14 @@ export class KnowledgeEngine {
947
1147
  if (bm25Results.length === 0)
948
1148
  continue;
949
1149
  let vecResults = [];
950
- const queryVec = await embedQuery(query);
951
- const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), candidateLimit);
1150
+ const queryVec = await embedQuery(query, signal);
1151
+ throwIfAborted(signal);
1152
+ const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), candidateLimit, signal);
952
1153
  vecResults = vectorResults.results;
953
1154
  for (const [chunkId, vector] of vectorResults.vectorsByChunkId)
954
1155
  vectorsByChunkId.set(chunkId, vector);
955
1156
  const fused = weightedScoreFusion(bm25Results, vecResults);
956
- allResults.push(...fused);
1157
+ allResults.push(...fused.map((result) => ({ ...result, score: result.score * kbTrustMultiplier(kb) })));
957
1158
  }
958
1159
  }
959
1160
  // Deduplicate and sort
@@ -966,6 +1167,7 @@ export class KnowledgeEngine {
966
1167
  });
967
1168
  const rankingByChunkId = new Map();
968
1169
  for (const result of unique) {
1170
+ throwIfAborted(signal);
969
1171
  const chunk = getChunkById(db, result.chunkId);
970
1172
  if (chunk) {
971
1173
  const ranking = scoreChunkForQuery(result.score, chunk, queryTokens);
@@ -1013,9 +1215,11 @@ export class KnowledgeEngine {
1013
1215
  return chunk ? { chunkId: r.chunkId, content: chunk.content } : null;
1014
1216
  })
1015
1217
  .filter(Boolean);
1016
- const reranked = await rerank(query, candidates, Math.max(limit * 3, limit));
1218
+ const reranked = await rerank(query, candidates, Math.max(limit * 3, limit), signal);
1219
+ throwIfAborted(signal);
1017
1220
  const ranked = [];
1018
1221
  for (const r of reranked) {
1222
+ throwIfAborted(signal);
1019
1223
  const chunk = getChunkById(db, r.chunkId);
1020
1224
  if (!chunk)
1021
1225
  continue;
@@ -1034,17 +1238,30 @@ export class KnowledgeEngine {
1034
1238
  }
1035
1239
  const diversified = interleaveByFile(diversifyRankedChunks(ranked, diversity, vectorsByChunkId), diversity);
1036
1240
  const page = diversified.slice(offset, offset + limit);
1037
- const results = page.map((r) => ({
1038
- content: r.content,
1039
- file_path: r.chunk.file_path,
1040
- file_type: r.chunk.file_type,
1041
- kb_name: r.kbName,
1042
- score: r.score,
1043
- ranking: r.ranking,
1044
- snippet: r.snippet,
1045
- start_line: r.startLine,
1046
- end_line: r.endLine,
1047
- }));
1241
+ const results = page.map((r) => {
1242
+ const kb = kbById.get(r.chunk.kb_id);
1243
+ const sourceMtime = sourceMtimeFor(kb, r.chunk.file_path);
1244
+ return {
1245
+ content: r.content,
1246
+ file_path: r.chunk.file_path,
1247
+ file_type: r.chunk.file_type,
1248
+ kb_name: r.kbName,
1249
+ score: r.score,
1250
+ ranking: r.ranking,
1251
+ snippet: r.snippet,
1252
+ start_line: r.startLine,
1253
+ end_line: r.endLine,
1254
+ provenance: {
1255
+ chunk_id: r.chunk.id,
1256
+ chunk_hash: r.chunk.content_hash,
1257
+ indexed_at: r.chunk.indexed_at,
1258
+ source_mtime: sourceMtime,
1259
+ stale: sourceMtime !== undefined ? sourceMtime > r.chunk.indexed_at : kb?.status === "stale",
1260
+ match_reason: matchReasonFor(mode),
1261
+ source_chunk_ids: r.sourceChunkIds,
1262
+ },
1263
+ };
1264
+ });
1048
1265
  return {
1049
1266
  results,
1050
1267
  total_count: diversified.length,
@@ -1055,6 +1272,7 @@ export class KnowledgeEngine {
1055
1272
  }
1056
1273
  const ranked = [];
1057
1274
  for (const r of filtered) {
1275
+ throwIfAborted(signal);
1058
1276
  const chunk = getChunkById(db, r.chunkId);
1059
1277
  if (!chunk)
1060
1278
  continue;
@@ -1101,6 +1319,19 @@ export class KnowledgeEngine {
1101
1319
  snippet: r.snippet,
1102
1320
  start_line: r.startLine,
1103
1321
  end_line: r.endLine,
1322
+ provenance: (() => {
1323
+ const kb = kbById.get(r.chunk.kb_id);
1324
+ const sourceMtime = sourceMtimeFor(kb, r.chunk.file_path);
1325
+ return {
1326
+ chunk_id: r.chunk.id,
1327
+ chunk_hash: r.chunk.content_hash,
1328
+ indexed_at: r.chunk.indexed_at,
1329
+ source_mtime: sourceMtime,
1330
+ stale: sourceMtime !== undefined ? sourceMtime > r.chunk.indexed_at : kb?.status === "stale",
1331
+ match_reason: matchReasonFor(mode),
1332
+ source_chunk_ids: r.sourceChunkIds,
1333
+ };
1334
+ })(),
1104
1335
  }));
1105
1336
  return {
1106
1337
  results,
@@ -1119,31 +1350,94 @@ export class KnowledgeEngine {
1119
1350
  remove(nameOrId) {
1120
1351
  if (!this.db)
1121
1352
  return false;
1353
+ if (this.disposing)
1354
+ throw new Error("Knowledge engine is shutting down");
1355
+ if (this.activeMutations.size > 0)
1356
+ throw new Error("A knowledge-base mutation is already running");
1122
1357
  const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
1123
1358
  if (!kb)
1124
1359
  return false;
1125
1360
  deleteKB(this.db, kb.id);
1361
+ this.deleteVectorFile(kb.id);
1126
1362
  return true;
1127
1363
  }
1128
- list() {
1364
+ list(signal) {
1129
1365
  if (!this.db)
1130
1366
  return [];
1367
+ throwIfAborted(signal);
1131
1368
  return listKBs(this.db);
1132
1369
  }
1133
1370
  clear() {
1134
1371
  if (!this.db)
1135
1372
  return;
1136
- for (const kb of listKBs(this.db))
1373
+ if (this.disposing)
1374
+ throw new Error("Knowledge engine is shutting down");
1375
+ if (this.activeMutations.size > 0)
1376
+ throw new Error("A knowledge-base mutation is already running");
1377
+ for (const kb of listKBs(this.db)) {
1137
1378
  deleteKB(this.db, kb.id);
1379
+ this.deleteVectorFile(kb.id);
1380
+ }
1138
1381
  }
1139
- diagnose() {
1382
+ diagnose(signal) {
1140
1383
  if (!this.db)
1141
1384
  return [];
1385
+ throwIfAborted(signal);
1386
+ const db = this.db;
1387
+ return listKBs(db).map((kb) => {
1388
+ throwIfAborted(signal);
1389
+ return diagnoseKB(db, kb, signal);
1390
+ });
1391
+ }
1392
+ symbolSearch(query, options = {}, signal) {
1393
+ if (!this.db)
1394
+ throw new Error("Engine not initialized");
1395
+ throwIfAborted(signal);
1142
1396
  const db = this.db;
1143
- return listKBs(db).map((kb) => diagnoseKB(db, kb));
1397
+ const selectedKB = options.kb_id ? (getKB(db, options.kb_id) ?? getKBByName(db, options.kb_id)) : undefined;
1398
+ if (options.kb_id && !selectedKB)
1399
+ throw new Error(`Knowledge base not found: ${options.kb_id}`);
1400
+ const limit = Math.max(1, Math.min(200, options.limit ?? 20));
1401
+ const offset = Math.max(0, options.offset ?? 0);
1402
+ const symbolOptions = {
1403
+ kbId: selectedKB?.id,
1404
+ kind: options.kind,
1405
+ filePattern: options.file_pattern,
1406
+ exact: options.exact,
1407
+ };
1408
+ const symbols = searchSymbols(db, query, {
1409
+ ...symbolOptions,
1410
+ limit: limit + 1,
1411
+ offset,
1412
+ });
1413
+ throwIfAborted(signal);
1414
+ const page = symbols.slice(0, limit);
1415
+ const total = countSymbols(db, query, symbolOptions);
1416
+ throwIfAborted(signal);
1417
+ return {
1418
+ results: page.map((symbol) => {
1419
+ const kb = getKB(db, symbol.kb_id);
1420
+ return {
1421
+ name: symbol.name,
1422
+ kind: symbol.kind,
1423
+ file_path: symbol.file_path,
1424
+ file_type: symbol.file_type,
1425
+ kb_name: kb?.name ?? "unknown",
1426
+ start_line: symbol.start_line,
1427
+ end_line: symbol.end_line,
1428
+ signature: symbol.signature ?? undefined,
1429
+ container_name: symbol.container_name ?? undefined,
1430
+ text: symbol.text,
1431
+ indexed_at: symbol.indexed_at,
1432
+ };
1433
+ }),
1434
+ total_count: total,
1435
+ has_more: symbols.length > limit,
1436
+ };
1144
1437
  }
1145
- doctor() {
1146
- const diagnostics = this.diagnose();
1438
+ doctor(signal) {
1439
+ throwIfAborted(signal);
1440
+ const diagnostics = this.diagnose(signal);
1147
1441
  const issues = [];
1148
1442
  const kbs = this.list();
1149
1443
  if (kbs.length === 0) {
@@ -1151,9 +1445,11 @@ export class KnowledgeEngine {
1151
1445
  severity: "blocking",
1152
1446
  message: "No knowledge bases are indexed.",
1153
1447
  action: "Run knowledge_add for the project root or relevant source/docs directory.",
1448
+ action_code: "rebuild_kb",
1154
1449
  });
1155
1450
  }
1156
1451
  for (const diagnostic of diagnostics) {
1452
+ throwIfAborted(signal);
1157
1453
  if (diagnostic.stuck_indexing) {
1158
1454
  const phase = diagnostic.job ? ` during ${diagnostic.job.phase}` : "";
1159
1455
  issues.push({
@@ -1161,6 +1457,7 @@ export class KnowledgeEngine {
1161
1457
  kb_name: diagnostic.kb_name,
1162
1458
  message: `Indexing appears stuck${phase} for ${formatDuration(diagnostic.last_progress_age_ms)}.`,
1163
1459
  action: "Check knowledge_status for the last progress message. If no Pi process is actively indexing it, remove and rebuild this KB.",
1460
+ action_code: "wait_for_indexing",
1164
1461
  });
1165
1462
  }
1166
1463
  if (diagnostic.status === "error") {
@@ -1169,6 +1466,7 @@ export class KnowledgeEngine {
1169
1466
  kb_name: diagnostic.kb_name,
1170
1467
  message: "Knowledge base is in error state and is skipped by search.",
1171
1468
  action: "Run knowledge_remove and knowledge_add to rebuild it from the source.",
1469
+ action_code: "rebuild_kb",
1172
1470
  });
1173
1471
  }
1174
1472
  if (diagnostic.coverage_percent < 80) {
@@ -1177,6 +1475,7 @@ export class KnowledgeEngine {
1177
1475
  kb_name: diagnostic.kb_name,
1178
1476
  message: `Coverage is ${diagnostic.coverage_percent}% (${diagnostic.indexed_files}/${diagnostic.total_source_files} files).`,
1179
1477
  action: "Review skipped files and source path. Rebuild if index-time rules changed.",
1478
+ action_code: "review_skipped_scope",
1180
1479
  });
1181
1480
  }
1182
1481
  if (diagnostic.stale_files.length > 0) {
@@ -1185,6 +1484,7 @@ export class KnowledgeEngine {
1185
1484
  kb_name: diagnostic.kb_name,
1186
1485
  message: `${diagnostic.stale_files.length} files changed after indexing.`,
1187
1486
  action: "Run knowledge_update for this KB.",
1487
+ action_code: "run_update",
1188
1488
  });
1189
1489
  }
1190
1490
  if (diagnostic.orphan_files.length > 0) {
@@ -1193,6 +1493,7 @@ export class KnowledgeEngine {
1193
1493
  kb_name: diagnostic.kb_name,
1194
1494
  message: `${diagnostic.orphan_files.length} indexed files no longer exist in the source.`,
1195
1495
  action: "Run knowledge_update or rebuild the KB.",
1496
+ action_code: "run_update",
1196
1497
  });
1197
1498
  }
1198
1499
  if (diagnostic.skipped_files.total > 0) {
@@ -1204,9 +1505,24 @@ export class KnowledgeEngine {
1204
1505
  .map(([reason, count]) => `${reason}: ${count}`)
1205
1506
  .join(", ")}).`,
1206
1507
  action: "Use skipped samples to confirm exclusions are expected. Adjust source path or ignore rules if needed.",
1508
+ action_code: "review_skipped_scope",
1207
1509
  });
1208
1510
  }
1209
1511
  }
1512
+ if (this.db) {
1513
+ for (const kb of kbs) {
1514
+ throwIfAborted(signal);
1515
+ if (kb.status === "ready" && kb.chunk_count > 0 && getSymbolCount(this.db, kb.id) === 0) {
1516
+ issues.push({
1517
+ severity: "info",
1518
+ kb_name: kb.name,
1519
+ message: "No lightweight symbols are indexed for this KB.",
1520
+ action: "Run knowledge_update to rebuild symbol/config/heading lookup metadata.",
1521
+ action_code: "run_update",
1522
+ });
1523
+ }
1524
+ }
1525
+ }
1210
1526
  const penalty = issues.reduce((score, issue) => {
1211
1527
  if (issue.severity === "blocking")
1212
1528
  return score + 35;
@@ -1225,135 +1541,193 @@ export class KnowledgeEngine {
1225
1541
  summary,
1226
1542
  issues,
1227
1543
  diagnostics,
1544
+ actions: issues.map((issue) => ({
1545
+ code: issue.action_code ?? "none",
1546
+ kb_name: issue.kb_name,
1547
+ target: issue.kb_name,
1548
+ description: issue.action,
1549
+ })),
1228
1550
  };
1229
1551
  }
1230
- async exportKB(nameOrId, outputPath) {
1552
+ async exportKB(nameOrId, outputPath, signal, onProgress) {
1231
1553
  if (!this.db)
1232
1554
  throw new Error("Engine not initialized");
1555
+ throwIfAborted(signal);
1233
1556
  const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
1234
1557
  if (!kb)
1235
1558
  throw new Error(`Knowledge base not found: ${nameOrId}`);
1236
- const chunks = getChunksByKB(this.db, kb.id);
1237
- const { writeFileSync } = await import("node:fs");
1559
+ return this.runExclusive(["global:mutation", `kb:${kb.id}`], `Operation for "${kb.name}"`, () => this.exportKBUnlocked(kb, outputPath, signal, onProgress));
1560
+ }
1561
+ async exportKBUnlocked(kb, outputPath, signal, onProgress) {
1562
+ if (!this.db)
1563
+ throw new Error("Engine not initialized");
1564
+ const { createWriteStream } = await import("node:fs");
1565
+ const tempOutputPath = tempVectorPath(outputPath);
1566
+ const stream = createWriteStream(tempOutputPath, { encoding: "utf-8" });
1567
+ let count = 0;
1238
1568
  const header = JSON.stringify({
1239
1569
  name: kb.name,
1240
1570
  description: kb.description,
1241
1571
  source_type: "text",
1242
- chunk_count: chunks.length,
1572
+ chunk_count: kb.chunk_count,
1243
1573
  });
1244
- const lines = [
1245
- header,
1246
- ...chunks.map((c) => JSON.stringify({
1247
- content: c.content,
1248
- file_path: c.file_path,
1249
- file_type: c.file_type,
1250
- start_line: c.start_line,
1251
- end_line: c.end_line,
1252
- metadata_json: c.metadata_json,
1253
- })),
1254
- ];
1255
- writeFileSync(outputPath, `${lines.join("\n")}\n`);
1256
- return chunks.length;
1574
+ try {
1575
+ await writeLine(stream, header);
1576
+ for (const chunk of iterateChunksByKB(this.db, kb.id)) {
1577
+ throwIfAborted(signal);
1578
+ await writeLine(stream, JSON.stringify({
1579
+ content: chunk.content,
1580
+ file_path: chunk.file_path,
1581
+ file_type: chunk.file_type,
1582
+ start_line: chunk.start_line,
1583
+ end_line: chunk.end_line,
1584
+ metadata_json: chunk.metadata_json,
1585
+ }));
1586
+ count++;
1587
+ if (count % INDEX_EMBED_BATCH_SIZE === 0)
1588
+ onProgress?.(`Exported ${count}/${kb.chunk_count} chunks...`);
1589
+ }
1590
+ await finishWriteStream(stream);
1591
+ throwIfAborted(signal);
1592
+ renameSync(tempOutputPath, outputPath);
1593
+ onProgress?.(`Exported ${count}/${kb.chunk_count} chunks.`);
1594
+ return count;
1595
+ }
1596
+ catch (error) {
1597
+ stream.destroy();
1598
+ rmSync(tempOutputPath, { force: true });
1599
+ throw error;
1600
+ }
1257
1601
  }
1258
1602
  async importKB(inputPath, onProgress, signal) {
1259
1603
  if (!this.db)
1260
1604
  throw new Error("Engine not initialized");
1261
- const { readFileSync } = await import("node:fs");
1262
- const lines = readFileSync(inputPath, "utf-8").trim().split("\n");
1263
- if (lines.length < 1)
1264
- throw new Error("Empty import file");
1265
- const header = JSON.parse(lines[0]);
1266
- const kb = createKB(this.db, { name: header.name, description: header.description, source_type: "text" });
1267
- updateKBStatus(this.db, kb.id, "indexing");
1268
- startIndexingJob(this.db, kb.id, "import", `Starting import for "${header.name}"`);
1605
+ return this.runExclusive("global:mutation", "Knowledge-base import", () => this.importKBUnlocked(inputPath, onProgress, signal));
1606
+ }
1607
+ async importKBUnlocked(inputPath, onProgress, signal) {
1608
+ if (!this.db)
1609
+ throw new Error("Engine not initialized");
1610
+ throwIfAborted(signal);
1611
+ const { createReadStream } = await import("node:fs");
1612
+ const { createInterface } = await import("node:readline");
1613
+ const stream = createReadStream(inputPath, { encoding: "utf-8" });
1614
+ const lines = createInterface({ input: stream, crlfDelay: Infinity });
1615
+ let header;
1616
+ let kb;
1269
1617
  let vectorWriter;
1270
1618
  let tempVectorFile;
1271
- try {
1272
- const chunkData = lines.slice(1).map((l) => JSON.parse(l));
1273
- const importMessage = `Importing ${chunkData.length} chunks...`;
1619
+ let inserted = 0;
1620
+ let lineNumber = 0;
1621
+ const pendingChunks = [];
1622
+ const pendingSymbols = [];
1623
+ const flushPending = async () => {
1624
+ if (!this.db || !kb || !vectorWriter || pendingChunks.length === 0)
1625
+ return;
1626
+ throwIfAborted(signal);
1627
+ const batch = pendingChunks.splice(0, INDEX_EMBED_BATCH_SIZE);
1628
+ const symbols = pendingSymbols.splice(0);
1629
+ const total = header?.chunk_count;
1630
+ const message = `Embedding import batch: ${inserted}/${total ?? "unknown"} chunks stored`;
1274
1631
  updateIndexingJob(this.db, kb.id, {
1275
- phase: "importing",
1276
- message: importMessage,
1277
- total_files: chunkData.length,
1632
+ phase: "embedding",
1633
+ message,
1634
+ processed_chunks: inserted,
1635
+ total_files: total,
1636
+ added_chunks: inserted,
1278
1637
  });
1279
- onProgress?.(importMessage);
1280
- const chunks = chunkData.map((c) => ({
1281
- content_hash: chunkIdentityHash({
1282
- content: c.content,
1283
- filePath: c.file_path,
1284
- fileType: c.file_type,
1285
- startLine: c.start_line,
1286
- endLine: c.end_line,
1287
- metadataJson: c.metadata_json || "{}",
1288
- }),
1289
- content: c.content,
1290
- content_tokenized: "",
1291
- file_path: c.file_path,
1292
- file_type: c.file_type,
1293
- start_line: c.start_line,
1294
- end_line: c.end_line,
1295
- metadata_json: c.metadata_json || "{}",
1296
- }));
1297
- const indexedChunks = chunks.map((chunk) => ({
1298
- ...chunk,
1299
- content_tokenized: preTokenizeForFTS(buildChunkEmbeddingText(chunk)),
1300
- }));
1301
- const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
1302
- tempVectorFile = tempVectorPath(vectorPath);
1303
- vectorWriter = openVectorWriter(tempVectorFile);
1304
- let inserted = 0;
1305
- for (let offset = 0; offset < indexedChunks.length; offset += INDEX_EMBED_BATCH_SIZE) {
1306
- if (signal?.aborted)
1307
- throw new Error("Cancelled");
1308
- const batch = indexedChunks.slice(offset, offset + INDEX_EMBED_BATCH_SIZE);
1309
- const message = `Embedding import batch ${Math.floor(offset / INDEX_EMBED_BATCH_SIZE) + 1}/${Math.ceil(indexedChunks.length / INDEX_EMBED_BATCH_SIZE)}: ${offset}/${indexedChunks.length} chunks`;
1310
- updateIndexingJob(this.db, kb.id, {
1311
- phase: "embedding",
1312
- message,
1313
- processed_chunks: offset,
1314
- total_files: indexedChunks.length,
1315
- added_chunks: inserted,
1316
- });
1317
- onProgress?.(message);
1318
- const vectors = await embedDocuments(batch.map((c) => buildChunkEmbeddingText(c)), signal);
1319
- if (signal?.aborted)
1320
- throw new Error("Cancelled");
1321
- insertChunks(this.db, kb.id, batch);
1322
- vectorWriter.append(vectors);
1323
- inserted += batch.length;
1324
- updateKBCounts(this.db, kb.id, inserted, new Set(indexedChunks.slice(0, inserted).map((c) => c.file_path)).size);
1325
- updateIndexingJob(this.db, kb.id, {
1326
- phase: "storing",
1327
- message: `Stored import batch: ${inserted}/${indexedChunks.length} chunks`,
1328
- processed_chunks: inserted,
1329
- total_files: indexedChunks.length,
1330
- added_chunks: inserted,
1331
- });
1638
+ onProgress?.(message);
1639
+ const vectors = await embedDocuments(batch.map((chunk) => buildChunkEmbeddingText(chunk)), signal);
1640
+ throwIfAborted(signal);
1641
+ insertChunks(this.db, kb.id, batch);
1642
+ insertSymbols(this.db, kb.id, symbols);
1643
+ vectorWriter.append(vectors);
1644
+ inserted += batch.length;
1645
+ updateKBCounts(this.db, kb.id, inserted, getFileCount(this.db, kb.id));
1646
+ updateIndexingJob(this.db, kb.id, {
1647
+ phase: "storing",
1648
+ message: `Stored import batch: ${inserted}/${total ?? "unknown"} chunks`,
1649
+ processed_chunks: inserted,
1650
+ total_files: total,
1651
+ added_chunks: inserted,
1652
+ });
1653
+ };
1654
+ try {
1655
+ for await (const rawLine of lines) {
1656
+ throwIfAborted(signal);
1657
+ lineNumber++;
1658
+ const line = rawLine.trim();
1659
+ if (!line)
1660
+ continue;
1661
+ if (!header) {
1662
+ header = JSON.parse(line);
1663
+ if (!header.name)
1664
+ throw new Error("Import header is missing a knowledge base name");
1665
+ const existingKB = getKBByName(this.db, header.name);
1666
+ if (existingKB) {
1667
+ throw new Error(`Knowledge base "${header.name}" already exists. Use knowledge_update to refresh it, or knowledge_remove before importing a replacement.`);
1668
+ }
1669
+ kb = createKB(this.db, { name: header.name, description: header.description, source_type: "text" });
1670
+ updateKBStatus(this.db, kb.id, "indexing");
1671
+ startIndexingJob(this.db, kb.id, "import", `Starting import for "${header.name}"`);
1672
+ const importMessage = `Importing ${header.chunk_count ?? "unknown"} chunks...`;
1673
+ updateIndexingJob(this.db, kb.id, {
1674
+ phase: "importing",
1675
+ message: importMessage,
1676
+ total_files: header.chunk_count,
1677
+ });
1678
+ onProgress?.(importMessage);
1679
+ const vectorPath = this.vectorPathFor(kb.id);
1680
+ tempVectorFile = tempVectorPath(vectorPath);
1681
+ vectorWriter = openVectorWriter(tempVectorFile);
1682
+ continue;
1683
+ }
1684
+ const imported = JSON.parse(line);
1685
+ const chunk = importedChunkToInsert(imported);
1686
+ chunk.content_tokenized = preTokenizeForFTS(buildChunkEmbeddingText(chunk));
1687
+ pendingChunks.push(chunk);
1688
+ pendingSymbols.push(...extractSymbols(imported.content, imported.file_path, imported.file_type));
1689
+ if (pendingChunks.length >= INDEX_EMBED_BATCH_SIZE)
1690
+ await flushPending();
1332
1691
  }
1692
+ if (!header)
1693
+ throw new Error("Empty import file");
1694
+ if (!kb || !vectorWriter || !tempVectorFile)
1695
+ throw new Error("Import did not create a knowledge base");
1696
+ await flushPending();
1333
1697
  vectorWriter.close();
1334
1698
  vectorWriter = undefined;
1335
- renameSync(tempVectorFile, vectorPath);
1699
+ renameSync(tempVectorFile, this.vectorPathFor(kb.id));
1336
1700
  tempVectorFile = undefined;
1337
- updateKBCounts(this.db, kb.id, indexedChunks.length, new Set(indexedChunks.map((c) => c.file_path)).size);
1701
+ updateKBCounts(this.db, kb.id, inserted, getFileCount(this.db, kb.id));
1338
1702
  updateKBStatus(this.db, kb.id, "ready");
1339
1703
  const savedKB = getKB(this.db, kb.id);
1340
1704
  if (!savedKB)
1341
1705
  throw new Error(`Knowledge base disappeared after import: ${kb.id}`);
1342
- finishIndexingJob(this.db, kb.id, "succeeded", `Ready: imported ${indexedChunks.length} chunks`);
1343
- return { kb: savedKB, chunkCount: chunks.length };
1706
+ finishIndexingJob(this.db, kb.id, "succeeded", `Ready: imported ${inserted} chunks`);
1707
+ return { kb: savedKB, chunkCount: inserted };
1344
1708
  }
1345
- catch (e) {
1709
+ catch (error) {
1710
+ lines.close();
1711
+ stream.destroy();
1346
1712
  vectorWriter?.close();
1347
1713
  if (tempVectorFile)
1348
1714
  rmSync(tempVectorFile, { force: true });
1349
- if (this.db) {
1350
- finishIndexingJob(this.db, kb.id, isCancellationError(e) ? "cancelled" : "failed", isCancellationError(e) ? "Import cancelled." : "Import failed.", e instanceof Error ? e.message : String(e));
1715
+ if (this.db && kb) {
1716
+ finishIndexingJob(this.db, kb.id, isCancellationError(error) ? "cancelled" : "failed", isCancellationError(error) ? "Import cancelled." : `Import failed near line ${lineNumber}.`, error instanceof Error ? error.message : String(error));
1717
+ deleteKB(this.db, kb.id);
1718
+ this.deleteVectorFile(kb.id);
1351
1719
  }
1352
- deleteKB(this.db, kb.id);
1353
- throw e;
1720
+ throw error;
1354
1721
  }
1355
1722
  }
1356
1723
  async dispose(options = {}) {
1724
+ this.disposing = true;
1725
+ const activeUpdates = [...this.activeUpdates.values()];
1726
+ if (activeUpdates.length > 0)
1727
+ await Promise.allSettled(activeUpdates);
1728
+ const activeMutations = [...this.activeMutations.values()];
1729
+ if (activeMutations.length > 0)
1730
+ await Promise.allSettled(activeMutations);
1357
1731
  const disposeModels = options.disposeModels ?? true;
1358
1732
  if (disposeModels) {
1359
1733
  await disposeEmbedding();