pi-knowledge 0.15.0 → 0.16.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/CHANGELOG.md +29 -0
  2. package/README.md +61 -23
  3. package/dist/index.d.ts +10 -4
  4. package/dist/index.js +145 -166
  5. package/dist/src/coverage-snapshot.d.ts +23 -0
  6. package/dist/src/coverage-snapshot.js +53 -0
  7. package/dist/src/daemon/client.d.ts +60 -0
  8. package/dist/src/daemon/client.js +360 -0
  9. package/dist/src/daemon/main.d.ts +2 -0
  10. package/dist/src/daemon/main.js +109 -0
  11. package/dist/src/daemon/protocol.d.ts +71 -0
  12. package/dist/src/daemon/protocol.js +170 -0
  13. package/dist/src/daemon/runtime.d.ts +26 -0
  14. package/dist/src/daemon/runtime.js +192 -0
  15. package/dist/src/daemon/server.d.ts +46 -0
  16. package/dist/src/daemon/server.js +500 -0
  17. package/dist/src/daemon/service.d.ts +49 -0
  18. package/dist/src/daemon/service.js +527 -0
  19. package/dist/src/diagnostics/health.d.ts +6 -0
  20. package/dist/src/diagnostics/health.js +135 -2
  21. package/dist/src/engine.d.ts +13 -8
  22. package/dist/src/engine.js +569 -100
  23. package/dist/src/filesystem-access.d.ts +9 -0
  24. package/dist/src/filesystem-access.js +100 -0
  25. package/dist/src/indexer/chunker.d.ts +12 -0
  26. package/dist/src/indexer/chunker.js +307 -9
  27. package/dist/src/mcp/http.d.ts +47 -0
  28. package/dist/src/mcp/http.js +564 -0
  29. package/dist/src/mcp/main.d.ts +2 -0
  30. package/dist/src/mcp/main.js +97 -0
  31. package/dist/src/mcp/protocol.d.ts +81 -0
  32. package/dist/src/mcp/protocol.js +444 -0
  33. package/dist/src/mcp/server.d.ts +42 -0
  34. package/dist/src/mcp/server.js +378 -0
  35. package/dist/src/model-worker-client.js +9 -0
  36. package/dist/src/model-worker-contract.d.ts +3 -0
  37. package/dist/src/model-worker-contract.js +8 -0
  38. package/dist/src/model-worker.js +76 -4
  39. package/dist/src/network/safe-fetch.d.ts +7 -0
  40. package/dist/src/network/safe-fetch.js +199 -0
  41. package/dist/src/node-runtime.d.ts +3 -0
  42. package/dist/src/node-runtime.js +78 -0
  43. package/dist/src/profiles.d.ts +15 -0
  44. package/dist/src/profiles.js +160 -0
  45. package/dist/src/runtime-config.js +5 -6
  46. package/dist/src/secure-files.d.ts +17 -0
  47. package/dist/src/secure-files.js +183 -0
  48. package/dist/src/status.js +12 -7
  49. package/dist/src/storage/sqlite.d.ts +4 -1
  50. package/dist/src/storage/sqlite.js +46 -3
  51. package/dist/src/type-guards.d.ts +1 -0
  52. package/dist/src/type-guards.js +3 -0
  53. package/dist/src/watcher/file-watcher.d.ts +3 -1
  54. package/dist/src/watcher/file-watcher.js +166 -32
  55. package/dist/src/windows-security.d.ts +4 -0
  56. package/dist/src/windows-security.js +67 -0
  57. package/docs/configuration.md +40 -9
  58. package/docs/daemon-profiles.md +302 -0
  59. package/docs/mcp-clients.md +266 -0
  60. package/docs/offline-mode.md +6 -6
  61. package/package.json +28 -2
@@ -1,13 +1,18 @@
1
- import { existsSync, readFileSync, renameSync, rmSync, statSync } from "node:fs";
2
- import { join, resolve } from "node:path";
3
- import { diagnoseKB } from "./diagnostics/health.js";
1
+ import { closeSync, constants, createReadStream, createWriteStream, existsSync, fstatSync, fsyncSync, lstatSync, openSync, renameSync, rmSync, statSync, } from "node:fs";
2
+ import { lstat as lstatFile, mkdtemp, open as openFile, realpath as realpathFile, rm as rmAsync, writeFile, } from "node:fs/promises";
3
+ import { tmpdir } from "node:os";
4
+ import { isAbsolute, join, relative, resolve, sep } from "node:path";
5
+ import { createInterface } from "node:readline";
6
+ import { diagnoseKB, diagnoseKBAsync } from "./diagnostics/health.js";
4
7
  import { describeEmbeddingCompatibility, describeEmbeddingDecoration, dispose as disposeEmbedding, embedDocuments, embeddingConfigLabel, embeddingSignature, embedQueryWithConfig, embedValidatedDocuments, inspectEmbeddingCompatibility, prepareForShutdown as prepareEmbeddingForShutdown, resolveEmbeddingConfig, } from "./embedding/provider.js";
5
8
  import { openVectorReader, openVectorWriter } from "./embedding/vectors.js";
6
- import { addSkippedScanEntry, analyzeIndexableContent, buildChunkEmbeddingText, chunkFile, chunkIdentityHash, chunkMarkdown, createSkippedScanStats, isReadableTextFile, isSupportedDocumentFile, iterateScannableFiles, preTokenizeForFTS, summarizeSkippedScan, } from "./indexer/chunker.js";
9
+ import { assertNetworkAccess, assertOpenedPathAuthorized, assertOutputPathAuthorized, authorizeExistingPath, authorizeOutputPath, } from "./filesystem-access.js";
10
+ import { addSkippedScanEntry, analyzeIndexableContent, buildChunkEmbeddingText, chunkFile, chunkIdentityHash, chunkMarkdown, createSkippedScanStats, isReadableTextFile, isSupportedDocumentFile, iterateScannableFiles, iterateScannableFilesAsync, preTokenizeForFTS, summarizeSkippedScan, } from "./indexer/chunker.js";
7
11
  import { finalizeChunksForEmbedding } from "./indexer/embedding-safe-chunks.js";
8
12
  import { htmlToStructuredMarkdown } from "./indexer/html.js";
9
13
  import { extractSymbols } from "./indexer/symbols.js";
10
14
  import { extractDocumentInModelWorker, shutdownModelWorker } from "./model-worker-client.js";
15
+ import { fetchPublicText } from "./network/safe-fetch.js";
11
16
  import { searchBM25 } from "./search/bm25.js";
12
17
  import { MAX_SEARCH_WINDOW } from "./search/contracts.js";
13
18
  import { diversifyRankedChunks, interleaveByFile } from "./search/diversity.js";
@@ -17,7 +22,8 @@ import { hasAnyLexicalEvidence, hasEnoughLexicalEvidence, normalizeFileTypeFilte
17
22
  import { disposeReranker, prepareRerankerForShutdown, rerank } from "./search/reranker.js";
18
23
  import { resolveSearchTuning, resolveSearchWindow, summarizeSearchTuning, } from "./search/tuning.js";
19
24
  import { searchVectorFile } from "./search/vector.js";
20
- import { countSymbols, createKB, deleteChunksByIds, deleteKB, deleteSymbolsByKB, finishIndexingJob, getChunkById, getChunkCount, getChunksByFile, getFileCount, getKB, getKBByName, getSymbolCount, insertChunks, insertSymbols, iterateChunkIdsByKB, iterateChunksByKB, listKBs, openDatabase, searchSymbols, startIndexingJob, updateIndexingJob, updateKBCounts, updateKBEmbeddingMetadata, updateKBSourceOptions, updateKBStatus, } from "./storage/sqlite.js";
25
+ import { ensureOwnerOnlyDirectory, prepareSafeNewFile, publishNewFile } from "./secure-files.js";
26
+ import { countSymbols, createKB, deleteChunksByIds, deleteKB, deleteSymbolsByKB, finishIndexingJob, getChunkById, getChunkCount, getChunksByFile, getFileCount, getKB, getKBByName, getSymbolCount, getSymbolsByFile, insertChunks, insertSymbols, iterateChunkIdsByKB, iterateChunksByKB, listKBs, markKBIndexed, openDatabase, searchSymbols, startIndexingJob, updateIndexingJob, updateKBCounts, updateKBEmbeddingMetadata, updateKBSourceOptions, updateKBStatus, } from "./storage/sqlite.js";
21
27
  export { CURRENT_EMBEDDING_MODEL } from "./embedding/provider.js";
22
28
  export const CURRENT_URL_STRUCTURE_POLICY = 1;
23
29
  export const CURRENT_EMBEDDING_INPUT_POLICY = 2;
@@ -63,11 +69,12 @@ function matchReasonFor(mode) {
63
69
  return "bm25";
64
70
  return "hybrid";
65
71
  }
66
- function toScanOptions(options = {}) {
72
+ function toScanOptions(options = {}, accessPolicy) {
67
73
  return {
68
74
  includeSuggestedText: options.include_suggested_text === true,
69
75
  includePaths: options.include_paths,
70
76
  excludePaths: options.exclude_paths,
77
+ accessPolicy,
71
78
  };
72
79
  }
73
80
  function serializeAddOptions(options = {}) {
@@ -131,16 +138,22 @@ function planDirectoryScan(dirPath, options = {}, signal) {
131
138
  skipped,
132
139
  };
133
140
  }
134
- function sourceMtimeFor(kb, filePath) {
135
- if (!kb?.source_path)
136
- return undefined;
137
- const absPath = kb.source_type === "directory" ? join(kb.source_path, filePath) : kb.source_path;
138
- try {
139
- return statSync(absPath).mtimeMs;
140
- }
141
- catch {
142
- return undefined;
141
+ async function planDirectoryScanAsync(dirPath, options = {}, signal) {
142
+ const skipped = createSkippedScanStats();
143
+ let files = 0;
144
+ let bytes = 0;
145
+ for await (const file of iterateScannableFilesAsync(dirPath, skipped, options, signal)) {
146
+ throwIfAborted(signal);
147
+ files++;
148
+ bytes += file.size;
143
149
  }
150
+ return {
151
+ files,
152
+ bytes,
153
+ skippedTotal: skipped.total,
154
+ skippedSummary: summarizeSkippedScan(skipped),
155
+ skipped,
156
+ };
144
157
  }
145
158
  function kbTrustMultiplier(kb) {
146
159
  let multiplier = kb.status === "stale" ? 0.92 : 1;
@@ -535,35 +548,146 @@ function isWeakAutoResponse(query, response, primaryMode, attemptMode) {
535
548
  }
536
549
  return false;
537
550
  }
551
+ const MAX_DIRECT_TEXT_BYTES = 10 * 1024 * 1024;
552
+ const MAX_DIRECT_DOCUMENT_BYTES = 64 * 1024 * 1024;
553
+ const MAX_IMPORT_BYTES = 256 * 1024 * 1024;
554
+ const MAX_IMPORT_LINE_BYTES = 16 * 1024 * 1024;
555
+ const MAX_IMPORT_LINES = 1_000_001;
556
+ const MAX_IMPORT_CHUNKS = 1_000_000;
557
+ const MAX_EXPORT_BYTES = MAX_IMPORT_BYTES;
558
+ const MAX_EXPORT_CHUNKS = MAX_IMPORT_CHUNKS;
559
+ const MAX_EXPORT_DURATION_MS = 60 * 60_000;
560
+ const MAX_KB_NAME_BYTES = 512;
561
+ const MAX_KB_DESCRIPTION_BYTES = 4 * 1024;
562
+ const MAX_IMPORT_CONTENT_BYTES = 12 * 1024 * 1024;
563
+ const MAX_IMPORT_FILE_PATH_BYTES = 4_096;
564
+ const MAX_IMPORT_FILE_TYPE_BYTES = 256;
565
+ const MAX_IMPORT_METADATA_BYTES = 64 * 1024;
566
+ const MAX_IMPORT_METADATA_CONTEXT_BYTES = 8 * 1024;
567
+ const MAX_IMPORT_METADATA_SCOPE_ITEMS = 128;
568
+ const MAX_IMPORT_LINE_NUMBER = 2_147_483_647;
569
+ const MAX_IMPORT_FIXED_CONTEXT_BYTES = 64 * 1024;
570
+ const IMPORTED_CHUNK_KEYS = new Set(["content", "file_path", "file_type", "start_line", "end_line", "metadata_json"]);
571
+ const IMPORTED_CONTEXT_STRING_KEYS = [
572
+ "heading",
573
+ "breadcrumb",
574
+ "language",
575
+ "symbol",
576
+ "function_name",
577
+ "symbol_kind",
578
+ "parent_symbol",
579
+ "signature",
580
+ ];
538
581
  async function chunkUrl(source, signal) {
539
- const res = await fetch(source, { signal });
540
- if (!res.ok)
541
- throw new Error(`Fetch failed: ${res.status}`);
542
- const html = await res.text();
582
+ const html = await fetchPublicText(source, signal);
543
583
  throwIfAborted(signal);
544
584
  const text = htmlToStructuredMarkdown(html);
545
585
  const structuredChunks = chunkMarkdown(text, source, "html");
546
586
  return structuredChunks.length > 0 ? structuredChunks : chunkFile(text, source);
547
587
  }
548
- async function extractSourceFileContent(filePath, signal) {
588
+ function validateKnowledgeBaseMetadata(name, description) {
589
+ if (typeof name !== "string" || !name.trim())
590
+ throw new Error("Knowledge base name must be a non-empty string");
591
+ if (Buffer.byteLength(name) > MAX_KB_NAME_BYTES) {
592
+ throw new Error(`Knowledge base name exceeds the ${MAX_KB_NAME_BYTES}-byte limit`);
593
+ }
594
+ const normalizedDescription = description === null ? undefined : description;
595
+ if (normalizedDescription !== undefined && typeof normalizedDescription !== "string") {
596
+ throw new Error("Knowledge base description must be a string");
597
+ }
598
+ if (typeof normalizedDescription === "string" &&
599
+ Buffer.byteLength(normalizedDescription) > MAX_KB_DESCRIPTION_BYTES) {
600
+ throw new Error(`Knowledge base description exceeds the ${MAX_KB_DESCRIPTION_BYTES}-byte limit`);
601
+ }
602
+ return { name, description: normalizedDescription };
603
+ }
604
+ async function readBoundedRegularFile(filePath, maxBytes, signal, expected, access) {
605
+ if (expected?.rootPath) {
606
+ const canonical = await realpathFile(filePath);
607
+ const rel = relative(expected.rootPath, canonical);
608
+ if (rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel)) {
609
+ throw new Error(`Source file escaped its scanned root: ${filePath}`);
610
+ }
611
+ }
612
+ const direct = lstatSync(filePath);
613
+ if (direct.isSymbolicLink() || !direct.isFile())
614
+ throw new Error(`Source is not a regular non-symlink file: ${filePath}`);
615
+ if (expected?.device !== undefined &&
616
+ expected.inode !== undefined &&
617
+ (direct.dev !== expected.device || direct.ino !== expected.inode)) {
618
+ throw new Error(`Source file changed after scanning: ${filePath}`);
619
+ }
620
+ const handle = await openFile(filePath, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
621
+ try {
622
+ const opened = await handle.stat();
623
+ if (!opened.isFile())
624
+ throw new Error(`Source is not a regular file: ${filePath}`);
625
+ assertOpenedPathAuthorized(filePath, opened.dev, opened.ino, access);
626
+ if (expected?.rootPath) {
627
+ const canonical = await realpathFile(filePath);
628
+ const current = await lstatFile(filePath);
629
+ const rel = relative(expected.rootPath, canonical);
630
+ if (rel === ".." ||
631
+ rel.startsWith(`..${sep}`) ||
632
+ isAbsolute(rel) ||
633
+ current.isSymbolicLink() ||
634
+ current.dev !== opened.dev ||
635
+ current.ino !== opened.ino) {
636
+ throw new Error(`Source file changed outside its scanned root: ${filePath}`);
637
+ }
638
+ }
639
+ if (expected?.device !== undefined &&
640
+ expected.inode !== undefined &&
641
+ (opened.dev !== expected.device || opened.ino !== expected.inode)) {
642
+ throw new Error(`Source file changed while opening: ${filePath}`);
643
+ }
644
+ if (opened.size > maxBytes) {
645
+ throw new Error(`Source file exceeds the ${maxBytes}-byte direct indexing limit: ${filePath}`);
646
+ }
647
+ const chunks = [];
648
+ let bytes = 0;
649
+ while (bytes <= maxBytes) {
650
+ throwIfAborted(signal);
651
+ const buffer = Buffer.allocUnsafe(Math.min(64 * 1024, maxBytes - bytes + 1));
652
+ const result = await handle.read(buffer, 0, buffer.length, null);
653
+ if (result.bytesRead === 0)
654
+ break;
655
+ bytes += result.bytesRead;
656
+ if (bytes > maxBytes) {
657
+ throw new Error(`Source file exceeds the ${maxBytes}-byte direct indexing limit: ${filePath}`);
658
+ }
659
+ chunks.push(buffer.subarray(0, result.bytesRead));
660
+ }
661
+ return Buffer.concat(chunks, bytes);
662
+ }
663
+ finally {
664
+ await handle.close();
665
+ }
666
+ }
667
+ async function extractSourceFileContent(filePath, signal, expected, access) {
668
+ filePath = authorizeExistingPath(filePath, access);
549
669
  const lowerPath = filePath.toLowerCase();
550
- if (lowerPath.endsWith(".pdf")) {
551
- throwIfAborted(signal);
552
- const extracted = await extractDocumentInModelWorker(filePath, "pdf", signal);
553
- throwIfAborted(signal);
554
- return extracted;
670
+ const isDocument = lowerPath.endsWith(".pdf") || lowerPath.endsWith(".docx") || lowerPath.endsWith(".doc");
671
+ const maxBytes = isDocument ? MAX_DIRECT_DOCUMENT_BYTES : MAX_DIRECT_TEXT_BYTES;
672
+ const content = await readBoundedRegularFile(filePath, maxBytes, signal, expected, access);
673
+ if (!isDocument) {
674
+ if (content.subarray(0, 8_000).includes(0x00)) {
675
+ throw new Error(`File is not readable text and has no supported extractor: ${filePath}`);
676
+ }
677
+ return { content: content.toString("utf8"), fileType: "text" };
555
678
  }
556
- if (lowerPath.endsWith(".docx") || lowerPath.endsWith(".doc")) {
557
- throwIfAborted(signal);
558
- const extracted = await extractDocumentInModelWorker(filePath, "docx", signal);
679
+ const tempDirectory = ensureOwnerOnlyDirectory(await mkdtemp(join(tmpdir(), "pi-knowledge-document-")));
680
+ const tempPath = join(tempDirectory, "source");
681
+ try {
682
+ await writeFile(tempPath, content, { flag: "wx", mode: 0o600 });
683
+ const fileType = lowerPath.endsWith(".pdf") ? "pdf" : "docx";
684
+ const extracted = await extractDocumentInModelWorker(tempPath, fileType, signal);
559
685
  throwIfAborted(signal);
560
686
  return extracted;
561
687
  }
562
- if (!isReadableTextFile(filePath)) {
563
- throw new Error(`File is not readable text and has no supported extractor: ${filePath}`);
688
+ finally {
689
+ await rmAsync(tempDirectory, { recursive: true, force: true });
564
690
  }
565
- throwIfAborted(signal);
566
- return { content: readFileSync(filePath, "utf-8"), fileType: "text" };
567
691
  }
568
692
  function looksLikeLocalPath(source) {
569
693
  const trimmed = source.trim();
@@ -577,11 +701,15 @@ function looksLikeLocalPath(source) {
577
701
  return true;
578
702
  return /\.[A-Za-z0-9][A-Za-z0-9_-]{0,15}$/.test(trimmed);
579
703
  }
580
- function classifySource(source) {
581
- const resolvedSource = resolve(source);
704
+ function classifySource(source, access) {
582
705
  const isUrl = source.startsWith("http://") || source.startsWith("https://");
583
- const isDir = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isDirectory();
584
- const isFile = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isFile();
706
+ if (isUrl)
707
+ assertNetworkAccess(access);
708
+ const requestedSource = resolve(source);
709
+ const exists = !isUrl && existsSync(requestedSource);
710
+ const resolvedSource = exists ? authorizeExistingPath(requestedSource, access) : requestedSource;
711
+ const isDir = exists && statSync(resolvedSource).isDirectory();
712
+ const isFile = exists && statSync(resolvedSource).isFile();
585
713
  if (!isUrl && !isDir && !isFile && looksLikeLocalPath(source)) {
586
714
  throw new Error(`Local path does not exist: ${source}. Pass inline text only when the source itself is text.`);
587
715
  }
@@ -594,7 +722,7 @@ function classifySource(source) {
594
722
  };
595
723
  }
596
724
  async function extractScannableFileContent(file, signal) {
597
- return extractSourceFileContent(file.path, signal);
725
+ return extractSourceFileContent(file.path, signal, file);
598
726
  }
599
727
  async function extractScannableFileContentOrSkip(file, skipped, signal) {
600
728
  try {
@@ -622,6 +750,100 @@ function throwIfAborted(signal) {
622
750
  if (signal?.aborted)
623
751
  throw new Error("Cancelled");
624
752
  }
753
+ function boundedImportString(value, field, maxBytes) {
754
+ if (typeof value !== "string")
755
+ throw new Error(`Imported chunk ${field} must be a string`);
756
+ if (Buffer.byteLength(value) > maxBytes) {
757
+ throw new Error(`Imported chunk ${field} exceeds the ${maxBytes}-byte limit`);
758
+ }
759
+ return value;
760
+ }
761
+ function validateImportedMetadata(metadataJson) {
762
+ let parsed;
763
+ try {
764
+ parsed = JSON.parse(metadataJson);
765
+ }
766
+ catch {
767
+ throw new Error("Imported chunk metadata_json must contain valid JSON");
768
+ }
769
+ if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
770
+ throw new Error("Imported chunk metadata_json must contain an object");
771
+ }
772
+ const metadata = parsed;
773
+ let contextBytes = 0;
774
+ for (const key of IMPORTED_CONTEXT_STRING_KEYS) {
775
+ const value = metadata[key];
776
+ if (value === undefined)
777
+ continue;
778
+ if (typeof value !== "string")
779
+ throw new Error(`Imported chunk metadata ${key} must be a string`);
780
+ contextBytes += Buffer.byteLength(value);
781
+ }
782
+ const scope = metadata.scope;
783
+ if (scope !== undefined) {
784
+ if (!Array.isArray(scope) || scope.length > MAX_IMPORT_METADATA_SCOPE_ITEMS) {
785
+ throw new Error("Imported chunk metadata scope is invalid");
786
+ }
787
+ for (const item of scope) {
788
+ if (typeof item !== "string")
789
+ throw new Error("Imported chunk metadata scope must contain strings");
790
+ contextBytes += Buffer.byteLength(item);
791
+ }
792
+ }
793
+ if (contextBytes > MAX_IMPORT_METADATA_CONTEXT_BYTES) {
794
+ throw new Error(`Imported chunk embedding metadata exceeds the ${MAX_IMPORT_METADATA_CONTEXT_BYTES}-byte limit`);
795
+ }
796
+ }
797
+ function parseImportedChunk(value) {
798
+ if (typeof value !== "object" || value === null || Array.isArray(value)) {
799
+ throw new Error("Imported chunk must be an object");
800
+ }
801
+ const record = value;
802
+ for (const key of Object.keys(record)) {
803
+ if (!IMPORTED_CHUNK_KEYS.has(key))
804
+ throw new Error(`Imported chunk contains unknown field: ${key}`);
805
+ }
806
+ const content = boundedImportString(record.content, "content", MAX_IMPORT_CONTENT_BYTES);
807
+ const filePath = boundedImportString(record.file_path, "file_path", MAX_IMPORT_FILE_PATH_BYTES);
808
+ const fileType = boundedImportString(record.file_type, "file_type", MAX_IMPORT_FILE_TYPE_BYTES);
809
+ if (!content)
810
+ throw new Error("Imported chunk content must not be empty");
811
+ if (!filePath)
812
+ throw new Error("Imported chunk file_path must not be empty");
813
+ if (!fileType)
814
+ throw new Error("Imported chunk file_type must not be empty");
815
+ const startLine = record.start_line;
816
+ const endLine = record.end_line;
817
+ if (!Number.isSafeInteger(startLine) ||
818
+ typeof startLine !== "number" ||
819
+ startLine < 0 ||
820
+ startLine > MAX_IMPORT_LINE_NUMBER) {
821
+ throw new Error("Imported chunk start_line is invalid");
822
+ }
823
+ if (!Number.isSafeInteger(endLine) ||
824
+ typeof endLine !== "number" ||
825
+ endLine < startLine ||
826
+ endLine > MAX_IMPORT_LINE_NUMBER) {
827
+ throw new Error("Imported chunk end_line is invalid");
828
+ }
829
+ const metadataJson = record.metadata_json === undefined
830
+ ? "{}"
831
+ : boundedImportString(record.metadata_json, "metadata_json", MAX_IMPORT_METADATA_BYTES);
832
+ validateImportedMetadata(metadataJson);
833
+ const imported = {
834
+ content,
835
+ file_path: filePath,
836
+ file_type: fileType,
837
+ start_line: startLine,
838
+ end_line: endLine,
839
+ metadata_json: metadataJson,
840
+ };
841
+ const fixedContext = buildChunkEmbeddingText({ ...imported, content: "" });
842
+ if (Buffer.byteLength(fixedContext) > MAX_IMPORT_FIXED_CONTEXT_BYTES) {
843
+ throw new Error(`Imported chunk fixed embedding context exceeds the ${MAX_IMPORT_FIXED_CONTEXT_BYTES}-byte limit`);
844
+ }
845
+ return imported;
846
+ }
625
847
  function importedChunkToInsert(chunk) {
626
848
  const metadataJson = chunk.metadata_json || "{}";
627
849
  return {
@@ -718,9 +940,12 @@ export class KnowledgeEngine {
718
940
  deleteVectorFile(kbId) {
719
941
  rmSync(this.vectorPathFor(kbId), { force: true });
720
942
  }
721
- plan(source, options = {}, signal) {
943
+ plan(source, options = {}, signal, access) {
722
944
  throwIfAborted(signal);
723
- const { resolvedSource, isDir, isFile, sourceType } = classifySource(source);
945
+ const { resolvedSource, isDir, isFile, sourceType } = classifySource(source, access);
946
+ if ((isDir || isFile) && lstatSync(resolvedSource).isSymbolicLink()) {
947
+ throw new Error(`Source is not a regular non-symlink path: ${resolvedSource}`);
948
+ }
724
949
  if (isDir) {
725
950
  const plan = planDirectoryScan(resolvedSource, toScanOptions(options), signal);
726
951
  return {
@@ -764,17 +989,53 @@ export class KnowledgeEngine {
764
989
  summary: `${sourceType === "url" ? "URL" : "Inline text"} plan: 1 source`,
765
990
  };
766
991
  }
767
- async add(source, name, onProgress, signal, options = {}) {
992
+ async planAsync(source, options = {}, signal, access) {
993
+ throwIfAborted(signal);
994
+ const { resolvedSource, isDir, isFile, sourceType } = classifySource(source, access);
995
+ if (isDir) {
996
+ const plan = await planDirectoryScanAsync(resolvedSource, toScanOptions(options, access), signal);
997
+ return {
998
+ source_type: "directory",
999
+ scannable_files: plan.files,
1000
+ scannable_bytes: plan.bytes,
1001
+ skipped: plan.skipped,
1002
+ summary: `Directory plan: ${plan.files} scannable files, ${formatBytes(plan.bytes)} scannable text, skipped ${plan.skippedTotal} (${plan.skippedSummary})`,
1003
+ };
1004
+ }
1005
+ if (isFile && lstatSync(resolvedSource).isSymbolicLink()) {
1006
+ throw new Error(`Source is not a regular non-symlink file: ${resolvedSource}`);
1007
+ }
1008
+ const skipped = createSkippedScanStats();
1009
+ const sourceBytes = isFile ? statSync(resolvedSource).size : Buffer.byteLength(source);
1010
+ const maxBytes = isFile && isSupportedDocumentFile(resolvedSource) ? MAX_DIRECT_DOCUMENT_BYTES : MAX_DIRECT_TEXT_BYTES;
1011
+ if (sourceType !== "url" && sourceBytes > maxBytes) {
1012
+ addSkippedScanEntry(skipped, {
1013
+ path: isFile ? resolvedSource : "<inline>",
1014
+ reason: "oversized",
1015
+ size: sourceBytes,
1016
+ });
1017
+ return {
1018
+ source_type: sourceType,
1019
+ scannable_files: 0,
1020
+ scannable_bytes: 0,
1021
+ skipped,
1022
+ summary: `${isFile ? "File" : "Inline text"} plan: 0 scannable sources; source exceeds ${formatBytes(maxBytes)}`,
1023
+ };
1024
+ }
1025
+ return this.plan(source, options, signal, access);
1026
+ }
1027
+ async add(source, name, onProgress, signal, options = {}, access) {
768
1028
  throwIfAborted(signal);
769
- return this.runExclusive(["global:mutation", `kb-name:${name}`], `Mutation for "${name}"`, () => this.addUnlocked(source, name, onProgress, signal, options));
1029
+ validateKnowledgeBaseMetadata(name, undefined);
1030
+ return this.runExclusive(["global:mutation", `kb-name:${name}`], `Mutation for "${name}"`, () => this.addUnlocked(source, name, onProgress, signal, options, access));
770
1031
  }
771
- async addUnlocked(source, name, onProgress, signal, options = {}) {
1032
+ async addUnlocked(source, name, onProgress, signal, options = {}, access) {
772
1033
  if (!this.db)
773
1034
  throw new Error("Engine not initialized");
774
1035
  throwIfAborted(signal);
775
1036
  const db = this.db;
776
- const { resolvedSource, isUrl, isDir, isFile, sourceType } = classifySource(source);
777
- const scanOptions = toScanOptions(options);
1037
+ const { resolvedSource, isUrl, isDir, isFile, sourceType } = classifySource(source, access);
1038
+ const scanOptions = toScanOptions(options, access);
778
1039
  const existingKB = getKBByName(db, name);
779
1040
  if (existingKB) {
780
1041
  throw new Error(`Knowledge base "${name}" already exists. Use knowledge_update to refresh it, or knowledge_remove before adding a replacement.`);
@@ -874,13 +1135,13 @@ export class KnowledgeEngine {
874
1135
  await addChunks(chunks);
875
1136
  }
876
1137
  else if (isFile) {
877
- const extracted = await extractSourceFileContent(resolvedSource, signal);
1138
+ const extracted = await extractSourceFileContent(resolvedSource, signal, undefined, access);
878
1139
  fileCount = 1;
879
1140
  const chunks = await analyzeAndAddSymbols(extracted.content, resolvedSource, extracted.fileType);
880
1141
  await addChunks(chunks);
881
1142
  }
882
1143
  else if (isDir) {
883
- const plan = planDirectoryScan(resolvedSource, scanOptions, signal);
1144
+ const plan = await planDirectoryScanAsync(resolvedSource, scanOptions, signal);
884
1145
  const planningMessage = `Planned directory scan: ${plan.files} files, ${formatBytes(plan.bytes)} scannable text, skipped ${plan.skippedTotal} (${plan.skippedSummary})`;
885
1146
  updateIndexingJob(this.db, kb.id, {
886
1147
  phase: "planning",
@@ -899,7 +1160,7 @@ export class KnowledgeEngine {
899
1160
  onProgress?.(scanningMessage);
900
1161
  const skipped = createSkippedScanStats();
901
1162
  let processedFiles = 0;
902
- for (const file of iterateScannableFiles(resolvedSource, skipped, scanOptions)) {
1163
+ for await (const file of iterateScannableFilesAsync(resolvedSource, skipped, scanOptions, signal)) {
903
1164
  if (signal?.aborted)
904
1165
  throw new Error("Cancelled");
905
1166
  const extracted = await extractScannableFileContentOrSkip(file, skipped, signal);
@@ -932,6 +1193,9 @@ export class KnowledgeEngine {
932
1193
  }
933
1194
  else {
934
1195
  fileCount = 1;
1196
+ if (Buffer.byteLength(source) > MAX_DIRECT_TEXT_BYTES) {
1197
+ throw new Error(`Inline text exceeds the ${MAX_DIRECT_TEXT_BYTES}-byte indexing limit`);
1198
+ }
935
1199
  const chunks = await analyzeAndAddSymbols(source, "inline-text", "text");
936
1200
  await addChunks(chunks);
937
1201
  }
@@ -942,6 +1206,7 @@ export class KnowledgeEngine {
942
1206
  tempVectorFile = undefined;
943
1207
  const savedFileCount = isDir ? getFileCount(this.db, kb.id) : fileCount;
944
1208
  updateKBCounts(this.db, kb.id, chunkCount, savedFileCount);
1209
+ markKBIndexed(this.db, kb.id);
945
1210
  updateKBStatus(this.db, kb.id, "ready");
946
1211
  const savedKB = getKB(this.db, kb.id);
947
1212
  if (!savedKB)
@@ -972,7 +1237,7 @@ export class KnowledgeEngine {
972
1237
  throw e;
973
1238
  }
974
1239
  }
975
- async update(nameOrId, onProgress, signal) {
1240
+ async update(nameOrId, onProgress, signal, access) {
976
1241
  if (!this.db)
977
1242
  throw new Error("Engine not initialized");
978
1243
  if (this.disposing)
@@ -984,24 +1249,32 @@ export class KnowledgeEngine {
984
1249
  if (!kb.source_path || (kb.source_type !== "url" && !existsSync(kb.source_path))) {
985
1250
  throw new Error(`Source path not available or missing: ${kb.source_path}`);
986
1251
  }
987
- const updateableKB = { ...kb, source_path: kb.source_path };
1252
+ let sourcePath;
1253
+ if (kb.source_type === "url") {
1254
+ assertNetworkAccess(access);
1255
+ sourcePath = kb.source_path;
1256
+ }
1257
+ else {
1258
+ sourcePath = authorizeExistingPath(kb.source_path, access);
1259
+ }
1260
+ const updateableKB = { ...kb, source_path: sourcePath };
988
1261
  const activeUpdate = this.activeUpdates.get(updateableKB.id);
989
1262
  if (activeUpdate) {
990
1263
  onProgress?.(`Update already running for "${updateableKB.name}"; waiting for the active update.`);
991
1264
  return activeUpdate;
992
1265
  }
993
- const updateRun = this.runExclusive(["global:mutation", `kb:${updateableKB.id}`], `Mutation for "${updateableKB.name}"`, () => this.runUpdate(updateableKB, onProgress, signal)).finally(() => {
1266
+ const updateRun = this.runExclusive(["global:mutation", `kb:${updateableKB.id}`], `Mutation for "${updateableKB.name}"`, () => this.runUpdate(updateableKB, onProgress, signal, access)).finally(() => {
994
1267
  if (this.activeUpdates.get(updateableKB.id) === updateRun)
995
1268
  this.activeUpdates.delete(updateableKB.id);
996
1269
  });
997
1270
  this.activeUpdates.set(updateableKB.id, updateRun);
998
1271
  return updateRun;
999
1272
  }
1000
- async runUpdate(kb, onProgress, signal) {
1273
+ async runUpdate(kb, onProgress, signal, access) {
1001
1274
  if (!this.db)
1002
1275
  throw new Error("Engine not initialized");
1003
1276
  throwIfAborted(signal);
1004
- const scanOptions = toScanOptions(parseAddOptions(kb.source_options));
1277
+ const scanOptions = toScanOptions(parseAddOptions(kb.source_options), access);
1005
1278
  const embeddingConfig = resolveEmbeddingConfig();
1006
1279
  const embeddingBatchSize = resolveIndexEmbedBatchSize();
1007
1280
  updateKBStatus(this.db, kb.id, "indexing");
@@ -1140,8 +1413,8 @@ export class KnowledgeEngine {
1140
1413
  stagedSymbols.push(...extractSymbols(chunks.map((chunk) => chunk.content).join("\n\n"), kb.source_path, "html"));
1141
1414
  await processChunks(chunks);
1142
1415
  }
1143
- else if (statSync(kb.source_path).isDirectory()) {
1144
- const plan = planDirectoryScan(kb.source_path, scanOptions, signal);
1416
+ else if (kb.source_type === "directory") {
1417
+ const plan = await planDirectoryScanAsync(kb.source_path, scanOptions, signal);
1145
1418
  plannedTotalFiles = plan.files;
1146
1419
  const planningMessage = `Planned directory scan: ${plan.files} files, ${formatBytes(plan.bytes)} scannable text, skipped ${plan.skippedTotal} (${plan.skippedSummary})`;
1147
1420
  updateIndexingJob(this.db, kb.id, {
@@ -1152,12 +1425,27 @@ export class KnowledgeEngine {
1152
1425
  });
1153
1426
  onProgress?.(planningMessage);
1154
1427
  const skipped = createSkippedScanStats();
1155
- for (const file of iterateScannableFiles(kb.source_path, skipped, scanOptions)) {
1428
+ for await (const file of iterateScannableFilesAsync(kb.source_path, skipped, scanOptions, signal)) {
1156
1429
  if (signal?.aborted)
1157
1430
  throw new Error("Cancelled");
1158
1431
  const extracted = await extractScannableFileContentOrSkip(file, skipped, signal);
1159
- if (!extracted)
1432
+ if (!extracted) {
1433
+ for (const chunk of getChunksByFile(this.db, kb.id, file.relPath, 0, Number.MAX_SAFE_INTEGER)) {
1434
+ const entries = existingHashes.get(chunk.content_hash);
1435
+ const retainedIndex = entries?.findIndex((entry) => entry.id === chunk.id) ?? -1;
1436
+ if (!entries || retainedIndex < 0)
1437
+ continue;
1438
+ const [retained] = entries.splice(retainedIndex, 1);
1439
+ if (canReuseExistingVectors && retained) {
1440
+ const indexes = oldVectorIndexByHash.get(chunk.content_hash) ?? [];
1441
+ indexes.push(retained.vectorIndex);
1442
+ oldVectorIndexByHash.set(chunk.content_hash, indexes);
1443
+ }
1444
+ unchanged++;
1445
+ }
1446
+ stagedSymbols.push(...getSymbolsByFile(this.db, kb.id, file.relPath));
1160
1447
  continue;
1448
+ }
1161
1449
  const analysis = await analyzeIndexableContent(extracted.content, file.relPath, extracted.fileType);
1162
1450
  scannedFiles++;
1163
1451
  stagedSymbols.push(...analysis.symbols);
@@ -1193,7 +1481,7 @@ export class KnowledgeEngine {
1193
1481
  onProgress?.(message);
1194
1482
  }
1195
1483
  else {
1196
- const extracted = await extractSourceFileContent(kb.source_path, signal);
1484
+ const extracted = await extractSourceFileContent(kb.source_path, signal, undefined, access);
1197
1485
  scannedFiles = 1;
1198
1486
  const analysis = await analyzeIndexableContent(extracted.content, kb.source_path, extracted.fileType);
1199
1487
  stagedSymbols.push(...analysis.symbols);
@@ -1300,6 +1588,7 @@ export class KnowledgeEngine {
1300
1588
  updateKBEmbeddingMetadata(this.db, kb.id, embeddingModel, finalEmbeddingDimension === undefined ? null : embeddingSignature(embeddingConfig, finalEmbeddingDimension), finalEmbeddingDimension ?? null);
1301
1589
  updateKBCounts(this.db, kb.id, finalChunkCount, getFileCount(this.db, kb.id));
1302
1590
  updateKBSourceOptions(this.db, kb.id, serializeAddOptions(parseAddOptions(kb.source_options)));
1591
+ markKBIndexed(this.db, kb.id);
1303
1592
  updateKBStatus(this.db, kb.id, "ready");
1304
1593
  const skippedReadySuffix = latestSkippedTotal > 0 ? `, skipped ${latestSkippedTotal} (${latestSkippedSummary})` : "";
1305
1594
  const readyMessage = `Ready: +${addedCount} -${idsToRemove.length} =${unchanged}${skippedReadySuffix}`;
@@ -1616,7 +1905,7 @@ export class KnowledgeEngine {
1616
1905
  const page = diversified.slice(offset, offset + limit);
1617
1906
  const results = page.map((r) => {
1618
1907
  const kb = kbById.get(r.chunk.kb_id);
1619
- const sourceMtime = sourceMtimeFor(kb, r.chunk.file_path);
1908
+ const stale = kb?.status === "stale";
1620
1909
  return {
1621
1910
  content: r.content,
1622
1911
  file_path: r.chunk.file_path,
@@ -1631,8 +1920,7 @@ export class KnowledgeEngine {
1631
1920
  chunk_id: r.chunk.id,
1632
1921
  chunk_hash: r.chunk.content_hash,
1633
1922
  indexed_at: r.chunk.indexed_at,
1634
- source_mtime: sourceMtime,
1635
- stale: sourceMtime !== undefined ? sourceMtime > r.chunk.indexed_at : kb?.status === "stale",
1923
+ stale,
1636
1924
  match_reason: matchReasonFor(mode),
1637
1925
  source_chunk_ids: r.sourceChunkIds,
1638
1926
  },
@@ -1730,13 +2018,12 @@ export class KnowledgeEngine {
1730
2018
  end_line: r.endLine,
1731
2019
  provenance: (() => {
1732
2020
  const kb = kbById.get(r.chunk.kb_id);
1733
- const sourceMtime = sourceMtimeFor(kb, r.chunk.file_path);
2021
+ const stale = kb?.status === "stale";
1734
2022
  return {
1735
2023
  chunk_id: r.chunk.id,
1736
2024
  chunk_hash: r.chunk.content_hash,
1737
2025
  indexed_at: r.chunk.indexed_at,
1738
- source_mtime: sourceMtime,
1739
- stale: sourceMtime !== undefined ? sourceMtime > r.chunk.indexed_at : kb?.status === "stale",
2026
+ stale,
1740
2027
  match_reason: matchReasonFor(mode),
1741
2028
  source_chunk_ids: r.sourceChunkIds,
1742
2029
  };
@@ -1794,11 +2081,11 @@ export class KnowledgeEngine {
1794
2081
  this.deleteVectorFile(kb.id);
1795
2082
  return true;
1796
2083
  }
1797
- list(signal) {
2084
+ list(signal, limit) {
1798
2085
  if (!this.db)
1799
2086
  return [];
1800
2087
  throwIfAborted(signal);
1801
- return listKBs(this.db);
2088
+ return listKBs(this.db, limit);
1802
2089
  }
1803
2090
  clear() {
1804
2091
  if (!this.db)
@@ -1822,16 +2109,32 @@ export class KnowledgeEngine {
1822
2109
  return diagnoseKB(db, kb, signal);
1823
2110
  });
1824
2111
  }
1825
- statusSnapshot(signal) {
2112
+ async diagnoseAsync(signal, knowledgeBasesOverride, maxDiagnosticPaths = Number.POSITIVE_INFINITY) {
1826
2113
  if (!this.db)
1827
- return { knowledge_bases: [], diagnostics: [], embeddings: [] };
2114
+ return [];
1828
2115
  throwIfAborted(signal);
1829
2116
  const db = this.db;
1830
- const knowledgeBases = listKBs(db);
1831
- const diagnostics = knowledgeBases.map((kb) => {
2117
+ const diagnostics = [];
2118
+ let remainingPaths = maxDiagnosticPaths;
2119
+ for (const kb of knowledgeBasesOverride ?? listKBs(db)) {
1832
2120
  throwIfAborted(signal);
1833
- return diagnoseKB(db, kb, signal);
1834
- });
2121
+ const diagnostic = await diagnoseKBAsync(db, kb, signal, Math.max(0, remainingPaths));
2122
+ diagnostics.push(diagnostic);
2123
+ remainingPaths = Math.max(0, remainingPaths - diagnostic.stale_files.length - diagnostic.orphan_files.length);
2124
+ }
2125
+ return diagnostics;
2126
+ }
2127
+ statusSnapshot(signal, diagnosticsOverride, knowledgeBasesOverride) {
2128
+ if (!this.db)
2129
+ return { knowledge_bases: [], diagnostics: [], embeddings: [] };
2130
+ throwIfAborted(signal);
2131
+ const db = this.db;
2132
+ const knowledgeBases = knowledgeBasesOverride ?? listKBs(db);
2133
+ const diagnostics = diagnosticsOverride ??
2134
+ knowledgeBases.map((kb) => {
2135
+ throwIfAborted(signal);
2136
+ return diagnoseKB(db, kb, signal);
2137
+ });
1835
2138
  try {
1836
2139
  const config = resolveEmbeddingConfig();
1837
2140
  const activeModel = embeddingConfigLabel(config);
@@ -1862,6 +2165,15 @@ export class KnowledgeEngine {
1862
2165
  };
1863
2166
  }
1864
2167
  }
2168
+ async statusSnapshotAsync(signal, maxKnowledgeBases = Number.POSITIVE_INFINITY, maxDiagnosticPaths = Number.POSITIVE_INFINITY) {
2169
+ if (!this.db)
2170
+ return { knowledge_bases: [], diagnostics: [], embeddings: [] };
2171
+ const knowledgeBases = listKBs(this.db, Number.isFinite(maxKnowledgeBases) ? maxKnowledgeBases + 1 : undefined);
2172
+ if (knowledgeBases.length > maxKnowledgeBases) {
2173
+ throw new Error(`Knowledge status exceeds the ${maxKnowledgeBases}-knowledge-base RPC limit`);
2174
+ }
2175
+ return this.statusSnapshot(signal, await this.diagnoseAsync(signal, knowledgeBases, maxDiagnosticPaths), knowledgeBases);
2176
+ }
1865
2177
  symbolSearch(query, options = {}, signal) {
1866
2178
  if (!this.db)
1867
2179
  throw new Error("Engine not initialized");
@@ -1952,11 +2264,11 @@ export class KnowledgeEngine {
1952
2264
  miss,
1953
2265
  };
1954
2266
  }
1955
- doctor(signal) {
2267
+ doctor(signal, diagnosticsOverride, knowledgeBasesOverride) {
1956
2268
  throwIfAborted(signal);
1957
- const diagnostics = this.diagnose(signal);
2269
+ const diagnostics = diagnosticsOverride ?? this.diagnose(signal);
1958
2270
  const issues = [];
1959
- const kbs = this.list();
2271
+ const kbs = knowledgeBasesOverride ?? this.list();
1960
2272
  if (kbs.length === 0) {
1961
2273
  issues.push({
1962
2274
  severity: "blocking",
@@ -1986,7 +2298,16 @@ export class KnowledgeEngine {
1986
2298
  action_code: "rebuild_kb",
1987
2299
  });
1988
2300
  }
1989
- if (diagnostic.coverage_percent < 80) {
2301
+ if (diagnostic.scan_error) {
2302
+ issues.push({
2303
+ severity: "warning",
2304
+ kb_name: diagnostic.kb_name,
2305
+ message: `Source diagnostics scan failed: ${diagnostic.scan_error}`,
2306
+ action: "Check source accessibility and scanner limits, then run knowledge_doctor again.",
2307
+ action_code: "check_source",
2308
+ });
2309
+ }
2310
+ else if (diagnostic.coverage_percent < 80) {
1990
2311
  issues.push({
1991
2312
  severity: "warning",
1992
2313
  kb_name: diagnostic.kb_name,
@@ -1995,20 +2316,22 @@ export class KnowledgeEngine {
1995
2316
  action_code: "review_skipped_scope",
1996
2317
  });
1997
2318
  }
1998
- if (diagnostic.stale_files.length > 0) {
2319
+ const staleFileCount = diagnostic.stale_file_count ?? diagnostic.stale_files.length;
2320
+ if (staleFileCount > 0) {
1999
2321
  issues.push({
2000
2322
  severity: "warning",
2001
2323
  kb_name: diagnostic.kb_name,
2002
- message: `${diagnostic.stale_files.length} files changed after indexing.`,
2324
+ message: `${staleFileCount} files changed after indexing.`,
2003
2325
  action: "Run knowledge_update for this KB.",
2004
2326
  action_code: "run_update",
2005
2327
  });
2006
2328
  }
2007
- if (diagnostic.orphan_files.length > 0) {
2329
+ const orphanFileCount = diagnostic.orphan_file_count ?? diagnostic.orphan_files.length;
2330
+ if (orphanFileCount > 0) {
2008
2331
  issues.push({
2009
2332
  severity: "warning",
2010
2333
  kb_name: diagnostic.kb_name,
2011
- message: `${diagnostic.orphan_files.length} indexed files no longer exist in the source.`,
2334
+ message: `${orphanFileCount} indexed files no longer exist in the source.`,
2012
2335
  action: "Run knowledge_update or rebuild the KB.",
2013
2336
  action_code: "run_update",
2014
2337
  });
@@ -2117,23 +2440,68 @@ export class KnowledgeEngine {
2117
2440
  })),
2118
2441
  };
2119
2442
  }
2120
- async exportKB(nameOrId, outputPath, signal, onProgress) {
2443
+ async doctorAsync(signal, maxKnowledgeBases = Number.POSITIVE_INFINITY, maxDiagnosticPaths = Number.POSITIVE_INFINITY) {
2444
+ if (!this.db)
2445
+ return this.doctor(signal, [], []);
2446
+ const knowledgeBases = listKBs(this.db, Number.isFinite(maxKnowledgeBases) ? maxKnowledgeBases + 1 : undefined);
2447
+ if (knowledgeBases.length > maxKnowledgeBases) {
2448
+ throw new Error(`Knowledge doctor exceeds the ${maxKnowledgeBases}-knowledge-base RPC limit`);
2449
+ }
2450
+ return this.doctor(signal, await this.diagnoseAsync(signal, knowledgeBases, maxDiagnosticPaths), knowledgeBases);
2451
+ }
2452
+ async exportKB(nameOrId, outputPath, signal, onProgress, access) {
2121
2453
  if (!this.db)
2122
2454
  throw new Error("Engine not initialized");
2123
2455
  throwIfAborted(signal);
2124
2456
  const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
2125
2457
  if (!kb)
2126
2458
  throw new Error(`Knowledge base not found: ${nameOrId}`);
2127
- return this.runExclusive(["global:mutation", `kb:${kb.id}`], `Operation for "${kb.name}"`, () => this.exportKBUnlocked(kb, outputPath, signal, onProgress));
2459
+ return this.runExclusive(["global:mutation", `kb:${kb.id}`], `Operation for "${kb.name}"`, () => this.exportKBUnlocked(kb, outputPath, signal, onProgress, access));
2128
2460
  }
2129
- async exportKBUnlocked(kb, outputPath, signal, onProgress) {
2461
+ async exportKBUnlocked(kb, outputPath, signal, onProgress, access) {
2130
2462
  if (!this.db)
2131
2463
  throw new Error("Engine not initialized");
2132
- const { createWriteStream } = await import("node:fs");
2133
- const tempOutputPath = tempVectorPath(outputPath);
2134
- const stream = createWriteStream(tempOutputPath, { encoding: "utf-8" });
2464
+ if (kb.chunk_count > MAX_EXPORT_CHUNKS) {
2465
+ throw new Error(`Export exceeds the ${MAX_EXPORT_CHUNKS}-chunk limit`);
2466
+ }
2467
+ const target = prepareSafeNewFile(authorizeOutputPath(outputPath, access));
2468
+ let tempFd;
2469
+ let tempDevice;
2470
+ let tempInode;
2471
+ try {
2472
+ tempFd = openSync(target.tempPath, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
2473
+ const opened = fstatSync(tempFd);
2474
+ if (!opened.isFile())
2475
+ throw new Error(`Export target is not a regular file: ${target.tempPath}`);
2476
+ tempDevice = opened.dev;
2477
+ tempInode = opened.ino;
2478
+ assertOpenedPathAuthorized(target.tempPath, tempDevice, tempInode, access);
2479
+ assertOutputPathAuthorized(target.outputPath, target.parentDevice, target.parentInode, access);
2480
+ }
2481
+ catch (error) {
2482
+ if (tempFd !== undefined)
2483
+ closeSync(tempFd);
2484
+ throw error;
2485
+ }
2486
+ const stream = createWriteStream("", {
2487
+ autoClose: false,
2488
+ encoding: "utf-8",
2489
+ fd: tempFd,
2490
+ });
2135
2491
  const progressBatchSize = resolveIndexEmbedBatchSize();
2136
2492
  let count = 0;
2493
+ let bytes = 0;
2494
+ const deadline = Date.now() + MAX_EXPORT_DURATION_MS;
2495
+ const writeBoundedLine = async (line) => {
2496
+ if (Date.now() > deadline)
2497
+ throw new Error(`Export exceeded ${MAX_EXPORT_DURATION_MS}ms`);
2498
+ const lineBytes = Buffer.byteLength(line) + 1;
2499
+ if (bytes + lineBytes > MAX_EXPORT_BYTES) {
2500
+ throw new Error(`Export exceeds the ${MAX_EXPORT_BYTES}-byte limit`);
2501
+ }
2502
+ bytes += lineBytes;
2503
+ await writeLine(stream, line);
2504
+ };
2137
2505
  const header = JSON.stringify({
2138
2506
  name: kb.name,
2139
2507
  description: kb.description,
@@ -2141,48 +2509,130 @@ export class KnowledgeEngine {
2141
2509
  chunk_count: kb.chunk_count,
2142
2510
  });
2143
2511
  try {
2144
- await writeLine(stream, header);
2512
+ await writeBoundedLine(header);
2145
2513
  for (const chunk of iterateChunksByKB(this.db, kb.id)) {
2146
2514
  throwIfAborted(signal);
2147
- await writeLine(stream, JSON.stringify({
2515
+ const line = JSON.stringify({
2148
2516
  content: chunk.content,
2149
2517
  file_path: chunk.file_path,
2150
2518
  file_type: chunk.file_type,
2151
2519
  start_line: chunk.start_line,
2152
2520
  end_line: chunk.end_line,
2153
2521
  metadata_json: chunk.metadata_json,
2154
- }));
2522
+ });
2523
+ await writeBoundedLine(line);
2155
2524
  count++;
2156
2525
  if (count % progressBatchSize === 0)
2157
2526
  onProgress?.(`Exported ${count}/${kb.chunk_count} chunks...`);
2158
2527
  }
2159
2528
  await finishWriteStream(stream);
2160
2529
  throwIfAborted(signal);
2161
- renameSync(tempOutputPath, outputPath);
2530
+ fsyncSync(tempFd);
2531
+ if (stream.fd === tempFd)
2532
+ stream.fd = null;
2533
+ closeSync(tempFd);
2534
+ tempFd = undefined;
2535
+ stream.destroy();
2536
+ assertOpenedPathAuthorized(target.tempPath, tempDevice, tempInode, access);
2537
+ assertOutputPathAuthorized(target.outputPath, target.parentDevice, target.parentInode, access);
2538
+ publishNewFile(target.tempPath, target);
2162
2539
  onProgress?.(`Exported ${count}/${kb.chunk_count} chunks.`);
2163
2540
  return count;
2164
2541
  }
2165
2542
  catch (error) {
2543
+ if (tempFd !== undefined) {
2544
+ const descriptor = tempFd;
2545
+ tempFd = undefined;
2546
+ if (stream.fd === descriptor)
2547
+ stream.fd = null;
2548
+ try {
2549
+ closeSync(descriptor);
2550
+ }
2551
+ catch (closeError) {
2552
+ if (closeError.code !== "EBADF") {
2553
+ console.error(`Failed to close export descriptor: ${String(closeError)}`);
2554
+ }
2555
+ }
2556
+ }
2166
2557
  stream.destroy();
2167
- rmSync(tempOutputPath, { force: true });
2558
+ try {
2559
+ const current = lstatSync(target.tempPath);
2560
+ if (!current.isSymbolicLink() && current.dev === tempDevice && current.ino === tempInode) {
2561
+ assertOpenedPathAuthorized(target.tempPath, tempDevice, tempInode, access);
2562
+ rmSync(target.tempPath, { force: true });
2563
+ }
2564
+ }
2565
+ catch {
2566
+ // Leave an unverified path untouched rather than deleting outside the authorized root.
2567
+ }
2168
2568
  throw error;
2169
2569
  }
2170
2570
  }
2171
- async importKB(inputPath, onProgress, signal) {
2571
+ async importKB(inputPath, onProgress, signal, access) {
2172
2572
  if (!this.db)
2173
2573
  throw new Error("Engine not initialized");
2174
- return this.runExclusive("global:mutation", "Knowledge-base import", () => this.importKBUnlocked(inputPath, onProgress, signal));
2574
+ return this.runExclusive("global:mutation", "Knowledge-base import", () => this.importKBUnlocked(inputPath, onProgress, signal, access));
2175
2575
  }
2176
- async importKBUnlocked(inputPath, onProgress, signal) {
2576
+ async importKBUnlocked(inputPath, onProgress, signal, access) {
2177
2577
  if (!this.db)
2178
2578
  throw new Error("Engine not initialized");
2179
2579
  throwIfAborted(signal);
2180
- const { createReadStream } = await import("node:fs");
2181
- const { createInterface } = await import("node:readline");
2182
2580
  const embeddingConfig = resolveEmbeddingConfig();
2183
2581
  const embeddingBatchSize = resolveIndexEmbedBatchSize();
2184
- const stream = createReadStream(inputPath, { encoding: "utf-8" });
2582
+ inputPath = authorizeExistingPath(inputPath, access);
2583
+ const directInput = lstatSync(inputPath);
2584
+ if (directInput.isSymbolicLink() || !directInput.isFile()) {
2585
+ throw new Error(`Import source is not a regular non-symlink file: ${inputPath}`);
2586
+ }
2587
+ const inputFd = openSync(inputPath, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
2588
+ const inputInfo = fstatSync(inputFd);
2589
+ if (inputInfo.dev !== directInput.dev || inputInfo.ino !== directInput.ino) {
2590
+ closeSync(inputFd);
2591
+ throw new Error(`Import source changed while opening: ${inputPath}`);
2592
+ }
2593
+ if (!inputInfo.isFile()) {
2594
+ closeSync(inputFd);
2595
+ throw new Error(`Import source is not a regular file: ${inputPath}`);
2596
+ }
2597
+ try {
2598
+ assertOpenedPathAuthorized(inputPath, inputInfo.dev, inputInfo.ino, access);
2599
+ }
2600
+ catch (error) {
2601
+ closeSync(inputFd);
2602
+ throw error;
2603
+ }
2604
+ if (inputInfo.size > MAX_IMPORT_BYTES) {
2605
+ closeSync(inputFd);
2606
+ throw new Error(`Import file exceeds the ${MAX_IMPORT_BYTES}-byte limit: ${inputPath}`);
2607
+ }
2608
+ const stream = createReadStream("", { autoClose: true, encoding: "utf-8", fd: inputFd });
2185
2609
  const lines = createInterface({ input: stream, crlfDelay: Infinity });
2610
+ let observedBytes = 0;
2611
+ let observedLineBytes = 0;
2612
+ stream.on("data", (chunk) => {
2613
+ observedBytes += Buffer.byteLength(String(chunk));
2614
+ if (observedBytes > MAX_IMPORT_BYTES) {
2615
+ stream.destroy(new Error(`Import exceeds the ${MAX_IMPORT_BYTES}-byte limit`));
2616
+ return;
2617
+ }
2618
+ const parts = String(chunk).split("\n");
2619
+ if (parts.length === 1) {
2620
+ observedLineBytes += Buffer.byteLength(parts[0] ?? "");
2621
+ }
2622
+ else {
2623
+ observedLineBytes += Buffer.byteLength(parts[0] ?? "");
2624
+ for (let index = 1; index < parts.length - 1; index++) {
2625
+ if (Buffer.byteLength(parts[index] ?? "") > MAX_IMPORT_LINE_BYTES) {
2626
+ stream.destroy(new Error(`Import line exceeds the ${MAX_IMPORT_LINE_BYTES}-byte limit`));
2627
+ return;
2628
+ }
2629
+ }
2630
+ observedLineBytes = Buffer.byteLength(parts.at(-1) ?? "");
2631
+ }
2632
+ if (observedLineBytes > MAX_IMPORT_LINE_BYTES) {
2633
+ stream.destroy(new Error(`Import line exceeds the ${MAX_IMPORT_LINE_BYTES}-byte limit`));
2634
+ }
2635
+ });
2186
2636
  let header;
2187
2637
  let kb;
2188
2638
  let vectorWriter;
@@ -2228,13 +2678,28 @@ export class KnowledgeEngine {
2228
2678
  for await (const rawLine of lines) {
2229
2679
  throwIfAborted(signal);
2230
2680
  lineNumber++;
2681
+ if (lineNumber > MAX_IMPORT_LINES)
2682
+ throw new Error(`Import exceeds the ${MAX_IMPORT_LINES}-line limit`);
2231
2683
  const line = rawLine.trim();
2684
+ if (Buffer.byteLength(rawLine) > MAX_IMPORT_LINE_BYTES) {
2685
+ throw new Error(`Import line ${lineNumber} exceeds the ${MAX_IMPORT_LINE_BYTES}-byte limit`);
2686
+ }
2232
2687
  if (!line)
2233
2688
  continue;
2234
2689
  if (!header) {
2235
- header = JSON.parse(line);
2236
- if (!header.name)
2237
- throw new Error("Import header is missing a knowledge base name");
2690
+ const parsedHeader = JSON.parse(line);
2691
+ if (typeof parsedHeader !== "object" || parsedHeader === null || Array.isArray(parsedHeader)) {
2692
+ throw new Error("Import header must be an object");
2693
+ }
2694
+ const headerRecord = parsedHeader;
2695
+ header = {
2696
+ ...validateKnowledgeBaseMetadata(headerRecord.name, headerRecord.description),
2697
+ chunk_count: headerRecord.chunk_count === undefined ? undefined : headerRecord.chunk_count,
2698
+ };
2699
+ if (header.chunk_count !== undefined &&
2700
+ (!Number.isInteger(header.chunk_count) || header.chunk_count < 0 || header.chunk_count > MAX_IMPORT_CHUNKS)) {
2701
+ throw new Error(`Import header chunk_count must be between 0 and ${MAX_IMPORT_CHUNKS}`);
2702
+ }
2238
2703
  const existingKB = getKBByName(this.db, header.name);
2239
2704
  if (existingKB) {
2240
2705
  throw new Error(`Knowledge base "${header.name}" already exists. Use knowledge_update to refresh it, or knowledge_remove before importing a replacement.`);
@@ -2260,11 +2725,14 @@ export class KnowledgeEngine {
2260
2725
  vectorWriter = openVectorWriter(tempVectorFile);
2261
2726
  continue;
2262
2727
  }
2263
- const imported = JSON.parse(line);
2728
+ const imported = parseImportedChunk(JSON.parse(line));
2264
2729
  const chunk = importedChunkToInsert(imported);
2265
2730
  chunk.content_tokenized = preTokenizeForFTS(buildChunkEmbeddingText(chunk));
2266
2731
  const finalizedChunks = await finalizeChunksForEmbedding([chunk], embeddingConfig, signal);
2267
2732
  pendingChunks.push(...finalizedChunks);
2733
+ if (inserted + pendingChunks.length > MAX_IMPORT_CHUNKS) {
2734
+ throw new Error(`Import exceeds the ${MAX_IMPORT_CHUNKS}-chunk limit`);
2735
+ }
2268
2736
  pendingSymbols.push(...extractSymbols(imported.content, imported.file_path, imported.file_type));
2269
2737
  while (pendingChunks.length >= embeddingBatchSize)
2270
2738
  await flushPending();
@@ -2279,6 +2747,7 @@ export class KnowledgeEngine {
2279
2747
  renameSync(tempVectorFile, this.vectorPathFor(kb.id));
2280
2748
  tempVectorFile = undefined;
2281
2749
  updateKBCounts(this.db, kb.id, inserted, getFileCount(this.db, kb.id));
2750
+ markKBIndexed(this.db, kb.id);
2282
2751
  updateKBStatus(this.db, kb.id, "ready");
2283
2752
  const savedKB = getKB(this.db, kb.id);
2284
2753
  if (!savedKB)