pi-knowledge 0.14.0 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/CHANGELOG.md +32 -0
  2. package/README.md +63 -24
  3. package/dist/index.d.ts +10 -4
  4. package/dist/index.js +300 -187
  5. package/dist/src/coverage-snapshot.d.ts +23 -0
  6. package/dist/src/coverage-snapshot.js +53 -0
  7. package/dist/src/daemon/client.d.ts +60 -0
  8. package/dist/src/daemon/client.js +360 -0
  9. package/dist/src/daemon/main.d.ts +2 -0
  10. package/dist/src/daemon/main.js +109 -0
  11. package/dist/src/daemon/protocol.d.ts +71 -0
  12. package/dist/src/daemon/protocol.js +170 -0
  13. package/dist/src/daemon/runtime.d.ts +26 -0
  14. package/dist/src/daemon/runtime.js +192 -0
  15. package/dist/src/daemon/server.d.ts +46 -0
  16. package/dist/src/daemon/server.js +500 -0
  17. package/dist/src/daemon/service.d.ts +49 -0
  18. package/dist/src/daemon/service.js +527 -0
  19. package/dist/src/diagnostics/health.d.ts +6 -0
  20. package/dist/src/diagnostics/health.js +135 -2
  21. package/dist/src/engine.d.ts +63 -9
  22. package/dist/src/engine.js +959 -170
  23. package/dist/src/filesystem-access.d.ts +9 -0
  24. package/dist/src/filesystem-access.js +100 -0
  25. package/dist/src/indexer/chunker.d.ts +10 -0
  26. package/dist/src/indexer/chunker.js +284 -9
  27. package/dist/src/mcp/http.d.ts +47 -0
  28. package/dist/src/mcp/http.js +564 -0
  29. package/dist/src/mcp/main.d.ts +2 -0
  30. package/dist/src/mcp/main.js +97 -0
  31. package/dist/src/mcp/protocol.d.ts +81 -0
  32. package/dist/src/mcp/protocol.js +444 -0
  33. package/dist/src/mcp/server.d.ts +42 -0
  34. package/dist/src/mcp/server.js +378 -0
  35. package/dist/src/model-worker-client.js +9 -0
  36. package/dist/src/model-worker-contract.d.ts +3 -0
  37. package/dist/src/model-worker-contract.js +8 -0
  38. package/dist/src/model-worker.js +76 -4
  39. package/dist/src/network/safe-fetch.d.ts +7 -0
  40. package/dist/src/network/safe-fetch.js +199 -0
  41. package/dist/src/node-runtime.d.ts +3 -0
  42. package/dist/src/node-runtime.js +78 -0
  43. package/dist/src/profiles.d.ts +15 -0
  44. package/dist/src/profiles.js +160 -0
  45. package/dist/src/runtime-config.js +5 -6
  46. package/dist/src/search/tuning.d.ts +2 -0
  47. package/dist/src/search/tuning.js +2 -0
  48. package/dist/src/secure-files.d.ts +17 -0
  49. package/dist/src/secure-files.js +184 -0
  50. package/dist/src/status.js +12 -7
  51. package/dist/src/storage/sqlite.d.ts +4 -1
  52. package/dist/src/storage/sqlite.js +46 -3
  53. package/dist/src/type-guards.d.ts +1 -0
  54. package/dist/src/type-guards.js +3 -0
  55. package/dist/src/watcher/file-watcher.d.ts +3 -1
  56. package/dist/src/watcher/file-watcher.js +166 -32
  57. package/dist/src/windows-security.d.ts +4 -0
  58. package/dist/src/windows-security.js +47 -0
  59. package/docs/configuration.md +42 -10
  60. package/docs/daemon-profiles.md +302 -0
  61. package/docs/mcp-clients.md +266 -0
  62. package/docs/offline-mode.md +6 -6
  63. package/package.json +28 -2
@@ -1,13 +1,18 @@
1
- import { existsSync, readFileSync, renameSync, rmSync, statSync } from "node:fs";
2
- import { join, resolve } from "node:path";
3
- import { diagnoseKB } from "./diagnostics/health.js";
1
+ import { closeSync, constants, createReadStream, createWriteStream, existsSync, fstatSync, fsyncSync, lstatSync, openSync, renameSync, rmSync, statSync, } from "node:fs";
2
+ import { lstat as lstatFile, mkdtemp, open as openFile, realpath as realpathFile, rm as rmAsync, writeFile, } from "node:fs/promises";
3
+ import { tmpdir } from "node:os";
4
+ import { isAbsolute, join, relative, resolve, sep } from "node:path";
5
+ import { createInterface } from "node:readline";
6
+ import { diagnoseKB, diagnoseKBAsync } from "./diagnostics/health.js";
4
7
  import { describeEmbeddingCompatibility, describeEmbeddingDecoration, dispose as disposeEmbedding, embedDocuments, embeddingConfigLabel, embeddingSignature, embedQueryWithConfig, embedValidatedDocuments, inspectEmbeddingCompatibility, prepareForShutdown as prepareEmbeddingForShutdown, resolveEmbeddingConfig, } from "./embedding/provider.js";
5
8
  import { openVectorReader, openVectorWriter } from "./embedding/vectors.js";
6
- import { addSkippedScanEntry, analyzeIndexableContent, buildChunkEmbeddingText, chunkFile, chunkIdentityHash, chunkMarkdown, createSkippedScanStats, isReadableTextFile, isSupportedDocumentFile, iterateScannableFiles, preTokenizeForFTS, summarizeSkippedScan, } from "./indexer/chunker.js";
9
+ import { assertNetworkAccess, assertOpenedPathAuthorized, assertOutputPathAuthorized, authorizeExistingPath, authorizeOutputPath, } from "./filesystem-access.js";
10
+ import { addSkippedScanEntry, analyzeIndexableContent, buildChunkEmbeddingText, chunkFile, chunkIdentityHash, chunkMarkdown, createSkippedScanStats, isReadableTextFile, isSupportedDocumentFile, iterateScannableFiles, iterateScannableFilesAsync, preTokenizeForFTS, summarizeSkippedScan, } from "./indexer/chunker.js";
7
11
  import { finalizeChunksForEmbedding } from "./indexer/embedding-safe-chunks.js";
8
12
  import { htmlToStructuredMarkdown } from "./indexer/html.js";
9
13
  import { extractSymbols } from "./indexer/symbols.js";
10
14
  import { extractDocumentInModelWorker, shutdownModelWorker } from "./model-worker-client.js";
15
+ import { fetchPublicText } from "./network/safe-fetch.js";
11
16
  import { searchBM25 } from "./search/bm25.js";
12
17
  import { MAX_SEARCH_WINDOW } from "./search/contracts.js";
13
18
  import { diversifyRankedChunks, interleaveByFile } from "./search/diversity.js";
@@ -17,7 +22,8 @@ import { hasAnyLexicalEvidence, hasEnoughLexicalEvidence, normalizeFileTypeFilte
17
22
  import { disposeReranker, prepareRerankerForShutdown, rerank } from "./search/reranker.js";
18
23
  import { resolveSearchTuning, resolveSearchWindow, summarizeSearchTuning, } from "./search/tuning.js";
19
24
  import { searchVectorFile } from "./search/vector.js";
20
- import { countSymbols, createKB, deleteChunksByIds, deleteKB, deleteSymbolsByKB, finishIndexingJob, getChunkById, getChunkCount, getChunksByFile, getFileCount, getKB, getKBByName, getSymbolCount, insertChunks, insertSymbols, iterateChunkIdsByKB, iterateChunksByKB, listKBs, openDatabase, searchSymbols, startIndexingJob, updateIndexingJob, updateKBCounts, updateKBEmbeddingMetadata, updateKBSourceOptions, updateKBStatus, } from "./storage/sqlite.js";
25
+ import { ensureOwnerOnlyDirectory, prepareSafeNewFile, publishNewFile } from "./secure-files.js";
26
+ import { countSymbols, createKB, deleteChunksByIds, deleteKB, deleteSymbolsByKB, finishIndexingJob, getChunkById, getChunkCount, getChunksByFile, getFileCount, getKB, getKBByName, getSymbolCount, getSymbolsByFile, insertChunks, insertSymbols, iterateChunkIdsByKB, iterateChunksByKB, listKBs, markKBIndexed, openDatabase, searchSymbols, startIndexingJob, updateIndexingJob, updateKBCounts, updateKBEmbeddingMetadata, updateKBSourceOptions, updateKBStatus, } from "./storage/sqlite.js";
21
27
  export { CURRENT_EMBEDDING_MODEL } from "./embedding/provider.js";
22
28
  export const CURRENT_URL_STRUCTURE_POLICY = 1;
23
29
  export const CURRENT_EMBEDDING_INPUT_POLICY = 2;
@@ -63,11 +69,12 @@ function matchReasonFor(mode) {
63
69
  return "bm25";
64
70
  return "hybrid";
65
71
  }
66
- function toScanOptions(options = {}) {
72
+ function toScanOptions(options = {}, accessPolicy) {
67
73
  return {
68
74
  includeSuggestedText: options.include_suggested_text === true,
69
75
  includePaths: options.include_paths,
70
76
  excludePaths: options.exclude_paths,
77
+ accessPolicy,
71
78
  };
72
79
  }
73
80
  function serializeAddOptions(options = {}) {
@@ -131,16 +138,22 @@ function planDirectoryScan(dirPath, options = {}, signal) {
131
138
  skipped,
132
139
  };
133
140
  }
134
- function sourceMtimeFor(kb, filePath) {
135
- if (!kb?.source_path)
136
- return undefined;
137
- const absPath = kb.source_type === "directory" ? join(kb.source_path, filePath) : kb.source_path;
138
- try {
139
- return statSync(absPath).mtimeMs;
140
- }
141
- catch {
142
- return undefined;
141
+ async function planDirectoryScanAsync(dirPath, options = {}, signal) {
142
+ const skipped = createSkippedScanStats();
143
+ let files = 0;
144
+ let bytes = 0;
145
+ for await (const file of iterateScannableFilesAsync(dirPath, skipped, options, signal)) {
146
+ throwIfAborted(signal);
147
+ files++;
148
+ bytes += file.size;
143
149
  }
150
+ return {
151
+ files,
152
+ bytes,
153
+ skippedTotal: skipped.total,
154
+ skippedSummary: summarizeSkippedScan(skipped),
155
+ skipped,
156
+ };
144
157
  }
145
158
  function kbTrustMultiplier(kb) {
146
159
  let multiplier = kb.status === "stale" ? 0.92 : 1;
@@ -150,6 +163,181 @@ function kbTrustMultiplier(kb) {
150
163
  multiplier *= 0.98;
151
164
  return multiplier;
152
165
  }
166
+ function summarizeKnowledgeBases(kbs) {
167
+ return kbs.map((kb) => ({
168
+ id: kb.id,
169
+ name: kb.name,
170
+ status: kb.status,
171
+ chunk_count: kb.chunk_count,
172
+ }));
173
+ }
174
+ function searchMissAction(reason, mode, availableKnowledgeBases) {
175
+ switch (reason) {
176
+ case "no_knowledge_bases":
177
+ return "add_knowledge_base";
178
+ case "knowledge_base_not_found":
179
+ return "fix_kb_id";
180
+ case "no_searchable_knowledge_bases":
181
+ return availableKnowledgeBases.length > 0 && availableKnowledgeBases.every((kb) => kb.status === "indexing")
182
+ ? "wait_for_indexing"
183
+ : "fix_knowledge_base";
184
+ case "no_indexed_chunks":
185
+ return availableKnowledgeBases.some((kb) => kb.status === "indexing")
186
+ ? "wait_for_indexing"
187
+ : "fix_knowledge_base";
188
+ case "embedding_incompatible":
189
+ return "fix_embedding";
190
+ case "no_retrieval_candidates":
191
+ return mode === "fast" || mode === "code" || mode === "config" || mode === "errors"
192
+ ? "try_semantic"
193
+ : mode === "semantic" || mode === "semantic_hybrid"
194
+ ? "rephrase"
195
+ : "try_semantic_hybrid";
196
+ case "threshold_filtered":
197
+ return mode === "semantic_hybrid" ? "rephrase" : "try_semantic_hybrid";
198
+ case "evidence_filtered":
199
+ return "try_semantic_hybrid";
200
+ case "metadata_filtered":
201
+ return "relax_filter";
202
+ case "page_out_of_range":
203
+ return "reset_offset";
204
+ case "reranker_empty":
205
+ case "no_match":
206
+ return "rephrase";
207
+ }
208
+ }
209
+ function classifySearchMiss(facts) {
210
+ const skippedKnowledgeBases = facts.availableKnowledgeBases.filter((kb) => kb.status !== "ready" && kb.status !== "stale");
211
+ let reason;
212
+ if (facts.allKnowledgeBases.length === 0) {
213
+ reason = "no_knowledge_bases";
214
+ }
215
+ else if (facts.requestedKbId &&
216
+ !facts.availableKnowledgeBases.some((kb) => kb.id === facts.requestedKbId || kb.name === facts.requestedKbId)) {
217
+ reason = "knowledge_base_not_found";
218
+ }
219
+ else if (facts.searchedKnowledgeBases.length === 0) {
220
+ reason = "no_searchable_knowledge_bases";
221
+ }
222
+ else if (facts.searchedKnowledgeBases.every((kb) => kb.chunk_count === 0)) {
223
+ reason = "no_indexed_chunks";
224
+ }
225
+ else if (facts.retrievedCandidates === 0 &&
226
+ facts.embeddingIncompatibleCount === facts.searchedKnowledgeBases.filter((kb) => kb.chunk_count > 0).length) {
227
+ reason = "embedding_incompatible";
228
+ }
229
+ else if (facts.retrievedCandidates === 0) {
230
+ reason = "no_retrieval_candidates";
231
+ }
232
+ else if (facts.thresholdCandidates === 0) {
233
+ reason = "threshold_filtered";
234
+ }
235
+ else if (facts.eligibleCandidates === 0) {
236
+ reason = "evidence_filtered";
237
+ }
238
+ else if (facts.filteredCandidates === 0 && (facts.fileType || facts.pathPattern)) {
239
+ reason = "metadata_filtered";
240
+ }
241
+ else if (facts.mode === "deep" && facts.rankedCandidates === 0) {
242
+ reason = "reranker_empty";
243
+ }
244
+ else if (facts.totalResults > 0 && facts.offset >= facts.totalResults) {
245
+ reason = "page_out_of_range";
246
+ }
247
+ else {
248
+ reason = "no_match";
249
+ }
250
+ return {
251
+ reason,
252
+ action: searchMissAction(reason, facts.mode, facts.availableKnowledgeBases),
253
+ requested_kb_id: facts.requestedKbId,
254
+ min_hybrid_score: facts.minimumScore,
255
+ file_type: facts.fileType,
256
+ path_pattern: facts.pathPattern,
257
+ searched_kbs: summarizeKnowledgeBases(facts.searchedKnowledgeBases),
258
+ skipped_kbs: summarizeKnowledgeBases(skippedKnowledgeBases),
259
+ retrieved_candidates: facts.retrievedCandidates,
260
+ threshold_candidates: facts.thresholdCandidates,
261
+ eligible_candidates: facts.eligibleCandidates,
262
+ filtered_candidates: facts.filteredCandidates,
263
+ ranked_candidates: facts.rankedCandidates,
264
+ total_results: facts.totalResults,
265
+ };
266
+ }
267
+ function searchAttemptSummary(mode, response) {
268
+ return {
269
+ mode,
270
+ reason: response.miss?.reason,
271
+ retrieved_candidates: response.tuning?.retrieved_candidates ?? 0,
272
+ threshold_candidates: response.tuning?.threshold_candidates ?? 0,
273
+ eligible_candidates: response.tuning?.eligible_candidates ?? 0,
274
+ filtered_candidates: response.tuning?.filtered_candidates ?? 0,
275
+ result_count: response.results.length,
276
+ };
277
+ }
278
+ function symbolMissAction(reason, availableKnowledgeBases) {
279
+ switch (reason) {
280
+ case "no_knowledge_bases":
281
+ return "add_knowledge_base";
282
+ case "knowledge_base_not_found":
283
+ return "fix_kb_id";
284
+ case "no_searchable_knowledge_bases":
285
+ return availableKnowledgeBases.length > 0 && availableKnowledgeBases.every((kb) => kb.status === "indexing")
286
+ ? "wait_for_indexing"
287
+ : "fix_knowledge_base";
288
+ case "symbol_index_empty":
289
+ return "rebuild_symbol_index";
290
+ case "exact_name_not_found":
291
+ return "broaden_symbol_search";
292
+ case "metadata_filtered":
293
+ return "relax_filter";
294
+ case "page_out_of_range":
295
+ return "reset_offset";
296
+ case "no_match":
297
+ return "use_text_search";
298
+ }
299
+ }
300
+ function classifySymbolSearchMiss(facts) {
301
+ const skippedKnowledgeBases = facts.availableKnowledgeBases.filter((kb) => kb.status !== "ready" && kb.status !== "stale");
302
+ let reason;
303
+ if (facts.allKnowledgeBases.length === 0) {
304
+ reason = "no_knowledge_bases";
305
+ }
306
+ else if (facts.requestedKbId &&
307
+ !facts.availableKnowledgeBases.some((kb) => kb.id === facts.requestedKbId || kb.name === facts.requestedKbId)) {
308
+ reason = "knowledge_base_not_found";
309
+ }
310
+ else if (facts.searchedKnowledgeBases.length === 0) {
311
+ reason = "no_searchable_knowledge_bases";
312
+ }
313
+ else if (facts.indexedSymbols === 0) {
314
+ reason = "symbol_index_empty";
315
+ }
316
+ else if (facts.totalResults > 0 && facts.offset >= facts.totalResults) {
317
+ reason = "page_out_of_range";
318
+ }
319
+ else if ((facts.kind || facts.filePattern) && facts.unfilteredMatches > 0) {
320
+ reason = "metadata_filtered";
321
+ }
322
+ else if (facts.exact) {
323
+ reason = "exact_name_not_found";
324
+ }
325
+ else {
326
+ reason = "no_match";
327
+ }
328
+ return {
329
+ reason,
330
+ action: symbolMissAction(reason, facts.availableKnowledgeBases),
331
+ requested_kb_id: facts.requestedKbId,
332
+ kind: facts.kind,
333
+ file_pattern: facts.filePattern,
334
+ exact: facts.exact,
335
+ searched_kbs: summarizeKnowledgeBases(facts.searchedKnowledgeBases),
336
+ skipped_kbs: summarizeKnowledgeBases(skippedKnowledgeBases),
337
+ indexed_symbols: facts.indexedSymbols,
338
+ unfiltered_matches: facts.unfilteredMatches,
339
+ };
340
+ }
153
341
  function formatBytes(bytes) {
154
342
  if (bytes < 1024)
155
343
  return `${bytes} B`;
@@ -360,35 +548,146 @@ function isWeakAutoResponse(query, response, primaryMode, attemptMode) {
360
548
  }
361
549
  return false;
362
550
  }
551
+ const MAX_DIRECT_TEXT_BYTES = 10 * 1024 * 1024;
552
+ const MAX_DIRECT_DOCUMENT_BYTES = 64 * 1024 * 1024;
553
+ const MAX_IMPORT_BYTES = 256 * 1024 * 1024;
554
+ const MAX_IMPORT_LINE_BYTES = 16 * 1024 * 1024;
555
+ const MAX_IMPORT_LINES = 1_000_001;
556
+ const MAX_IMPORT_CHUNKS = 1_000_000;
557
+ const MAX_EXPORT_BYTES = MAX_IMPORT_BYTES;
558
+ const MAX_EXPORT_CHUNKS = MAX_IMPORT_CHUNKS;
559
+ const MAX_EXPORT_DURATION_MS = 60 * 60_000;
560
+ const MAX_KB_NAME_BYTES = 512;
561
+ const MAX_KB_DESCRIPTION_BYTES = 4 * 1024;
562
+ const MAX_IMPORT_CONTENT_BYTES = 12 * 1024 * 1024;
563
+ const MAX_IMPORT_FILE_PATH_BYTES = 4_096;
564
+ const MAX_IMPORT_FILE_TYPE_BYTES = 256;
565
+ const MAX_IMPORT_METADATA_BYTES = 64 * 1024;
566
+ const MAX_IMPORT_METADATA_CONTEXT_BYTES = 8 * 1024;
567
+ const MAX_IMPORT_METADATA_SCOPE_ITEMS = 128;
568
+ const MAX_IMPORT_LINE_NUMBER = 2_147_483_647;
569
+ const MAX_IMPORT_FIXED_CONTEXT_BYTES = 64 * 1024;
570
+ const IMPORTED_CHUNK_KEYS = new Set(["content", "file_path", "file_type", "start_line", "end_line", "metadata_json"]);
571
+ const IMPORTED_CONTEXT_STRING_KEYS = [
572
+ "heading",
573
+ "breadcrumb",
574
+ "language",
575
+ "symbol",
576
+ "function_name",
577
+ "symbol_kind",
578
+ "parent_symbol",
579
+ "signature",
580
+ ];
363
581
  async function chunkUrl(source, signal) {
364
- const res = await fetch(source, { signal });
365
- if (!res.ok)
366
- throw new Error(`Fetch failed: ${res.status}`);
367
- const html = await res.text();
582
+ const html = await fetchPublicText(source, signal);
368
583
  throwIfAborted(signal);
369
584
  const text = htmlToStructuredMarkdown(html);
370
585
  const structuredChunks = chunkMarkdown(text, source, "html");
371
586
  return structuredChunks.length > 0 ? structuredChunks : chunkFile(text, source);
372
587
  }
373
- async function extractSourceFileContent(filePath, signal) {
588
+ function validateKnowledgeBaseMetadata(name, description) {
589
+ if (typeof name !== "string" || !name.trim())
590
+ throw new Error("Knowledge base name must be a non-empty string");
591
+ if (Buffer.byteLength(name) > MAX_KB_NAME_BYTES) {
592
+ throw new Error(`Knowledge base name exceeds the ${MAX_KB_NAME_BYTES}-byte limit`);
593
+ }
594
+ const normalizedDescription = description === null ? undefined : description;
595
+ if (normalizedDescription !== undefined && typeof normalizedDescription !== "string") {
596
+ throw new Error("Knowledge base description must be a string");
597
+ }
598
+ if (typeof normalizedDescription === "string" &&
599
+ Buffer.byteLength(normalizedDescription) > MAX_KB_DESCRIPTION_BYTES) {
600
+ throw new Error(`Knowledge base description exceeds the ${MAX_KB_DESCRIPTION_BYTES}-byte limit`);
601
+ }
602
+ return { name, description: normalizedDescription };
603
+ }
604
+ async function readBoundedRegularFile(filePath, maxBytes, signal, expected, access) {
605
+ if (expected?.rootPath) {
606
+ const canonical = await realpathFile(filePath);
607
+ const rel = relative(expected.rootPath, canonical);
608
+ if (rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel)) {
609
+ throw new Error(`Source file escaped its scanned root: ${filePath}`);
610
+ }
611
+ }
612
+ const direct = lstatSync(filePath);
613
+ if (direct.isSymbolicLink() || !direct.isFile())
614
+ throw new Error(`Source is not a regular non-symlink file: ${filePath}`);
615
+ if (expected?.device !== undefined &&
616
+ expected.inode !== undefined &&
617
+ (direct.dev !== expected.device || direct.ino !== expected.inode)) {
618
+ throw new Error(`Source file changed after scanning: ${filePath}`);
619
+ }
620
+ const handle = await openFile(filePath, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
621
+ try {
622
+ const opened = await handle.stat();
623
+ if (!opened.isFile())
624
+ throw new Error(`Source is not a regular file: ${filePath}`);
625
+ assertOpenedPathAuthorized(filePath, opened.dev, opened.ino, access);
626
+ if (expected?.rootPath) {
627
+ const canonical = await realpathFile(filePath);
628
+ const current = await lstatFile(filePath);
629
+ const rel = relative(expected.rootPath, canonical);
630
+ if (rel === ".." ||
631
+ rel.startsWith(`..${sep}`) ||
632
+ isAbsolute(rel) ||
633
+ current.isSymbolicLink() ||
634
+ current.dev !== opened.dev ||
635
+ current.ino !== opened.ino) {
636
+ throw new Error(`Source file changed outside its scanned root: ${filePath}`);
637
+ }
638
+ }
639
+ if (expected?.device !== undefined &&
640
+ expected.inode !== undefined &&
641
+ (opened.dev !== expected.device || opened.ino !== expected.inode)) {
642
+ throw new Error(`Source file changed while opening: ${filePath}`);
643
+ }
644
+ if (opened.size > maxBytes) {
645
+ throw new Error(`Source file exceeds the ${maxBytes}-byte direct indexing limit: ${filePath}`);
646
+ }
647
+ const chunks = [];
648
+ let bytes = 0;
649
+ while (bytes <= maxBytes) {
650
+ throwIfAborted(signal);
651
+ const buffer = Buffer.allocUnsafe(Math.min(64 * 1024, maxBytes - bytes + 1));
652
+ const result = await handle.read(buffer, 0, buffer.length, null);
653
+ if (result.bytesRead === 0)
654
+ break;
655
+ bytes += result.bytesRead;
656
+ if (bytes > maxBytes) {
657
+ throw new Error(`Source file exceeds the ${maxBytes}-byte direct indexing limit: ${filePath}`);
658
+ }
659
+ chunks.push(buffer.subarray(0, result.bytesRead));
660
+ }
661
+ return Buffer.concat(chunks, bytes);
662
+ }
663
+ finally {
664
+ await handle.close();
665
+ }
666
+ }
667
+ async function extractSourceFileContent(filePath, signal, expected, access) {
668
+ filePath = authorizeExistingPath(filePath, access);
374
669
  const lowerPath = filePath.toLowerCase();
375
- if (lowerPath.endsWith(".pdf")) {
376
- throwIfAborted(signal);
377
- const extracted = await extractDocumentInModelWorker(filePath, "pdf", signal);
378
- throwIfAborted(signal);
379
- return extracted;
670
+ const isDocument = lowerPath.endsWith(".pdf") || lowerPath.endsWith(".docx") || lowerPath.endsWith(".doc");
671
+ const maxBytes = isDocument ? MAX_DIRECT_DOCUMENT_BYTES : MAX_DIRECT_TEXT_BYTES;
672
+ const content = await readBoundedRegularFile(filePath, maxBytes, signal, expected, access);
673
+ if (!isDocument) {
674
+ if (content.subarray(0, 8_000).includes(0x00)) {
675
+ throw new Error(`File is not readable text and has no supported extractor: ${filePath}`);
676
+ }
677
+ return { content: content.toString("utf8"), fileType: "text" };
380
678
  }
381
- if (lowerPath.endsWith(".docx") || lowerPath.endsWith(".doc")) {
382
- throwIfAborted(signal);
383
- const extracted = await extractDocumentInModelWorker(filePath, "docx", signal);
679
+ const tempDirectory = ensureOwnerOnlyDirectory(await mkdtemp(join(tmpdir(), "pi-knowledge-document-")));
680
+ const tempPath = join(tempDirectory, "source");
681
+ try {
682
+ await writeFile(tempPath, content, { flag: "wx", mode: 0o600 });
683
+ const fileType = lowerPath.endsWith(".pdf") ? "pdf" : "docx";
684
+ const extracted = await extractDocumentInModelWorker(tempPath, fileType, signal);
384
685
  throwIfAborted(signal);
385
686
  return extracted;
386
687
  }
387
- if (!isReadableTextFile(filePath)) {
388
- throw new Error(`File is not readable text and has no supported extractor: ${filePath}`);
688
+ finally {
689
+ await rmAsync(tempDirectory, { recursive: true, force: true });
389
690
  }
390
- throwIfAborted(signal);
391
- return { content: readFileSync(filePath, "utf-8"), fileType: "text" };
392
691
  }
393
692
  function looksLikeLocalPath(source) {
394
693
  const trimmed = source.trim();
@@ -402,11 +701,15 @@ function looksLikeLocalPath(source) {
402
701
  return true;
403
702
  return /\.[A-Za-z0-9][A-Za-z0-9_-]{0,15}$/.test(trimmed);
404
703
  }
405
- function classifySource(source) {
406
- const resolvedSource = resolve(source);
704
+ function classifySource(source, access) {
407
705
  const isUrl = source.startsWith("http://") || source.startsWith("https://");
408
- const isDir = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isDirectory();
409
- const isFile = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isFile();
706
+ if (isUrl)
707
+ assertNetworkAccess(access);
708
+ const requestedSource = resolve(source);
709
+ const exists = !isUrl && existsSync(requestedSource);
710
+ const resolvedSource = exists ? authorizeExistingPath(requestedSource, access) : requestedSource;
711
+ const isDir = exists && statSync(resolvedSource).isDirectory();
712
+ const isFile = exists && statSync(resolvedSource).isFile();
410
713
  if (!isUrl && !isDir && !isFile && looksLikeLocalPath(source)) {
411
714
  throw new Error(`Local path does not exist: ${source}. Pass inline text only when the source itself is text.`);
412
715
  }
@@ -419,7 +722,7 @@ function classifySource(source) {
419
722
  };
420
723
  }
421
724
  async function extractScannableFileContent(file, signal) {
422
- return extractSourceFileContent(file.path, signal);
725
+ return extractSourceFileContent(file.path, signal, file);
423
726
  }
424
727
  async function extractScannableFileContentOrSkip(file, skipped, signal) {
425
728
  try {
@@ -447,6 +750,100 @@ function throwIfAborted(signal) {
447
750
  if (signal?.aborted)
448
751
  throw new Error("Cancelled");
449
752
  }
753
+ function boundedImportString(value, field, maxBytes) {
754
+ if (typeof value !== "string")
755
+ throw new Error(`Imported chunk ${field} must be a string`);
756
+ if (Buffer.byteLength(value) > maxBytes) {
757
+ throw new Error(`Imported chunk ${field} exceeds the ${maxBytes}-byte limit`);
758
+ }
759
+ return value;
760
+ }
761
+ function validateImportedMetadata(metadataJson) {
762
+ let parsed;
763
+ try {
764
+ parsed = JSON.parse(metadataJson);
765
+ }
766
+ catch {
767
+ throw new Error("Imported chunk metadata_json must contain valid JSON");
768
+ }
769
+ if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
770
+ throw new Error("Imported chunk metadata_json must contain an object");
771
+ }
772
+ const metadata = parsed;
773
+ let contextBytes = 0;
774
+ for (const key of IMPORTED_CONTEXT_STRING_KEYS) {
775
+ const value = metadata[key];
776
+ if (value === undefined)
777
+ continue;
778
+ if (typeof value !== "string")
779
+ throw new Error(`Imported chunk metadata ${key} must be a string`);
780
+ contextBytes += Buffer.byteLength(value);
781
+ }
782
+ const scope = metadata.scope;
783
+ if (scope !== undefined) {
784
+ if (!Array.isArray(scope) || scope.length > MAX_IMPORT_METADATA_SCOPE_ITEMS) {
785
+ throw new Error("Imported chunk metadata scope is invalid");
786
+ }
787
+ for (const item of scope) {
788
+ if (typeof item !== "string")
789
+ throw new Error("Imported chunk metadata scope must contain strings");
790
+ contextBytes += Buffer.byteLength(item);
791
+ }
792
+ }
793
+ if (contextBytes > MAX_IMPORT_METADATA_CONTEXT_BYTES) {
794
+ throw new Error(`Imported chunk embedding metadata exceeds the ${MAX_IMPORT_METADATA_CONTEXT_BYTES}-byte limit`);
795
+ }
796
+ }
797
+ function parseImportedChunk(value) {
798
+ if (typeof value !== "object" || value === null || Array.isArray(value)) {
799
+ throw new Error("Imported chunk must be an object");
800
+ }
801
+ const record = value;
802
+ for (const key of Object.keys(record)) {
803
+ if (!IMPORTED_CHUNK_KEYS.has(key))
804
+ throw new Error(`Imported chunk contains unknown field: ${key}`);
805
+ }
806
+ const content = boundedImportString(record.content, "content", MAX_IMPORT_CONTENT_BYTES);
807
+ const filePath = boundedImportString(record.file_path, "file_path", MAX_IMPORT_FILE_PATH_BYTES);
808
+ const fileType = boundedImportString(record.file_type, "file_type", MAX_IMPORT_FILE_TYPE_BYTES);
809
+ if (!content)
810
+ throw new Error("Imported chunk content must not be empty");
811
+ if (!filePath)
812
+ throw new Error("Imported chunk file_path must not be empty");
813
+ if (!fileType)
814
+ throw new Error("Imported chunk file_type must not be empty");
815
+ const startLine = record.start_line;
816
+ const endLine = record.end_line;
817
+ if (!Number.isSafeInteger(startLine) ||
818
+ typeof startLine !== "number" ||
819
+ startLine < 0 ||
820
+ startLine > MAX_IMPORT_LINE_NUMBER) {
821
+ throw new Error("Imported chunk start_line is invalid");
822
+ }
823
+ if (!Number.isSafeInteger(endLine) ||
824
+ typeof endLine !== "number" ||
825
+ endLine < startLine ||
826
+ endLine > MAX_IMPORT_LINE_NUMBER) {
827
+ throw new Error("Imported chunk end_line is invalid");
828
+ }
829
+ const metadataJson = record.metadata_json === undefined
830
+ ? "{}"
831
+ : boundedImportString(record.metadata_json, "metadata_json", MAX_IMPORT_METADATA_BYTES);
832
+ validateImportedMetadata(metadataJson);
833
+ const imported = {
834
+ content,
835
+ file_path: filePath,
836
+ file_type: fileType,
837
+ start_line: startLine,
838
+ end_line: endLine,
839
+ metadata_json: metadataJson,
840
+ };
841
+ const fixedContext = buildChunkEmbeddingText({ ...imported, content: "" });
842
+ if (Buffer.byteLength(fixedContext) > MAX_IMPORT_FIXED_CONTEXT_BYTES) {
843
+ throw new Error(`Imported chunk fixed embedding context exceeds the ${MAX_IMPORT_FIXED_CONTEXT_BYTES}-byte limit`);
844
+ }
845
+ return imported;
846
+ }
450
847
  function importedChunkToInsert(chunk) {
451
848
  const metadataJson = chunk.metadata_json || "{}";
452
849
  return {
@@ -543,9 +940,12 @@ export class KnowledgeEngine {
543
940
  deleteVectorFile(kbId) {
544
941
  rmSync(this.vectorPathFor(kbId), { force: true });
545
942
  }
546
- plan(source, options = {}, signal) {
943
+ plan(source, options = {}, signal, access) {
547
944
  throwIfAborted(signal);
548
- const { resolvedSource, isDir, isFile, sourceType } = classifySource(source);
945
+ const { resolvedSource, isDir, isFile, sourceType } = classifySource(source, access);
946
+ if ((isDir || isFile) && lstatSync(resolvedSource).isSymbolicLink()) {
947
+ throw new Error(`Source is not a regular non-symlink path: ${resolvedSource}`);
948
+ }
549
949
  if (isDir) {
550
950
  const plan = planDirectoryScan(resolvedSource, toScanOptions(options), signal);
551
951
  return {
@@ -589,17 +989,53 @@ export class KnowledgeEngine {
589
989
  summary: `${sourceType === "url" ? "URL" : "Inline text"} plan: 1 source`,
590
990
  };
591
991
  }
592
- async add(source, name, onProgress, signal, options = {}) {
992
+ async planAsync(source, options = {}, signal, access) {
593
993
  throwIfAborted(signal);
594
- return this.runExclusive(["global:mutation", `kb-name:${name}`], `Mutation for "${name}"`, () => this.addUnlocked(source, name, onProgress, signal, options));
994
+ const { resolvedSource, isDir, isFile, sourceType } = classifySource(source, access);
995
+ if (isDir) {
996
+ const plan = await planDirectoryScanAsync(resolvedSource, toScanOptions(options, access), signal);
997
+ return {
998
+ source_type: "directory",
999
+ scannable_files: plan.files,
1000
+ scannable_bytes: plan.bytes,
1001
+ skipped: plan.skipped,
1002
+ summary: `Directory plan: ${plan.files} scannable files, ${formatBytes(plan.bytes)} scannable text, skipped ${plan.skippedTotal} (${plan.skippedSummary})`,
1003
+ };
1004
+ }
1005
+ if (isFile && lstatSync(resolvedSource).isSymbolicLink()) {
1006
+ throw new Error(`Source is not a regular non-symlink file: ${resolvedSource}`);
1007
+ }
1008
+ const skipped = createSkippedScanStats();
1009
+ const sourceBytes = isFile ? statSync(resolvedSource).size : Buffer.byteLength(source);
1010
+ const maxBytes = isFile && isSupportedDocumentFile(resolvedSource) ? MAX_DIRECT_DOCUMENT_BYTES : MAX_DIRECT_TEXT_BYTES;
1011
+ if (sourceType !== "url" && sourceBytes > maxBytes) {
1012
+ addSkippedScanEntry(skipped, {
1013
+ path: isFile ? resolvedSource : "<inline>",
1014
+ reason: "oversized",
1015
+ size: sourceBytes,
1016
+ });
1017
+ return {
1018
+ source_type: sourceType,
1019
+ scannable_files: 0,
1020
+ scannable_bytes: 0,
1021
+ skipped,
1022
+ summary: `${isFile ? "File" : "Inline text"} plan: 0 scannable sources; source exceeds ${formatBytes(maxBytes)}`,
1023
+ };
1024
+ }
1025
+ return this.plan(source, options, signal, access);
1026
+ }
1027
+ async add(source, name, onProgress, signal, options = {}, access) {
1028
+ throwIfAborted(signal);
1029
+ validateKnowledgeBaseMetadata(name, undefined);
1030
+ return this.runExclusive(["global:mutation", `kb-name:${name}`], `Mutation for "${name}"`, () => this.addUnlocked(source, name, onProgress, signal, options, access));
595
1031
  }
596
- async addUnlocked(source, name, onProgress, signal, options = {}) {
1032
+ async addUnlocked(source, name, onProgress, signal, options = {}, access) {
597
1033
  if (!this.db)
598
1034
  throw new Error("Engine not initialized");
599
1035
  throwIfAborted(signal);
600
1036
  const db = this.db;
601
- const { resolvedSource, isUrl, isDir, isFile, sourceType } = classifySource(source);
602
- const scanOptions = toScanOptions(options);
1037
+ const { resolvedSource, isUrl, isDir, isFile, sourceType } = classifySource(source, access);
1038
+ const scanOptions = toScanOptions(options, access);
603
1039
  const existingKB = getKBByName(db, name);
604
1040
  if (existingKB) {
605
1041
  throw new Error(`Knowledge base "${name}" already exists. Use knowledge_update to refresh it, or knowledge_remove before adding a replacement.`);
@@ -699,13 +1135,13 @@ export class KnowledgeEngine {
699
1135
  await addChunks(chunks);
700
1136
  }
701
1137
  else if (isFile) {
702
- const extracted = await extractSourceFileContent(resolvedSource, signal);
1138
+ const extracted = await extractSourceFileContent(resolvedSource, signal, undefined, access);
703
1139
  fileCount = 1;
704
1140
  const chunks = await analyzeAndAddSymbols(extracted.content, resolvedSource, extracted.fileType);
705
1141
  await addChunks(chunks);
706
1142
  }
707
1143
  else if (isDir) {
708
- const plan = planDirectoryScan(resolvedSource, scanOptions, signal);
1144
+ const plan = await planDirectoryScanAsync(resolvedSource, scanOptions, signal);
709
1145
  const planningMessage = `Planned directory scan: ${plan.files} files, ${formatBytes(plan.bytes)} scannable text, skipped ${plan.skippedTotal} (${plan.skippedSummary})`;
710
1146
  updateIndexingJob(this.db, kb.id, {
711
1147
  phase: "planning",
@@ -724,7 +1160,7 @@ export class KnowledgeEngine {
724
1160
  onProgress?.(scanningMessage);
725
1161
  const skipped = createSkippedScanStats();
726
1162
  let processedFiles = 0;
727
- for (const file of iterateScannableFiles(resolvedSource, skipped, scanOptions)) {
1163
+ for await (const file of iterateScannableFilesAsync(resolvedSource, skipped, scanOptions, signal)) {
728
1164
  if (signal?.aborted)
729
1165
  throw new Error("Cancelled");
730
1166
  const extracted = await extractScannableFileContentOrSkip(file, skipped, signal);
@@ -757,6 +1193,9 @@ export class KnowledgeEngine {
757
1193
  }
758
1194
  else {
759
1195
  fileCount = 1;
1196
+ if (Buffer.byteLength(source) > MAX_DIRECT_TEXT_BYTES) {
1197
+ throw new Error(`Inline text exceeds the ${MAX_DIRECT_TEXT_BYTES}-byte indexing limit`);
1198
+ }
760
1199
  const chunks = await analyzeAndAddSymbols(source, "inline-text", "text");
761
1200
  await addChunks(chunks);
762
1201
  }
@@ -767,6 +1206,7 @@ export class KnowledgeEngine {
767
1206
  tempVectorFile = undefined;
768
1207
  const savedFileCount = isDir ? getFileCount(this.db, kb.id) : fileCount;
769
1208
  updateKBCounts(this.db, kb.id, chunkCount, savedFileCount);
1209
+ markKBIndexed(this.db, kb.id);
770
1210
  updateKBStatus(this.db, kb.id, "ready");
771
1211
  const savedKB = getKB(this.db, kb.id);
772
1212
  if (!savedKB)
@@ -797,7 +1237,7 @@ export class KnowledgeEngine {
797
1237
  throw e;
798
1238
  }
799
1239
  }
800
- async update(nameOrId, onProgress, signal) {
1240
+ async update(nameOrId, onProgress, signal, access) {
801
1241
  if (!this.db)
802
1242
  throw new Error("Engine not initialized");
803
1243
  if (this.disposing)
@@ -809,24 +1249,32 @@ export class KnowledgeEngine {
809
1249
  if (!kb.source_path || (kb.source_type !== "url" && !existsSync(kb.source_path))) {
810
1250
  throw new Error(`Source path not available or missing: ${kb.source_path}`);
811
1251
  }
812
- const updateableKB = { ...kb, source_path: kb.source_path };
1252
+ let sourcePath;
1253
+ if (kb.source_type === "url") {
1254
+ assertNetworkAccess(access);
1255
+ sourcePath = kb.source_path;
1256
+ }
1257
+ else {
1258
+ sourcePath = authorizeExistingPath(kb.source_path, access);
1259
+ }
1260
+ const updateableKB = { ...kb, source_path: sourcePath };
813
1261
  const activeUpdate = this.activeUpdates.get(updateableKB.id);
814
1262
  if (activeUpdate) {
815
1263
  onProgress?.(`Update already running for "${updateableKB.name}"; waiting for the active update.`);
816
1264
  return activeUpdate;
817
1265
  }
818
- const updateRun = this.runExclusive(["global:mutation", `kb:${updateableKB.id}`], `Mutation for "${updateableKB.name}"`, () => this.runUpdate(updateableKB, onProgress, signal)).finally(() => {
1266
+ const updateRun = this.runExclusive(["global:mutation", `kb:${updateableKB.id}`], `Mutation for "${updateableKB.name}"`, () => this.runUpdate(updateableKB, onProgress, signal, access)).finally(() => {
819
1267
  if (this.activeUpdates.get(updateableKB.id) === updateRun)
820
1268
  this.activeUpdates.delete(updateableKB.id);
821
1269
  });
822
1270
  this.activeUpdates.set(updateableKB.id, updateRun);
823
1271
  return updateRun;
824
1272
  }
825
- async runUpdate(kb, onProgress, signal) {
1273
+ async runUpdate(kb, onProgress, signal, access) {
826
1274
  if (!this.db)
827
1275
  throw new Error("Engine not initialized");
828
1276
  throwIfAborted(signal);
829
- const scanOptions = toScanOptions(parseAddOptions(kb.source_options));
1277
+ const scanOptions = toScanOptions(parseAddOptions(kb.source_options), access);
830
1278
  const embeddingConfig = resolveEmbeddingConfig();
831
1279
  const embeddingBatchSize = resolveIndexEmbedBatchSize();
832
1280
  updateKBStatus(this.db, kb.id, "indexing");
@@ -965,8 +1413,8 @@ export class KnowledgeEngine {
965
1413
  stagedSymbols.push(...extractSymbols(chunks.map((chunk) => chunk.content).join("\n\n"), kb.source_path, "html"));
966
1414
  await processChunks(chunks);
967
1415
  }
968
- else if (statSync(kb.source_path).isDirectory()) {
969
- const plan = planDirectoryScan(kb.source_path, scanOptions, signal);
1416
+ else if (kb.source_type === "directory") {
1417
+ const plan = await planDirectoryScanAsync(kb.source_path, scanOptions, signal);
970
1418
  plannedTotalFiles = plan.files;
971
1419
  const planningMessage = `Planned directory scan: ${plan.files} files, ${formatBytes(plan.bytes)} scannable text, skipped ${plan.skippedTotal} (${plan.skippedSummary})`;
972
1420
  updateIndexingJob(this.db, kb.id, {
@@ -977,12 +1425,27 @@ export class KnowledgeEngine {
977
1425
  });
978
1426
  onProgress?.(planningMessage);
979
1427
  const skipped = createSkippedScanStats();
980
- for (const file of iterateScannableFiles(kb.source_path, skipped, scanOptions)) {
1428
+ for await (const file of iterateScannableFilesAsync(kb.source_path, skipped, scanOptions, signal)) {
981
1429
  if (signal?.aborted)
982
1430
  throw new Error("Cancelled");
983
1431
  const extracted = await extractScannableFileContentOrSkip(file, skipped, signal);
984
- if (!extracted)
1432
+ if (!extracted) {
1433
+ for (const chunk of getChunksByFile(this.db, kb.id, file.relPath, 0, Number.MAX_SAFE_INTEGER)) {
1434
+ const entries = existingHashes.get(chunk.content_hash);
1435
+ const retainedIndex = entries?.findIndex((entry) => entry.id === chunk.id) ?? -1;
1436
+ if (!entries || retainedIndex < 0)
1437
+ continue;
1438
+ const [retained] = entries.splice(retainedIndex, 1);
1439
+ if (canReuseExistingVectors && retained) {
1440
+ const indexes = oldVectorIndexByHash.get(chunk.content_hash) ?? [];
1441
+ indexes.push(retained.vectorIndex);
1442
+ oldVectorIndexByHash.set(chunk.content_hash, indexes);
1443
+ }
1444
+ unchanged++;
1445
+ }
1446
+ stagedSymbols.push(...getSymbolsByFile(this.db, kb.id, file.relPath));
985
1447
  continue;
1448
+ }
986
1449
  const analysis = await analyzeIndexableContent(extracted.content, file.relPath, extracted.fileType);
987
1450
  scannedFiles++;
988
1451
  stagedSymbols.push(...analysis.symbols);
@@ -1018,7 +1481,7 @@ export class KnowledgeEngine {
1018
1481
  onProgress?.(message);
1019
1482
  }
1020
1483
  else {
1021
- const extracted = await extractSourceFileContent(kb.source_path, signal);
1484
+ const extracted = await extractSourceFileContent(kb.source_path, signal, undefined, access);
1022
1485
  scannedFiles = 1;
1023
1486
  const analysis = await analyzeIndexableContent(extracted.content, kb.source_path, extracted.fileType);
1024
1487
  stagedSymbols.push(...analysis.symbols);
@@ -1125,6 +1588,7 @@ export class KnowledgeEngine {
1125
1588
  updateKBEmbeddingMetadata(this.db, kb.id, embeddingModel, finalEmbeddingDimension === undefined ? null : embeddingSignature(embeddingConfig, finalEmbeddingDimension), finalEmbeddingDimension ?? null);
1126
1589
  updateKBCounts(this.db, kb.id, finalChunkCount, getFileCount(this.db, kb.id));
1127
1590
  updateKBSourceOptions(this.db, kb.id, serializeAddOptions(parseAddOptions(kb.source_options)));
1591
+ markKBIndexed(this.db, kb.id);
1128
1592
  updateKBStatus(this.db, kb.id, "ready");
1129
1593
  const skippedReadySuffix = latestSkippedTotal > 0 ? `, skipped ${latestSkippedTotal} (${latestSkippedSummary})` : "";
1130
1594
  const readyMessage = `Ready: +${addedCount} -${idsToRemove.length} =${unchanged}${skippedReadySuffix}`;
@@ -1163,43 +1627,82 @@ export class KnowledgeEngine {
1163
1627
  const requestedMode = options.mode ?? "hybrid";
1164
1628
  if (requestedMode === "auto") {
1165
1629
  const primaryMode = chooseAutoMode(query);
1166
- const attempts = [primaryMode, ...fallbackModesFor(primaryMode)];
1630
+ const modes = [primaryMode, ...fallbackModesFor(primaryMode)];
1167
1631
  const tried = [];
1632
+ const attemptSummaries = [];
1168
1633
  const warnings = [];
1169
- let lastTuning;
1170
- for (const mode of attempts) {
1634
+ let lastMissResponse;
1635
+ let bestWeakResponse;
1636
+ for (const mode of modes) {
1171
1637
  if (tried.includes(mode))
1172
1638
  continue;
1173
1639
  tried.push(mode);
1174
1640
  throwIfAborted(signal);
1175
1641
  const response = await this.search(query, { ...options, mode }, signal);
1176
- lastTuning = response.tuning;
1642
+ attemptSummaries.push(searchAttemptSummary(mode, response));
1177
1643
  if (response.warnings)
1178
1644
  warnings.push(...response.warnings);
1645
+ if (response.miss?.reason === "no_knowledge_bases" ||
1646
+ response.miss?.reason === "knowledge_base_not_found" ||
1647
+ response.miss?.reason === "no_searchable_knowledge_bases" ||
1648
+ response.miss?.reason === "no_indexed_chunks") {
1649
+ return {
1650
+ ...response,
1651
+ warnings: warnings.length > 0 ? [...new Set(warnings)] : response.warnings,
1652
+ mode_used: mode,
1653
+ retry_modes: tried.slice(0, -1),
1654
+ miss: { ...response.miss, attempts: attemptSummaries },
1655
+ };
1656
+ }
1179
1657
  if (!isWeakAutoResponse(query, response, primaryMode, mode)) {
1180
1658
  return {
1181
1659
  ...response,
1182
1660
  warnings: warnings.length > 0 ? [...new Set(warnings)] : response.warnings,
1183
1661
  mode_used: mode,
1184
1662
  retry_modes: tried.slice(0, -1),
1185
- suggestions: response.suggestions,
1186
1663
  };
1187
1664
  }
1665
+ if (response.results.length > 0) {
1666
+ if (!bestWeakResponse || response.results.length > bestWeakResponse.response.results.length) {
1667
+ bestWeakResponse = { mode, response };
1668
+ }
1669
+ }
1670
+ else {
1671
+ lastMissResponse = { mode, response };
1672
+ }
1188
1673
  if (tried.length >= 3)
1189
1674
  break;
1190
1675
  }
1191
- return {
1192
- results: [],
1193
- total_count: 0,
1194
- has_more: false,
1195
- warnings: warnings.length > 0 ? [...new Set(warnings)] : undefined,
1196
- mode_used: tried.at(-1),
1197
- retry_modes: tried.slice(0, -1),
1198
- tuning: lastTuning,
1199
- suggestions: [
1200
- "No results after auto mode fallback. Check knowledge_status, try a more exact term, or rebuild the KB if indexing rules changed.",
1201
- ],
1202
- };
1676
+ if (lastMissResponse?.response.miss) {
1677
+ const attemptedModes = new Set(attemptSummaries.map((attempt) => attempt.mode));
1678
+ let action = lastMissResponse.response.miss.action;
1679
+ if (action === "try_semantic" && attemptedModes.has("semantic")) {
1680
+ action = attemptedModes.has("semantic_hybrid") ? "rephrase" : "try_semantic_hybrid";
1681
+ }
1682
+ else if (action === "try_semantic_hybrid" && attemptedModes.has("semantic_hybrid")) {
1683
+ action = "rephrase";
1684
+ }
1685
+ return {
1686
+ ...lastMissResponse.response,
1687
+ warnings: warnings.length > 0 ? [...new Set(warnings)] : lastMissResponse.response.warnings,
1688
+ mode_used: lastMissResponse.mode,
1689
+ retry_modes: tried.filter((mode) => mode !== lastMissResponse.mode),
1690
+ miss: {
1691
+ ...lastMissResponse.response.miss,
1692
+ action,
1693
+ attempts: attemptSummaries,
1694
+ },
1695
+ };
1696
+ }
1697
+ if (bestWeakResponse) {
1698
+ return {
1699
+ ...bestWeakResponse.response,
1700
+ warnings: warnings.length > 0 ? [...new Set(warnings)] : bestWeakResponse.response.warnings,
1701
+ mode_used: bestWeakResponse.mode,
1702
+ retry_modes: tried.filter((mode) => mode !== bestWeakResponse.mode),
1703
+ };
1704
+ }
1705
+ throw new Error("Auto search exhausted without a result or miss classification");
1203
1706
  }
1204
1707
  const db = this.db;
1205
1708
  const { mode = "hybrid", offset: requestedOffset, kb_id, filters, diversity = "balanced" } = options;
@@ -1209,8 +1712,9 @@ export class KnowledgeEngine {
1209
1712
  const queryTokens = tokenizeForSimilarity(query);
1210
1713
  const normalizedFileType = normalizeFileTypeFilter(filters?.file_type);
1211
1714
  const warnings = [];
1715
+ const allKBs = listKBs(db);
1212
1716
  const selectedKB = kb_id ? (getKB(db, kb_id) ?? getKBByName(db, kb_id)) : undefined;
1213
- const availableKBs = kb_id ? [selectedKB].filter(Boolean) : listKBs(db);
1717
+ const availableKBs = kb_id ? [selectedKB].filter(Boolean) : allKBs;
1214
1718
  const kbs = availableKBs.filter((kb) => kb.status === "ready" || kb.status === "stale");
1215
1719
  const kbById = new Map(kbs.map((kb) => [kb.id, kb]));
1216
1720
  for (const kb of availableKBs) {
@@ -1236,11 +1740,30 @@ export class KnowledgeEngine {
1236
1740
  warnings: warnings.length > 0 ? warnings : undefined,
1237
1741
  mode_used: mode,
1238
1742
  tuning: tuningSummary,
1743
+ miss: classifySearchMiss({
1744
+ allKnowledgeBases: allKBs,
1745
+ availableKnowledgeBases: availableKBs,
1746
+ searchedKnowledgeBases: kbs,
1747
+ requestedKbId: kb_id,
1748
+ mode,
1749
+ fileType: filters?.file_type,
1750
+ pathPattern: filters?.path_pattern,
1751
+ minimumScore: tuning.minHybridScore,
1752
+ offset,
1753
+ retrievedCandidates: 0,
1754
+ thresholdCandidates: 0,
1755
+ eligibleCandidates: 0,
1756
+ filteredCandidates: 0,
1757
+ rankedCandidates: 0,
1758
+ totalResults: 0,
1759
+ embeddingIncompatibleCount: 0,
1760
+ }),
1239
1761
  };
1240
1762
  }
1241
1763
  const allResults = [];
1242
1764
  const vectorsByChunkId = new Map();
1243
1765
  let queryEmbedding;
1766
+ const embeddingIncompatibleKbIds = new Set();
1244
1767
  for (const kb of kbs) {
1245
1768
  throwIfAborted(signal);
1246
1769
  if (kb.chunk_count === 0)
@@ -1258,6 +1781,7 @@ export class KnowledgeEngine {
1258
1781
  throwIfAborted(signal);
1259
1782
  const compatibility = inspectEmbeddingCompatibility(kb.embedding_signature, kb.embedding_dimension, queryEmbeddingConfig, queryVec.length);
1260
1783
  if (!compatibility.compatible) {
1784
+ embeddingIncompatibleKbIds.add(kb.id);
1261
1785
  warnings.push(embeddingMismatchWarning(kb, compatibility));
1262
1786
  continue;
1263
1787
  }
@@ -1283,6 +1807,7 @@ export class KnowledgeEngine {
1283
1807
  vectorsByChunkId.set(chunkId, vector);
1284
1808
  }
1285
1809
  else {
1810
+ embeddingIncompatibleKbIds.add(kb.id);
1286
1811
  warnings.push(embeddingMismatchWarning(kb, compatibility));
1287
1812
  }
1288
1813
  const fused = weightedScoreFusion(bm25Results, vecResults);
@@ -1307,30 +1832,30 @@ export class KnowledgeEngine {
1307
1832
  result.score = ranking.adjusted_score;
1308
1833
  }
1309
1834
  }
1310
- let scored = unique;
1835
+ let thresholded = unique.filter((result) => rankingByChunkId.has(result.chunkId));
1836
+ if (retrievalMode !== "fast" && retrievalMode !== "semantic") {
1837
+ thresholded = thresholded.filter((result) => result.score >= tuning.minHybridScore);
1838
+ }
1839
+ const thresholdCandidates = thresholded.length;
1840
+ let eligible = thresholded;
1311
1841
  if (retrievalMode === "fast") {
1312
- scored = unique.filter((result) => {
1842
+ eligible = thresholded.filter((result) => {
1313
1843
  const chunk = getChunkById(db, result.chunkId);
1314
- if (!chunk)
1315
- return false;
1316
- return hasAnyLexicalEvidence(buildChunkEmbeddingText(chunk), queryTokens);
1844
+ return chunk ? hasAnyLexicalEvidence(buildChunkEmbeddingText(chunk), queryTokens) : false;
1317
1845
  });
1318
1846
  }
1319
- else if (retrievalMode !== "semantic") {
1320
- scored = unique.filter((result) => {
1847
+ else if (retrievalMode !== "semantic" && retrievalMode !== "semantic_hybrid") {
1848
+ eligible = thresholded.filter((result) => {
1321
1849
  const chunk = getChunkById(db, result.chunkId);
1322
- if (!chunk)
1323
- return false;
1324
- if (retrievalMode === "semantic_hybrid")
1325
- return result.score >= tuning.minHybridScore;
1326
- return result.score >= tuning.minHybridScore && hasEnoughLexicalEvidence(chunk, queryTokens);
1850
+ return chunk ? hasEnoughLexicalEvidence(chunk, queryTokens) : false;
1327
1851
  });
1328
1852
  }
1329
- scored.sort((a, b) => b.score - a.score);
1853
+ eligible.sort((a, b) => b.score - a.score);
1854
+ const eligibleCandidates = eligible.length;
1330
1855
  // Apply metadata filters post-retrieval
1331
- let filtered = scored;
1856
+ let filtered = eligible;
1332
1857
  if (normalizedFileType || filters?.path_pattern) {
1333
- filtered = scored.filter((r) => {
1858
+ filtered = eligible.filter((r) => {
1334
1859
  const chunk = getChunkById(db, r.chunkId);
1335
1860
  if (!chunk)
1336
1861
  return false;
@@ -1380,7 +1905,7 @@ export class KnowledgeEngine {
1380
1905
  const page = diversified.slice(offset, offset + limit);
1381
1906
  const results = page.map((r) => {
1382
1907
  const kb = kbById.get(r.chunk.kb_id);
1383
- const sourceMtime = sourceMtimeFor(kb, r.chunk.file_path);
1908
+ const stale = kb?.status === "stale";
1384
1909
  return {
1385
1910
  content: r.content,
1386
1911
  file_path: r.chunk.file_path,
@@ -1395,26 +1920,48 @@ export class KnowledgeEngine {
1395
1920
  chunk_id: r.chunk.id,
1396
1921
  chunk_hash: r.chunk.content_hash,
1397
1922
  indexed_at: r.chunk.indexed_at,
1398
- source_mtime: sourceMtime,
1399
- stale: sourceMtime !== undefined ? sourceMtime > r.chunk.indexed_at : kb?.status === "stale",
1923
+ stale,
1400
1924
  match_reason: matchReasonFor(mode),
1401
1925
  source_chunk_ids: r.sourceChunkIds,
1402
1926
  },
1403
1927
  };
1404
1928
  });
1929
+ const deepTuning = {
1930
+ ...tuningSummary,
1931
+ retrieved_candidates: unique.length,
1932
+ threshold_candidates: thresholdCandidates,
1933
+ eligible_candidates: eligibleCandidates,
1934
+ filtered_candidates: filtered.length,
1935
+ diversity_candidates: ranked.length,
1936
+ candidate_pool_truncated: candidatePoolTruncated,
1937
+ };
1405
1938
  return {
1406
1939
  results,
1407
1940
  total_count: diversified.length,
1408
1941
  has_more: offset + limit < diversified.length,
1409
1942
  warnings: warnings.length > 0 ? warnings : undefined,
1410
1943
  mode_used: mode,
1411
- tuning: {
1412
- ...tuningSummary,
1413
- retrieved_candidates: unique.length,
1414
- filtered_candidates: filtered.length,
1415
- diversity_candidates: ranked.length,
1416
- candidate_pool_truncated: candidatePoolTruncated,
1417
- },
1944
+ tuning: deepTuning,
1945
+ miss: results.length === 0
1946
+ ? classifySearchMiss({
1947
+ allKnowledgeBases: allKBs,
1948
+ availableKnowledgeBases: availableKBs,
1949
+ searchedKnowledgeBases: kbs,
1950
+ requestedKbId: kb_id,
1951
+ mode,
1952
+ fileType: filters?.file_type,
1953
+ pathPattern: filters?.path_pattern,
1954
+ minimumScore: tuning.minHybridScore,
1955
+ offset,
1956
+ retrievedCandidates: unique.length,
1957
+ thresholdCandidates,
1958
+ eligibleCandidates,
1959
+ filteredCandidates: filtered.length,
1960
+ rankedCandidates: ranked.length,
1961
+ totalResults: diversified.length,
1962
+ embeddingIncompatibleCount: embeddingIncompatibleKbIds.size,
1963
+ })
1964
+ : undefined,
1418
1965
  };
1419
1966
  }
1420
1967
  const ranked = [];
@@ -1471,36 +2018,52 @@ export class KnowledgeEngine {
1471
2018
  end_line: r.endLine,
1472
2019
  provenance: (() => {
1473
2020
  const kb = kbById.get(r.chunk.kb_id);
1474
- const sourceMtime = sourceMtimeFor(kb, r.chunk.file_path);
2021
+ const stale = kb?.status === "stale";
1475
2022
  return {
1476
2023
  chunk_id: r.chunk.id,
1477
2024
  chunk_hash: r.chunk.content_hash,
1478
2025
  indexed_at: r.chunk.indexed_at,
1479
- source_mtime: sourceMtime,
1480
- stale: sourceMtime !== undefined ? sourceMtime > r.chunk.indexed_at : kb?.status === "stale",
2026
+ stale,
1481
2027
  match_reason: matchReasonFor(mode),
1482
2028
  source_chunk_ids: r.sourceChunkIds,
1483
2029
  };
1484
2030
  })(),
1485
2031
  }));
2032
+ const responseTuning = {
2033
+ ...tuningSummary,
2034
+ retrieved_candidates: unique.length,
2035
+ threshold_candidates: thresholdCandidates,
2036
+ eligible_candidates: eligibleCandidates,
2037
+ filtered_candidates: filtered.length,
2038
+ diversity_candidates: ranked.length,
2039
+ candidate_pool_truncated: candidatePoolTruncated,
2040
+ };
1486
2041
  return {
1487
2042
  results,
1488
2043
  total_count: total,
1489
2044
  has_more: offset + limit < total,
1490
2045
  warnings: warnings.length > 0 ? warnings : undefined,
1491
2046
  mode_used: mode,
1492
- tuning: {
1493
- ...tuningSummary,
1494
- retrieved_candidates: unique.length,
1495
- filtered_candidates: filtered.length,
1496
- diversity_candidates: ranked.length,
1497
- candidate_pool_truncated: candidatePoolTruncated,
1498
- },
1499
- suggestions: results.length === 0
1500
- ? [
1501
- "Try mode 'fast' for exact symbols, mode 'semantic' for pure vector search, or mode 'semantic_hybrid' when weak lexical overlap should not block fused vector results.",
1502
- "Run knowledge_status if the KB should contain this answer.",
1503
- ]
2047
+ tuning: responseTuning,
2048
+ miss: results.length === 0
2049
+ ? classifySearchMiss({
2050
+ allKnowledgeBases: allKBs,
2051
+ availableKnowledgeBases: availableKBs,
2052
+ searchedKnowledgeBases: kbs,
2053
+ requestedKbId: kb_id,
2054
+ mode,
2055
+ fileType: filters?.file_type,
2056
+ pathPattern: filters?.path_pattern,
2057
+ minimumScore: tuning.minHybridScore,
2058
+ offset,
2059
+ retrievedCandidates: unique.length,
2060
+ thresholdCandidates,
2061
+ eligibleCandidates,
2062
+ filteredCandidates: filtered.length,
2063
+ rankedCandidates: ranked.length,
2064
+ totalResults: total,
2065
+ embeddingIncompatibleCount: embeddingIncompatibleKbIds.size,
2066
+ })
1504
2067
  : undefined,
1505
2068
  };
1506
2069
  }
@@ -1518,11 +2081,11 @@ export class KnowledgeEngine {
1518
2081
  this.deleteVectorFile(kb.id);
1519
2082
  return true;
1520
2083
  }
1521
- list(signal) {
2084
+ list(signal, limit) {
1522
2085
  if (!this.db)
1523
2086
  return [];
1524
2087
  throwIfAborted(signal);
1525
- return listKBs(this.db);
2088
+ return listKBs(this.db, limit);
1526
2089
  }
1527
2090
  clear() {
1528
2091
  if (!this.db)
@@ -1546,16 +2109,32 @@ export class KnowledgeEngine {
1546
2109
  return diagnoseKB(db, kb, signal);
1547
2110
  });
1548
2111
  }
1549
- statusSnapshot(signal) {
2112
+ async diagnoseAsync(signal, knowledgeBasesOverride, maxDiagnosticPaths = Number.POSITIVE_INFINITY) {
1550
2113
  if (!this.db)
1551
- return { knowledge_bases: [], diagnostics: [], embeddings: [] };
2114
+ return [];
1552
2115
  throwIfAborted(signal);
1553
2116
  const db = this.db;
1554
- const knowledgeBases = listKBs(db);
1555
- const diagnostics = knowledgeBases.map((kb) => {
2117
+ const diagnostics = [];
2118
+ let remainingPaths = maxDiagnosticPaths;
2119
+ for (const kb of knowledgeBasesOverride ?? listKBs(db)) {
1556
2120
  throwIfAborted(signal);
1557
- return diagnoseKB(db, kb, signal);
1558
- });
2121
+ const diagnostic = await diagnoseKBAsync(db, kb, signal, Math.max(0, remainingPaths));
2122
+ diagnostics.push(diagnostic);
2123
+ remainingPaths = Math.max(0, remainingPaths - diagnostic.stale_files.length - diagnostic.orphan_files.length);
2124
+ }
2125
+ return diagnostics;
2126
+ }
2127
+ statusSnapshot(signal, diagnosticsOverride, knowledgeBasesOverride) {
2128
+ if (!this.db)
2129
+ return { knowledge_bases: [], diagnostics: [], embeddings: [] };
2130
+ throwIfAborted(signal);
2131
+ const db = this.db;
2132
+ const knowledgeBases = knowledgeBasesOverride ?? listKBs(db);
2133
+ const diagnostics = diagnosticsOverride ??
2134
+ knowledgeBases.map((kb) => {
2135
+ throwIfAborted(signal);
2136
+ return diagnoseKB(db, kb, signal);
2137
+ });
1559
2138
  try {
1560
2139
  const config = resolveEmbeddingConfig();
1561
2140
  const activeModel = embeddingConfigLabel(config);
@@ -1586,16 +2165,46 @@ export class KnowledgeEngine {
1586
2165
  };
1587
2166
  }
1588
2167
  }
2168
+ async statusSnapshotAsync(signal, maxKnowledgeBases = Number.POSITIVE_INFINITY, maxDiagnosticPaths = Number.POSITIVE_INFINITY) {
2169
+ if (!this.db)
2170
+ return { knowledge_bases: [], diagnostics: [], embeddings: [] };
2171
+ const knowledgeBases = listKBs(this.db, Number.isFinite(maxKnowledgeBases) ? maxKnowledgeBases + 1 : undefined);
2172
+ if (knowledgeBases.length > maxKnowledgeBases) {
2173
+ throw new Error(`Knowledge status exceeds the ${maxKnowledgeBases}-knowledge-base RPC limit`);
2174
+ }
2175
+ return this.statusSnapshot(signal, await this.diagnoseAsync(signal, knowledgeBases, maxDiagnosticPaths), knowledgeBases);
2176
+ }
1589
2177
  symbolSearch(query, options = {}, signal) {
1590
2178
  if (!this.db)
1591
2179
  throw new Error("Engine not initialized");
1592
2180
  throwIfAborted(signal);
1593
2181
  const db = this.db;
2182
+ const allKBs = listKBs(db);
1594
2183
  const selectedKB = options.kb_id ? (getKB(db, options.kb_id) ?? getKBByName(db, options.kb_id)) : undefined;
1595
- if (options.kb_id && !selectedKB)
1596
- throw new Error(`Knowledge base not found: ${options.kb_id}`);
2184
+ const availableKBs = options.kb_id ? [selectedKB].filter(Boolean) : allKBs;
2185
+ const searchedKBs = availableKBs.filter((kb) => kb.status === "ready" || kb.status === "stale");
1597
2186
  const limit = Math.max(1, Math.min(200, options.limit ?? 20));
1598
2187
  const offset = Math.max(0, options.offset ?? 0);
2188
+ if (searchedKBs.length === 0) {
2189
+ return {
2190
+ results: [],
2191
+ total_count: 0,
2192
+ has_more: false,
2193
+ miss: classifySymbolSearchMiss({
2194
+ allKnowledgeBases: allKBs,
2195
+ availableKnowledgeBases: availableKBs,
2196
+ searchedKnowledgeBases: searchedKBs,
2197
+ requestedKbId: options.kb_id,
2198
+ kind: options.kind,
2199
+ filePattern: options.file_pattern,
2200
+ exact: options.exact === true,
2201
+ offset,
2202
+ totalResults: 0,
2203
+ indexedSymbols: 0,
2204
+ unfilteredMatches: 0,
2205
+ }),
2206
+ };
2207
+ }
1599
2208
  const symbolOptions = {
1600
2209
  kbId: selectedKB?.id,
1601
2210
  kind: options.kind,
@@ -1611,32 +2220,55 @@ export class KnowledgeEngine {
1611
2220
  const page = symbols.slice(0, limit);
1612
2221
  const total = countSymbols(db, query, symbolOptions);
1613
2222
  throwIfAborted(signal);
2223
+ const results = page.map((symbol) => {
2224
+ const kb = getKB(db, symbol.kb_id);
2225
+ return {
2226
+ name: symbol.name,
2227
+ kind: symbol.kind,
2228
+ file_path: symbol.file_path,
2229
+ file_type: symbol.file_type,
2230
+ kb_name: kb?.name ?? "unknown",
2231
+ start_line: symbol.start_line,
2232
+ end_line: symbol.end_line,
2233
+ signature: symbol.signature ?? undefined,
2234
+ container_name: symbol.container_name ?? undefined,
2235
+ text: symbol.text,
2236
+ indexed_at: symbol.indexed_at,
2237
+ };
2238
+ });
2239
+ let miss;
2240
+ if (results.length === 0) {
2241
+ const indexedSymbols = countSymbols(db, "", { kbId: selectedKB?.id });
2242
+ const unfilteredMatches = total === 0 && (options.kind || options.file_pattern)
2243
+ ? countSymbols(db, query, { kbId: selectedKB?.id, exact: options.exact })
2244
+ : total;
2245
+ throwIfAborted(signal);
2246
+ miss = classifySymbolSearchMiss({
2247
+ allKnowledgeBases: allKBs,
2248
+ availableKnowledgeBases: availableKBs,
2249
+ searchedKnowledgeBases: searchedKBs,
2250
+ requestedKbId: options.kb_id,
2251
+ kind: options.kind,
2252
+ filePattern: options.file_pattern,
2253
+ exact: options.exact === true,
2254
+ offset,
2255
+ totalResults: total,
2256
+ indexedSymbols,
2257
+ unfilteredMatches,
2258
+ });
2259
+ }
1614
2260
  return {
1615
- results: page.map((symbol) => {
1616
- const kb = getKB(db, symbol.kb_id);
1617
- return {
1618
- name: symbol.name,
1619
- kind: symbol.kind,
1620
- file_path: symbol.file_path,
1621
- file_type: symbol.file_type,
1622
- kb_name: kb?.name ?? "unknown",
1623
- start_line: symbol.start_line,
1624
- end_line: symbol.end_line,
1625
- signature: symbol.signature ?? undefined,
1626
- container_name: symbol.container_name ?? undefined,
1627
- text: symbol.text,
1628
- indexed_at: symbol.indexed_at,
1629
- };
1630
- }),
2261
+ results,
1631
2262
  total_count: total,
1632
2263
  has_more: symbols.length > limit,
2264
+ miss,
1633
2265
  };
1634
2266
  }
1635
- doctor(signal) {
2267
+ doctor(signal, diagnosticsOverride, knowledgeBasesOverride) {
1636
2268
  throwIfAborted(signal);
1637
- const diagnostics = this.diagnose(signal);
2269
+ const diagnostics = diagnosticsOverride ?? this.diagnose(signal);
1638
2270
  const issues = [];
1639
- const kbs = this.list();
2271
+ const kbs = knowledgeBasesOverride ?? this.list();
1640
2272
  if (kbs.length === 0) {
1641
2273
  issues.push({
1642
2274
  severity: "blocking",
@@ -1666,7 +2298,16 @@ export class KnowledgeEngine {
1666
2298
  action_code: "rebuild_kb",
1667
2299
  });
1668
2300
  }
1669
- if (diagnostic.coverage_percent < 80) {
2301
+ if (diagnostic.scan_error) {
2302
+ issues.push({
2303
+ severity: "warning",
2304
+ kb_name: diagnostic.kb_name,
2305
+ message: `Source diagnostics scan failed: ${diagnostic.scan_error}`,
2306
+ action: "Check source accessibility and scanner limits, then run knowledge_doctor again.",
2307
+ action_code: "check_source",
2308
+ });
2309
+ }
2310
+ else if (diagnostic.coverage_percent < 80) {
1670
2311
  issues.push({
1671
2312
  severity: "warning",
1672
2313
  kb_name: diagnostic.kb_name,
@@ -1675,20 +2316,22 @@ export class KnowledgeEngine {
1675
2316
  action_code: "review_skipped_scope",
1676
2317
  });
1677
2318
  }
1678
- if (diagnostic.stale_files.length > 0) {
2319
+ const staleFileCount = diagnostic.stale_file_count ?? diagnostic.stale_files.length;
2320
+ if (staleFileCount > 0) {
1679
2321
  issues.push({
1680
2322
  severity: "warning",
1681
2323
  kb_name: diagnostic.kb_name,
1682
- message: `${diagnostic.stale_files.length} files changed after indexing.`,
2324
+ message: `${staleFileCount} files changed after indexing.`,
1683
2325
  action: "Run knowledge_update for this KB.",
1684
2326
  action_code: "run_update",
1685
2327
  });
1686
2328
  }
1687
- if (diagnostic.orphan_files.length > 0) {
2329
+ const orphanFileCount = diagnostic.orphan_file_count ?? diagnostic.orphan_files.length;
2330
+ if (orphanFileCount > 0) {
1688
2331
  issues.push({
1689
2332
  severity: "warning",
1690
2333
  kb_name: diagnostic.kb_name,
1691
- message: `${diagnostic.orphan_files.length} indexed files no longer exist in the source.`,
2334
+ message: `${orphanFileCount} indexed files no longer exist in the source.`,
1692
2335
  action: "Run knowledge_update or rebuild the KB.",
1693
2336
  action_code: "run_update",
1694
2337
  });
@@ -1797,23 +2440,68 @@ export class KnowledgeEngine {
1797
2440
  })),
1798
2441
  };
1799
2442
  }
1800
- async exportKB(nameOrId, outputPath, signal, onProgress) {
2443
+ async doctorAsync(signal, maxKnowledgeBases = Number.POSITIVE_INFINITY, maxDiagnosticPaths = Number.POSITIVE_INFINITY) {
2444
+ if (!this.db)
2445
+ return this.doctor(signal, [], []);
2446
+ const knowledgeBases = listKBs(this.db, Number.isFinite(maxKnowledgeBases) ? maxKnowledgeBases + 1 : undefined);
2447
+ if (knowledgeBases.length > maxKnowledgeBases) {
2448
+ throw new Error(`Knowledge doctor exceeds the ${maxKnowledgeBases}-knowledge-base RPC limit`);
2449
+ }
2450
+ return this.doctor(signal, await this.diagnoseAsync(signal, knowledgeBases, maxDiagnosticPaths), knowledgeBases);
2451
+ }
2452
+ async exportKB(nameOrId, outputPath, signal, onProgress, access) {
1801
2453
  if (!this.db)
1802
2454
  throw new Error("Engine not initialized");
1803
2455
  throwIfAborted(signal);
1804
2456
  const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
1805
2457
  if (!kb)
1806
2458
  throw new Error(`Knowledge base not found: ${nameOrId}`);
1807
- return this.runExclusive(["global:mutation", `kb:${kb.id}`], `Operation for "${kb.name}"`, () => this.exportKBUnlocked(kb, outputPath, signal, onProgress));
2459
+ return this.runExclusive(["global:mutation", `kb:${kb.id}`], `Operation for "${kb.name}"`, () => this.exportKBUnlocked(kb, outputPath, signal, onProgress, access));
1808
2460
  }
1809
- async exportKBUnlocked(kb, outputPath, signal, onProgress) {
2461
+ async exportKBUnlocked(kb, outputPath, signal, onProgress, access) {
1810
2462
  if (!this.db)
1811
2463
  throw new Error("Engine not initialized");
1812
- const { createWriteStream } = await import("node:fs");
1813
- const tempOutputPath = tempVectorPath(outputPath);
1814
- const stream = createWriteStream(tempOutputPath, { encoding: "utf-8" });
2464
+ if (kb.chunk_count > MAX_EXPORT_CHUNKS) {
2465
+ throw new Error(`Export exceeds the ${MAX_EXPORT_CHUNKS}-chunk limit`);
2466
+ }
2467
+ const target = prepareSafeNewFile(authorizeOutputPath(outputPath, access));
2468
+ let tempFd;
2469
+ let tempDevice;
2470
+ let tempInode;
2471
+ try {
2472
+ tempFd = openSync(target.tempPath, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
2473
+ const opened = fstatSync(tempFd);
2474
+ if (!opened.isFile())
2475
+ throw new Error(`Export target is not a regular file: ${target.tempPath}`);
2476
+ tempDevice = opened.dev;
2477
+ tempInode = opened.ino;
2478
+ assertOpenedPathAuthorized(target.tempPath, tempDevice, tempInode, access);
2479
+ assertOutputPathAuthorized(target.outputPath, target.parentDevice, target.parentInode, access);
2480
+ }
2481
+ catch (error) {
2482
+ if (tempFd !== undefined)
2483
+ closeSync(tempFd);
2484
+ throw error;
2485
+ }
2486
+ const stream = createWriteStream("", {
2487
+ autoClose: false,
2488
+ encoding: "utf-8",
2489
+ fd: tempFd,
2490
+ });
1815
2491
  const progressBatchSize = resolveIndexEmbedBatchSize();
1816
2492
  let count = 0;
2493
+ let bytes = 0;
2494
+ const deadline = Date.now() + MAX_EXPORT_DURATION_MS;
2495
+ const writeBoundedLine = async (line) => {
2496
+ if (Date.now() > deadline)
2497
+ throw new Error(`Export exceeded ${MAX_EXPORT_DURATION_MS}ms`);
2498
+ const lineBytes = Buffer.byteLength(line) + 1;
2499
+ if (bytes + lineBytes > MAX_EXPORT_BYTES) {
2500
+ throw new Error(`Export exceeds the ${MAX_EXPORT_BYTES}-byte limit`);
2501
+ }
2502
+ bytes += lineBytes;
2503
+ await writeLine(stream, line);
2504
+ };
1817
2505
  const header = JSON.stringify({
1818
2506
  name: kb.name,
1819
2507
  description: kb.description,
@@ -1821,48 +2509,130 @@ export class KnowledgeEngine {
1821
2509
  chunk_count: kb.chunk_count,
1822
2510
  });
1823
2511
  try {
1824
- await writeLine(stream, header);
2512
+ await writeBoundedLine(header);
1825
2513
  for (const chunk of iterateChunksByKB(this.db, kb.id)) {
1826
2514
  throwIfAborted(signal);
1827
- await writeLine(stream, JSON.stringify({
2515
+ const line = JSON.stringify({
1828
2516
  content: chunk.content,
1829
2517
  file_path: chunk.file_path,
1830
2518
  file_type: chunk.file_type,
1831
2519
  start_line: chunk.start_line,
1832
2520
  end_line: chunk.end_line,
1833
2521
  metadata_json: chunk.metadata_json,
1834
- }));
2522
+ });
2523
+ await writeBoundedLine(line);
1835
2524
  count++;
1836
2525
  if (count % progressBatchSize === 0)
1837
2526
  onProgress?.(`Exported ${count}/${kb.chunk_count} chunks...`);
1838
2527
  }
1839
2528
  await finishWriteStream(stream);
1840
2529
  throwIfAborted(signal);
1841
- renameSync(tempOutputPath, outputPath);
2530
+ fsyncSync(tempFd);
2531
+ if (stream.fd === tempFd)
2532
+ stream.fd = null;
2533
+ closeSync(tempFd);
2534
+ tempFd = undefined;
2535
+ stream.destroy();
2536
+ assertOpenedPathAuthorized(target.tempPath, tempDevice, tempInode, access);
2537
+ assertOutputPathAuthorized(target.outputPath, target.parentDevice, target.parentInode, access);
2538
+ publishNewFile(target.tempPath, target);
1842
2539
  onProgress?.(`Exported ${count}/${kb.chunk_count} chunks.`);
1843
2540
  return count;
1844
2541
  }
1845
2542
  catch (error) {
2543
+ if (tempFd !== undefined) {
2544
+ const descriptor = tempFd;
2545
+ tempFd = undefined;
2546
+ if (stream.fd === descriptor)
2547
+ stream.fd = null;
2548
+ try {
2549
+ closeSync(descriptor);
2550
+ }
2551
+ catch (closeError) {
2552
+ if (closeError.code !== "EBADF") {
2553
+ console.error(`Failed to close export descriptor: ${String(closeError)}`);
2554
+ }
2555
+ }
2556
+ }
1846
2557
  stream.destroy();
1847
- rmSync(tempOutputPath, { force: true });
2558
+ try {
2559
+ const current = lstatSync(target.tempPath);
2560
+ if (!current.isSymbolicLink() && current.dev === tempDevice && current.ino === tempInode) {
2561
+ assertOpenedPathAuthorized(target.tempPath, tempDevice, tempInode, access);
2562
+ rmSync(target.tempPath, { force: true });
2563
+ }
2564
+ }
2565
+ catch {
2566
+ // Leave an unverified path untouched rather than deleting outside the authorized root.
2567
+ }
1848
2568
  throw error;
1849
2569
  }
1850
2570
  }
1851
- async importKB(inputPath, onProgress, signal) {
2571
+ async importKB(inputPath, onProgress, signal, access) {
1852
2572
  if (!this.db)
1853
2573
  throw new Error("Engine not initialized");
1854
- return this.runExclusive("global:mutation", "Knowledge-base import", () => this.importKBUnlocked(inputPath, onProgress, signal));
2574
+ return this.runExclusive("global:mutation", "Knowledge-base import", () => this.importKBUnlocked(inputPath, onProgress, signal, access));
1855
2575
  }
1856
- async importKBUnlocked(inputPath, onProgress, signal) {
2576
+ async importKBUnlocked(inputPath, onProgress, signal, access) {
1857
2577
  if (!this.db)
1858
2578
  throw new Error("Engine not initialized");
1859
2579
  throwIfAborted(signal);
1860
- const { createReadStream } = await import("node:fs");
1861
- const { createInterface } = await import("node:readline");
1862
2580
  const embeddingConfig = resolveEmbeddingConfig();
1863
2581
  const embeddingBatchSize = resolveIndexEmbedBatchSize();
1864
- const stream = createReadStream(inputPath, { encoding: "utf-8" });
2582
+ inputPath = authorizeExistingPath(inputPath, access);
2583
+ const directInput = lstatSync(inputPath);
2584
+ if (directInput.isSymbolicLink() || !directInput.isFile()) {
2585
+ throw new Error(`Import source is not a regular non-symlink file: ${inputPath}`);
2586
+ }
2587
+ const inputFd = openSync(inputPath, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
2588
+ const inputInfo = fstatSync(inputFd);
2589
+ if (inputInfo.dev !== directInput.dev || inputInfo.ino !== directInput.ino) {
2590
+ closeSync(inputFd);
2591
+ throw new Error(`Import source changed while opening: ${inputPath}`);
2592
+ }
2593
+ if (!inputInfo.isFile()) {
2594
+ closeSync(inputFd);
2595
+ throw new Error(`Import source is not a regular file: ${inputPath}`);
2596
+ }
2597
+ try {
2598
+ assertOpenedPathAuthorized(inputPath, inputInfo.dev, inputInfo.ino, access);
2599
+ }
2600
+ catch (error) {
2601
+ closeSync(inputFd);
2602
+ throw error;
2603
+ }
2604
+ if (inputInfo.size > MAX_IMPORT_BYTES) {
2605
+ closeSync(inputFd);
2606
+ throw new Error(`Import file exceeds the ${MAX_IMPORT_BYTES}-byte limit: ${inputPath}`);
2607
+ }
2608
+ const stream = createReadStream("", { autoClose: true, encoding: "utf-8", fd: inputFd });
1865
2609
  const lines = createInterface({ input: stream, crlfDelay: Infinity });
2610
+ let observedBytes = 0;
2611
+ let observedLineBytes = 0;
2612
+ stream.on("data", (chunk) => {
2613
+ observedBytes += Buffer.byteLength(String(chunk));
2614
+ if (observedBytes > MAX_IMPORT_BYTES) {
2615
+ stream.destroy(new Error(`Import exceeds the ${MAX_IMPORT_BYTES}-byte limit`));
2616
+ return;
2617
+ }
2618
+ const parts = String(chunk).split("\n");
2619
+ if (parts.length === 1) {
2620
+ observedLineBytes += Buffer.byteLength(parts[0] ?? "");
2621
+ }
2622
+ else {
2623
+ observedLineBytes += Buffer.byteLength(parts[0] ?? "");
2624
+ for (let index = 1; index < parts.length - 1; index++) {
2625
+ if (Buffer.byteLength(parts[index] ?? "") > MAX_IMPORT_LINE_BYTES) {
2626
+ stream.destroy(new Error(`Import line exceeds the ${MAX_IMPORT_LINE_BYTES}-byte limit`));
2627
+ return;
2628
+ }
2629
+ }
2630
+ observedLineBytes = Buffer.byteLength(parts.at(-1) ?? "");
2631
+ }
2632
+ if (observedLineBytes > MAX_IMPORT_LINE_BYTES) {
2633
+ stream.destroy(new Error(`Import line exceeds the ${MAX_IMPORT_LINE_BYTES}-byte limit`));
2634
+ }
2635
+ });
1866
2636
  let header;
1867
2637
  let kb;
1868
2638
  let vectorWriter;
@@ -1908,13 +2678,28 @@ export class KnowledgeEngine {
1908
2678
  for await (const rawLine of lines) {
1909
2679
  throwIfAborted(signal);
1910
2680
  lineNumber++;
2681
+ if (lineNumber > MAX_IMPORT_LINES)
2682
+ throw new Error(`Import exceeds the ${MAX_IMPORT_LINES}-line limit`);
1911
2683
  const line = rawLine.trim();
2684
+ if (Buffer.byteLength(rawLine) > MAX_IMPORT_LINE_BYTES) {
2685
+ throw new Error(`Import line ${lineNumber} exceeds the ${MAX_IMPORT_LINE_BYTES}-byte limit`);
2686
+ }
1912
2687
  if (!line)
1913
2688
  continue;
1914
2689
  if (!header) {
1915
- header = JSON.parse(line);
1916
- if (!header.name)
1917
- throw new Error("Import header is missing a knowledge base name");
2690
+ const parsedHeader = JSON.parse(line);
2691
+ if (typeof parsedHeader !== "object" || parsedHeader === null || Array.isArray(parsedHeader)) {
2692
+ throw new Error("Import header must be an object");
2693
+ }
2694
+ const headerRecord = parsedHeader;
2695
+ header = {
2696
+ ...validateKnowledgeBaseMetadata(headerRecord.name, headerRecord.description),
2697
+ chunk_count: headerRecord.chunk_count === undefined ? undefined : headerRecord.chunk_count,
2698
+ };
2699
+ if (header.chunk_count !== undefined &&
2700
+ (!Number.isInteger(header.chunk_count) || header.chunk_count < 0 || header.chunk_count > MAX_IMPORT_CHUNKS)) {
2701
+ throw new Error(`Import header chunk_count must be between 0 and ${MAX_IMPORT_CHUNKS}`);
2702
+ }
1918
2703
  const existingKB = getKBByName(this.db, header.name);
1919
2704
  if (existingKB) {
1920
2705
  throw new Error(`Knowledge base "${header.name}" already exists. Use knowledge_update to refresh it, or knowledge_remove before importing a replacement.`);
@@ -1940,11 +2725,14 @@ export class KnowledgeEngine {
1940
2725
  vectorWriter = openVectorWriter(tempVectorFile);
1941
2726
  continue;
1942
2727
  }
1943
- const imported = JSON.parse(line);
2728
+ const imported = parseImportedChunk(JSON.parse(line));
1944
2729
  const chunk = importedChunkToInsert(imported);
1945
2730
  chunk.content_tokenized = preTokenizeForFTS(buildChunkEmbeddingText(chunk));
1946
2731
  const finalizedChunks = await finalizeChunksForEmbedding([chunk], embeddingConfig, signal);
1947
2732
  pendingChunks.push(...finalizedChunks);
2733
+ if (inserted + pendingChunks.length > MAX_IMPORT_CHUNKS) {
2734
+ throw new Error(`Import exceeds the ${MAX_IMPORT_CHUNKS}-chunk limit`);
2735
+ }
1948
2736
  pendingSymbols.push(...extractSymbols(imported.content, imported.file_path, imported.file_type));
1949
2737
  while (pendingChunks.length >= embeddingBatchSize)
1950
2738
  await flushPending();
@@ -1959,6 +2747,7 @@ export class KnowledgeEngine {
1959
2747
  renameSync(tempVectorFile, this.vectorPathFor(kb.id));
1960
2748
  tempVectorFile = undefined;
1961
2749
  updateKBCounts(this.db, kb.id, inserted, getFileCount(this.db, kb.id));
2750
+ markKBIndexed(this.db, kb.id);
1962
2751
  updateKBStatus(this.db, kb.id, "ready");
1963
2752
  const savedKB = getKB(this.db, kb.id);
1964
2753
  if (!savedKB)