pi-knowledge 0.13.0 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,13 +1,13 @@
1
1
  import { existsSync, readFileSync, renameSync, rmSync, statSync } from "node:fs";
2
2
  import { join, resolve } from "node:path";
3
3
  import { diagnoseKB } from "./diagnostics/health.js";
4
- import { dispose as disposeEmbedding, embedDocuments, embeddingConfigLabel, embeddingSignature, embedQueryWithConfig, embedValidatedDocuments, prepareForShutdown as prepareEmbeddingForShutdown, resolveEmbeddingConfig, } from "./embedding/provider.js";
4
+ import { describeEmbeddingCompatibility, describeEmbeddingDecoration, dispose as disposeEmbedding, embedDocuments, embeddingConfigLabel, embeddingSignature, embedQueryWithConfig, embedValidatedDocuments, inspectEmbeddingCompatibility, prepareForShutdown as prepareEmbeddingForShutdown, resolveEmbeddingConfig, } from "./embedding/provider.js";
5
5
  import { openVectorReader, openVectorWriter } from "./embedding/vectors.js";
6
6
  import { addSkippedScanEntry, analyzeIndexableContent, buildChunkEmbeddingText, chunkFile, chunkIdentityHash, chunkMarkdown, createSkippedScanStats, isReadableTextFile, isSupportedDocumentFile, iterateScannableFiles, preTokenizeForFTS, summarizeSkippedScan, } from "./indexer/chunker.js";
7
7
  import { finalizeChunksForEmbedding } from "./indexer/embedding-safe-chunks.js";
8
8
  import { htmlToStructuredMarkdown } from "./indexer/html.js";
9
9
  import { extractSymbols } from "./indexer/symbols.js";
10
- import { shutdownModelWorker } from "./model-worker-client.js";
10
+ import { extractDocumentInModelWorker, shutdownModelWorker } from "./model-worker-client.js";
11
11
  import { searchBM25 } from "./search/bm25.js";
12
12
  import { MAX_SEARCH_WINDOW } from "./search/contracts.js";
13
13
  import { diversifyRankedChunks, interleaveByFile } from "./search/diversity.js";
@@ -150,6 +150,181 @@ function kbTrustMultiplier(kb) {
150
150
  multiplier *= 0.98;
151
151
  return multiplier;
152
152
  }
153
+ function summarizeKnowledgeBases(kbs) {
154
+ return kbs.map((kb) => ({
155
+ id: kb.id,
156
+ name: kb.name,
157
+ status: kb.status,
158
+ chunk_count: kb.chunk_count,
159
+ }));
160
+ }
161
+ function searchMissAction(reason, mode, availableKnowledgeBases) {
162
+ switch (reason) {
163
+ case "no_knowledge_bases":
164
+ return "add_knowledge_base";
165
+ case "knowledge_base_not_found":
166
+ return "fix_kb_id";
167
+ case "no_searchable_knowledge_bases":
168
+ return availableKnowledgeBases.length > 0 && availableKnowledgeBases.every((kb) => kb.status === "indexing")
169
+ ? "wait_for_indexing"
170
+ : "fix_knowledge_base";
171
+ case "no_indexed_chunks":
172
+ return availableKnowledgeBases.some((kb) => kb.status === "indexing")
173
+ ? "wait_for_indexing"
174
+ : "fix_knowledge_base";
175
+ case "embedding_incompatible":
176
+ return "fix_embedding";
177
+ case "no_retrieval_candidates":
178
+ return mode === "fast" || mode === "code" || mode === "config" || mode === "errors"
179
+ ? "try_semantic"
180
+ : mode === "semantic" || mode === "semantic_hybrid"
181
+ ? "rephrase"
182
+ : "try_semantic_hybrid";
183
+ case "threshold_filtered":
184
+ return mode === "semantic_hybrid" ? "rephrase" : "try_semantic_hybrid";
185
+ case "evidence_filtered":
186
+ return "try_semantic_hybrid";
187
+ case "metadata_filtered":
188
+ return "relax_filter";
189
+ case "page_out_of_range":
190
+ return "reset_offset";
191
+ case "reranker_empty":
192
+ case "no_match":
193
+ return "rephrase";
194
+ }
195
+ }
196
+ function classifySearchMiss(facts) {
197
+ const skippedKnowledgeBases = facts.availableKnowledgeBases.filter((kb) => kb.status !== "ready" && kb.status !== "stale");
198
+ let reason;
199
+ if (facts.allKnowledgeBases.length === 0) {
200
+ reason = "no_knowledge_bases";
201
+ }
202
+ else if (facts.requestedKbId &&
203
+ !facts.availableKnowledgeBases.some((kb) => kb.id === facts.requestedKbId || kb.name === facts.requestedKbId)) {
204
+ reason = "knowledge_base_not_found";
205
+ }
206
+ else if (facts.searchedKnowledgeBases.length === 0) {
207
+ reason = "no_searchable_knowledge_bases";
208
+ }
209
+ else if (facts.searchedKnowledgeBases.every((kb) => kb.chunk_count === 0)) {
210
+ reason = "no_indexed_chunks";
211
+ }
212
+ else if (facts.retrievedCandidates === 0 &&
213
+ facts.embeddingIncompatibleCount === facts.searchedKnowledgeBases.filter((kb) => kb.chunk_count > 0).length) {
214
+ reason = "embedding_incompatible";
215
+ }
216
+ else if (facts.retrievedCandidates === 0) {
217
+ reason = "no_retrieval_candidates";
218
+ }
219
+ else if (facts.thresholdCandidates === 0) {
220
+ reason = "threshold_filtered";
221
+ }
222
+ else if (facts.eligibleCandidates === 0) {
223
+ reason = "evidence_filtered";
224
+ }
225
+ else if (facts.filteredCandidates === 0 && (facts.fileType || facts.pathPattern)) {
226
+ reason = "metadata_filtered";
227
+ }
228
+ else if (facts.mode === "deep" && facts.rankedCandidates === 0) {
229
+ reason = "reranker_empty";
230
+ }
231
+ else if (facts.totalResults > 0 && facts.offset >= facts.totalResults) {
232
+ reason = "page_out_of_range";
233
+ }
234
+ else {
235
+ reason = "no_match";
236
+ }
237
+ return {
238
+ reason,
239
+ action: searchMissAction(reason, facts.mode, facts.availableKnowledgeBases),
240
+ requested_kb_id: facts.requestedKbId,
241
+ min_hybrid_score: facts.minimumScore,
242
+ file_type: facts.fileType,
243
+ path_pattern: facts.pathPattern,
244
+ searched_kbs: summarizeKnowledgeBases(facts.searchedKnowledgeBases),
245
+ skipped_kbs: summarizeKnowledgeBases(skippedKnowledgeBases),
246
+ retrieved_candidates: facts.retrievedCandidates,
247
+ threshold_candidates: facts.thresholdCandidates,
248
+ eligible_candidates: facts.eligibleCandidates,
249
+ filtered_candidates: facts.filteredCandidates,
250
+ ranked_candidates: facts.rankedCandidates,
251
+ total_results: facts.totalResults,
252
+ };
253
+ }
254
+ function searchAttemptSummary(mode, response) {
255
+ return {
256
+ mode,
257
+ reason: response.miss?.reason,
258
+ retrieved_candidates: response.tuning?.retrieved_candidates ?? 0,
259
+ threshold_candidates: response.tuning?.threshold_candidates ?? 0,
260
+ eligible_candidates: response.tuning?.eligible_candidates ?? 0,
261
+ filtered_candidates: response.tuning?.filtered_candidates ?? 0,
262
+ result_count: response.results.length,
263
+ };
264
+ }
265
+ function symbolMissAction(reason, availableKnowledgeBases) {
266
+ switch (reason) {
267
+ case "no_knowledge_bases":
268
+ return "add_knowledge_base";
269
+ case "knowledge_base_not_found":
270
+ return "fix_kb_id";
271
+ case "no_searchable_knowledge_bases":
272
+ return availableKnowledgeBases.length > 0 && availableKnowledgeBases.every((kb) => kb.status === "indexing")
273
+ ? "wait_for_indexing"
274
+ : "fix_knowledge_base";
275
+ case "symbol_index_empty":
276
+ return "rebuild_symbol_index";
277
+ case "exact_name_not_found":
278
+ return "broaden_symbol_search";
279
+ case "metadata_filtered":
280
+ return "relax_filter";
281
+ case "page_out_of_range":
282
+ return "reset_offset";
283
+ case "no_match":
284
+ return "use_text_search";
285
+ }
286
+ }
287
+ function classifySymbolSearchMiss(facts) {
288
+ const skippedKnowledgeBases = facts.availableKnowledgeBases.filter((kb) => kb.status !== "ready" && kb.status !== "stale");
289
+ let reason;
290
+ if (facts.allKnowledgeBases.length === 0) {
291
+ reason = "no_knowledge_bases";
292
+ }
293
+ else if (facts.requestedKbId &&
294
+ !facts.availableKnowledgeBases.some((kb) => kb.id === facts.requestedKbId || kb.name === facts.requestedKbId)) {
295
+ reason = "knowledge_base_not_found";
296
+ }
297
+ else if (facts.searchedKnowledgeBases.length === 0) {
298
+ reason = "no_searchable_knowledge_bases";
299
+ }
300
+ else if (facts.indexedSymbols === 0) {
301
+ reason = "symbol_index_empty";
302
+ }
303
+ else if (facts.totalResults > 0 && facts.offset >= facts.totalResults) {
304
+ reason = "page_out_of_range";
305
+ }
306
+ else if ((facts.kind || facts.filePattern) && facts.unfilteredMatches > 0) {
307
+ reason = "metadata_filtered";
308
+ }
309
+ else if (facts.exact) {
310
+ reason = "exact_name_not_found";
311
+ }
312
+ else {
313
+ reason = "no_match";
314
+ }
315
+ return {
316
+ reason,
317
+ action: symbolMissAction(reason, facts.availableKnowledgeBases),
318
+ requested_kb_id: facts.requestedKbId,
319
+ kind: facts.kind,
320
+ file_pattern: facts.filePattern,
321
+ exact: facts.exact,
322
+ searched_kbs: summarizeKnowledgeBases(facts.searchedKnowledgeBases),
323
+ skipped_kbs: summarizeKnowledgeBases(skippedKnowledgeBases),
324
+ indexed_symbols: facts.indexedSymbols,
325
+ unfiltered_matches: facts.unfilteredMatches,
326
+ };
327
+ }
153
328
  function formatBytes(bytes) {
154
329
  if (bytes < 1024)
155
330
  return `${bytes} B`;
@@ -370,30 +545,19 @@ async function chunkUrl(source, signal) {
370
545
  const structuredChunks = chunkMarkdown(text, source, "html");
371
546
  return structuredChunks.length > 0 ? structuredChunks : chunkFile(text, source);
372
547
  }
373
- function normalizeExtractedText(text) {
374
- return Array.isArray(text) ? text.join("\n\n") : text;
375
- }
376
548
  async function extractSourceFileContent(filePath, signal) {
377
549
  const lowerPath = filePath.toLowerCase();
378
550
  if (lowerPath.endsWith(".pdf")) {
379
551
  throwIfAborted(signal);
380
- // PDF extraction is an optional heavy runtime path; keep parser loading out of extension startup.
381
- const { extractText } = await import("unpdf");
382
- throwIfAborted(signal);
383
- const buf = readFileSync(filePath);
552
+ const extracted = await extractDocumentInModelWorker(filePath, "pdf", signal);
384
553
  throwIfAborted(signal);
385
- const { text } = await extractText(new Uint8Array(buf));
386
- throwIfAborted(signal);
387
- return { content: normalizeExtractedText(text), fileType: "pdf" };
554
+ return extracted;
388
555
  }
389
556
  if (lowerPath.endsWith(".docx") || lowerPath.endsWith(".doc")) {
390
557
  throwIfAborted(signal);
391
- // DOCX extraction is an optional heavy runtime path; keep parser loading out of extension startup.
392
- const mammoth = await import("mammoth");
393
- throwIfAborted(signal);
394
- const result = await mammoth.extractRawText({ path: filePath });
558
+ const extracted = await extractDocumentInModelWorker(filePath, "docx", signal);
395
559
  throwIfAborted(signal);
396
- return { content: result.value, fileType: "docx" };
560
+ return extracted;
397
561
  }
398
562
  if (!isReadableTextFile(filePath)) {
399
563
  throw new Error(`File is not readable text and has no supported extractor: ${filePath}`);
@@ -451,15 +615,8 @@ function persistEmbeddingMetadata(db, kbId, config, vectors) {
451
615
  updateKBEmbeddingMetadata(db, kbId, embeddingConfigLabel(config), embeddingSignature(config, dimension), dimension);
452
616
  return dimension;
453
617
  }
454
- function embeddingMismatchWarning(kb) {
455
- return `"${kb.name}" has incompatible embedding metadata; vector retrieval was skipped and knowledge_update should rebuild it`;
456
- }
457
- function canSearchVectors(kb, config, queryDimension) {
458
- if (kb.embedding_signature === null || kb.embedding_dimension === null)
459
- return false;
460
- if (kb.embedding_dimension !== queryDimension)
461
- return false;
462
- return kb.embedding_signature === embeddingSignature(config, queryDimension);
618
+ function embeddingMismatchWarning(kb, compatibility) {
619
+ return `"${kb.name}" has incompatible embedding vectors (${describeEmbeddingCompatibility(compatibility)}); vector retrieval was skipped and knowledge_update should rebuild them`;
463
620
  }
464
621
  function throwIfAborted(signal) {
465
622
  if (signal?.aborted)
@@ -844,14 +1001,17 @@ export class KnowledgeEngine {
844
1001
  if (!this.db)
845
1002
  throw new Error("Engine not initialized");
846
1003
  throwIfAborted(signal);
847
- updateKBStatus(this.db, kb.id, "indexing");
848
- startIndexingJob(this.db, kb.id, "update", `Starting update for "${kb.name}"`);
849
1004
  const scanOptions = toScanOptions(parseAddOptions(kb.source_options));
850
1005
  const embeddingConfig = resolveEmbeddingConfig();
851
1006
  const embeddingBatchSize = resolveIndexEmbedBatchSize();
1007
+ updateKBStatus(this.db, kb.id, "indexing");
1008
+ startIndexingJob(this.db, kb.id, "update", `Starting update for "${kb.name}"`);
852
1009
  const embeddingModel = embeddingConfigLabel(embeddingConfig);
853
- const currentSignature = kb.embedding_dimension === null ? undefined : embeddingSignature(embeddingConfig, kb.embedding_dimension);
854
- let canReuseExistingVectors = kb.embedding_signature !== null && kb.embedding_dimension !== null && kb.embedding_signature === currentSignature;
1010
+ const compatibility = inspectEmbeddingCompatibility(kb.embedding_signature, kb.embedding_dimension, embeddingConfig, kb.embedding_dimension ?? 0);
1011
+ let canReuseExistingVectors = compatibility.compatible;
1012
+ let rebuildMessage = compatibility.compatible
1013
+ ? undefined
1014
+ : `Embedding vectors for "${kb.name}" are incompatible (${describeEmbeddingCompatibility(compatibility)}); rebuilding all vectors`;
855
1015
  let replacementVectorPath;
856
1016
  let addedVectorPath;
857
1017
  let addedVectorWriter;
@@ -880,9 +1040,7 @@ export class KnowledgeEngine {
880
1040
  canReuseExistingVectors = false;
881
1041
  const vectorCount = oldVectorReader?.count ?? 0;
882
1042
  const vectorDimension = oldVectorReader?.dim ?? 0;
883
- const message = `Stored vector file for "${kb.name}" is incomplete or incompatible (${vectorCount} vectors/${vectorDimension}d for ${existingIndex} chunks/${kb.embedding_dimension}d); rebuilding all vectors`;
884
- updateIndexingJob(this.db, kb.id, { phase: "embedding", message });
885
- onProgress?.(message);
1043
+ rebuildMessage = `Stored vector file for "${kb.name}" is incomplete or incompatible (${vectorCount} vectors/${vectorDimension}d for ${existingIndex} chunks/${kb.embedding_dimension}d); rebuilding all vectors`;
886
1044
  }
887
1045
  }
888
1046
  finally {
@@ -892,10 +1050,9 @@ export class KnowledgeEngine {
892
1050
  const reusableHashes = canReuseExistingVectors
893
1051
  ? existingHashes
894
1052
  : new Map();
895
- if (!canReuseExistingVectors) {
896
- const message = `Embedding metadata changed or missing for "${kb.name}"; rebuilding all vectors`;
897
- updateIndexingJob(this.db, kb.id, { phase: "embedding", message });
898
- onProgress?.(message);
1053
+ if (!canReuseExistingVectors && rebuildMessage) {
1054
+ updateIndexingJob(this.db, kb.id, { phase: "embedding", message: rebuildMessage });
1055
+ onProgress?.(rebuildMessage);
899
1056
  }
900
1057
  const oldVectorIndexByHash = new Map();
901
1058
  const newVectorIndexByHash = new Map();
@@ -1181,43 +1338,82 @@ export class KnowledgeEngine {
1181
1338
  const requestedMode = options.mode ?? "hybrid";
1182
1339
  if (requestedMode === "auto") {
1183
1340
  const primaryMode = chooseAutoMode(query);
1184
- const attempts = [primaryMode, ...fallbackModesFor(primaryMode)];
1341
+ const modes = [primaryMode, ...fallbackModesFor(primaryMode)];
1185
1342
  const tried = [];
1343
+ const attemptSummaries = [];
1186
1344
  const warnings = [];
1187
- let lastTuning;
1188
- for (const mode of attempts) {
1345
+ let lastMissResponse;
1346
+ let bestWeakResponse;
1347
+ for (const mode of modes) {
1189
1348
  if (tried.includes(mode))
1190
1349
  continue;
1191
1350
  tried.push(mode);
1192
1351
  throwIfAborted(signal);
1193
1352
  const response = await this.search(query, { ...options, mode }, signal);
1194
- lastTuning = response.tuning;
1353
+ attemptSummaries.push(searchAttemptSummary(mode, response));
1195
1354
  if (response.warnings)
1196
1355
  warnings.push(...response.warnings);
1356
+ if (response.miss?.reason === "no_knowledge_bases" ||
1357
+ response.miss?.reason === "knowledge_base_not_found" ||
1358
+ response.miss?.reason === "no_searchable_knowledge_bases" ||
1359
+ response.miss?.reason === "no_indexed_chunks") {
1360
+ return {
1361
+ ...response,
1362
+ warnings: warnings.length > 0 ? [...new Set(warnings)] : response.warnings,
1363
+ mode_used: mode,
1364
+ retry_modes: tried.slice(0, -1),
1365
+ miss: { ...response.miss, attempts: attemptSummaries },
1366
+ };
1367
+ }
1197
1368
  if (!isWeakAutoResponse(query, response, primaryMode, mode)) {
1198
1369
  return {
1199
1370
  ...response,
1200
1371
  warnings: warnings.length > 0 ? [...new Set(warnings)] : response.warnings,
1201
1372
  mode_used: mode,
1202
1373
  retry_modes: tried.slice(0, -1),
1203
- suggestions: response.suggestions,
1204
1374
  };
1205
1375
  }
1376
+ if (response.results.length > 0) {
1377
+ if (!bestWeakResponse || response.results.length > bestWeakResponse.response.results.length) {
1378
+ bestWeakResponse = { mode, response };
1379
+ }
1380
+ }
1381
+ else {
1382
+ lastMissResponse = { mode, response };
1383
+ }
1206
1384
  if (tried.length >= 3)
1207
1385
  break;
1208
1386
  }
1209
- return {
1210
- results: [],
1211
- total_count: 0,
1212
- has_more: false,
1213
- warnings: warnings.length > 0 ? [...new Set(warnings)] : undefined,
1214
- mode_used: tried.at(-1),
1215
- retry_modes: tried.slice(0, -1),
1216
- tuning: lastTuning,
1217
- suggestions: [
1218
- "No results after auto mode fallback. Check knowledge_status, try a more exact term, or rebuild the KB if indexing rules changed.",
1219
- ],
1220
- };
1387
+ if (lastMissResponse?.response.miss) {
1388
+ const attemptedModes = new Set(attemptSummaries.map((attempt) => attempt.mode));
1389
+ let action = lastMissResponse.response.miss.action;
1390
+ if (action === "try_semantic" && attemptedModes.has("semantic")) {
1391
+ action = attemptedModes.has("semantic_hybrid") ? "rephrase" : "try_semantic_hybrid";
1392
+ }
1393
+ else if (action === "try_semantic_hybrid" && attemptedModes.has("semantic_hybrid")) {
1394
+ action = "rephrase";
1395
+ }
1396
+ return {
1397
+ ...lastMissResponse.response,
1398
+ warnings: warnings.length > 0 ? [...new Set(warnings)] : lastMissResponse.response.warnings,
1399
+ mode_used: lastMissResponse.mode,
1400
+ retry_modes: tried.filter((mode) => mode !== lastMissResponse.mode),
1401
+ miss: {
1402
+ ...lastMissResponse.response.miss,
1403
+ action,
1404
+ attempts: attemptSummaries,
1405
+ },
1406
+ };
1407
+ }
1408
+ if (bestWeakResponse) {
1409
+ return {
1410
+ ...bestWeakResponse.response,
1411
+ warnings: warnings.length > 0 ? [...new Set(warnings)] : bestWeakResponse.response.warnings,
1412
+ mode_used: bestWeakResponse.mode,
1413
+ retry_modes: tried.filter((mode) => mode !== bestWeakResponse.mode),
1414
+ };
1415
+ }
1416
+ throw new Error("Auto search exhausted without a result or miss classification");
1221
1417
  }
1222
1418
  const db = this.db;
1223
1419
  const { mode = "hybrid", offset: requestedOffset, kb_id, filters, diversity = "balanced" } = options;
@@ -1227,8 +1423,9 @@ export class KnowledgeEngine {
1227
1423
  const queryTokens = tokenizeForSimilarity(query);
1228
1424
  const normalizedFileType = normalizeFileTypeFilter(filters?.file_type);
1229
1425
  const warnings = [];
1426
+ const allKBs = listKBs(db);
1230
1427
  const selectedKB = kb_id ? (getKB(db, kb_id) ?? getKBByName(db, kb_id)) : undefined;
1231
- const availableKBs = kb_id ? [selectedKB].filter(Boolean) : listKBs(db);
1428
+ const availableKBs = kb_id ? [selectedKB].filter(Boolean) : allKBs;
1232
1429
  const kbs = availableKBs.filter((kb) => kb.status === "ready" || kb.status === "stale");
1233
1430
  const kbById = new Map(kbs.map((kb) => [kb.id, kb]));
1234
1431
  for (const kb of availableKBs) {
@@ -1254,11 +1451,30 @@ export class KnowledgeEngine {
1254
1451
  warnings: warnings.length > 0 ? warnings : undefined,
1255
1452
  mode_used: mode,
1256
1453
  tuning: tuningSummary,
1454
+ miss: classifySearchMiss({
1455
+ allKnowledgeBases: allKBs,
1456
+ availableKnowledgeBases: availableKBs,
1457
+ searchedKnowledgeBases: kbs,
1458
+ requestedKbId: kb_id,
1459
+ mode,
1460
+ fileType: filters?.file_type,
1461
+ pathPattern: filters?.path_pattern,
1462
+ minimumScore: tuning.minHybridScore,
1463
+ offset,
1464
+ retrievedCandidates: 0,
1465
+ thresholdCandidates: 0,
1466
+ eligibleCandidates: 0,
1467
+ filteredCandidates: 0,
1468
+ rankedCandidates: 0,
1469
+ totalResults: 0,
1470
+ embeddingIncompatibleCount: 0,
1471
+ }),
1257
1472
  };
1258
1473
  }
1259
1474
  const allResults = [];
1260
1475
  const vectorsByChunkId = new Map();
1261
1476
  let queryEmbedding;
1477
+ const embeddingIncompatibleKbIds = new Set();
1262
1478
  for (const kb of kbs) {
1263
1479
  throwIfAborted(signal);
1264
1480
  if (kb.chunk_count === 0)
@@ -1274,8 +1490,10 @@ export class KnowledgeEngine {
1274
1490
  queryEmbedding ??= await embedQueryWithConfig(query, signal);
1275
1491
  const { vector: queryVec, config: queryEmbeddingConfig } = queryEmbedding;
1276
1492
  throwIfAborted(signal);
1277
- if (!canSearchVectors(kb, queryEmbeddingConfig, queryVec.length)) {
1278
- warnings.push(embeddingMismatchWarning(kb));
1493
+ const compatibility = inspectEmbeddingCompatibility(kb.embedding_signature, kb.embedding_dimension, queryEmbeddingConfig, queryVec.length);
1494
+ if (!compatibility.compatible) {
1495
+ embeddingIncompatibleKbIds.add(kb.id);
1496
+ warnings.push(embeddingMismatchWarning(kb, compatibility));
1279
1497
  continue;
1280
1498
  }
1281
1499
  const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), retrievalLimitPerKb, signal);
@@ -1292,14 +1510,16 @@ export class KnowledgeEngine {
1292
1510
  queryEmbedding ??= await embedQueryWithConfig(query, signal);
1293
1511
  const { vector: queryVec, config: queryEmbeddingConfig } = queryEmbedding;
1294
1512
  throwIfAborted(signal);
1295
- if (canSearchVectors(kb, queryEmbeddingConfig, queryVec.length)) {
1513
+ const compatibility = inspectEmbeddingCompatibility(kb.embedding_signature, kb.embedding_dimension, queryEmbeddingConfig, queryVec.length);
1514
+ if (compatibility.compatible) {
1296
1515
  const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), retrievalLimitPerKb, signal);
1297
1516
  vecResults = vectorResults.results;
1298
1517
  for (const [chunkId, vector] of vectorResults.vectorsByChunkId)
1299
1518
  vectorsByChunkId.set(chunkId, vector);
1300
1519
  }
1301
1520
  else {
1302
- warnings.push(embeddingMismatchWarning(kb));
1521
+ embeddingIncompatibleKbIds.add(kb.id);
1522
+ warnings.push(embeddingMismatchWarning(kb, compatibility));
1303
1523
  }
1304
1524
  const fused = weightedScoreFusion(bm25Results, vecResults);
1305
1525
  allResults.push(...fused.map((result) => ({ ...result, score: result.score * kbTrustMultiplier(kb) })));
@@ -1323,30 +1543,30 @@ export class KnowledgeEngine {
1323
1543
  result.score = ranking.adjusted_score;
1324
1544
  }
1325
1545
  }
1326
- let scored = unique;
1546
+ let thresholded = unique.filter((result) => rankingByChunkId.has(result.chunkId));
1547
+ if (retrievalMode !== "fast" && retrievalMode !== "semantic") {
1548
+ thresholded = thresholded.filter((result) => result.score >= tuning.minHybridScore);
1549
+ }
1550
+ const thresholdCandidates = thresholded.length;
1551
+ let eligible = thresholded;
1327
1552
  if (retrievalMode === "fast") {
1328
- scored = unique.filter((result) => {
1553
+ eligible = thresholded.filter((result) => {
1329
1554
  const chunk = getChunkById(db, result.chunkId);
1330
- if (!chunk)
1331
- return false;
1332
- return hasAnyLexicalEvidence(buildChunkEmbeddingText(chunk), queryTokens);
1555
+ return chunk ? hasAnyLexicalEvidence(buildChunkEmbeddingText(chunk), queryTokens) : false;
1333
1556
  });
1334
1557
  }
1335
- else if (retrievalMode !== "semantic") {
1336
- scored = unique.filter((result) => {
1558
+ else if (retrievalMode !== "semantic" && retrievalMode !== "semantic_hybrid") {
1559
+ eligible = thresholded.filter((result) => {
1337
1560
  const chunk = getChunkById(db, result.chunkId);
1338
- if (!chunk)
1339
- return false;
1340
- if (retrievalMode === "semantic_hybrid")
1341
- return result.score >= tuning.minHybridScore;
1342
- return result.score >= tuning.minHybridScore && hasEnoughLexicalEvidence(chunk, queryTokens);
1561
+ return chunk ? hasEnoughLexicalEvidence(chunk, queryTokens) : false;
1343
1562
  });
1344
1563
  }
1345
- scored.sort((a, b) => b.score - a.score);
1564
+ eligible.sort((a, b) => b.score - a.score);
1565
+ const eligibleCandidates = eligible.length;
1346
1566
  // Apply metadata filters post-retrieval
1347
- let filtered = scored;
1567
+ let filtered = eligible;
1348
1568
  if (normalizedFileType || filters?.path_pattern) {
1349
- filtered = scored.filter((r) => {
1569
+ filtered = eligible.filter((r) => {
1350
1570
  const chunk = getChunkById(db, r.chunkId);
1351
1571
  if (!chunk)
1352
1572
  return false;
@@ -1418,19 +1638,42 @@ export class KnowledgeEngine {
1418
1638
  },
1419
1639
  };
1420
1640
  });
1641
+ const deepTuning = {
1642
+ ...tuningSummary,
1643
+ retrieved_candidates: unique.length,
1644
+ threshold_candidates: thresholdCandidates,
1645
+ eligible_candidates: eligibleCandidates,
1646
+ filtered_candidates: filtered.length,
1647
+ diversity_candidates: ranked.length,
1648
+ candidate_pool_truncated: candidatePoolTruncated,
1649
+ };
1421
1650
  return {
1422
1651
  results,
1423
1652
  total_count: diversified.length,
1424
1653
  has_more: offset + limit < diversified.length,
1425
1654
  warnings: warnings.length > 0 ? warnings : undefined,
1426
1655
  mode_used: mode,
1427
- tuning: {
1428
- ...tuningSummary,
1429
- retrieved_candidates: unique.length,
1430
- filtered_candidates: filtered.length,
1431
- diversity_candidates: ranked.length,
1432
- candidate_pool_truncated: candidatePoolTruncated,
1433
- },
1656
+ tuning: deepTuning,
1657
+ miss: results.length === 0
1658
+ ? classifySearchMiss({
1659
+ allKnowledgeBases: allKBs,
1660
+ availableKnowledgeBases: availableKBs,
1661
+ searchedKnowledgeBases: kbs,
1662
+ requestedKbId: kb_id,
1663
+ mode,
1664
+ fileType: filters?.file_type,
1665
+ pathPattern: filters?.path_pattern,
1666
+ minimumScore: tuning.minHybridScore,
1667
+ offset,
1668
+ retrievedCandidates: unique.length,
1669
+ thresholdCandidates,
1670
+ eligibleCandidates,
1671
+ filteredCandidates: filtered.length,
1672
+ rankedCandidates: ranked.length,
1673
+ totalResults: diversified.length,
1674
+ embeddingIncompatibleCount: embeddingIncompatibleKbIds.size,
1675
+ })
1676
+ : undefined,
1434
1677
  };
1435
1678
  }
1436
1679
  const ranked = [];
@@ -1499,24 +1742,41 @@ export class KnowledgeEngine {
1499
1742
  };
1500
1743
  })(),
1501
1744
  }));
1745
+ const responseTuning = {
1746
+ ...tuningSummary,
1747
+ retrieved_candidates: unique.length,
1748
+ threshold_candidates: thresholdCandidates,
1749
+ eligible_candidates: eligibleCandidates,
1750
+ filtered_candidates: filtered.length,
1751
+ diversity_candidates: ranked.length,
1752
+ candidate_pool_truncated: candidatePoolTruncated,
1753
+ };
1502
1754
  return {
1503
1755
  results,
1504
1756
  total_count: total,
1505
1757
  has_more: offset + limit < total,
1506
1758
  warnings: warnings.length > 0 ? warnings : undefined,
1507
1759
  mode_used: mode,
1508
- tuning: {
1509
- ...tuningSummary,
1510
- retrieved_candidates: unique.length,
1511
- filtered_candidates: filtered.length,
1512
- diversity_candidates: ranked.length,
1513
- candidate_pool_truncated: candidatePoolTruncated,
1514
- },
1515
- suggestions: results.length === 0
1516
- ? [
1517
- "Try mode 'fast' for exact symbols, mode 'semantic' for pure vector search, or mode 'semantic_hybrid' when weak lexical overlap should not block fused vector results.",
1518
- "Run knowledge_status if the KB should contain this answer.",
1519
- ]
1760
+ tuning: responseTuning,
1761
+ miss: results.length === 0
1762
+ ? classifySearchMiss({
1763
+ allKnowledgeBases: allKBs,
1764
+ availableKnowledgeBases: availableKBs,
1765
+ searchedKnowledgeBases: kbs,
1766
+ requestedKbId: kb_id,
1767
+ mode,
1768
+ fileType: filters?.file_type,
1769
+ pathPattern: filters?.path_pattern,
1770
+ minimumScore: tuning.minHybridScore,
1771
+ offset,
1772
+ retrievedCandidates: unique.length,
1773
+ thresholdCandidates,
1774
+ eligibleCandidates,
1775
+ filteredCandidates: filtered.length,
1776
+ rankedCandidates: ranked.length,
1777
+ totalResults: total,
1778
+ embeddingIncompatibleCount: embeddingIncompatibleKbIds.size,
1779
+ })
1520
1780
  : undefined,
1521
1781
  };
1522
1782
  }
@@ -1562,16 +1822,77 @@ export class KnowledgeEngine {
1562
1822
  return diagnoseKB(db, kb, signal);
1563
1823
  });
1564
1824
  }
1825
+ statusSnapshot(signal) {
1826
+ if (!this.db)
1827
+ return { knowledge_bases: [], diagnostics: [], embeddings: [] };
1828
+ throwIfAborted(signal);
1829
+ const db = this.db;
1830
+ const knowledgeBases = listKBs(db);
1831
+ const diagnostics = knowledgeBases.map((kb) => {
1832
+ throwIfAborted(signal);
1833
+ return diagnoseKB(db, kb, signal);
1834
+ });
1835
+ try {
1836
+ const config = resolveEmbeddingConfig();
1837
+ const activeModel = embeddingConfigLabel(config);
1838
+ const activeDecoration = describeEmbeddingDecoration(config);
1839
+ return {
1840
+ knowledge_bases: knowledgeBases,
1841
+ diagnostics,
1842
+ embeddings: knowledgeBases.map((kb) => {
1843
+ const compatibility = inspectEmbeddingCompatibility(kb.embedding_signature, kb.embedding_dimension, config, kb.embedding_dimension ?? 0);
1844
+ return {
1845
+ kb_id: kb.id,
1846
+ kb_name: kb.name,
1847
+ indexed_model: kb.embedding_model,
1848
+ active_model: activeModel,
1849
+ active_decoration: activeDecoration,
1850
+ compatibility,
1851
+ detail: describeEmbeddingCompatibility(compatibility),
1852
+ };
1853
+ }),
1854
+ };
1855
+ }
1856
+ catch (error) {
1857
+ return {
1858
+ knowledge_bases: knowledgeBases,
1859
+ diagnostics,
1860
+ embeddings: [],
1861
+ embedding_error: error instanceof Error ? error.message : String(error),
1862
+ };
1863
+ }
1864
+ }
1565
1865
  symbolSearch(query, options = {}, signal) {
1566
1866
  if (!this.db)
1567
1867
  throw new Error("Engine not initialized");
1568
1868
  throwIfAborted(signal);
1569
1869
  const db = this.db;
1870
+ const allKBs = listKBs(db);
1570
1871
  const selectedKB = options.kb_id ? (getKB(db, options.kb_id) ?? getKBByName(db, options.kb_id)) : undefined;
1571
- if (options.kb_id && !selectedKB)
1572
- throw new Error(`Knowledge base not found: ${options.kb_id}`);
1872
+ const availableKBs = options.kb_id ? [selectedKB].filter(Boolean) : allKBs;
1873
+ const searchedKBs = availableKBs.filter((kb) => kb.status === "ready" || kb.status === "stale");
1573
1874
  const limit = Math.max(1, Math.min(200, options.limit ?? 20));
1574
1875
  const offset = Math.max(0, options.offset ?? 0);
1876
+ if (searchedKBs.length === 0) {
1877
+ return {
1878
+ results: [],
1879
+ total_count: 0,
1880
+ has_more: false,
1881
+ miss: classifySymbolSearchMiss({
1882
+ allKnowledgeBases: allKBs,
1883
+ availableKnowledgeBases: availableKBs,
1884
+ searchedKnowledgeBases: searchedKBs,
1885
+ requestedKbId: options.kb_id,
1886
+ kind: options.kind,
1887
+ filePattern: options.file_pattern,
1888
+ exact: options.exact === true,
1889
+ offset,
1890
+ totalResults: 0,
1891
+ indexedSymbols: 0,
1892
+ unfilteredMatches: 0,
1893
+ }),
1894
+ };
1895
+ }
1575
1896
  const symbolOptions = {
1576
1897
  kbId: selectedKB?.id,
1577
1898
  kind: options.kind,
@@ -1587,25 +1908,48 @@ export class KnowledgeEngine {
1587
1908
  const page = symbols.slice(0, limit);
1588
1909
  const total = countSymbols(db, query, symbolOptions);
1589
1910
  throwIfAborted(signal);
1911
+ const results = page.map((symbol) => {
1912
+ const kb = getKB(db, symbol.kb_id);
1913
+ return {
1914
+ name: symbol.name,
1915
+ kind: symbol.kind,
1916
+ file_path: symbol.file_path,
1917
+ file_type: symbol.file_type,
1918
+ kb_name: kb?.name ?? "unknown",
1919
+ start_line: symbol.start_line,
1920
+ end_line: symbol.end_line,
1921
+ signature: symbol.signature ?? undefined,
1922
+ container_name: symbol.container_name ?? undefined,
1923
+ text: symbol.text,
1924
+ indexed_at: symbol.indexed_at,
1925
+ };
1926
+ });
1927
+ let miss;
1928
+ if (results.length === 0) {
1929
+ const indexedSymbols = countSymbols(db, "", { kbId: selectedKB?.id });
1930
+ const unfilteredMatches = total === 0 && (options.kind || options.file_pattern)
1931
+ ? countSymbols(db, query, { kbId: selectedKB?.id, exact: options.exact })
1932
+ : total;
1933
+ throwIfAborted(signal);
1934
+ miss = classifySymbolSearchMiss({
1935
+ allKnowledgeBases: allKBs,
1936
+ availableKnowledgeBases: availableKBs,
1937
+ searchedKnowledgeBases: searchedKBs,
1938
+ requestedKbId: options.kb_id,
1939
+ kind: options.kind,
1940
+ filePattern: options.file_pattern,
1941
+ exact: options.exact === true,
1942
+ offset,
1943
+ totalResults: total,
1944
+ indexedSymbols,
1945
+ unfilteredMatches,
1946
+ });
1947
+ }
1590
1948
  return {
1591
- results: page.map((symbol) => {
1592
- const kb = getKB(db, symbol.kb_id);
1593
- return {
1594
- name: symbol.name,
1595
- kind: symbol.kind,
1596
- file_path: symbol.file_path,
1597
- file_type: symbol.file_type,
1598
- kb_name: kb?.name ?? "unknown",
1599
- start_line: symbol.start_line,
1600
- end_line: symbol.end_line,
1601
- signature: symbol.signature ?? undefined,
1602
- container_name: symbol.container_name ?? undefined,
1603
- text: symbol.text,
1604
- indexed_at: symbol.indexed_at,
1605
- };
1606
- }),
1949
+ results,
1607
1950
  total_count: total,
1608
1951
  has_more: symbols.length > limit,
1952
+ miss,
1609
1953
  };
1610
1954
  }
1611
1955
  doctor(signal) {
@@ -1683,23 +2027,34 @@ export class KnowledgeEngine {
1683
2027
  }
1684
2028
  }
1685
2029
  if (this.db) {
1686
- const activeEmbeddingConfig = resolveEmbeddingConfig();
2030
+ let activeEmbeddingConfig;
2031
+ try {
2032
+ activeEmbeddingConfig = resolveEmbeddingConfig();
2033
+ }
2034
+ catch (error) {
2035
+ issues.push({
2036
+ severity: "blocking",
2037
+ message: `Active embedding configuration is invalid: ${error instanceof Error ? error.message : String(error)}`,
2038
+ action: "Correct the embedding environment variables, then run knowledge_doctor again.",
2039
+ action_code: "fix_embedding_config",
2040
+ });
2041
+ }
1687
2042
  for (const kb of kbs) {
1688
2043
  throwIfAborted(signal);
1689
- if ((kb.status === "ready" || kb.status === "stale") &&
1690
- kb.chunk_count > 0 &&
1691
- kb.embedding_dimension !== null &&
1692
- kb.embedding_signature !== embeddingSignature(activeEmbeddingConfig, kb.embedding_dimension)) {
1693
- const canUpdate = kb.source_path !== null && kb.source_type !== "text";
1694
- issues.push({
1695
- severity: "warning",
1696
- kb_name: kb.name,
1697
- message: "This KB's vectors are incompatible with the active embedding provider or normalization policy.",
1698
- action: canUpdate
1699
- ? "Run knowledge_update to rebuild vectors with the active embedding configuration."
1700
- : "Re-add inline text or re-import the original source with the active embedding configuration.",
1701
- action_code: "rebuild_vectors",
1702
- });
2044
+ if (activeEmbeddingConfig && (kb.status === "ready" || kb.status === "stale") && kb.chunk_count > 0) {
2045
+ const compatibility = inspectEmbeddingCompatibility(kb.embedding_signature, kb.embedding_dimension, activeEmbeddingConfig, kb.embedding_dimension ?? 0);
2046
+ if (!compatibility.compatible) {
2047
+ const canUpdate = kb.source_path !== null && kb.source_type !== "text";
2048
+ issues.push({
2049
+ severity: "warning",
2050
+ kb_name: kb.name,
2051
+ message: `This KB's vectors are incompatible with the active embedding configuration: ${describeEmbeddingCompatibility(compatibility)}.`,
2052
+ action: canUpdate
2053
+ ? "Run knowledge_update to rebuild vectors with the active embedding configuration."
2054
+ : "Re-add inline text or re-import the original source with the active embedding configuration.",
2055
+ action_code: "rebuild_vectors",
2056
+ });
2057
+ }
1703
2058
  }
1704
2059
  if ((kb.status === "ready" || kb.status === "stale") &&
1705
2060
  (storedSourcePolicyVersion(kb.source_options, "embedding_input_policy") ?? 0) < CURRENT_EMBEDDING_INPUT_POLICY) {
@@ -1824,10 +2179,10 @@ export class KnowledgeEngine {
1824
2179
  throwIfAborted(signal);
1825
2180
  const { createReadStream } = await import("node:fs");
1826
2181
  const { createInterface } = await import("node:readline");
1827
- const stream = createReadStream(inputPath, { encoding: "utf-8" });
1828
- const lines = createInterface({ input: stream, crlfDelay: Infinity });
1829
2182
  const embeddingConfig = resolveEmbeddingConfig();
1830
2183
  const embeddingBatchSize = resolveIndexEmbedBatchSize();
2184
+ const stream = createReadStream(inputPath, { encoding: "utf-8" });
2185
+ const lines = createInterface({ input: stream, crlfDelay: Infinity });
1831
2186
  let header;
1832
2187
  let kb;
1833
2188
  let vectorWriter;