pi-knowledge 0.12.0 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,26 +1,27 @@
1
1
  import { existsSync, readFileSync, renameSync, rmSync, statSync } from "node:fs";
2
2
  import { join, resolve } from "node:path";
3
3
  import { diagnoseKB } from "./diagnostics/health.js";
4
- import { dispose as disposeEmbedding, embedDocuments, embeddingConfigLabel, embeddingSignature, embedQueryWithConfig, embedValidatedDocuments, prepareForShutdown as prepareEmbeddingForShutdown, resolveEmbeddingConfig, } from "./embedding/provider.js";
4
+ import { describeEmbeddingCompatibility, describeEmbeddingDecoration, dispose as disposeEmbedding, embedDocuments, embeddingConfigLabel, embeddingSignature, embedQueryWithConfig, embedValidatedDocuments, inspectEmbeddingCompatibility, prepareForShutdown as prepareEmbeddingForShutdown, resolveEmbeddingConfig, } from "./embedding/provider.js";
5
5
  import { openVectorReader, openVectorWriter } from "./embedding/vectors.js";
6
6
  import { addSkippedScanEntry, analyzeIndexableContent, buildChunkEmbeddingText, chunkFile, chunkIdentityHash, chunkMarkdown, createSkippedScanStats, isReadableTextFile, isSupportedDocumentFile, iterateScannableFiles, preTokenizeForFTS, summarizeSkippedScan, } from "./indexer/chunker.js";
7
7
  import { finalizeChunksForEmbedding } from "./indexer/embedding-safe-chunks.js";
8
8
  import { htmlToStructuredMarkdown } from "./indexer/html.js";
9
9
  import { extractSymbols } from "./indexer/symbols.js";
10
- import { shutdownModelWorker } from "./model-worker-client.js";
10
+ import { extractDocumentInModelWorker, shutdownModelWorker } from "./model-worker-client.js";
11
11
  import { searchBM25 } from "./search/bm25.js";
12
+ import { MAX_SEARCH_WINDOW } from "./search/contracts.js";
13
+ import { diversifyRankedChunks, interleaveByFile } from "./search/diversity.js";
12
14
  import { weightedScoreFusion } from "./search/fusion.js";
13
15
  import { normalizedQueryText, tokenizeForSearch } from "./search/query.js";
14
16
  import { hasAnyLexicalEvidence, hasEnoughLexicalEvidence, normalizeFileTypeFilter, queryCoverage, scoreChunkForQuery, } from "./search/ranking.js";
15
17
  import { disposeReranker, prepareRerankerForShutdown, rerank } from "./search/reranker.js";
16
- import { resolveSearchTuning, summarizeSearchTuning, } from "./search/tuning.js";
18
+ import { resolveSearchTuning, resolveSearchWindow, summarizeSearchTuning, } from "./search/tuning.js";
17
19
  import { searchVectorFile } from "./search/vector.js";
18
20
  import { countSymbols, createKB, deleteChunksByIds, deleteKB, deleteSymbolsByKB, finishIndexingJob, getChunkById, getChunkCount, getChunksByFile, getFileCount, getKB, getKBByName, getSymbolCount, insertChunks, insertSymbols, iterateChunkIdsByKB, iterateChunksByKB, listKBs, openDatabase, searchSymbols, startIndexingJob, updateIndexingJob, updateKBCounts, updateKBEmbeddingMetadata, updateKBSourceOptions, updateKBStatus, } from "./storage/sqlite.js";
19
21
  export { CURRENT_EMBEDDING_MODEL } from "./embedding/provider.js";
20
22
  export const CURRENT_URL_STRUCTURE_POLICY = 1;
21
23
  export const CURRENT_EMBEDDING_INPUT_POLICY = 2;
22
24
  const DEFAULT_INDEX_EMBED_BATCH_SIZE = 64;
23
- const VECTOR_REDUNDANCY_WEIGHT = 0.35;
24
25
  function resolveIndexEmbedBatchSize(env = process.env) {
25
26
  const raw = env.PI_KNOWLEDGE_EMBEDDING_BATCH_SIZE?.trim();
26
27
  if (!raw)
@@ -160,116 +161,9 @@ function formatBytes(bytes) {
160
161
  return `${mb.toFixed(1)} MB`;
161
162
  return `${(mb / 1024).toFixed(2)} GB`;
162
163
  }
163
- function cosineSimilarity(a, b) {
164
- if (!a || !b || a.length !== b.length)
165
- return 0;
166
- let dot = 0;
167
- for (let i = 0; i < a.length; i++)
168
- dot += a[i] * b[i];
169
- return dot;
170
- }
171
164
  function tokenizeForSimilarity(text) {
172
165
  return tokenizeForSearch(text);
173
166
  }
174
- function jaccardSimilarity(a, b) {
175
- if (a.size === 0 || b.size === 0)
176
- return 0;
177
- let intersection = 0;
178
- for (const token of a) {
179
- if (b.has(token))
180
- intersection++;
181
- }
182
- return intersection / (a.size + b.size - intersection);
183
- }
184
- function lineProximity(a, b) {
185
- if (a.chunk.file_path !== b.chunk.file_path || a.chunk.kb_id !== b.chunk.kb_id)
186
- return 0;
187
- if (a.sourceChunkIds.some((id) => b.sourceChunkIds.includes(id)))
188
- return 1;
189
- const overlap = Math.max(0, Math.min(a.endLine, b.endLine) - Math.max(a.startLine, b.startLine) + 1);
190
- if (overlap > 0)
191
- return 1;
192
- const gap = Math.max(a.startLine, b.startLine) - Math.min(a.endLine, b.endLine);
193
- if (gap <= 20)
194
- return 0.8;
195
- if (gap <= 80)
196
- return 0.45;
197
- return 0.2;
198
- }
199
- function normalizeScores(candidates) {
200
- const scores = candidates.map((candidate) => candidate.score);
201
- const min = Math.min(...scores);
202
- const max = Math.max(...scores);
203
- const normalized = new Map();
204
- for (const candidate of candidates) {
205
- const value = max === min ? 1 : (candidate.score - min) / (max - min);
206
- normalized.set(candidate.chunk.id, value);
207
- }
208
- return normalized;
209
- }
210
- function diversifyRankedChunks(candidates, diversity, vectorsByChunkId = new Map()) {
211
- if (diversity === "off" || candidates.length <= 2)
212
- return candidates;
213
- const lambda = diversity === "strong" ? 0.62 : 0.76;
214
- const normalized = normalizeScores(candidates);
215
- const tokenSets = new Map(candidates.map((candidate) => [candidate.chunk.id, tokenizeForSimilarity(candidate.content)]));
216
- const selected = [];
217
- const remaining = [...candidates];
218
- while (remaining.length > 0) {
219
- let bestIndex = 0;
220
- let bestScore = Number.NEGATIVE_INFINITY;
221
- for (let i = 0; i < remaining.length; i++) {
222
- const candidate = remaining[i];
223
- let redundancy = 0;
224
- for (const chosen of selected) {
225
- const lexical = jaccardSimilarity(tokenSets.get(candidate.chunk.id) ?? new Set(), tokenSets.get(chosen.chunk.id) ?? new Set());
226
- const vector = Math.max(0, cosineSimilarity(vectorsByChunkId.get(candidate.chunk.id), vectorsByChunkId.get(chosen.chunk.id)));
227
- redundancy = Math.max(redundancy, Math.max(lexical, lineProximity(candidate, chosen), vector * VECTOR_REDUNDANCY_WEIGHT));
228
- }
229
- const relevance = normalized.get(candidate.chunk.id) ?? 0;
230
- const mmrScore = lambda * relevance - (1 - lambda) * redundancy;
231
- if (mmrScore > bestScore) {
232
- bestScore = mmrScore;
233
- bestIndex = i;
234
- }
235
- }
236
- selected.push(remaining.splice(bestIndex, 1)[0]);
237
- }
238
- return selected;
239
- }
240
- function interleaveByFile(candidates, diversity) {
241
- if (diversity === "off" || candidates.length <= 2)
242
- return candidates;
243
- const buckets = new Map();
244
- const fileOrder = [];
245
- for (const candidate of candidates) {
246
- const key = `${candidate.chunk.kb_id}:${candidate.chunk.file_path}`;
247
- if (!buckets.has(key)) {
248
- buckets.set(key, []);
249
- fileOrder.push(key);
250
- }
251
- buckets.get(key)?.push(candidate);
252
- }
253
- if (fileOrder.length <= 1)
254
- return candidates;
255
- const result = [];
256
- let round = 0;
257
- while (result.length < candidates.length) {
258
- let added = false;
259
- for (const key of fileOrder) {
260
- const bucket = buckets.get(key);
261
- const item = bucket?.[round];
262
- if (!item)
263
- continue;
264
- result.push(item);
265
- added = true;
266
- }
267
- if (!added)
268
- break;
269
- round++;
270
- }
271
- return result;
272
- }
273
167
  function buildQuerySnippet(content, query, maxLength = 240) {
274
168
  const terms = [...tokenizeForSimilarity(query)].sort((a, b) => b.length - a.length);
275
169
  const lower = content.toLowerCase();
@@ -476,30 +370,19 @@ async function chunkUrl(source, signal) {
476
370
  const structuredChunks = chunkMarkdown(text, source, "html");
477
371
  return structuredChunks.length > 0 ? structuredChunks : chunkFile(text, source);
478
372
  }
479
- function normalizeExtractedText(text) {
480
- return Array.isArray(text) ? text.join("\n\n") : text;
481
- }
482
373
  async function extractSourceFileContent(filePath, signal) {
483
374
  const lowerPath = filePath.toLowerCase();
484
375
  if (lowerPath.endsWith(".pdf")) {
485
376
  throwIfAborted(signal);
486
- // PDF extraction is an optional heavy runtime path; keep parser loading out of extension startup.
487
- const { extractText } = await import("unpdf");
377
+ const extracted = await extractDocumentInModelWorker(filePath, "pdf", signal);
488
378
  throwIfAborted(signal);
489
- const buf = readFileSync(filePath);
490
- throwIfAborted(signal);
491
- const { text } = await extractText(new Uint8Array(buf));
492
- throwIfAborted(signal);
493
- return { content: normalizeExtractedText(text), fileType: "pdf" };
379
+ return extracted;
494
380
  }
495
381
  if (lowerPath.endsWith(".docx") || lowerPath.endsWith(".doc")) {
496
382
  throwIfAborted(signal);
497
- // DOCX extraction is an optional heavy runtime path; keep parser loading out of extension startup.
498
- const mammoth = await import("mammoth");
383
+ const extracted = await extractDocumentInModelWorker(filePath, "docx", signal);
499
384
  throwIfAborted(signal);
500
- const result = await mammoth.extractRawText({ path: filePath });
501
- throwIfAborted(signal);
502
- return { content: result.value, fileType: "docx" };
385
+ return extracted;
503
386
  }
504
387
  if (!isReadableTextFile(filePath)) {
505
388
  throw new Error(`File is not readable text and has no supported extractor: ${filePath}`);
@@ -557,15 +440,8 @@ function persistEmbeddingMetadata(db, kbId, config, vectors) {
557
440
  updateKBEmbeddingMetadata(db, kbId, embeddingConfigLabel(config), embeddingSignature(config, dimension), dimension);
558
441
  return dimension;
559
442
  }
560
- function embeddingMismatchWarning(kb) {
561
- return `"${kb.name}" has incompatible embedding metadata; vector retrieval was skipped and knowledge_update should rebuild it`;
562
- }
563
- function canSearchVectors(kb, config, queryDimension) {
564
- if (kb.embedding_signature === null || kb.embedding_dimension === null)
565
- return false;
566
- if (kb.embedding_dimension !== queryDimension)
567
- return false;
568
- return kb.embedding_signature === embeddingSignature(config, queryDimension);
443
+ function embeddingMismatchWarning(kb, compatibility) {
444
+ return `"${kb.name}" has incompatible embedding vectors (${describeEmbeddingCompatibility(compatibility)}); vector retrieval was skipped and knowledge_update should rebuild them`;
569
445
  }
570
446
  function throwIfAborted(signal) {
571
447
  if (signal?.aborted)
@@ -950,14 +826,17 @@ export class KnowledgeEngine {
950
826
  if (!this.db)
951
827
  throw new Error("Engine not initialized");
952
828
  throwIfAborted(signal);
953
- updateKBStatus(this.db, kb.id, "indexing");
954
- startIndexingJob(this.db, kb.id, "update", `Starting update for "${kb.name}"`);
955
829
  const scanOptions = toScanOptions(parseAddOptions(kb.source_options));
956
830
  const embeddingConfig = resolveEmbeddingConfig();
957
831
  const embeddingBatchSize = resolveIndexEmbedBatchSize();
832
+ updateKBStatus(this.db, kb.id, "indexing");
833
+ startIndexingJob(this.db, kb.id, "update", `Starting update for "${kb.name}"`);
958
834
  const embeddingModel = embeddingConfigLabel(embeddingConfig);
959
- const currentSignature = kb.embedding_dimension === null ? undefined : embeddingSignature(embeddingConfig, kb.embedding_dimension);
960
- let canReuseExistingVectors = kb.embedding_signature !== null && kb.embedding_dimension !== null && kb.embedding_signature === currentSignature;
835
+ const compatibility = inspectEmbeddingCompatibility(kb.embedding_signature, kb.embedding_dimension, embeddingConfig, kb.embedding_dimension ?? 0);
836
+ let canReuseExistingVectors = compatibility.compatible;
837
+ let rebuildMessage = compatibility.compatible
838
+ ? undefined
839
+ : `Embedding vectors for "${kb.name}" are incompatible (${describeEmbeddingCompatibility(compatibility)}); rebuilding all vectors`;
961
840
  let replacementVectorPath;
962
841
  let addedVectorPath;
963
842
  let addedVectorWriter;
@@ -986,9 +865,7 @@ export class KnowledgeEngine {
986
865
  canReuseExistingVectors = false;
987
866
  const vectorCount = oldVectorReader?.count ?? 0;
988
867
  const vectorDimension = oldVectorReader?.dim ?? 0;
989
- const message = `Stored vector file for "${kb.name}" is incomplete or incompatible (${vectorCount} vectors/${vectorDimension}d for ${existingIndex} chunks/${kb.embedding_dimension}d); rebuilding all vectors`;
990
- updateIndexingJob(this.db, kb.id, { phase: "embedding", message });
991
- onProgress?.(message);
868
+ rebuildMessage = `Stored vector file for "${kb.name}" is incomplete or incompatible (${vectorCount} vectors/${vectorDimension}d for ${existingIndex} chunks/${kb.embedding_dimension}d); rebuilding all vectors`;
992
869
  }
993
870
  }
994
871
  finally {
@@ -998,10 +875,9 @@ export class KnowledgeEngine {
998
875
  const reusableHashes = canReuseExistingVectors
999
876
  ? existingHashes
1000
877
  : new Map();
1001
- if (!canReuseExistingVectors) {
1002
- const message = `Embedding metadata changed or missing for "${kb.name}"; rebuilding all vectors`;
1003
- updateIndexingJob(this.db, kb.id, { phase: "embedding", message });
1004
- onProgress?.(message);
878
+ if (!canReuseExistingVectors && rebuildMessage) {
879
+ updateIndexingJob(this.db, kb.id, { phase: "embedding", message: rebuildMessage });
880
+ onProgress?.(rebuildMessage);
1005
881
  }
1006
882
  const oldVectorIndexByHash = new Map();
1007
883
  const newVectorIndexByHash = new Map();
@@ -1326,7 +1202,7 @@ export class KnowledgeEngine {
1326
1202
  };
1327
1203
  }
1328
1204
  const db = this.db;
1329
- const { mode = "hybrid", offset = 0, kb_id, filters, diversity = "balanced" } = options;
1205
+ const { mode = "hybrid", offset: requestedOffset, kb_id, filters, diversity = "balanced" } = options;
1330
1206
  const resolvedMode = retrievalModeFor(mode);
1331
1207
  const retrievalMode = resolvedMode === "adaptive" ? "hybrid" : resolvedMode;
1332
1208
  const normalizedQuery = normalizedQueryText(query);
@@ -1348,10 +1224,9 @@ export class KnowledgeEngine {
1348
1224
  profile: options.profile,
1349
1225
  kbSourceTypes: kbs.map((kb) => kb.source_type),
1350
1226
  });
1351
- const limit = typeof options.limit === "number" && Number.isFinite(options.limit) && options.limit > 0
1352
- ? Math.trunc(options.limit)
1353
- : tuning.defaultLimit;
1354
- const candidateLimit = Math.max(tuning.candidateMin, offset + limit * tuning.candidateMultiplier);
1227
+ const { limit, offset } = resolveSearchWindow(options.limit, requestedOffset, tuning.defaultLimit);
1228
+ const candidateLimit = Math.min(MAX_SEARCH_WINDOW, Math.max(tuning.candidateMin, offset + limit * tuning.candidateMultiplier));
1229
+ const retrievalLimitPerKb = candidateLimit;
1355
1230
  const tuningSummary = summarizeSearchTuning(tuning, limit, candidateLimit);
1356
1231
  if (kbs.length === 0) {
1357
1232
  return {
@@ -1365,45 +1240,50 @@ export class KnowledgeEngine {
1365
1240
  }
1366
1241
  const allResults = [];
1367
1242
  const vectorsByChunkId = new Map();
1243
+ let queryEmbedding;
1368
1244
  for (const kb of kbs) {
1369
1245
  throwIfAborted(signal);
1370
1246
  if (kb.chunk_count === 0)
1371
1247
  continue;
1372
1248
  const vectorPath = this.vectorPathFor(kb.id);
1373
1249
  if (retrievalMode === "fast") {
1374
- allResults.push(...searchBM25(db, normalizedQuery || query, candidateLimit, kb.id, { allowOrFallback: false }).map((result) => ({
1250
+ allResults.push(...searchBM25(db, normalizedQuery || query, retrievalLimitPerKb, kb.id, { allowOrFallback: false }).map((result) => ({
1375
1251
  ...result,
1376
1252
  score: result.score * kbTrustMultiplier(kb),
1377
1253
  })));
1378
1254
  }
1379
1255
  else if (retrievalMode === "semantic") {
1380
- const { vector: queryVec, config: queryEmbeddingConfig } = await embedQueryWithConfig(query, signal);
1256
+ queryEmbedding ??= await embedQueryWithConfig(query, signal);
1257
+ const { vector: queryVec, config: queryEmbeddingConfig } = queryEmbedding;
1381
1258
  throwIfAborted(signal);
1382
- if (!canSearchVectors(kb, queryEmbeddingConfig, queryVec.length)) {
1383
- warnings.push(embeddingMismatchWarning(kb));
1259
+ const compatibility = inspectEmbeddingCompatibility(kb.embedding_signature, kb.embedding_dimension, queryEmbeddingConfig, queryVec.length);
1260
+ if (!compatibility.compatible) {
1261
+ warnings.push(embeddingMismatchWarning(kb, compatibility));
1384
1262
  continue;
1385
1263
  }
1386
- const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), candidateLimit, signal);
1264
+ const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), retrievalLimitPerKb, signal);
1387
1265
  allResults.push(...vectorResults.results.map((result) => ({ ...result, score: result.score * kbTrustMultiplier(kb) })));
1388
1266
  for (const [chunkId, vector] of vectorResults.vectorsByChunkId)
1389
1267
  vectorsByChunkId.set(chunkId, vector);
1390
1268
  }
1391
1269
  else {
1392
1270
  const semanticHybrid = retrievalMode === "semantic_hybrid";
1393
- const bm25Results = searchBM25(db, normalizedQuery || query, candidateLimit, kb.id);
1271
+ const bm25Results = searchBM25(db, normalizedQuery || query, retrievalLimitPerKb, kb.id);
1394
1272
  if (bm25Results.length === 0 && !semanticHybrid)
1395
1273
  continue;
1396
1274
  let vecResults = [];
1397
- const { vector: queryVec, config: queryEmbeddingConfig } = await embedQueryWithConfig(query, signal);
1275
+ queryEmbedding ??= await embedQueryWithConfig(query, signal);
1276
+ const { vector: queryVec, config: queryEmbeddingConfig } = queryEmbedding;
1398
1277
  throwIfAborted(signal);
1399
- if (canSearchVectors(kb, queryEmbeddingConfig, queryVec.length)) {
1400
- const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), candidateLimit, signal);
1278
+ const compatibility = inspectEmbeddingCompatibility(kb.embedding_signature, kb.embedding_dimension, queryEmbeddingConfig, queryVec.length);
1279
+ if (compatibility.compatible) {
1280
+ const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), retrievalLimitPerKb, signal);
1401
1281
  vecResults = vectorResults.results;
1402
1282
  for (const [chunkId, vector] of vectorResults.vectorsByChunkId)
1403
1283
  vectorsByChunkId.set(chunkId, vector);
1404
1284
  }
1405
1285
  else {
1406
- warnings.push(embeddingMismatchWarning(kb));
1286
+ warnings.push(embeddingMismatchWarning(kb, compatibility));
1407
1287
  }
1408
1288
  const fused = weightedScoreFusion(bm25Results, vecResults);
1409
1289
  allResults.push(...fused.map((result) => ({ ...result, score: result.score * kbTrustMultiplier(kb) })));
@@ -1461,8 +1341,15 @@ export class KnowledgeEngine {
1461
1341
  return true;
1462
1342
  });
1463
1343
  }
1464
- if (mode === "deep" && filtered.length > 0) {
1465
- const candidates = filtered
1344
+ const candidatePool = filtered.slice(0, candidateLimit);
1345
+ const candidatePoolTruncated = candidatePool.length < filtered.length;
1346
+ const candidateIds = new Set(candidatePool.map((candidate) => candidate.chunkId));
1347
+ for (const chunkId of vectorsByChunkId.keys()) {
1348
+ if (!candidateIds.has(chunkId))
1349
+ vectorsByChunkId.delete(chunkId);
1350
+ }
1351
+ if (mode === "deep" && candidatePool.length > 0) {
1352
+ const candidates = candidatePool
1466
1353
  .slice(0, tuning.deepRerankCandidates)
1467
1354
  .map((r) => {
1468
1355
  const chunk = getChunkById(db, r.chunkId);
@@ -1489,7 +1376,7 @@ export class KnowledgeEngine {
1489
1376
  sourceChunkIds: [chunk.id],
1490
1377
  });
1491
1378
  }
1492
- const diversified = interleaveByFile(diversifyRankedChunks(ranked, diversity, vectorsByChunkId), diversity);
1379
+ const diversified = interleaveByFile(await diversifyRankedChunks(ranked, diversity, vectorsByChunkId, signal), diversity, signal);
1493
1380
  const page = diversified.slice(offset, offset + limit);
1494
1381
  const results = page.map((r) => {
1495
1382
  const kb = kbById.get(r.chunk.kb_id);
@@ -1521,11 +1408,17 @@ export class KnowledgeEngine {
1521
1408
  has_more: offset + limit < diversified.length,
1522
1409
  warnings: warnings.length > 0 ? warnings : undefined,
1523
1410
  mode_used: mode,
1524
- tuning: tuningSummary,
1411
+ tuning: {
1412
+ ...tuningSummary,
1413
+ retrieved_candidates: unique.length,
1414
+ filtered_candidates: filtered.length,
1415
+ diversity_candidates: ranked.length,
1416
+ candidate_pool_truncated: candidatePoolTruncated,
1417
+ },
1525
1418
  };
1526
1419
  }
1527
1420
  const ranked = [];
1528
- for (const r of filtered) {
1421
+ for (const r of candidatePool) {
1529
1422
  throwIfAborted(signal);
1530
1423
  const chunk = getChunkById(db, r.chunkId);
1531
1424
  if (!chunk)
@@ -1563,7 +1456,7 @@ export class KnowledgeEngine {
1563
1456
  });
1564
1457
  }
1565
1458
  }
1566
- const diversified = interleaveByFile(diversifyRankedChunks(ranked, diversity, vectorsByChunkId), diversity);
1459
+ const diversified = interleaveByFile(await diversifyRankedChunks(ranked, diversity, vectorsByChunkId, signal), diversity, signal);
1567
1460
  const total = diversified.length;
1568
1461
  const page = diversified.slice(offset, offset + limit);
1569
1462
  const results = page.map((r) => ({
@@ -1596,7 +1489,13 @@ export class KnowledgeEngine {
1596
1489
  has_more: offset + limit < total,
1597
1490
  warnings: warnings.length > 0 ? warnings : undefined,
1598
1491
  mode_used: mode,
1599
- tuning: tuningSummary,
1492
+ tuning: {
1493
+ ...tuningSummary,
1494
+ retrieved_candidates: unique.length,
1495
+ filtered_candidates: filtered.length,
1496
+ diversity_candidates: ranked.length,
1497
+ candidate_pool_truncated: candidatePoolTruncated,
1498
+ },
1600
1499
  suggestions: results.length === 0
1601
1500
  ? [
1602
1501
  "Try mode 'fast' for exact symbols, mode 'semantic' for pure vector search, or mode 'semantic_hybrid' when weak lexical overlap should not block fused vector results.",
@@ -1647,6 +1546,46 @@ export class KnowledgeEngine {
1647
1546
  return diagnoseKB(db, kb, signal);
1648
1547
  });
1649
1548
  }
1549
+ statusSnapshot(signal) {
1550
+ if (!this.db)
1551
+ return { knowledge_bases: [], diagnostics: [], embeddings: [] };
1552
+ throwIfAborted(signal);
1553
+ const db = this.db;
1554
+ const knowledgeBases = listKBs(db);
1555
+ const diagnostics = knowledgeBases.map((kb) => {
1556
+ throwIfAborted(signal);
1557
+ return diagnoseKB(db, kb, signal);
1558
+ });
1559
+ try {
1560
+ const config = resolveEmbeddingConfig();
1561
+ const activeModel = embeddingConfigLabel(config);
1562
+ const activeDecoration = describeEmbeddingDecoration(config);
1563
+ return {
1564
+ knowledge_bases: knowledgeBases,
1565
+ diagnostics,
1566
+ embeddings: knowledgeBases.map((kb) => {
1567
+ const compatibility = inspectEmbeddingCompatibility(kb.embedding_signature, kb.embedding_dimension, config, kb.embedding_dimension ?? 0);
1568
+ return {
1569
+ kb_id: kb.id,
1570
+ kb_name: kb.name,
1571
+ indexed_model: kb.embedding_model,
1572
+ active_model: activeModel,
1573
+ active_decoration: activeDecoration,
1574
+ compatibility,
1575
+ detail: describeEmbeddingCompatibility(compatibility),
1576
+ };
1577
+ }),
1578
+ };
1579
+ }
1580
+ catch (error) {
1581
+ return {
1582
+ knowledge_bases: knowledgeBases,
1583
+ diagnostics,
1584
+ embeddings: [],
1585
+ embedding_error: error instanceof Error ? error.message : String(error),
1586
+ };
1587
+ }
1588
+ }
1650
1589
  symbolSearch(query, options = {}, signal) {
1651
1590
  if (!this.db)
1652
1591
  throw new Error("Engine not initialized");
@@ -1768,8 +1707,35 @@ export class KnowledgeEngine {
1768
1707
  }
1769
1708
  }
1770
1709
  if (this.db) {
1710
+ let activeEmbeddingConfig;
1711
+ try {
1712
+ activeEmbeddingConfig = resolveEmbeddingConfig();
1713
+ }
1714
+ catch (error) {
1715
+ issues.push({
1716
+ severity: "blocking",
1717
+ message: `Active embedding configuration is invalid: ${error instanceof Error ? error.message : String(error)}`,
1718
+ action: "Correct the embedding environment variables, then run knowledge_doctor again.",
1719
+ action_code: "fix_embedding_config",
1720
+ });
1721
+ }
1771
1722
  for (const kb of kbs) {
1772
1723
  throwIfAborted(signal);
1724
+ if (activeEmbeddingConfig && (kb.status === "ready" || kb.status === "stale") && kb.chunk_count > 0) {
1725
+ const compatibility = inspectEmbeddingCompatibility(kb.embedding_signature, kb.embedding_dimension, activeEmbeddingConfig, kb.embedding_dimension ?? 0);
1726
+ if (!compatibility.compatible) {
1727
+ const canUpdate = kb.source_path !== null && kb.source_type !== "text";
1728
+ issues.push({
1729
+ severity: "warning",
1730
+ kb_name: kb.name,
1731
+ message: `This KB's vectors are incompatible with the active embedding configuration: ${describeEmbeddingCompatibility(compatibility)}.`,
1732
+ action: canUpdate
1733
+ ? "Run knowledge_update to rebuild vectors with the active embedding configuration."
1734
+ : "Re-add inline text or re-import the original source with the active embedding configuration.",
1735
+ action_code: "rebuild_vectors",
1736
+ });
1737
+ }
1738
+ }
1773
1739
  if ((kb.status === "ready" || kb.status === "stale") &&
1774
1740
  (storedSourcePolicyVersion(kb.source_options, "embedding_input_policy") ?? 0) < CURRENT_EMBEDDING_INPUT_POLICY) {
1775
1741
  const canUpdate = kb.source_path !== null && kb.source_type !== "text";
@@ -1893,10 +1859,10 @@ export class KnowledgeEngine {
1893
1859
  throwIfAborted(signal);
1894
1860
  const { createReadStream } = await import("node:fs");
1895
1861
  const { createInterface } = await import("node:readline");
1896
- const stream = createReadStream(inputPath, { encoding: "utf-8" });
1897
- const lines = createInterface({ input: stream, crlfDelay: Infinity });
1898
1862
  const embeddingConfig = resolveEmbeddingConfig();
1899
1863
  const embeddingBatchSize = resolveIndexEmbedBatchSize();
1864
+ const stream = createReadStream(inputPath, { encoding: "utf-8" });
1865
+ const lines = createInterface({ input: stream, crlfDelay: Infinity });
1900
1866
  let header;
1901
1867
  let kb;
1902
1868
  let vectorWriter;
@@ -1,3 +1,4 @@
1
+ import { embeddingInstructionLength } from "../embedding/instructions.js";
1
2
  import { LOCAL_EMBEDDING_PREFERRED_TOKENS, measureEmbeddingInputs } from "../embedding/provider.js";
2
3
  import { buildChunkEmbeddingText, chunkIdentityHash, preTokenizeForFTS } from "./chunker.js";
3
4
  const SENTENCE_BOUNDARIES = {
@@ -153,8 +154,8 @@ function withPartMetadata(chunk, part, partCount) {
153
154
  function isProvablySafeLocalAsciiInput(text, config) {
154
155
  if (config.provider !== "local")
155
156
  return false;
156
- const prefixLength = config.prefixes === "on" ? "passage: ".length : 0;
157
- if (text.length + prefixLength + 2 > LOCAL_EMBEDDING_PREFERRED_TOKENS)
157
+ const instructionLength = embeddingInstructionLength(config.instructions, "passage");
158
+ if (text.length + instructionLength + 2 > LOCAL_EMBEDDING_PREFERRED_TOKENS)
158
159
  return false;
159
160
  for (let index = 0; index < text.length; index++) {
160
161
  if (text.charCodeAt(index) > 0x7f)
@@ -7,6 +7,10 @@ export declare function measureEmbeddingInputInModelWorker(texts: string[], pref
7
7
  counts: number[];
8
8
  maximum: number;
9
9
  }>;
10
+ export declare function extractDocumentInModelWorker(filePath: string, fileType: "pdf" | "docx", signal?: AbortSignal): Promise<{
11
+ content: string;
12
+ fileType: "pdf" | "docx";
13
+ }>;
10
14
  export interface RerankWorkerCandidate {
11
15
  chunkId: string;
12
16
  content: string;
@@ -39,7 +39,7 @@ function validateNodeExecPath(candidate) {
39
39
  }).trim();
40
40
  const major = parseNodeMajor(version);
41
41
  if (major === null || major < MIN_NODE_MAJOR) {
42
- throw new Error(`Node ${MIN_NODE_MAJOR}+ is required for the model worker; ${candidate} reported ${version}`);
42
+ throw new Error(`Node ${MIN_NODE_MAJOR}+ is required for the isolated runtime worker; ${candidate} reported ${version}`);
43
43
  }
44
44
  return candidate;
45
45
  }
@@ -149,7 +149,7 @@ function getNodeExecPath() {
149
149
  // Try the next candidate before surfacing a deterministic error below.
150
150
  }
151
151
  }
152
- throw new Error(`Node ${MIN_NODE_MAJOR}+ is required for local pi-knowledge embeddings, but no usable node executable was found. Install Node ${MIN_NODE_MAJOR}+ or set PI_KNOWLEDGE_NODE_PATH to the full node.exe path.`);
152
+ throw new Error(`Node ${MIN_NODE_MAJOR}+ is required for isolated pi-knowledge worker operations such as local models and PDF/DOCX extraction, but no usable node executable was found. Install Node ${MIN_NODE_MAJOR}+ or set PI_KNOWLEDGE_NODE_PATH to the full node.exe path.`);
153
153
  }
154
154
  export function validateModelWorkerNodePath(nodePath) {
155
155
  return validateNodeExecPath(normalizeExecutableCandidate(nodePath));
@@ -413,6 +413,18 @@ export async function measureEmbeddingInputInModelWorker(texts, prefix, prefixes
413
413
  }
414
414
  return { counts: result.counts, maximum: result.maximum };
415
415
  }
416
+ export async function extractDocumentInModelWorker(filePath, fileType, signal) {
417
+ const result = await getWorker().request({ type: "extract_document", filePath, fileType }, signal);
418
+ if (typeof result !== "object" ||
419
+ result === null ||
420
+ !("content" in result) ||
421
+ typeof result.content !== "string" ||
422
+ !("fileType" in result) ||
423
+ result.fileType !== fileType) {
424
+ throw new Error("Invalid document extraction worker response");
425
+ }
426
+ return { content: result.content, fileType };
427
+ }
416
428
  export async function rerankInModelWorker(query, candidates, topK, reranker, signal) {
417
429
  const result = await getWorker().request({ type: "rerank", query, candidates, topK, reranker }, signal);
418
430
  if (!Array.isArray(result))
@@ -1,5 +1,7 @@
1
+ import { readFile } from "node:fs/promises";
1
2
  import { join } from "node:path";
2
3
  import { createInterface } from "node:readline";
4
+ import { decorateEmbeddingInput, E5_INSTRUCTIONS, NO_INSTRUCTIONS, } from "./embedding/instructions.js";
3
5
  import { DEFAULT_RERANKER_REMOTE_HOST, DEFAULT_RERANKER_REMOTE_PATH_TEMPLATE, isHfRerankerConfig, rerankerCacheKey, resolveRerankerConfig, } from "./search/reranker-config.js";
4
6
  import { rerankWithSingleRawLogit } from "./search/reranker-logits.js";
5
7
  import { getDefaultKnowledgeDir } from "./storage/sqlite.js";
@@ -76,8 +78,8 @@ async function loadRerankerPipeline(config) {
76
78
  rerankerPipeline = { key, pipe: loaded };
77
79
  return loaded;
78
80
  }
79
- function applyEmbeddingPrefix(text, prefix, prefixes) {
80
- return prefixes === "on" ? `${prefix}: ${text}` : text;
81
+ function applyEmbeddingPrefix(text, role, prefixes) {
82
+ return decorateEmbeddingInput(text, role, prefixes === "on" ? E5_INSTRUCTIONS : NO_INSTRUCTIONS);
81
83
  }
82
84
  async function handleEmbed(request) {
83
85
  const pipe = await loadEmbeddingPipeline();
@@ -130,6 +132,24 @@ async function handleRerank(request) {
130
132
  results.sort((a, b) => b.score - a.score);
131
133
  return results.slice(0, request.topK);
132
134
  }
135
+ const PDFJS_ERRORS_ONLY_VERBOSITY = 0;
136
+ async function handleExtractDocument(request) {
137
+ if (request.fileType === "pdf") {
138
+ const { extractText, getDocumentProxy } = await import("unpdf");
139
+ const bytes = await readFile(request.filePath);
140
+ const pdf = await getDocumentProxy(new Uint8Array(bytes), { verbosity: PDFJS_ERRORS_ONLY_VERBOSITY });
141
+ try {
142
+ const { text } = await extractText(pdf);
143
+ return { content: Array.isArray(text) ? text.join("\n\n") : text, fileType: "pdf" };
144
+ }
145
+ finally {
146
+ await pdf.destroy();
147
+ }
148
+ }
149
+ const mammoth = await import("mammoth");
150
+ const result = await mammoth.extractRawText({ path: request.filePath });
151
+ return { content: result.value, fileType: "docx" };
152
+ }
133
153
  function isStringArray(value) {
134
154
  return Array.isArray(value) && value.every((item) => typeof item === "string");
135
155
  }
@@ -166,6 +186,15 @@ function parseWorkerRequest(message) {
166
186
  prefixes,
167
187
  };
168
188
  }
189
+ if (message.type === "extract_document") {
190
+ if (!("filePath" in message) || typeof message.filePath !== "string") {
191
+ throw new Error("Invalid document file path");
192
+ }
193
+ if (!("fileType" in message) || (message.fileType !== "pdf" && message.fileType !== "docx")) {
194
+ throw new Error("Invalid document file type");
195
+ }
196
+ return { id: message.id, type: "extract_document", filePath: message.filePath, fileType: message.fileType };
197
+ }
169
198
  if (message.type === "rerank") {
170
199
  if (!("query" in message) || typeof message.query !== "string")
171
200
  throw new Error("Invalid rerank query");
@@ -199,7 +228,9 @@ async function handleRequestMessage(message) {
199
228
  ? await handleEmbed(request)
200
229
  : request.type === "measure_embedding_input"
201
230
  ? await handleMeasureEmbeddingInput(request)
202
- : await handleRerank(request);
231
+ : request.type === "extract_document"
232
+ ? await handleExtractDocument(request)
233
+ : await handleRerank(request);
203
234
  return { id: request.id, result };
204
235
  }
205
236
  catch (error) {
@@ -0,0 +1,2 @@
1
+ export declare const MAX_SEARCH_RESULTS = 50;
2
+ export declare const MAX_SEARCH_WINDOW = 800;
@@ -0,0 +1,2 @@
1
+ export const MAX_SEARCH_RESULTS = 50;
2
+ export const MAX_SEARCH_WINDOW = 800;
@@ -0,0 +1,16 @@
1
+ import type { Chunk } from "../storage/sqlite.ts";
2
+ import type { RankingDiagnostics } from "./ranking.ts";
3
+ export type DiversityMode = "off" | "balanced" | "strong" | undefined;
4
+ export interface RankedChunk {
5
+ chunk: Chunk;
6
+ kbName: string;
7
+ ranking?: RankingDiagnostics;
8
+ score: number;
9
+ content: string;
10
+ snippet: string;
11
+ startLine: number;
12
+ endLine: number;
13
+ sourceChunkIds: string[];
14
+ }
15
+ export declare function diversifyRankedChunks(candidates: RankedChunk[], diversity: DiversityMode, vectorsByChunkId?: Map<string, Float32Array>, signal?: AbortSignal): Promise<RankedChunk[]>;
16
+ export declare function interleaveByFile(candidates: RankedChunk[], diversity: DiversityMode, signal?: AbortSignal): RankedChunk[];