pi-knowledge 0.11.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,22 +1,27 @@
1
1
  import { existsSync, readFileSync, renameSync, rmSync, statSync } from "node:fs";
2
2
  import { join, resolve } from "node:path";
3
3
  import { diagnoseKB } from "./diagnostics/health.js";
4
- import { dispose as disposeEmbedding, embedDocuments, embeddingConfigLabel, embeddingSignature, embedQueryWithConfig, prepareForShutdown as prepareEmbeddingForShutdown, resolveEmbeddingConfig, } from "./embedding/provider.js";
4
+ import { dispose as disposeEmbedding, embedDocuments, embeddingConfigLabel, embeddingSignature, embedQueryWithConfig, embedValidatedDocuments, prepareForShutdown as prepareEmbeddingForShutdown, resolveEmbeddingConfig, } from "./embedding/provider.js";
5
5
  import { openVectorReader, openVectorWriter } from "./embedding/vectors.js";
6
- import { addSkippedScanEntry, analyzeIndexableContent, buildChunkEmbeddingText, chunkFile, chunkIdentityHash, createSkippedScanStats, isReadableTextFile, isSupportedDocumentFile, iterateScannableFiles, preTokenizeForFTS, summarizeSkippedScan, } from "./indexer/chunker.js";
6
+ import { addSkippedScanEntry, analyzeIndexableContent, buildChunkEmbeddingText, chunkFile, chunkIdentityHash, chunkMarkdown, createSkippedScanStats, isReadableTextFile, isSupportedDocumentFile, iterateScannableFiles, preTokenizeForFTS, summarizeSkippedScan, } from "./indexer/chunker.js";
7
+ import { finalizeChunksForEmbedding } from "./indexer/embedding-safe-chunks.js";
8
+ import { htmlToStructuredMarkdown } from "./indexer/html.js";
7
9
  import { extractSymbols } from "./indexer/symbols.js";
8
10
  import { shutdownModelWorker } from "./model-worker-client.js";
9
11
  import { searchBM25 } from "./search/bm25.js";
12
+ import { MAX_SEARCH_WINDOW } from "./search/contracts.js";
13
+ import { diversifyRankedChunks, interleaveByFile } from "./search/diversity.js";
10
14
  import { weightedScoreFusion } from "./search/fusion.js";
11
15
  import { normalizedQueryText, tokenizeForSearch } from "./search/query.js";
12
16
  import { hasAnyLexicalEvidence, hasEnoughLexicalEvidence, normalizeFileTypeFilter, queryCoverage, scoreChunkForQuery, } from "./search/ranking.js";
13
17
  import { disposeReranker, prepareRerankerForShutdown, rerank } from "./search/reranker.js";
14
- import { resolveSearchTuning, summarizeSearchTuning, } from "./search/tuning.js";
18
+ import { resolveSearchTuning, resolveSearchWindow, summarizeSearchTuning, } from "./search/tuning.js";
15
19
  import { searchVectorFile } from "./search/vector.js";
16
- import { countSymbols, createKB, deleteChunksByIds, deleteKB, deleteSymbolsByKB, finishIndexingJob, getChunkById, getChunkCount, getChunksByFile, getFileCount, getKB, getKBByName, getSymbolCount, insertChunks, insertSymbols, iterateChunkIdsByKB, iterateChunksByKB, listKBs, openDatabase, searchSymbols, startIndexingJob, updateIndexingJob, updateKBCounts, updateKBEmbeddingMetadata, updateKBStatus, } from "./storage/sqlite.js";
20
+ import { countSymbols, createKB, deleteChunksByIds, deleteKB, deleteSymbolsByKB, finishIndexingJob, getChunkById, getChunkCount, getChunksByFile, getFileCount, getKB, getKBByName, getSymbolCount, insertChunks, insertSymbols, iterateChunkIdsByKB, iterateChunksByKB, listKBs, openDatabase, searchSymbols, startIndexingJob, updateIndexingJob, updateKBCounts, updateKBEmbeddingMetadata, updateKBSourceOptions, updateKBStatus, } from "./storage/sqlite.js";
17
21
  export { CURRENT_EMBEDDING_MODEL } from "./embedding/provider.js";
22
+ export const CURRENT_URL_STRUCTURE_POLICY = 1;
23
+ export const CURRENT_EMBEDDING_INPUT_POLICY = 2;
18
24
  const DEFAULT_INDEX_EMBED_BATCH_SIZE = 64;
19
- const VECTOR_REDUNDANCY_WEIGHT = 0.35;
20
25
  function resolveIndexEmbedBatchSize(env = process.env) {
21
26
  const raw = env.PI_KNOWLEDGE_EMBEDDING_BATCH_SIZE?.trim();
22
27
  if (!raw)
@@ -66,14 +71,17 @@ function toScanOptions(options = {}) {
66
71
  };
67
72
  }
68
73
  function serializeAddOptions(options = {}) {
69
- const normalized = {};
74
+ const normalized = {
75
+ embedding_input_policy: CURRENT_EMBEDDING_INPUT_POLICY,
76
+ url_structure_policy: CURRENT_URL_STRUCTURE_POLICY,
77
+ };
70
78
  if (options.include_suggested_text === true)
71
79
  normalized.include_suggested_text = true;
72
80
  if (options.include_paths && options.include_paths.length > 0)
73
81
  normalized.include_paths = options.include_paths;
74
82
  if (options.exclude_paths && options.exclude_paths.length > 0)
75
83
  normalized.exclude_paths = options.exclude_paths;
76
- return Object.keys(normalized).length > 0 ? JSON.stringify(normalized) : undefined;
84
+ return JSON.stringify(normalized);
77
85
  }
78
86
  function parseAddOptions(raw) {
79
87
  if (!raw)
@@ -94,6 +102,18 @@ function parseAddOptions(raw) {
94
102
  return {};
95
103
  }
96
104
  }
105
+ function storedSourcePolicyVersion(raw, key) {
106
+ if (!raw)
107
+ return undefined;
108
+ try {
109
+ const parsed = JSON.parse(raw);
110
+ const version = parsed[key];
111
+ return typeof version === "number" && Number.isInteger(version) ? version : undefined;
112
+ }
113
+ catch {
114
+ return undefined;
115
+ }
116
+ }
97
117
  function planDirectoryScan(dirPath, options = {}, signal) {
98
118
  const skipped = createSkippedScanStats();
99
119
  let files = 0;
@@ -141,116 +161,9 @@ function formatBytes(bytes) {
141
161
  return `${mb.toFixed(1)} MB`;
142
162
  return `${(mb / 1024).toFixed(2)} GB`;
143
163
  }
144
- function cosineSimilarity(a, b) {
145
- if (!a || !b || a.length !== b.length)
146
- return 0;
147
- let dot = 0;
148
- for (let i = 0; i < a.length; i++)
149
- dot += a[i] * b[i];
150
- return dot;
151
- }
152
164
  function tokenizeForSimilarity(text) {
153
165
  return tokenizeForSearch(text);
154
166
  }
155
- function jaccardSimilarity(a, b) {
156
- if (a.size === 0 || b.size === 0)
157
- return 0;
158
- let intersection = 0;
159
- for (const token of a) {
160
- if (b.has(token))
161
- intersection++;
162
- }
163
- return intersection / (a.size + b.size - intersection);
164
- }
165
- function lineProximity(a, b) {
166
- if (a.chunk.file_path !== b.chunk.file_path || a.chunk.kb_id !== b.chunk.kb_id)
167
- return 0;
168
- if (a.sourceChunkIds.some((id) => b.sourceChunkIds.includes(id)))
169
- return 1;
170
- const overlap = Math.max(0, Math.min(a.endLine, b.endLine) - Math.max(a.startLine, b.startLine) + 1);
171
- if (overlap > 0)
172
- return 1;
173
- const gap = Math.max(a.startLine, b.startLine) - Math.min(a.endLine, b.endLine);
174
- if (gap <= 20)
175
- return 0.8;
176
- if (gap <= 80)
177
- return 0.45;
178
- return 0.2;
179
- }
180
- function normalizeScores(candidates) {
181
- const scores = candidates.map((candidate) => candidate.score);
182
- const min = Math.min(...scores);
183
- const max = Math.max(...scores);
184
- const normalized = new Map();
185
- for (const candidate of candidates) {
186
- const value = max === min ? 1 : (candidate.score - min) / (max - min);
187
- normalized.set(candidate.chunk.id, value);
188
- }
189
- return normalized;
190
- }
191
- function diversifyRankedChunks(candidates, diversity, vectorsByChunkId = new Map()) {
192
- if (diversity === "off" || candidates.length <= 2)
193
- return candidates;
194
- const lambda = diversity === "strong" ? 0.62 : 0.76;
195
- const normalized = normalizeScores(candidates);
196
- const tokenSets = new Map(candidates.map((candidate) => [candidate.chunk.id, tokenizeForSimilarity(candidate.content)]));
197
- const selected = [];
198
- const remaining = [...candidates];
199
- while (remaining.length > 0) {
200
- let bestIndex = 0;
201
- let bestScore = Number.NEGATIVE_INFINITY;
202
- for (let i = 0; i < remaining.length; i++) {
203
- const candidate = remaining[i];
204
- let redundancy = 0;
205
- for (const chosen of selected) {
206
- const lexical = jaccardSimilarity(tokenSets.get(candidate.chunk.id) ?? new Set(), tokenSets.get(chosen.chunk.id) ?? new Set());
207
- const vector = Math.max(0, cosineSimilarity(vectorsByChunkId.get(candidate.chunk.id), vectorsByChunkId.get(chosen.chunk.id)));
208
- redundancy = Math.max(redundancy, Math.max(lexical, lineProximity(candidate, chosen), vector * VECTOR_REDUNDANCY_WEIGHT));
209
- }
210
- const relevance = normalized.get(candidate.chunk.id) ?? 0;
211
- const mmrScore = lambda * relevance - (1 - lambda) * redundancy;
212
- if (mmrScore > bestScore) {
213
- bestScore = mmrScore;
214
- bestIndex = i;
215
- }
216
- }
217
- selected.push(remaining.splice(bestIndex, 1)[0]);
218
- }
219
- return selected;
220
- }
221
- function interleaveByFile(candidates, diversity) {
222
- if (diversity === "off" || candidates.length <= 2)
223
- return candidates;
224
- const buckets = new Map();
225
- const fileOrder = [];
226
- for (const candidate of candidates) {
227
- const key = `${candidate.chunk.kb_id}:${candidate.chunk.file_path}`;
228
- if (!buckets.has(key)) {
229
- buckets.set(key, []);
230
- fileOrder.push(key);
231
- }
232
- buckets.get(key)?.push(candidate);
233
- }
234
- if (fileOrder.length <= 1)
235
- return candidates;
236
- const result = [];
237
- let round = 0;
238
- while (result.length < candidates.length) {
239
- let added = false;
240
- for (const key of fileOrder) {
241
- const bucket = buckets.get(key);
242
- const item = bucket?.[round];
243
- if (!item)
244
- continue;
245
- result.push(item);
246
- added = true;
247
- }
248
- if (!added)
249
- break;
250
- round++;
251
- }
252
- return result;
253
- }
254
167
  function buildQuerySnippet(content, query, maxLength = 240) {
255
168
  const terms = [...tokenizeForSimilarity(query)].sort((a, b) => b.length - a.length);
256
169
  const lower = content.toLowerCase();
@@ -452,14 +365,10 @@ async function chunkUrl(source, signal) {
452
365
  if (!res.ok)
453
366
  throw new Error(`Fetch failed: ${res.status}`);
454
367
  const html = await res.text();
455
- const text = html
456
- .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "")
457
- .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "")
458
- .replace(/<[^>]+>/g, " ")
459
- .replace(/&[a-z]+;/gi, " ")
460
- .replace(/\s+/g, " ")
461
- .trim();
462
- return chunkFile(text, source);
368
+ throwIfAborted(signal);
369
+ const text = htmlToStructuredMarkdown(html);
370
+ const structuredChunks = chunkMarkdown(text, source, "html");
371
+ return structuredChunks.length > 0 ? structuredChunks : chunkFile(text, source);
463
372
  }
464
373
  function normalizeExtractedText(text) {
465
374
  return Array.isArray(text) ? text.join("\n\n") : text;
@@ -772,7 +681,7 @@ export class KnowledgeEngine {
772
681
  throw new Error("Cancelled");
773
682
  const batch = pendingChunks.splice(0, embeddingBatchSize);
774
683
  reportProgress(`Embedding batch of ${batch.length}`, processedFiles, totalFiles);
775
- const vectors = await embedDocuments(batch.map((chunk) => buildChunkEmbeddingText(chunk)), signal);
684
+ const vectors = await embedValidatedDocuments(batch.map((chunk) => buildChunkEmbeddingText(chunk)), signal);
776
685
  if (signal?.aborted)
777
686
  throw new Error("Cancelled");
778
687
  assertEmbeddingBatchSize(vectors, batch.length, "add");
@@ -784,7 +693,8 @@ export class KnowledgeEngine {
784
693
  reportProgress("Stored batch", processedFiles, totalFiles);
785
694
  };
786
695
  const addChunks = async (chunks, processedFiles, totalFiles) => {
787
- pendingChunks.push(...chunks);
696
+ const finalizedChunks = await finalizeChunksForEmbedding(chunks, embeddingConfig, signal);
697
+ pendingChunks.push(...finalizedChunks);
788
698
  while (pendingChunks.length >= embeddingBatchSize) {
789
699
  await flushPending(processedFiles, totalFiles);
790
700
  }
@@ -1016,7 +926,7 @@ export class KnowledgeEngine {
1016
926
  unchanged_chunks: unchanged,
1017
927
  });
1018
928
  onProgress?.(message);
1019
- const newVectors = await embedDocuments(batch.map((c) => buildChunkEmbeddingText(c)), signal);
929
+ const newVectors = await embedValidatedDocuments(batch.map((c) => buildChunkEmbeddingText(c)), signal);
1020
930
  if (signal?.aborted)
1021
931
  throw new Error("Cancelled");
1022
932
  assertEmbeddingBatchSize(newVectors, batch.length, "update");
@@ -1043,7 +953,8 @@ export class KnowledgeEngine {
1043
953
  onProgress?.(storedMessage);
1044
954
  };
1045
955
  const processChunks = async (chunks) => {
1046
- for (const chunk of chunks) {
956
+ const finalizedChunks = await finalizeChunksForEmbedding(chunks, embeddingConfig, signal);
957
+ for (const chunk of finalizedChunks) {
1047
958
  scannedChunks++;
1048
959
  const existing = reusableHashes.get(chunk.content_hash);
1049
960
  if (existing && existing.length > 0) {
@@ -1231,6 +1142,7 @@ export class KnowledgeEngine {
1231
1142
  insertSymbols(this.db, kb.id, stagedSymbols);
1232
1143
  updateKBEmbeddingMetadata(this.db, kb.id, embeddingModel, finalEmbeddingDimension === undefined ? null : embeddingSignature(embeddingConfig, finalEmbeddingDimension), finalEmbeddingDimension ?? null);
1233
1144
  updateKBCounts(this.db, kb.id, finalChunkCount, getFileCount(this.db, kb.id));
1145
+ updateKBSourceOptions(this.db, kb.id, serializeAddOptions(parseAddOptions(kb.source_options)));
1234
1146
  updateKBStatus(this.db, kb.id, "ready");
1235
1147
  const skippedReadySuffix = latestSkippedTotal > 0 ? `, skipped ${latestSkippedTotal} (${latestSkippedSummary})` : "";
1236
1148
  const readyMessage = `Ready: +${addedCount} -${idsToRemove.length} =${unchanged}${skippedReadySuffix}`;
@@ -1308,7 +1220,7 @@ export class KnowledgeEngine {
1308
1220
  };
1309
1221
  }
1310
1222
  const db = this.db;
1311
- const { mode = "hybrid", offset = 0, kb_id, filters, diversity = "balanced" } = options;
1223
+ const { mode = "hybrid", offset: requestedOffset, kb_id, filters, diversity = "balanced" } = options;
1312
1224
  const resolvedMode = retrievalModeFor(mode);
1313
1225
  const retrievalMode = resolvedMode === "adaptive" ? "hybrid" : resolvedMode;
1314
1226
  const normalizedQuery = normalizedQueryText(query);
@@ -1330,10 +1242,9 @@ export class KnowledgeEngine {
1330
1242
  profile: options.profile,
1331
1243
  kbSourceTypes: kbs.map((kb) => kb.source_type),
1332
1244
  });
1333
- const limit = typeof options.limit === "number" && Number.isFinite(options.limit) && options.limit > 0
1334
- ? Math.trunc(options.limit)
1335
- : tuning.defaultLimit;
1336
- const candidateLimit = Math.max(tuning.candidateMin, offset + limit * tuning.candidateMultiplier);
1245
+ const { limit, offset } = resolveSearchWindow(options.limit, requestedOffset, tuning.defaultLimit);
1246
+ const candidateLimit = Math.min(MAX_SEARCH_WINDOW, Math.max(tuning.candidateMin, offset + limit * tuning.candidateMultiplier));
1247
+ const retrievalLimitPerKb = candidateLimit;
1337
1248
  const tuningSummary = summarizeSearchTuning(tuning, limit, candidateLimit);
1338
1249
  if (kbs.length === 0) {
1339
1250
  return {
@@ -1347,39 +1258,42 @@ export class KnowledgeEngine {
1347
1258
  }
1348
1259
  const allResults = [];
1349
1260
  const vectorsByChunkId = new Map();
1261
+ let queryEmbedding;
1350
1262
  for (const kb of kbs) {
1351
1263
  throwIfAborted(signal);
1352
1264
  if (kb.chunk_count === 0)
1353
1265
  continue;
1354
1266
  const vectorPath = this.vectorPathFor(kb.id);
1355
1267
  if (retrievalMode === "fast") {
1356
- allResults.push(...searchBM25(db, normalizedQuery || query, candidateLimit, kb.id, { allowOrFallback: false }).map((result) => ({
1268
+ allResults.push(...searchBM25(db, normalizedQuery || query, retrievalLimitPerKb, kb.id, { allowOrFallback: false }).map((result) => ({
1357
1269
  ...result,
1358
1270
  score: result.score * kbTrustMultiplier(kb),
1359
1271
  })));
1360
1272
  }
1361
1273
  else if (retrievalMode === "semantic") {
1362
- const { vector: queryVec, config: queryEmbeddingConfig } = await embedQueryWithConfig(query, signal);
1274
+ queryEmbedding ??= await embedQueryWithConfig(query, signal);
1275
+ const { vector: queryVec, config: queryEmbeddingConfig } = queryEmbedding;
1363
1276
  throwIfAborted(signal);
1364
1277
  if (!canSearchVectors(kb, queryEmbeddingConfig, queryVec.length)) {
1365
1278
  warnings.push(embeddingMismatchWarning(kb));
1366
1279
  continue;
1367
1280
  }
1368
- const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), candidateLimit, signal);
1281
+ const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), retrievalLimitPerKb, signal);
1369
1282
  allResults.push(...vectorResults.results.map((result) => ({ ...result, score: result.score * kbTrustMultiplier(kb) })));
1370
1283
  for (const [chunkId, vector] of vectorResults.vectorsByChunkId)
1371
1284
  vectorsByChunkId.set(chunkId, vector);
1372
1285
  }
1373
1286
  else {
1374
1287
  const semanticHybrid = retrievalMode === "semantic_hybrid";
1375
- const bm25Results = searchBM25(db, normalizedQuery || query, candidateLimit, kb.id);
1288
+ const bm25Results = searchBM25(db, normalizedQuery || query, retrievalLimitPerKb, kb.id);
1376
1289
  if (bm25Results.length === 0 && !semanticHybrid)
1377
1290
  continue;
1378
1291
  let vecResults = [];
1379
- const { vector: queryVec, config: queryEmbeddingConfig } = await embedQueryWithConfig(query, signal);
1292
+ queryEmbedding ??= await embedQueryWithConfig(query, signal);
1293
+ const { vector: queryVec, config: queryEmbeddingConfig } = queryEmbedding;
1380
1294
  throwIfAborted(signal);
1381
1295
  if (canSearchVectors(kb, queryEmbeddingConfig, queryVec.length)) {
1382
- const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), candidateLimit, signal);
1296
+ const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), retrievalLimitPerKb, signal);
1383
1297
  vecResults = vectorResults.results;
1384
1298
  for (const [chunkId, vector] of vectorResults.vectorsByChunkId)
1385
1299
  vectorsByChunkId.set(chunkId, vector);
@@ -1443,8 +1357,15 @@ export class KnowledgeEngine {
1443
1357
  return true;
1444
1358
  });
1445
1359
  }
1446
- if (mode === "deep" && filtered.length > 0) {
1447
- const candidates = filtered
1360
+ const candidatePool = filtered.slice(0, candidateLimit);
1361
+ const candidatePoolTruncated = candidatePool.length < filtered.length;
1362
+ const candidateIds = new Set(candidatePool.map((candidate) => candidate.chunkId));
1363
+ for (const chunkId of vectorsByChunkId.keys()) {
1364
+ if (!candidateIds.has(chunkId))
1365
+ vectorsByChunkId.delete(chunkId);
1366
+ }
1367
+ if (mode === "deep" && candidatePool.length > 0) {
1368
+ const candidates = candidatePool
1448
1369
  .slice(0, tuning.deepRerankCandidates)
1449
1370
  .map((r) => {
1450
1371
  const chunk = getChunkById(db, r.chunkId);
@@ -1471,7 +1392,7 @@ export class KnowledgeEngine {
1471
1392
  sourceChunkIds: [chunk.id],
1472
1393
  });
1473
1394
  }
1474
- const diversified = interleaveByFile(diversifyRankedChunks(ranked, diversity, vectorsByChunkId), diversity);
1395
+ const diversified = interleaveByFile(await diversifyRankedChunks(ranked, diversity, vectorsByChunkId, signal), diversity, signal);
1475
1396
  const page = diversified.slice(offset, offset + limit);
1476
1397
  const results = page.map((r) => {
1477
1398
  const kb = kbById.get(r.chunk.kb_id);
@@ -1503,11 +1424,17 @@ export class KnowledgeEngine {
1503
1424
  has_more: offset + limit < diversified.length,
1504
1425
  warnings: warnings.length > 0 ? warnings : undefined,
1505
1426
  mode_used: mode,
1506
- tuning: tuningSummary,
1427
+ tuning: {
1428
+ ...tuningSummary,
1429
+ retrieved_candidates: unique.length,
1430
+ filtered_candidates: filtered.length,
1431
+ diversity_candidates: ranked.length,
1432
+ candidate_pool_truncated: candidatePoolTruncated,
1433
+ },
1507
1434
  };
1508
1435
  }
1509
1436
  const ranked = [];
1510
- for (const r of filtered) {
1437
+ for (const r of candidatePool) {
1511
1438
  throwIfAborted(signal);
1512
1439
  const chunk = getChunkById(db, r.chunkId);
1513
1440
  if (!chunk)
@@ -1545,7 +1472,7 @@ export class KnowledgeEngine {
1545
1472
  });
1546
1473
  }
1547
1474
  }
1548
- const diversified = interleaveByFile(diversifyRankedChunks(ranked, diversity, vectorsByChunkId), diversity);
1475
+ const diversified = interleaveByFile(await diversifyRankedChunks(ranked, diversity, vectorsByChunkId, signal), diversity, signal);
1549
1476
  const total = diversified.length;
1550
1477
  const page = diversified.slice(offset, offset + limit);
1551
1478
  const results = page.map((r) => ({
@@ -1578,7 +1505,13 @@ export class KnowledgeEngine {
1578
1505
  has_more: offset + limit < total,
1579
1506
  warnings: warnings.length > 0 ? warnings : undefined,
1580
1507
  mode_used: mode,
1581
- tuning: tuningSummary,
1508
+ tuning: {
1509
+ ...tuningSummary,
1510
+ retrieved_candidates: unique.length,
1511
+ filtered_candidates: filtered.length,
1512
+ diversity_candidates: ranked.length,
1513
+ candidate_pool_truncated: candidatePoolTruncated,
1514
+ },
1582
1515
  suggestions: results.length === 0
1583
1516
  ? [
1584
1517
  "Try mode 'fast' for exact symbols, mode 'semantic' for pure vector search, or mode 'semantic_hybrid' when weak lexical overlap should not block fused vector results.",
@@ -1750,8 +1683,48 @@ export class KnowledgeEngine {
1750
1683
  }
1751
1684
  }
1752
1685
  if (this.db) {
1686
+ const activeEmbeddingConfig = resolveEmbeddingConfig();
1753
1687
  for (const kb of kbs) {
1754
1688
  throwIfAborted(signal);
1689
+ if ((kb.status === "ready" || kb.status === "stale") &&
1690
+ kb.chunk_count > 0 &&
1691
+ kb.embedding_dimension !== null &&
1692
+ kb.embedding_signature !== embeddingSignature(activeEmbeddingConfig, kb.embedding_dimension)) {
1693
+ const canUpdate = kb.source_path !== null && kb.source_type !== "text";
1694
+ issues.push({
1695
+ severity: "warning",
1696
+ kb_name: kb.name,
1697
+ message: "This KB's vectors are incompatible with the active embedding provider or normalization policy.",
1698
+ action: canUpdate
1699
+ ? "Run knowledge_update to rebuild vectors with the active embedding configuration."
1700
+ : "Re-add inline text or re-import the original source with the active embedding configuration.",
1701
+ action_code: "rebuild_vectors",
1702
+ });
1703
+ }
1704
+ if ((kb.status === "ready" || kb.status === "stale") &&
1705
+ (storedSourcePolicyVersion(kb.source_options, "embedding_input_policy") ?? 0) < CURRENT_EMBEDDING_INPUT_POLICY) {
1706
+ const canUpdate = kb.source_path !== null && kb.source_type !== "text";
1707
+ issues.push({
1708
+ severity: "warning",
1709
+ kb_name: kb.name,
1710
+ message: "This KB predates embedding-safe chunk limits, so oversized chunk tails may be missing from vector retrieval.",
1711
+ action: canUpdate
1712
+ ? "Run knowledge_update to re-chunk oversized inputs with the current embedding budget."
1713
+ : "Re-add inline text or re-import the original source to rebuild chunks with the current embedding budget.",
1714
+ action_code: "reindex_for_embedding_budget",
1715
+ });
1716
+ }
1717
+ if (kb.source_type === "url" &&
1718
+ (kb.status === "ready" || kb.status === "stale") &&
1719
+ (storedSourcePolicyVersion(kb.source_options, "url_structure_policy") ?? 0) < CURRENT_URL_STRUCTURE_POLICY) {
1720
+ issues.push({
1721
+ severity: "warning",
1722
+ kb_name: kb.name,
1723
+ message: "This URL KB predates structure-preserving HTML normalization.",
1724
+ action: "Run knowledge_update to rebuild URL headings, sections, lists, tables, and entities.",
1725
+ action_code: "run_update",
1726
+ });
1727
+ }
1755
1728
  if (kb.status === "ready" && kb.chunk_count > 0 && getSymbolCount(this.db, kb.id) === 0) {
1756
1729
  issues.push({
1757
1730
  severity: "info",
@@ -1879,7 +1852,7 @@ export class KnowledgeEngine {
1879
1852
  added_chunks: inserted,
1880
1853
  });
1881
1854
  onProgress?.(message);
1882
- const vectors = await embedDocuments(batch.map((chunk) => buildChunkEmbeddingText(chunk)), signal);
1855
+ const vectors = await embedValidatedDocuments(batch.map((chunk) => buildChunkEmbeddingText(chunk)), signal);
1883
1856
  throwIfAborted(signal);
1884
1857
  assertEmbeddingBatchSize(vectors, batch.length, "import");
1885
1858
  persistEmbeddingMetadata(this.db, kb.id, embeddingConfig, vectors);
@@ -1915,6 +1888,7 @@ export class KnowledgeEngine {
1915
1888
  name: header.name,
1916
1889
  description: header.description,
1917
1890
  source_type: "text",
1891
+ source_options: serializeAddOptions(),
1918
1892
  embedding_model: embeddingConfigLabel(embeddingConfig),
1919
1893
  });
1920
1894
  updateKBStatus(this.db, kb.id, "indexing");
@@ -1934,9 +1908,10 @@ export class KnowledgeEngine {
1934
1908
  const imported = JSON.parse(line);
1935
1909
  const chunk = importedChunkToInsert(imported);
1936
1910
  chunk.content_tokenized = preTokenizeForFTS(buildChunkEmbeddingText(chunk));
1937
- pendingChunks.push(chunk);
1911
+ const finalizedChunks = await finalizeChunksForEmbedding([chunk], embeddingConfig, signal);
1912
+ pendingChunks.push(...finalizedChunks);
1938
1913
  pendingSymbols.push(...extractSymbols(imported.content, imported.file_path, imported.file_type));
1939
- if (pendingChunks.length >= embeddingBatchSize)
1914
+ while (pendingChunks.length >= embeddingBatchSize)
1940
1915
  await flushPending();
1941
1916
  }
1942
1917
  if (!header)
@@ -50,7 +50,7 @@ export declare function chunkIdentityHash(opts: {
50
50
  metadataJson: string;
51
51
  }): string;
52
52
  export declare function buildChunkEmbeddingText(chunk: Pick<ChunkInsert, "content" | "file_path" | "file_type" | "metadata_json">): string;
53
- export declare function chunkMarkdown(content: string, filePath: string): Omit<ChunkInsert, "kb_id">[];
53
+ export declare function chunkMarkdown(content: string, filePath: string, fileType?: string): Omit<ChunkInsert, "kb_id">[];
54
54
  export declare function chunkText(content: string, filePath: string): Omit<ChunkInsert, "kb_id">[];
55
55
  export declare function isCodeFileType(fileType: string): boolean;
56
56
  export declare function analyzeIndexableContent(content: string, filePath: string, fileType?: string): Promise<{
@@ -397,7 +397,7 @@ function makeChunk(content, filePath, fileType, startLine, endLine, metadata = {
397
397
  };
398
398
  return { ...chunk, content_tokenized: preTokenizeForFTS(buildChunkEmbeddingText(chunk)) };
399
399
  }
400
- export function chunkMarkdown(content, filePath) {
400
+ export function chunkMarkdown(content, filePath, fileType = "markdown") {
401
401
  const lines = content.split("\n");
402
402
  const chunks = [];
403
403
  const headingStack = [];
@@ -410,7 +410,7 @@ export function chunkMarkdown(content, filePath) {
410
410
  function pushMarkdownChunk(text, start, end) {
411
411
  if (text.trim().length < 50)
412
412
  return;
413
- chunks.push(makeChunk(text.trim(), filePath, "markdown", start, end, {
413
+ chunks.push(makeChunk(text.trim(), filePath, fileType, start, end, {
414
414
  heading: currentHeading,
415
415
  breadcrumb: currentBreadcrumb(),
416
416
  }));
@@ -0,0 +1,5 @@
1
+ import type { EmbeddingConfig } from "../embedding/provider.ts";
2
+ import type { ChunkInsert } from "../storage/sqlite.ts";
3
+ type IndexableChunk = Omit<ChunkInsert, "kb_id">;
4
+ export declare function finalizeChunksForEmbedding(chunks: IndexableChunk[], config: EmbeddingConfig, signal?: AbortSignal): Promise<IndexableChunk[]>;
5
+ export {};