pi-knowledge 0.13.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/README.md +30 -8
- package/dist/index.js +176 -84
- package/dist/src/embedding/instructions.d.ts +33 -0
- package/dist/src/embedding/instructions.js +146 -0
- package/dist/src/embedding/provider.d.ts +25 -8
- package/dist/src/embedding/provider.js +155 -63
- package/dist/src/engine.d.ts +68 -2
- package/dist/src/engine.js +483 -128
- package/dist/src/indexer/embedding-safe-chunks.js +3 -2
- package/dist/src/model-worker-client.d.ts +4 -0
- package/dist/src/model-worker-client.js +14 -2
- package/dist/src/model-worker.js +34 -3
- package/dist/src/search/tuning.d.ts +2 -0
- package/dist/src/search/tuning.js +2 -0
- package/dist/src/status.d.ts +8 -0
- package/dist/src/status.js +93 -0
- package/docs/configuration.md +16 -11
- package/package.json +1 -1
package/dist/src/engine.js
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import { existsSync, readFileSync, renameSync, rmSync, statSync } from "node:fs";
|
|
2
2
|
import { join, resolve } from "node:path";
|
|
3
3
|
import { diagnoseKB } from "./diagnostics/health.js";
|
|
4
|
-
import { dispose as disposeEmbedding, embedDocuments, embeddingConfigLabel, embeddingSignature, embedQueryWithConfig, embedValidatedDocuments, prepareForShutdown as prepareEmbeddingForShutdown, resolveEmbeddingConfig, } from "./embedding/provider.js";
|
|
4
|
+
import { describeEmbeddingCompatibility, describeEmbeddingDecoration, dispose as disposeEmbedding, embedDocuments, embeddingConfigLabel, embeddingSignature, embedQueryWithConfig, embedValidatedDocuments, inspectEmbeddingCompatibility, prepareForShutdown as prepareEmbeddingForShutdown, resolveEmbeddingConfig, } from "./embedding/provider.js";
|
|
5
5
|
import { openVectorReader, openVectorWriter } from "./embedding/vectors.js";
|
|
6
6
|
import { addSkippedScanEntry, analyzeIndexableContent, buildChunkEmbeddingText, chunkFile, chunkIdentityHash, chunkMarkdown, createSkippedScanStats, isReadableTextFile, isSupportedDocumentFile, iterateScannableFiles, preTokenizeForFTS, summarizeSkippedScan, } from "./indexer/chunker.js";
|
|
7
7
|
import { finalizeChunksForEmbedding } from "./indexer/embedding-safe-chunks.js";
|
|
8
8
|
import { htmlToStructuredMarkdown } from "./indexer/html.js";
|
|
9
9
|
import { extractSymbols } from "./indexer/symbols.js";
|
|
10
|
-
import { shutdownModelWorker } from "./model-worker-client.js";
|
|
10
|
+
import { extractDocumentInModelWorker, shutdownModelWorker } from "./model-worker-client.js";
|
|
11
11
|
import { searchBM25 } from "./search/bm25.js";
|
|
12
12
|
import { MAX_SEARCH_WINDOW } from "./search/contracts.js";
|
|
13
13
|
import { diversifyRankedChunks, interleaveByFile } from "./search/diversity.js";
|
|
@@ -150,6 +150,181 @@ function kbTrustMultiplier(kb) {
|
|
|
150
150
|
multiplier *= 0.98;
|
|
151
151
|
return multiplier;
|
|
152
152
|
}
|
|
153
|
+
function summarizeKnowledgeBases(kbs) {
|
|
154
|
+
return kbs.map((kb) => ({
|
|
155
|
+
id: kb.id,
|
|
156
|
+
name: kb.name,
|
|
157
|
+
status: kb.status,
|
|
158
|
+
chunk_count: kb.chunk_count,
|
|
159
|
+
}));
|
|
160
|
+
}
|
|
161
|
+
function searchMissAction(reason, mode, availableKnowledgeBases) {
|
|
162
|
+
switch (reason) {
|
|
163
|
+
case "no_knowledge_bases":
|
|
164
|
+
return "add_knowledge_base";
|
|
165
|
+
case "knowledge_base_not_found":
|
|
166
|
+
return "fix_kb_id";
|
|
167
|
+
case "no_searchable_knowledge_bases":
|
|
168
|
+
return availableKnowledgeBases.length > 0 && availableKnowledgeBases.every((kb) => kb.status === "indexing")
|
|
169
|
+
? "wait_for_indexing"
|
|
170
|
+
: "fix_knowledge_base";
|
|
171
|
+
case "no_indexed_chunks":
|
|
172
|
+
return availableKnowledgeBases.some((kb) => kb.status === "indexing")
|
|
173
|
+
? "wait_for_indexing"
|
|
174
|
+
: "fix_knowledge_base";
|
|
175
|
+
case "embedding_incompatible":
|
|
176
|
+
return "fix_embedding";
|
|
177
|
+
case "no_retrieval_candidates":
|
|
178
|
+
return mode === "fast" || mode === "code" || mode === "config" || mode === "errors"
|
|
179
|
+
? "try_semantic"
|
|
180
|
+
: mode === "semantic" || mode === "semantic_hybrid"
|
|
181
|
+
? "rephrase"
|
|
182
|
+
: "try_semantic_hybrid";
|
|
183
|
+
case "threshold_filtered":
|
|
184
|
+
return mode === "semantic_hybrid" ? "rephrase" : "try_semantic_hybrid";
|
|
185
|
+
case "evidence_filtered":
|
|
186
|
+
return "try_semantic_hybrid";
|
|
187
|
+
case "metadata_filtered":
|
|
188
|
+
return "relax_filter";
|
|
189
|
+
case "page_out_of_range":
|
|
190
|
+
return "reset_offset";
|
|
191
|
+
case "reranker_empty":
|
|
192
|
+
case "no_match":
|
|
193
|
+
return "rephrase";
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
function classifySearchMiss(facts) {
|
|
197
|
+
const skippedKnowledgeBases = facts.availableKnowledgeBases.filter((kb) => kb.status !== "ready" && kb.status !== "stale");
|
|
198
|
+
let reason;
|
|
199
|
+
if (facts.allKnowledgeBases.length === 0) {
|
|
200
|
+
reason = "no_knowledge_bases";
|
|
201
|
+
}
|
|
202
|
+
else if (facts.requestedKbId &&
|
|
203
|
+
!facts.availableKnowledgeBases.some((kb) => kb.id === facts.requestedKbId || kb.name === facts.requestedKbId)) {
|
|
204
|
+
reason = "knowledge_base_not_found";
|
|
205
|
+
}
|
|
206
|
+
else if (facts.searchedKnowledgeBases.length === 0) {
|
|
207
|
+
reason = "no_searchable_knowledge_bases";
|
|
208
|
+
}
|
|
209
|
+
else if (facts.searchedKnowledgeBases.every((kb) => kb.chunk_count === 0)) {
|
|
210
|
+
reason = "no_indexed_chunks";
|
|
211
|
+
}
|
|
212
|
+
else if (facts.retrievedCandidates === 0 &&
|
|
213
|
+
facts.embeddingIncompatibleCount === facts.searchedKnowledgeBases.filter((kb) => kb.chunk_count > 0).length) {
|
|
214
|
+
reason = "embedding_incompatible";
|
|
215
|
+
}
|
|
216
|
+
else if (facts.retrievedCandidates === 0) {
|
|
217
|
+
reason = "no_retrieval_candidates";
|
|
218
|
+
}
|
|
219
|
+
else if (facts.thresholdCandidates === 0) {
|
|
220
|
+
reason = "threshold_filtered";
|
|
221
|
+
}
|
|
222
|
+
else if (facts.eligibleCandidates === 0) {
|
|
223
|
+
reason = "evidence_filtered";
|
|
224
|
+
}
|
|
225
|
+
else if (facts.filteredCandidates === 0 && (facts.fileType || facts.pathPattern)) {
|
|
226
|
+
reason = "metadata_filtered";
|
|
227
|
+
}
|
|
228
|
+
else if (facts.mode === "deep" && facts.rankedCandidates === 0) {
|
|
229
|
+
reason = "reranker_empty";
|
|
230
|
+
}
|
|
231
|
+
else if (facts.totalResults > 0 && facts.offset >= facts.totalResults) {
|
|
232
|
+
reason = "page_out_of_range";
|
|
233
|
+
}
|
|
234
|
+
else {
|
|
235
|
+
reason = "no_match";
|
|
236
|
+
}
|
|
237
|
+
return {
|
|
238
|
+
reason,
|
|
239
|
+
action: searchMissAction(reason, facts.mode, facts.availableKnowledgeBases),
|
|
240
|
+
requested_kb_id: facts.requestedKbId,
|
|
241
|
+
min_hybrid_score: facts.minimumScore,
|
|
242
|
+
file_type: facts.fileType,
|
|
243
|
+
path_pattern: facts.pathPattern,
|
|
244
|
+
searched_kbs: summarizeKnowledgeBases(facts.searchedKnowledgeBases),
|
|
245
|
+
skipped_kbs: summarizeKnowledgeBases(skippedKnowledgeBases),
|
|
246
|
+
retrieved_candidates: facts.retrievedCandidates,
|
|
247
|
+
threshold_candidates: facts.thresholdCandidates,
|
|
248
|
+
eligible_candidates: facts.eligibleCandidates,
|
|
249
|
+
filtered_candidates: facts.filteredCandidates,
|
|
250
|
+
ranked_candidates: facts.rankedCandidates,
|
|
251
|
+
total_results: facts.totalResults,
|
|
252
|
+
};
|
|
253
|
+
}
|
|
254
|
+
function searchAttemptSummary(mode, response) {
|
|
255
|
+
return {
|
|
256
|
+
mode,
|
|
257
|
+
reason: response.miss?.reason,
|
|
258
|
+
retrieved_candidates: response.tuning?.retrieved_candidates ?? 0,
|
|
259
|
+
threshold_candidates: response.tuning?.threshold_candidates ?? 0,
|
|
260
|
+
eligible_candidates: response.tuning?.eligible_candidates ?? 0,
|
|
261
|
+
filtered_candidates: response.tuning?.filtered_candidates ?? 0,
|
|
262
|
+
result_count: response.results.length,
|
|
263
|
+
};
|
|
264
|
+
}
|
|
265
|
+
function symbolMissAction(reason, availableKnowledgeBases) {
|
|
266
|
+
switch (reason) {
|
|
267
|
+
case "no_knowledge_bases":
|
|
268
|
+
return "add_knowledge_base";
|
|
269
|
+
case "knowledge_base_not_found":
|
|
270
|
+
return "fix_kb_id";
|
|
271
|
+
case "no_searchable_knowledge_bases":
|
|
272
|
+
return availableKnowledgeBases.length > 0 && availableKnowledgeBases.every((kb) => kb.status === "indexing")
|
|
273
|
+
? "wait_for_indexing"
|
|
274
|
+
: "fix_knowledge_base";
|
|
275
|
+
case "symbol_index_empty":
|
|
276
|
+
return "rebuild_symbol_index";
|
|
277
|
+
case "exact_name_not_found":
|
|
278
|
+
return "broaden_symbol_search";
|
|
279
|
+
case "metadata_filtered":
|
|
280
|
+
return "relax_filter";
|
|
281
|
+
case "page_out_of_range":
|
|
282
|
+
return "reset_offset";
|
|
283
|
+
case "no_match":
|
|
284
|
+
return "use_text_search";
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
function classifySymbolSearchMiss(facts) {
|
|
288
|
+
const skippedKnowledgeBases = facts.availableKnowledgeBases.filter((kb) => kb.status !== "ready" && kb.status !== "stale");
|
|
289
|
+
let reason;
|
|
290
|
+
if (facts.allKnowledgeBases.length === 0) {
|
|
291
|
+
reason = "no_knowledge_bases";
|
|
292
|
+
}
|
|
293
|
+
else if (facts.requestedKbId &&
|
|
294
|
+
!facts.availableKnowledgeBases.some((kb) => kb.id === facts.requestedKbId || kb.name === facts.requestedKbId)) {
|
|
295
|
+
reason = "knowledge_base_not_found";
|
|
296
|
+
}
|
|
297
|
+
else if (facts.searchedKnowledgeBases.length === 0) {
|
|
298
|
+
reason = "no_searchable_knowledge_bases";
|
|
299
|
+
}
|
|
300
|
+
else if (facts.indexedSymbols === 0) {
|
|
301
|
+
reason = "symbol_index_empty";
|
|
302
|
+
}
|
|
303
|
+
else if (facts.totalResults > 0 && facts.offset >= facts.totalResults) {
|
|
304
|
+
reason = "page_out_of_range";
|
|
305
|
+
}
|
|
306
|
+
else if ((facts.kind || facts.filePattern) && facts.unfilteredMatches > 0) {
|
|
307
|
+
reason = "metadata_filtered";
|
|
308
|
+
}
|
|
309
|
+
else if (facts.exact) {
|
|
310
|
+
reason = "exact_name_not_found";
|
|
311
|
+
}
|
|
312
|
+
else {
|
|
313
|
+
reason = "no_match";
|
|
314
|
+
}
|
|
315
|
+
return {
|
|
316
|
+
reason,
|
|
317
|
+
action: symbolMissAction(reason, facts.availableKnowledgeBases),
|
|
318
|
+
requested_kb_id: facts.requestedKbId,
|
|
319
|
+
kind: facts.kind,
|
|
320
|
+
file_pattern: facts.filePattern,
|
|
321
|
+
exact: facts.exact,
|
|
322
|
+
searched_kbs: summarizeKnowledgeBases(facts.searchedKnowledgeBases),
|
|
323
|
+
skipped_kbs: summarizeKnowledgeBases(skippedKnowledgeBases),
|
|
324
|
+
indexed_symbols: facts.indexedSymbols,
|
|
325
|
+
unfiltered_matches: facts.unfilteredMatches,
|
|
326
|
+
};
|
|
327
|
+
}
|
|
153
328
|
function formatBytes(bytes) {
|
|
154
329
|
if (bytes < 1024)
|
|
155
330
|
return `${bytes} B`;
|
|
@@ -370,30 +545,19 @@ async function chunkUrl(source, signal) {
|
|
|
370
545
|
const structuredChunks = chunkMarkdown(text, source, "html");
|
|
371
546
|
return structuredChunks.length > 0 ? structuredChunks : chunkFile(text, source);
|
|
372
547
|
}
|
|
373
|
-
function normalizeExtractedText(text) {
|
|
374
|
-
return Array.isArray(text) ? text.join("\n\n") : text;
|
|
375
|
-
}
|
|
376
548
|
async function extractSourceFileContent(filePath, signal) {
|
|
377
549
|
const lowerPath = filePath.toLowerCase();
|
|
378
550
|
if (lowerPath.endsWith(".pdf")) {
|
|
379
551
|
throwIfAborted(signal);
|
|
380
|
-
|
|
381
|
-
const { extractText } = await import("unpdf");
|
|
382
|
-
throwIfAborted(signal);
|
|
383
|
-
const buf = readFileSync(filePath);
|
|
552
|
+
const extracted = await extractDocumentInModelWorker(filePath, "pdf", signal);
|
|
384
553
|
throwIfAborted(signal);
|
|
385
|
-
|
|
386
|
-
throwIfAborted(signal);
|
|
387
|
-
return { content: normalizeExtractedText(text), fileType: "pdf" };
|
|
554
|
+
return extracted;
|
|
388
555
|
}
|
|
389
556
|
if (lowerPath.endsWith(".docx") || lowerPath.endsWith(".doc")) {
|
|
390
557
|
throwIfAborted(signal);
|
|
391
|
-
|
|
392
|
-
const mammoth = await import("mammoth");
|
|
393
|
-
throwIfAborted(signal);
|
|
394
|
-
const result = await mammoth.extractRawText({ path: filePath });
|
|
558
|
+
const extracted = await extractDocumentInModelWorker(filePath, "docx", signal);
|
|
395
559
|
throwIfAborted(signal);
|
|
396
|
-
return
|
|
560
|
+
return extracted;
|
|
397
561
|
}
|
|
398
562
|
if (!isReadableTextFile(filePath)) {
|
|
399
563
|
throw new Error(`File is not readable text and has no supported extractor: ${filePath}`);
|
|
@@ -451,15 +615,8 @@ function persistEmbeddingMetadata(db, kbId, config, vectors) {
|
|
|
451
615
|
updateKBEmbeddingMetadata(db, kbId, embeddingConfigLabel(config), embeddingSignature(config, dimension), dimension);
|
|
452
616
|
return dimension;
|
|
453
617
|
}
|
|
454
|
-
function embeddingMismatchWarning(kb) {
|
|
455
|
-
return `"${kb.name}" has incompatible embedding
|
|
456
|
-
}
|
|
457
|
-
function canSearchVectors(kb, config, queryDimension) {
|
|
458
|
-
if (kb.embedding_signature === null || kb.embedding_dimension === null)
|
|
459
|
-
return false;
|
|
460
|
-
if (kb.embedding_dimension !== queryDimension)
|
|
461
|
-
return false;
|
|
462
|
-
return kb.embedding_signature === embeddingSignature(config, queryDimension);
|
|
618
|
+
function embeddingMismatchWarning(kb, compatibility) {
|
|
619
|
+
return `"${kb.name}" has incompatible embedding vectors (${describeEmbeddingCompatibility(compatibility)}); vector retrieval was skipped and knowledge_update should rebuild them`;
|
|
463
620
|
}
|
|
464
621
|
function throwIfAborted(signal) {
|
|
465
622
|
if (signal?.aborted)
|
|
@@ -844,14 +1001,17 @@ export class KnowledgeEngine {
|
|
|
844
1001
|
if (!this.db)
|
|
845
1002
|
throw new Error("Engine not initialized");
|
|
846
1003
|
throwIfAborted(signal);
|
|
847
|
-
updateKBStatus(this.db, kb.id, "indexing");
|
|
848
|
-
startIndexingJob(this.db, kb.id, "update", `Starting update for "${kb.name}"`);
|
|
849
1004
|
const scanOptions = toScanOptions(parseAddOptions(kb.source_options));
|
|
850
1005
|
const embeddingConfig = resolveEmbeddingConfig();
|
|
851
1006
|
const embeddingBatchSize = resolveIndexEmbedBatchSize();
|
|
1007
|
+
updateKBStatus(this.db, kb.id, "indexing");
|
|
1008
|
+
startIndexingJob(this.db, kb.id, "update", `Starting update for "${kb.name}"`);
|
|
852
1009
|
const embeddingModel = embeddingConfigLabel(embeddingConfig);
|
|
853
|
-
const
|
|
854
|
-
let canReuseExistingVectors =
|
|
1010
|
+
const compatibility = inspectEmbeddingCompatibility(kb.embedding_signature, kb.embedding_dimension, embeddingConfig, kb.embedding_dimension ?? 0);
|
|
1011
|
+
let canReuseExistingVectors = compatibility.compatible;
|
|
1012
|
+
let rebuildMessage = compatibility.compatible
|
|
1013
|
+
? undefined
|
|
1014
|
+
: `Embedding vectors for "${kb.name}" are incompatible (${describeEmbeddingCompatibility(compatibility)}); rebuilding all vectors`;
|
|
855
1015
|
let replacementVectorPath;
|
|
856
1016
|
let addedVectorPath;
|
|
857
1017
|
let addedVectorWriter;
|
|
@@ -880,9 +1040,7 @@ export class KnowledgeEngine {
|
|
|
880
1040
|
canReuseExistingVectors = false;
|
|
881
1041
|
const vectorCount = oldVectorReader?.count ?? 0;
|
|
882
1042
|
const vectorDimension = oldVectorReader?.dim ?? 0;
|
|
883
|
-
|
|
884
|
-
updateIndexingJob(this.db, kb.id, { phase: "embedding", message });
|
|
885
|
-
onProgress?.(message);
|
|
1043
|
+
rebuildMessage = `Stored vector file for "${kb.name}" is incomplete or incompatible (${vectorCount} vectors/${vectorDimension}d for ${existingIndex} chunks/${kb.embedding_dimension}d); rebuilding all vectors`;
|
|
886
1044
|
}
|
|
887
1045
|
}
|
|
888
1046
|
finally {
|
|
@@ -892,10 +1050,9 @@ export class KnowledgeEngine {
|
|
|
892
1050
|
const reusableHashes = canReuseExistingVectors
|
|
893
1051
|
? existingHashes
|
|
894
1052
|
: new Map();
|
|
895
|
-
if (!canReuseExistingVectors) {
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
onProgress?.(message);
|
|
1053
|
+
if (!canReuseExistingVectors && rebuildMessage) {
|
|
1054
|
+
updateIndexingJob(this.db, kb.id, { phase: "embedding", message: rebuildMessage });
|
|
1055
|
+
onProgress?.(rebuildMessage);
|
|
899
1056
|
}
|
|
900
1057
|
const oldVectorIndexByHash = new Map();
|
|
901
1058
|
const newVectorIndexByHash = new Map();
|
|
@@ -1181,43 +1338,82 @@ export class KnowledgeEngine {
|
|
|
1181
1338
|
const requestedMode = options.mode ?? "hybrid";
|
|
1182
1339
|
if (requestedMode === "auto") {
|
|
1183
1340
|
const primaryMode = chooseAutoMode(query);
|
|
1184
|
-
const
|
|
1341
|
+
const modes = [primaryMode, ...fallbackModesFor(primaryMode)];
|
|
1185
1342
|
const tried = [];
|
|
1343
|
+
const attemptSummaries = [];
|
|
1186
1344
|
const warnings = [];
|
|
1187
|
-
let
|
|
1188
|
-
|
|
1345
|
+
let lastMissResponse;
|
|
1346
|
+
let bestWeakResponse;
|
|
1347
|
+
for (const mode of modes) {
|
|
1189
1348
|
if (tried.includes(mode))
|
|
1190
1349
|
continue;
|
|
1191
1350
|
tried.push(mode);
|
|
1192
1351
|
throwIfAborted(signal);
|
|
1193
1352
|
const response = await this.search(query, { ...options, mode }, signal);
|
|
1194
|
-
|
|
1353
|
+
attemptSummaries.push(searchAttemptSummary(mode, response));
|
|
1195
1354
|
if (response.warnings)
|
|
1196
1355
|
warnings.push(...response.warnings);
|
|
1356
|
+
if (response.miss?.reason === "no_knowledge_bases" ||
|
|
1357
|
+
response.miss?.reason === "knowledge_base_not_found" ||
|
|
1358
|
+
response.miss?.reason === "no_searchable_knowledge_bases" ||
|
|
1359
|
+
response.miss?.reason === "no_indexed_chunks") {
|
|
1360
|
+
return {
|
|
1361
|
+
...response,
|
|
1362
|
+
warnings: warnings.length > 0 ? [...new Set(warnings)] : response.warnings,
|
|
1363
|
+
mode_used: mode,
|
|
1364
|
+
retry_modes: tried.slice(0, -1),
|
|
1365
|
+
miss: { ...response.miss, attempts: attemptSummaries },
|
|
1366
|
+
};
|
|
1367
|
+
}
|
|
1197
1368
|
if (!isWeakAutoResponse(query, response, primaryMode, mode)) {
|
|
1198
1369
|
return {
|
|
1199
1370
|
...response,
|
|
1200
1371
|
warnings: warnings.length > 0 ? [...new Set(warnings)] : response.warnings,
|
|
1201
1372
|
mode_used: mode,
|
|
1202
1373
|
retry_modes: tried.slice(0, -1),
|
|
1203
|
-
suggestions: response.suggestions,
|
|
1204
1374
|
};
|
|
1205
1375
|
}
|
|
1376
|
+
if (response.results.length > 0) {
|
|
1377
|
+
if (!bestWeakResponse || response.results.length > bestWeakResponse.response.results.length) {
|
|
1378
|
+
bestWeakResponse = { mode, response };
|
|
1379
|
+
}
|
|
1380
|
+
}
|
|
1381
|
+
else {
|
|
1382
|
+
lastMissResponse = { mode, response };
|
|
1383
|
+
}
|
|
1206
1384
|
if (tried.length >= 3)
|
|
1207
1385
|
break;
|
|
1208
1386
|
}
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1387
|
+
if (lastMissResponse?.response.miss) {
|
|
1388
|
+
const attemptedModes = new Set(attemptSummaries.map((attempt) => attempt.mode));
|
|
1389
|
+
let action = lastMissResponse.response.miss.action;
|
|
1390
|
+
if (action === "try_semantic" && attemptedModes.has("semantic")) {
|
|
1391
|
+
action = attemptedModes.has("semantic_hybrid") ? "rephrase" : "try_semantic_hybrid";
|
|
1392
|
+
}
|
|
1393
|
+
else if (action === "try_semantic_hybrid" && attemptedModes.has("semantic_hybrid")) {
|
|
1394
|
+
action = "rephrase";
|
|
1395
|
+
}
|
|
1396
|
+
return {
|
|
1397
|
+
...lastMissResponse.response,
|
|
1398
|
+
warnings: warnings.length > 0 ? [...new Set(warnings)] : lastMissResponse.response.warnings,
|
|
1399
|
+
mode_used: lastMissResponse.mode,
|
|
1400
|
+
retry_modes: tried.filter((mode) => mode !== lastMissResponse.mode),
|
|
1401
|
+
miss: {
|
|
1402
|
+
...lastMissResponse.response.miss,
|
|
1403
|
+
action,
|
|
1404
|
+
attempts: attemptSummaries,
|
|
1405
|
+
},
|
|
1406
|
+
};
|
|
1407
|
+
}
|
|
1408
|
+
if (bestWeakResponse) {
|
|
1409
|
+
return {
|
|
1410
|
+
...bestWeakResponse.response,
|
|
1411
|
+
warnings: warnings.length > 0 ? [...new Set(warnings)] : bestWeakResponse.response.warnings,
|
|
1412
|
+
mode_used: bestWeakResponse.mode,
|
|
1413
|
+
retry_modes: tried.filter((mode) => mode !== bestWeakResponse.mode),
|
|
1414
|
+
};
|
|
1415
|
+
}
|
|
1416
|
+
throw new Error("Auto search exhausted without a result or miss classification");
|
|
1221
1417
|
}
|
|
1222
1418
|
const db = this.db;
|
|
1223
1419
|
const { mode = "hybrid", offset: requestedOffset, kb_id, filters, diversity = "balanced" } = options;
|
|
@@ -1227,8 +1423,9 @@ export class KnowledgeEngine {
|
|
|
1227
1423
|
const queryTokens = tokenizeForSimilarity(query);
|
|
1228
1424
|
const normalizedFileType = normalizeFileTypeFilter(filters?.file_type);
|
|
1229
1425
|
const warnings = [];
|
|
1426
|
+
const allKBs = listKBs(db);
|
|
1230
1427
|
const selectedKB = kb_id ? (getKB(db, kb_id) ?? getKBByName(db, kb_id)) : undefined;
|
|
1231
|
-
const availableKBs = kb_id ? [selectedKB].filter(Boolean) :
|
|
1428
|
+
const availableKBs = kb_id ? [selectedKB].filter(Boolean) : allKBs;
|
|
1232
1429
|
const kbs = availableKBs.filter((kb) => kb.status === "ready" || kb.status === "stale");
|
|
1233
1430
|
const kbById = new Map(kbs.map((kb) => [kb.id, kb]));
|
|
1234
1431
|
for (const kb of availableKBs) {
|
|
@@ -1254,11 +1451,30 @@ export class KnowledgeEngine {
|
|
|
1254
1451
|
warnings: warnings.length > 0 ? warnings : undefined,
|
|
1255
1452
|
mode_used: mode,
|
|
1256
1453
|
tuning: tuningSummary,
|
|
1454
|
+
miss: classifySearchMiss({
|
|
1455
|
+
allKnowledgeBases: allKBs,
|
|
1456
|
+
availableKnowledgeBases: availableKBs,
|
|
1457
|
+
searchedKnowledgeBases: kbs,
|
|
1458
|
+
requestedKbId: kb_id,
|
|
1459
|
+
mode,
|
|
1460
|
+
fileType: filters?.file_type,
|
|
1461
|
+
pathPattern: filters?.path_pattern,
|
|
1462
|
+
minimumScore: tuning.minHybridScore,
|
|
1463
|
+
offset,
|
|
1464
|
+
retrievedCandidates: 0,
|
|
1465
|
+
thresholdCandidates: 0,
|
|
1466
|
+
eligibleCandidates: 0,
|
|
1467
|
+
filteredCandidates: 0,
|
|
1468
|
+
rankedCandidates: 0,
|
|
1469
|
+
totalResults: 0,
|
|
1470
|
+
embeddingIncompatibleCount: 0,
|
|
1471
|
+
}),
|
|
1257
1472
|
};
|
|
1258
1473
|
}
|
|
1259
1474
|
const allResults = [];
|
|
1260
1475
|
const vectorsByChunkId = new Map();
|
|
1261
1476
|
let queryEmbedding;
|
|
1477
|
+
const embeddingIncompatibleKbIds = new Set();
|
|
1262
1478
|
for (const kb of kbs) {
|
|
1263
1479
|
throwIfAborted(signal);
|
|
1264
1480
|
if (kb.chunk_count === 0)
|
|
@@ -1274,8 +1490,10 @@ export class KnowledgeEngine {
|
|
|
1274
1490
|
queryEmbedding ??= await embedQueryWithConfig(query, signal);
|
|
1275
1491
|
const { vector: queryVec, config: queryEmbeddingConfig } = queryEmbedding;
|
|
1276
1492
|
throwIfAborted(signal);
|
|
1277
|
-
|
|
1278
|
-
|
|
1493
|
+
const compatibility = inspectEmbeddingCompatibility(kb.embedding_signature, kb.embedding_dimension, queryEmbeddingConfig, queryVec.length);
|
|
1494
|
+
if (!compatibility.compatible) {
|
|
1495
|
+
embeddingIncompatibleKbIds.add(kb.id);
|
|
1496
|
+
warnings.push(embeddingMismatchWarning(kb, compatibility));
|
|
1279
1497
|
continue;
|
|
1280
1498
|
}
|
|
1281
1499
|
const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), retrievalLimitPerKb, signal);
|
|
@@ -1292,14 +1510,16 @@ export class KnowledgeEngine {
|
|
|
1292
1510
|
queryEmbedding ??= await embedQueryWithConfig(query, signal);
|
|
1293
1511
|
const { vector: queryVec, config: queryEmbeddingConfig } = queryEmbedding;
|
|
1294
1512
|
throwIfAborted(signal);
|
|
1295
|
-
|
|
1513
|
+
const compatibility = inspectEmbeddingCompatibility(kb.embedding_signature, kb.embedding_dimension, queryEmbeddingConfig, queryVec.length);
|
|
1514
|
+
if (compatibility.compatible) {
|
|
1296
1515
|
const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), retrievalLimitPerKb, signal);
|
|
1297
1516
|
vecResults = vectorResults.results;
|
|
1298
1517
|
for (const [chunkId, vector] of vectorResults.vectorsByChunkId)
|
|
1299
1518
|
vectorsByChunkId.set(chunkId, vector);
|
|
1300
1519
|
}
|
|
1301
1520
|
else {
|
|
1302
|
-
|
|
1521
|
+
embeddingIncompatibleKbIds.add(kb.id);
|
|
1522
|
+
warnings.push(embeddingMismatchWarning(kb, compatibility));
|
|
1303
1523
|
}
|
|
1304
1524
|
const fused = weightedScoreFusion(bm25Results, vecResults);
|
|
1305
1525
|
allResults.push(...fused.map((result) => ({ ...result, score: result.score * kbTrustMultiplier(kb) })));
|
|
@@ -1323,30 +1543,30 @@ export class KnowledgeEngine {
|
|
|
1323
1543
|
result.score = ranking.adjusted_score;
|
|
1324
1544
|
}
|
|
1325
1545
|
}
|
|
1326
|
-
let
|
|
1546
|
+
let thresholded = unique.filter((result) => rankingByChunkId.has(result.chunkId));
|
|
1547
|
+
if (retrievalMode !== "fast" && retrievalMode !== "semantic") {
|
|
1548
|
+
thresholded = thresholded.filter((result) => result.score >= tuning.minHybridScore);
|
|
1549
|
+
}
|
|
1550
|
+
const thresholdCandidates = thresholded.length;
|
|
1551
|
+
let eligible = thresholded;
|
|
1327
1552
|
if (retrievalMode === "fast") {
|
|
1328
|
-
|
|
1553
|
+
eligible = thresholded.filter((result) => {
|
|
1329
1554
|
const chunk = getChunkById(db, result.chunkId);
|
|
1330
|
-
|
|
1331
|
-
return false;
|
|
1332
|
-
return hasAnyLexicalEvidence(buildChunkEmbeddingText(chunk), queryTokens);
|
|
1555
|
+
return chunk ? hasAnyLexicalEvidence(buildChunkEmbeddingText(chunk), queryTokens) : false;
|
|
1333
1556
|
});
|
|
1334
1557
|
}
|
|
1335
|
-
else if (retrievalMode !== "semantic") {
|
|
1336
|
-
|
|
1558
|
+
else if (retrievalMode !== "semantic" && retrievalMode !== "semantic_hybrid") {
|
|
1559
|
+
eligible = thresholded.filter((result) => {
|
|
1337
1560
|
const chunk = getChunkById(db, result.chunkId);
|
|
1338
|
-
|
|
1339
|
-
return false;
|
|
1340
|
-
if (retrievalMode === "semantic_hybrid")
|
|
1341
|
-
return result.score >= tuning.minHybridScore;
|
|
1342
|
-
return result.score >= tuning.minHybridScore && hasEnoughLexicalEvidence(chunk, queryTokens);
|
|
1561
|
+
return chunk ? hasEnoughLexicalEvidence(chunk, queryTokens) : false;
|
|
1343
1562
|
});
|
|
1344
1563
|
}
|
|
1345
|
-
|
|
1564
|
+
eligible.sort((a, b) => b.score - a.score);
|
|
1565
|
+
const eligibleCandidates = eligible.length;
|
|
1346
1566
|
// Apply metadata filters post-retrieval
|
|
1347
|
-
let filtered =
|
|
1567
|
+
let filtered = eligible;
|
|
1348
1568
|
if (normalizedFileType || filters?.path_pattern) {
|
|
1349
|
-
filtered =
|
|
1569
|
+
filtered = eligible.filter((r) => {
|
|
1350
1570
|
const chunk = getChunkById(db, r.chunkId);
|
|
1351
1571
|
if (!chunk)
|
|
1352
1572
|
return false;
|
|
@@ -1418,19 +1638,42 @@ export class KnowledgeEngine {
|
|
|
1418
1638
|
},
|
|
1419
1639
|
};
|
|
1420
1640
|
});
|
|
1641
|
+
const deepTuning = {
|
|
1642
|
+
...tuningSummary,
|
|
1643
|
+
retrieved_candidates: unique.length,
|
|
1644
|
+
threshold_candidates: thresholdCandidates,
|
|
1645
|
+
eligible_candidates: eligibleCandidates,
|
|
1646
|
+
filtered_candidates: filtered.length,
|
|
1647
|
+
diversity_candidates: ranked.length,
|
|
1648
|
+
candidate_pool_truncated: candidatePoolTruncated,
|
|
1649
|
+
};
|
|
1421
1650
|
return {
|
|
1422
1651
|
results,
|
|
1423
1652
|
total_count: diversified.length,
|
|
1424
1653
|
has_more: offset + limit < diversified.length,
|
|
1425
1654
|
warnings: warnings.length > 0 ? warnings : undefined,
|
|
1426
1655
|
mode_used: mode,
|
|
1427
|
-
tuning:
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1656
|
+
tuning: deepTuning,
|
|
1657
|
+
miss: results.length === 0
|
|
1658
|
+
? classifySearchMiss({
|
|
1659
|
+
allKnowledgeBases: allKBs,
|
|
1660
|
+
availableKnowledgeBases: availableKBs,
|
|
1661
|
+
searchedKnowledgeBases: kbs,
|
|
1662
|
+
requestedKbId: kb_id,
|
|
1663
|
+
mode,
|
|
1664
|
+
fileType: filters?.file_type,
|
|
1665
|
+
pathPattern: filters?.path_pattern,
|
|
1666
|
+
minimumScore: tuning.minHybridScore,
|
|
1667
|
+
offset,
|
|
1668
|
+
retrievedCandidates: unique.length,
|
|
1669
|
+
thresholdCandidates,
|
|
1670
|
+
eligibleCandidates,
|
|
1671
|
+
filteredCandidates: filtered.length,
|
|
1672
|
+
rankedCandidates: ranked.length,
|
|
1673
|
+
totalResults: diversified.length,
|
|
1674
|
+
embeddingIncompatibleCount: embeddingIncompatibleKbIds.size,
|
|
1675
|
+
})
|
|
1676
|
+
: undefined,
|
|
1434
1677
|
};
|
|
1435
1678
|
}
|
|
1436
1679
|
const ranked = [];
|
|
@@ -1499,24 +1742,41 @@ export class KnowledgeEngine {
|
|
|
1499
1742
|
};
|
|
1500
1743
|
})(),
|
|
1501
1744
|
}));
|
|
1745
|
+
const responseTuning = {
|
|
1746
|
+
...tuningSummary,
|
|
1747
|
+
retrieved_candidates: unique.length,
|
|
1748
|
+
threshold_candidates: thresholdCandidates,
|
|
1749
|
+
eligible_candidates: eligibleCandidates,
|
|
1750
|
+
filtered_candidates: filtered.length,
|
|
1751
|
+
diversity_candidates: ranked.length,
|
|
1752
|
+
candidate_pool_truncated: candidatePoolTruncated,
|
|
1753
|
+
};
|
|
1502
1754
|
return {
|
|
1503
1755
|
results,
|
|
1504
1756
|
total_count: total,
|
|
1505
1757
|
has_more: offset + limit < total,
|
|
1506
1758
|
warnings: warnings.length > 0 ? warnings : undefined,
|
|
1507
1759
|
mode_used: mode,
|
|
1508
|
-
tuning:
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1760
|
+
tuning: responseTuning,
|
|
1761
|
+
miss: results.length === 0
|
|
1762
|
+
? classifySearchMiss({
|
|
1763
|
+
allKnowledgeBases: allKBs,
|
|
1764
|
+
availableKnowledgeBases: availableKBs,
|
|
1765
|
+
searchedKnowledgeBases: kbs,
|
|
1766
|
+
requestedKbId: kb_id,
|
|
1767
|
+
mode,
|
|
1768
|
+
fileType: filters?.file_type,
|
|
1769
|
+
pathPattern: filters?.path_pattern,
|
|
1770
|
+
minimumScore: tuning.minHybridScore,
|
|
1771
|
+
offset,
|
|
1772
|
+
retrievedCandidates: unique.length,
|
|
1773
|
+
thresholdCandidates,
|
|
1774
|
+
eligibleCandidates,
|
|
1775
|
+
filteredCandidates: filtered.length,
|
|
1776
|
+
rankedCandidates: ranked.length,
|
|
1777
|
+
totalResults: total,
|
|
1778
|
+
embeddingIncompatibleCount: embeddingIncompatibleKbIds.size,
|
|
1779
|
+
})
|
|
1520
1780
|
: undefined,
|
|
1521
1781
|
};
|
|
1522
1782
|
}
|
|
@@ -1562,16 +1822,77 @@ export class KnowledgeEngine {
|
|
|
1562
1822
|
return diagnoseKB(db, kb, signal);
|
|
1563
1823
|
});
|
|
1564
1824
|
}
|
|
1825
|
+
statusSnapshot(signal) {
|
|
1826
|
+
if (!this.db)
|
|
1827
|
+
return { knowledge_bases: [], diagnostics: [], embeddings: [] };
|
|
1828
|
+
throwIfAborted(signal);
|
|
1829
|
+
const db = this.db;
|
|
1830
|
+
const knowledgeBases = listKBs(db);
|
|
1831
|
+
const diagnostics = knowledgeBases.map((kb) => {
|
|
1832
|
+
throwIfAborted(signal);
|
|
1833
|
+
return diagnoseKB(db, kb, signal);
|
|
1834
|
+
});
|
|
1835
|
+
try {
|
|
1836
|
+
const config = resolveEmbeddingConfig();
|
|
1837
|
+
const activeModel = embeddingConfigLabel(config);
|
|
1838
|
+
const activeDecoration = describeEmbeddingDecoration(config);
|
|
1839
|
+
return {
|
|
1840
|
+
knowledge_bases: knowledgeBases,
|
|
1841
|
+
diagnostics,
|
|
1842
|
+
embeddings: knowledgeBases.map((kb) => {
|
|
1843
|
+
const compatibility = inspectEmbeddingCompatibility(kb.embedding_signature, kb.embedding_dimension, config, kb.embedding_dimension ?? 0);
|
|
1844
|
+
return {
|
|
1845
|
+
kb_id: kb.id,
|
|
1846
|
+
kb_name: kb.name,
|
|
1847
|
+
indexed_model: kb.embedding_model,
|
|
1848
|
+
active_model: activeModel,
|
|
1849
|
+
active_decoration: activeDecoration,
|
|
1850
|
+
compatibility,
|
|
1851
|
+
detail: describeEmbeddingCompatibility(compatibility),
|
|
1852
|
+
};
|
|
1853
|
+
}),
|
|
1854
|
+
};
|
|
1855
|
+
}
|
|
1856
|
+
catch (error) {
|
|
1857
|
+
return {
|
|
1858
|
+
knowledge_bases: knowledgeBases,
|
|
1859
|
+
diagnostics,
|
|
1860
|
+
embeddings: [],
|
|
1861
|
+
embedding_error: error instanceof Error ? error.message : String(error),
|
|
1862
|
+
};
|
|
1863
|
+
}
|
|
1864
|
+
}
|
|
1565
1865
|
symbolSearch(query, options = {}, signal) {
|
|
1566
1866
|
if (!this.db)
|
|
1567
1867
|
throw new Error("Engine not initialized");
|
|
1568
1868
|
throwIfAborted(signal);
|
|
1569
1869
|
const db = this.db;
|
|
1870
|
+
const allKBs = listKBs(db);
|
|
1570
1871
|
const selectedKB = options.kb_id ? (getKB(db, options.kb_id) ?? getKBByName(db, options.kb_id)) : undefined;
|
|
1571
|
-
|
|
1572
|
-
|
|
1872
|
+
const availableKBs = options.kb_id ? [selectedKB].filter(Boolean) : allKBs;
|
|
1873
|
+
const searchedKBs = availableKBs.filter((kb) => kb.status === "ready" || kb.status === "stale");
|
|
1573
1874
|
const limit = Math.max(1, Math.min(200, options.limit ?? 20));
|
|
1574
1875
|
const offset = Math.max(0, options.offset ?? 0);
|
|
1876
|
+
if (searchedKBs.length === 0) {
|
|
1877
|
+
return {
|
|
1878
|
+
results: [],
|
|
1879
|
+
total_count: 0,
|
|
1880
|
+
has_more: false,
|
|
1881
|
+
miss: classifySymbolSearchMiss({
|
|
1882
|
+
allKnowledgeBases: allKBs,
|
|
1883
|
+
availableKnowledgeBases: availableKBs,
|
|
1884
|
+
searchedKnowledgeBases: searchedKBs,
|
|
1885
|
+
requestedKbId: options.kb_id,
|
|
1886
|
+
kind: options.kind,
|
|
1887
|
+
filePattern: options.file_pattern,
|
|
1888
|
+
exact: options.exact === true,
|
|
1889
|
+
offset,
|
|
1890
|
+
totalResults: 0,
|
|
1891
|
+
indexedSymbols: 0,
|
|
1892
|
+
unfilteredMatches: 0,
|
|
1893
|
+
}),
|
|
1894
|
+
};
|
|
1895
|
+
}
|
|
1575
1896
|
const symbolOptions = {
|
|
1576
1897
|
kbId: selectedKB?.id,
|
|
1577
1898
|
kind: options.kind,
|
|
@@ -1587,25 +1908,48 @@ export class KnowledgeEngine {
|
|
|
1587
1908
|
const page = symbols.slice(0, limit);
|
|
1588
1909
|
const total = countSymbols(db, query, symbolOptions);
|
|
1589
1910
|
throwIfAborted(signal);
|
|
1911
|
+
const results = page.map((symbol) => {
|
|
1912
|
+
const kb = getKB(db, symbol.kb_id);
|
|
1913
|
+
return {
|
|
1914
|
+
name: symbol.name,
|
|
1915
|
+
kind: symbol.kind,
|
|
1916
|
+
file_path: symbol.file_path,
|
|
1917
|
+
file_type: symbol.file_type,
|
|
1918
|
+
kb_name: kb?.name ?? "unknown",
|
|
1919
|
+
start_line: symbol.start_line,
|
|
1920
|
+
end_line: symbol.end_line,
|
|
1921
|
+
signature: symbol.signature ?? undefined,
|
|
1922
|
+
container_name: symbol.container_name ?? undefined,
|
|
1923
|
+
text: symbol.text,
|
|
1924
|
+
indexed_at: symbol.indexed_at,
|
|
1925
|
+
};
|
|
1926
|
+
});
|
|
1927
|
+
let miss;
|
|
1928
|
+
if (results.length === 0) {
|
|
1929
|
+
const indexedSymbols = countSymbols(db, "", { kbId: selectedKB?.id });
|
|
1930
|
+
const unfilteredMatches = total === 0 && (options.kind || options.file_pattern)
|
|
1931
|
+
? countSymbols(db, query, { kbId: selectedKB?.id, exact: options.exact })
|
|
1932
|
+
: total;
|
|
1933
|
+
throwIfAborted(signal);
|
|
1934
|
+
miss = classifySymbolSearchMiss({
|
|
1935
|
+
allKnowledgeBases: allKBs,
|
|
1936
|
+
availableKnowledgeBases: availableKBs,
|
|
1937
|
+
searchedKnowledgeBases: searchedKBs,
|
|
1938
|
+
requestedKbId: options.kb_id,
|
|
1939
|
+
kind: options.kind,
|
|
1940
|
+
filePattern: options.file_pattern,
|
|
1941
|
+
exact: options.exact === true,
|
|
1942
|
+
offset,
|
|
1943
|
+
totalResults: total,
|
|
1944
|
+
indexedSymbols,
|
|
1945
|
+
unfilteredMatches,
|
|
1946
|
+
});
|
|
1947
|
+
}
|
|
1590
1948
|
return {
|
|
1591
|
-
results
|
|
1592
|
-
const kb = getKB(db, symbol.kb_id);
|
|
1593
|
-
return {
|
|
1594
|
-
name: symbol.name,
|
|
1595
|
-
kind: symbol.kind,
|
|
1596
|
-
file_path: symbol.file_path,
|
|
1597
|
-
file_type: symbol.file_type,
|
|
1598
|
-
kb_name: kb?.name ?? "unknown",
|
|
1599
|
-
start_line: symbol.start_line,
|
|
1600
|
-
end_line: symbol.end_line,
|
|
1601
|
-
signature: symbol.signature ?? undefined,
|
|
1602
|
-
container_name: symbol.container_name ?? undefined,
|
|
1603
|
-
text: symbol.text,
|
|
1604
|
-
indexed_at: symbol.indexed_at,
|
|
1605
|
-
};
|
|
1606
|
-
}),
|
|
1949
|
+
results,
|
|
1607
1950
|
total_count: total,
|
|
1608
1951
|
has_more: symbols.length > limit,
|
|
1952
|
+
miss,
|
|
1609
1953
|
};
|
|
1610
1954
|
}
|
|
1611
1955
|
doctor(signal) {
|
|
@@ -1683,23 +2027,34 @@ export class KnowledgeEngine {
|
|
|
1683
2027
|
}
|
|
1684
2028
|
}
|
|
1685
2029
|
if (this.db) {
|
|
1686
|
-
|
|
2030
|
+
let activeEmbeddingConfig;
|
|
2031
|
+
try {
|
|
2032
|
+
activeEmbeddingConfig = resolveEmbeddingConfig();
|
|
2033
|
+
}
|
|
2034
|
+
catch (error) {
|
|
2035
|
+
issues.push({
|
|
2036
|
+
severity: "blocking",
|
|
2037
|
+
message: `Active embedding configuration is invalid: ${error instanceof Error ? error.message : String(error)}`,
|
|
2038
|
+
action: "Correct the embedding environment variables, then run knowledge_doctor again.",
|
|
2039
|
+
action_code: "fix_embedding_config",
|
|
2040
|
+
});
|
|
2041
|
+
}
|
|
1687
2042
|
for (const kb of kbs) {
|
|
1688
2043
|
throwIfAborted(signal);
|
|
1689
|
-
if ((kb.status === "ready" || kb.status === "stale") &&
|
|
1690
|
-
kb.
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
: "
|
|
1701
|
-
|
|
1702
|
-
}
|
|
2044
|
+
if (activeEmbeddingConfig && (kb.status === "ready" || kb.status === "stale") && kb.chunk_count > 0) {
|
|
2045
|
+
const compatibility = inspectEmbeddingCompatibility(kb.embedding_signature, kb.embedding_dimension, activeEmbeddingConfig, kb.embedding_dimension ?? 0);
|
|
2046
|
+
if (!compatibility.compatible) {
|
|
2047
|
+
const canUpdate = kb.source_path !== null && kb.source_type !== "text";
|
|
2048
|
+
issues.push({
|
|
2049
|
+
severity: "warning",
|
|
2050
|
+
kb_name: kb.name,
|
|
2051
|
+
message: `This KB's vectors are incompatible with the active embedding configuration: ${describeEmbeddingCompatibility(compatibility)}.`,
|
|
2052
|
+
action: canUpdate
|
|
2053
|
+
? "Run knowledge_update to rebuild vectors with the active embedding configuration."
|
|
2054
|
+
: "Re-add inline text or re-import the original source with the active embedding configuration.",
|
|
2055
|
+
action_code: "rebuild_vectors",
|
|
2056
|
+
});
|
|
2057
|
+
}
|
|
1703
2058
|
}
|
|
1704
2059
|
if ((kb.status === "ready" || kb.status === "stale") &&
|
|
1705
2060
|
(storedSourcePolicyVersion(kb.source_options, "embedding_input_policy") ?? 0) < CURRENT_EMBEDDING_INPUT_POLICY) {
|
|
@@ -1824,10 +2179,10 @@ export class KnowledgeEngine {
|
|
|
1824
2179
|
throwIfAborted(signal);
|
|
1825
2180
|
const { createReadStream } = await import("node:fs");
|
|
1826
2181
|
const { createInterface } = await import("node:readline");
|
|
1827
|
-
const stream = createReadStream(inputPath, { encoding: "utf-8" });
|
|
1828
|
-
const lines = createInterface({ input: stream, crlfDelay: Infinity });
|
|
1829
2182
|
const embeddingConfig = resolveEmbeddingConfig();
|
|
1830
2183
|
const embeddingBatchSize = resolveIndexEmbedBatchSize();
|
|
2184
|
+
const stream = createReadStream(inputPath, { encoding: "utf-8" });
|
|
2185
|
+
const lines = createInterface({ input: stream, crlfDelay: Infinity });
|
|
1831
2186
|
let header;
|
|
1832
2187
|
let kb;
|
|
1833
2188
|
let vectorWriter;
|