pi-knowledge 0.11.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/README.md +147 -33
- package/dist/index.js +15 -4
- package/dist/src/embedding/provider.d.ts +14 -0
- package/dist/src/embedding/provider.js +90 -8
- package/dist/src/engine.d.ts +3 -1
- package/dist/src/engine.js +122 -147
- package/dist/src/indexer/chunker.d.ts +1 -1
- package/dist/src/indexer/chunker.js +2 -2
- package/dist/src/indexer/embedding-safe-chunks.d.ts +5 -0
- package/dist/src/indexer/embedding-safe-chunks.js +215 -0
- package/dist/src/indexer/html.d.ts +1 -0
- package/dist/src/indexer/html.js +49 -0
- package/dist/src/model-worker-client.d.ts +4 -0
- package/dist/src/model-worker-client.js +17 -1
- package/dist/src/model-worker.js +63 -15
- package/dist/src/search/contracts.d.ts +2 -0
- package/dist/src/search/contracts.js +2 -0
- package/dist/src/search/diversity.d.ts +16 -0
- package/dist/src/search/diversity.js +183 -0
- package/dist/src/search/tuning.d.ts +10 -0
- package/dist/src/search/tuning.js +20 -0
- package/dist/src/storage/sqlite.d.ts +1 -0
- package/dist/src/storage/sqlite.js +3 -0
- package/docs/configuration.md +3 -3
- package/package.json +1 -1
package/dist/src/engine.js
CHANGED
|
@@ -1,22 +1,27 @@
|
|
|
1
1
|
import { existsSync, readFileSync, renameSync, rmSync, statSync } from "node:fs";
|
|
2
2
|
import { join, resolve } from "node:path";
|
|
3
3
|
import { diagnoseKB } from "./diagnostics/health.js";
|
|
4
|
-
import { dispose as disposeEmbedding, embedDocuments, embeddingConfigLabel, embeddingSignature, embedQueryWithConfig, prepareForShutdown as prepareEmbeddingForShutdown, resolveEmbeddingConfig, } from "./embedding/provider.js";
|
|
4
|
+
import { dispose as disposeEmbedding, embedDocuments, embeddingConfigLabel, embeddingSignature, embedQueryWithConfig, embedValidatedDocuments, prepareForShutdown as prepareEmbeddingForShutdown, resolveEmbeddingConfig, } from "./embedding/provider.js";
|
|
5
5
|
import { openVectorReader, openVectorWriter } from "./embedding/vectors.js";
|
|
6
|
-
import { addSkippedScanEntry, analyzeIndexableContent, buildChunkEmbeddingText, chunkFile, chunkIdentityHash, createSkippedScanStats, isReadableTextFile, isSupportedDocumentFile, iterateScannableFiles, preTokenizeForFTS, summarizeSkippedScan, } from "./indexer/chunker.js";
|
|
6
|
+
import { addSkippedScanEntry, analyzeIndexableContent, buildChunkEmbeddingText, chunkFile, chunkIdentityHash, chunkMarkdown, createSkippedScanStats, isReadableTextFile, isSupportedDocumentFile, iterateScannableFiles, preTokenizeForFTS, summarizeSkippedScan, } from "./indexer/chunker.js";
|
|
7
|
+
import { finalizeChunksForEmbedding } from "./indexer/embedding-safe-chunks.js";
|
|
8
|
+
import { htmlToStructuredMarkdown } from "./indexer/html.js";
|
|
7
9
|
import { extractSymbols } from "./indexer/symbols.js";
|
|
8
10
|
import { shutdownModelWorker } from "./model-worker-client.js";
|
|
9
11
|
import { searchBM25 } from "./search/bm25.js";
|
|
12
|
+
import { MAX_SEARCH_WINDOW } from "./search/contracts.js";
|
|
13
|
+
import { diversifyRankedChunks, interleaveByFile } from "./search/diversity.js";
|
|
10
14
|
import { weightedScoreFusion } from "./search/fusion.js";
|
|
11
15
|
import { normalizedQueryText, tokenizeForSearch } from "./search/query.js";
|
|
12
16
|
import { hasAnyLexicalEvidence, hasEnoughLexicalEvidence, normalizeFileTypeFilter, queryCoverage, scoreChunkForQuery, } from "./search/ranking.js";
|
|
13
17
|
import { disposeReranker, prepareRerankerForShutdown, rerank } from "./search/reranker.js";
|
|
14
|
-
import { resolveSearchTuning, summarizeSearchTuning, } from "./search/tuning.js";
|
|
18
|
+
import { resolveSearchTuning, resolveSearchWindow, summarizeSearchTuning, } from "./search/tuning.js";
|
|
15
19
|
import { searchVectorFile } from "./search/vector.js";
|
|
16
|
-
import { countSymbols, createKB, deleteChunksByIds, deleteKB, deleteSymbolsByKB, finishIndexingJob, getChunkById, getChunkCount, getChunksByFile, getFileCount, getKB, getKBByName, getSymbolCount, insertChunks, insertSymbols, iterateChunkIdsByKB, iterateChunksByKB, listKBs, openDatabase, searchSymbols, startIndexingJob, updateIndexingJob, updateKBCounts, updateKBEmbeddingMetadata, updateKBStatus, } from "./storage/sqlite.js";
|
|
20
|
+
import { countSymbols, createKB, deleteChunksByIds, deleteKB, deleteSymbolsByKB, finishIndexingJob, getChunkById, getChunkCount, getChunksByFile, getFileCount, getKB, getKBByName, getSymbolCount, insertChunks, insertSymbols, iterateChunkIdsByKB, iterateChunksByKB, listKBs, openDatabase, searchSymbols, startIndexingJob, updateIndexingJob, updateKBCounts, updateKBEmbeddingMetadata, updateKBSourceOptions, updateKBStatus, } from "./storage/sqlite.js";
|
|
17
21
|
export { CURRENT_EMBEDDING_MODEL } from "./embedding/provider.js";
|
|
22
|
+
export const CURRENT_URL_STRUCTURE_POLICY = 1;
|
|
23
|
+
export const CURRENT_EMBEDDING_INPUT_POLICY = 2;
|
|
18
24
|
const DEFAULT_INDEX_EMBED_BATCH_SIZE = 64;
|
|
19
|
-
const VECTOR_REDUNDANCY_WEIGHT = 0.35;
|
|
20
25
|
function resolveIndexEmbedBatchSize(env = process.env) {
|
|
21
26
|
const raw = env.PI_KNOWLEDGE_EMBEDDING_BATCH_SIZE?.trim();
|
|
22
27
|
if (!raw)
|
|
@@ -66,14 +71,17 @@ function toScanOptions(options = {}) {
|
|
|
66
71
|
};
|
|
67
72
|
}
|
|
68
73
|
function serializeAddOptions(options = {}) {
|
|
69
|
-
const normalized = {
|
|
74
|
+
const normalized = {
|
|
75
|
+
embedding_input_policy: CURRENT_EMBEDDING_INPUT_POLICY,
|
|
76
|
+
url_structure_policy: CURRENT_URL_STRUCTURE_POLICY,
|
|
77
|
+
};
|
|
70
78
|
if (options.include_suggested_text === true)
|
|
71
79
|
normalized.include_suggested_text = true;
|
|
72
80
|
if (options.include_paths && options.include_paths.length > 0)
|
|
73
81
|
normalized.include_paths = options.include_paths;
|
|
74
82
|
if (options.exclude_paths && options.exclude_paths.length > 0)
|
|
75
83
|
normalized.exclude_paths = options.exclude_paths;
|
|
76
|
-
return
|
|
84
|
+
return JSON.stringify(normalized);
|
|
77
85
|
}
|
|
78
86
|
function parseAddOptions(raw) {
|
|
79
87
|
if (!raw)
|
|
@@ -94,6 +102,18 @@ function parseAddOptions(raw) {
|
|
|
94
102
|
return {};
|
|
95
103
|
}
|
|
96
104
|
}
|
|
105
|
+
function storedSourcePolicyVersion(raw, key) {
|
|
106
|
+
if (!raw)
|
|
107
|
+
return undefined;
|
|
108
|
+
try {
|
|
109
|
+
const parsed = JSON.parse(raw);
|
|
110
|
+
const version = parsed[key];
|
|
111
|
+
return typeof version === "number" && Number.isInteger(version) ? version : undefined;
|
|
112
|
+
}
|
|
113
|
+
catch {
|
|
114
|
+
return undefined;
|
|
115
|
+
}
|
|
116
|
+
}
|
|
97
117
|
function planDirectoryScan(dirPath, options = {}, signal) {
|
|
98
118
|
const skipped = createSkippedScanStats();
|
|
99
119
|
let files = 0;
|
|
@@ -141,116 +161,9 @@ function formatBytes(bytes) {
|
|
|
141
161
|
return `${mb.toFixed(1)} MB`;
|
|
142
162
|
return `${(mb / 1024).toFixed(2)} GB`;
|
|
143
163
|
}
|
|
144
|
-
function cosineSimilarity(a, b) {
|
|
145
|
-
if (!a || !b || a.length !== b.length)
|
|
146
|
-
return 0;
|
|
147
|
-
let dot = 0;
|
|
148
|
-
for (let i = 0; i < a.length; i++)
|
|
149
|
-
dot += a[i] * b[i];
|
|
150
|
-
return dot;
|
|
151
|
-
}
|
|
152
164
|
function tokenizeForSimilarity(text) {
|
|
153
165
|
return tokenizeForSearch(text);
|
|
154
166
|
}
|
|
155
|
-
function jaccardSimilarity(a, b) {
|
|
156
|
-
if (a.size === 0 || b.size === 0)
|
|
157
|
-
return 0;
|
|
158
|
-
let intersection = 0;
|
|
159
|
-
for (const token of a) {
|
|
160
|
-
if (b.has(token))
|
|
161
|
-
intersection++;
|
|
162
|
-
}
|
|
163
|
-
return intersection / (a.size + b.size - intersection);
|
|
164
|
-
}
|
|
165
|
-
function lineProximity(a, b) {
|
|
166
|
-
if (a.chunk.file_path !== b.chunk.file_path || a.chunk.kb_id !== b.chunk.kb_id)
|
|
167
|
-
return 0;
|
|
168
|
-
if (a.sourceChunkIds.some((id) => b.sourceChunkIds.includes(id)))
|
|
169
|
-
return 1;
|
|
170
|
-
const overlap = Math.max(0, Math.min(a.endLine, b.endLine) - Math.max(a.startLine, b.startLine) + 1);
|
|
171
|
-
if (overlap > 0)
|
|
172
|
-
return 1;
|
|
173
|
-
const gap = Math.max(a.startLine, b.startLine) - Math.min(a.endLine, b.endLine);
|
|
174
|
-
if (gap <= 20)
|
|
175
|
-
return 0.8;
|
|
176
|
-
if (gap <= 80)
|
|
177
|
-
return 0.45;
|
|
178
|
-
return 0.2;
|
|
179
|
-
}
|
|
180
|
-
function normalizeScores(candidates) {
|
|
181
|
-
const scores = candidates.map((candidate) => candidate.score);
|
|
182
|
-
const min = Math.min(...scores);
|
|
183
|
-
const max = Math.max(...scores);
|
|
184
|
-
const normalized = new Map();
|
|
185
|
-
for (const candidate of candidates) {
|
|
186
|
-
const value = max === min ? 1 : (candidate.score - min) / (max - min);
|
|
187
|
-
normalized.set(candidate.chunk.id, value);
|
|
188
|
-
}
|
|
189
|
-
return normalized;
|
|
190
|
-
}
|
|
191
|
-
function diversifyRankedChunks(candidates, diversity, vectorsByChunkId = new Map()) {
|
|
192
|
-
if (diversity === "off" || candidates.length <= 2)
|
|
193
|
-
return candidates;
|
|
194
|
-
const lambda = diversity === "strong" ? 0.62 : 0.76;
|
|
195
|
-
const normalized = normalizeScores(candidates);
|
|
196
|
-
const tokenSets = new Map(candidates.map((candidate) => [candidate.chunk.id, tokenizeForSimilarity(candidate.content)]));
|
|
197
|
-
const selected = [];
|
|
198
|
-
const remaining = [...candidates];
|
|
199
|
-
while (remaining.length > 0) {
|
|
200
|
-
let bestIndex = 0;
|
|
201
|
-
let bestScore = Number.NEGATIVE_INFINITY;
|
|
202
|
-
for (let i = 0; i < remaining.length; i++) {
|
|
203
|
-
const candidate = remaining[i];
|
|
204
|
-
let redundancy = 0;
|
|
205
|
-
for (const chosen of selected) {
|
|
206
|
-
const lexical = jaccardSimilarity(tokenSets.get(candidate.chunk.id) ?? new Set(), tokenSets.get(chosen.chunk.id) ?? new Set());
|
|
207
|
-
const vector = Math.max(0, cosineSimilarity(vectorsByChunkId.get(candidate.chunk.id), vectorsByChunkId.get(chosen.chunk.id)));
|
|
208
|
-
redundancy = Math.max(redundancy, Math.max(lexical, lineProximity(candidate, chosen), vector * VECTOR_REDUNDANCY_WEIGHT));
|
|
209
|
-
}
|
|
210
|
-
const relevance = normalized.get(candidate.chunk.id) ?? 0;
|
|
211
|
-
const mmrScore = lambda * relevance - (1 - lambda) * redundancy;
|
|
212
|
-
if (mmrScore > bestScore) {
|
|
213
|
-
bestScore = mmrScore;
|
|
214
|
-
bestIndex = i;
|
|
215
|
-
}
|
|
216
|
-
}
|
|
217
|
-
selected.push(remaining.splice(bestIndex, 1)[0]);
|
|
218
|
-
}
|
|
219
|
-
return selected;
|
|
220
|
-
}
|
|
221
|
-
function interleaveByFile(candidates, diversity) {
|
|
222
|
-
if (diversity === "off" || candidates.length <= 2)
|
|
223
|
-
return candidates;
|
|
224
|
-
const buckets = new Map();
|
|
225
|
-
const fileOrder = [];
|
|
226
|
-
for (const candidate of candidates) {
|
|
227
|
-
const key = `${candidate.chunk.kb_id}:${candidate.chunk.file_path}`;
|
|
228
|
-
if (!buckets.has(key)) {
|
|
229
|
-
buckets.set(key, []);
|
|
230
|
-
fileOrder.push(key);
|
|
231
|
-
}
|
|
232
|
-
buckets.get(key)?.push(candidate);
|
|
233
|
-
}
|
|
234
|
-
if (fileOrder.length <= 1)
|
|
235
|
-
return candidates;
|
|
236
|
-
const result = [];
|
|
237
|
-
let round = 0;
|
|
238
|
-
while (result.length < candidates.length) {
|
|
239
|
-
let added = false;
|
|
240
|
-
for (const key of fileOrder) {
|
|
241
|
-
const bucket = buckets.get(key);
|
|
242
|
-
const item = bucket?.[round];
|
|
243
|
-
if (!item)
|
|
244
|
-
continue;
|
|
245
|
-
result.push(item);
|
|
246
|
-
added = true;
|
|
247
|
-
}
|
|
248
|
-
if (!added)
|
|
249
|
-
break;
|
|
250
|
-
round++;
|
|
251
|
-
}
|
|
252
|
-
return result;
|
|
253
|
-
}
|
|
254
167
|
function buildQuerySnippet(content, query, maxLength = 240) {
|
|
255
168
|
const terms = [...tokenizeForSimilarity(query)].sort((a, b) => b.length - a.length);
|
|
256
169
|
const lower = content.toLowerCase();
|
|
@@ -452,14 +365,10 @@ async function chunkUrl(source, signal) {
|
|
|
452
365
|
if (!res.ok)
|
|
453
366
|
throw new Error(`Fetch failed: ${res.status}`);
|
|
454
367
|
const html = await res.text();
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
.replace(/&[a-z]+;/gi, " ")
|
|
460
|
-
.replace(/\s+/g, " ")
|
|
461
|
-
.trim();
|
|
462
|
-
return chunkFile(text, source);
|
|
368
|
+
throwIfAborted(signal);
|
|
369
|
+
const text = htmlToStructuredMarkdown(html);
|
|
370
|
+
const structuredChunks = chunkMarkdown(text, source, "html");
|
|
371
|
+
return structuredChunks.length > 0 ? structuredChunks : chunkFile(text, source);
|
|
463
372
|
}
|
|
464
373
|
function normalizeExtractedText(text) {
|
|
465
374
|
return Array.isArray(text) ? text.join("\n\n") : text;
|
|
@@ -772,7 +681,7 @@ export class KnowledgeEngine {
|
|
|
772
681
|
throw new Error("Cancelled");
|
|
773
682
|
const batch = pendingChunks.splice(0, embeddingBatchSize);
|
|
774
683
|
reportProgress(`Embedding batch of ${batch.length}`, processedFiles, totalFiles);
|
|
775
|
-
const vectors = await
|
|
684
|
+
const vectors = await embedValidatedDocuments(batch.map((chunk) => buildChunkEmbeddingText(chunk)), signal);
|
|
776
685
|
if (signal?.aborted)
|
|
777
686
|
throw new Error("Cancelled");
|
|
778
687
|
assertEmbeddingBatchSize(vectors, batch.length, "add");
|
|
@@ -784,7 +693,8 @@ export class KnowledgeEngine {
|
|
|
784
693
|
reportProgress("Stored batch", processedFiles, totalFiles);
|
|
785
694
|
};
|
|
786
695
|
const addChunks = async (chunks, processedFiles, totalFiles) => {
|
|
787
|
-
|
|
696
|
+
const finalizedChunks = await finalizeChunksForEmbedding(chunks, embeddingConfig, signal);
|
|
697
|
+
pendingChunks.push(...finalizedChunks);
|
|
788
698
|
while (pendingChunks.length >= embeddingBatchSize) {
|
|
789
699
|
await flushPending(processedFiles, totalFiles);
|
|
790
700
|
}
|
|
@@ -1016,7 +926,7 @@ export class KnowledgeEngine {
|
|
|
1016
926
|
unchanged_chunks: unchanged,
|
|
1017
927
|
});
|
|
1018
928
|
onProgress?.(message);
|
|
1019
|
-
const newVectors = await
|
|
929
|
+
const newVectors = await embedValidatedDocuments(batch.map((c) => buildChunkEmbeddingText(c)), signal);
|
|
1020
930
|
if (signal?.aborted)
|
|
1021
931
|
throw new Error("Cancelled");
|
|
1022
932
|
assertEmbeddingBatchSize(newVectors, batch.length, "update");
|
|
@@ -1043,7 +953,8 @@ export class KnowledgeEngine {
|
|
|
1043
953
|
onProgress?.(storedMessage);
|
|
1044
954
|
};
|
|
1045
955
|
const processChunks = async (chunks) => {
|
|
1046
|
-
|
|
956
|
+
const finalizedChunks = await finalizeChunksForEmbedding(chunks, embeddingConfig, signal);
|
|
957
|
+
for (const chunk of finalizedChunks) {
|
|
1047
958
|
scannedChunks++;
|
|
1048
959
|
const existing = reusableHashes.get(chunk.content_hash);
|
|
1049
960
|
if (existing && existing.length > 0) {
|
|
@@ -1231,6 +1142,7 @@ export class KnowledgeEngine {
|
|
|
1231
1142
|
insertSymbols(this.db, kb.id, stagedSymbols);
|
|
1232
1143
|
updateKBEmbeddingMetadata(this.db, kb.id, embeddingModel, finalEmbeddingDimension === undefined ? null : embeddingSignature(embeddingConfig, finalEmbeddingDimension), finalEmbeddingDimension ?? null);
|
|
1233
1144
|
updateKBCounts(this.db, kb.id, finalChunkCount, getFileCount(this.db, kb.id));
|
|
1145
|
+
updateKBSourceOptions(this.db, kb.id, serializeAddOptions(parseAddOptions(kb.source_options)));
|
|
1234
1146
|
updateKBStatus(this.db, kb.id, "ready");
|
|
1235
1147
|
const skippedReadySuffix = latestSkippedTotal > 0 ? `, skipped ${latestSkippedTotal} (${latestSkippedSummary})` : "";
|
|
1236
1148
|
const readyMessage = `Ready: +${addedCount} -${idsToRemove.length} =${unchanged}${skippedReadySuffix}`;
|
|
@@ -1308,7 +1220,7 @@ export class KnowledgeEngine {
|
|
|
1308
1220
|
};
|
|
1309
1221
|
}
|
|
1310
1222
|
const db = this.db;
|
|
1311
|
-
const { mode = "hybrid", offset
|
|
1223
|
+
const { mode = "hybrid", offset: requestedOffset, kb_id, filters, diversity = "balanced" } = options;
|
|
1312
1224
|
const resolvedMode = retrievalModeFor(mode);
|
|
1313
1225
|
const retrievalMode = resolvedMode === "adaptive" ? "hybrid" : resolvedMode;
|
|
1314
1226
|
const normalizedQuery = normalizedQueryText(query);
|
|
@@ -1330,10 +1242,9 @@ export class KnowledgeEngine {
|
|
|
1330
1242
|
profile: options.profile,
|
|
1331
1243
|
kbSourceTypes: kbs.map((kb) => kb.source_type),
|
|
1332
1244
|
});
|
|
1333
|
-
const
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
const candidateLimit = Math.max(tuning.candidateMin, offset + limit * tuning.candidateMultiplier);
|
|
1245
|
+
const { limit, offset } = resolveSearchWindow(options.limit, requestedOffset, tuning.defaultLimit);
|
|
1246
|
+
const candidateLimit = Math.min(MAX_SEARCH_WINDOW, Math.max(tuning.candidateMin, offset + limit * tuning.candidateMultiplier));
|
|
1247
|
+
const retrievalLimitPerKb = candidateLimit;
|
|
1337
1248
|
const tuningSummary = summarizeSearchTuning(tuning, limit, candidateLimit);
|
|
1338
1249
|
if (kbs.length === 0) {
|
|
1339
1250
|
return {
|
|
@@ -1347,39 +1258,42 @@ export class KnowledgeEngine {
|
|
|
1347
1258
|
}
|
|
1348
1259
|
const allResults = [];
|
|
1349
1260
|
const vectorsByChunkId = new Map();
|
|
1261
|
+
let queryEmbedding;
|
|
1350
1262
|
for (const kb of kbs) {
|
|
1351
1263
|
throwIfAborted(signal);
|
|
1352
1264
|
if (kb.chunk_count === 0)
|
|
1353
1265
|
continue;
|
|
1354
1266
|
const vectorPath = this.vectorPathFor(kb.id);
|
|
1355
1267
|
if (retrievalMode === "fast") {
|
|
1356
|
-
allResults.push(...searchBM25(db, normalizedQuery || query,
|
|
1268
|
+
allResults.push(...searchBM25(db, normalizedQuery || query, retrievalLimitPerKb, kb.id, { allowOrFallback: false }).map((result) => ({
|
|
1357
1269
|
...result,
|
|
1358
1270
|
score: result.score * kbTrustMultiplier(kb),
|
|
1359
1271
|
})));
|
|
1360
1272
|
}
|
|
1361
1273
|
else if (retrievalMode === "semantic") {
|
|
1362
|
-
|
|
1274
|
+
queryEmbedding ??= await embedQueryWithConfig(query, signal);
|
|
1275
|
+
const { vector: queryVec, config: queryEmbeddingConfig } = queryEmbedding;
|
|
1363
1276
|
throwIfAborted(signal);
|
|
1364
1277
|
if (!canSearchVectors(kb, queryEmbeddingConfig, queryVec.length)) {
|
|
1365
1278
|
warnings.push(embeddingMismatchWarning(kb));
|
|
1366
1279
|
continue;
|
|
1367
1280
|
}
|
|
1368
|
-
const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id),
|
|
1281
|
+
const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), retrievalLimitPerKb, signal);
|
|
1369
1282
|
allResults.push(...vectorResults.results.map((result) => ({ ...result, score: result.score * kbTrustMultiplier(kb) })));
|
|
1370
1283
|
for (const [chunkId, vector] of vectorResults.vectorsByChunkId)
|
|
1371
1284
|
vectorsByChunkId.set(chunkId, vector);
|
|
1372
1285
|
}
|
|
1373
1286
|
else {
|
|
1374
1287
|
const semanticHybrid = retrievalMode === "semantic_hybrid";
|
|
1375
|
-
const bm25Results = searchBM25(db, normalizedQuery || query,
|
|
1288
|
+
const bm25Results = searchBM25(db, normalizedQuery || query, retrievalLimitPerKb, kb.id);
|
|
1376
1289
|
if (bm25Results.length === 0 && !semanticHybrid)
|
|
1377
1290
|
continue;
|
|
1378
1291
|
let vecResults = [];
|
|
1379
|
-
|
|
1292
|
+
queryEmbedding ??= await embedQueryWithConfig(query, signal);
|
|
1293
|
+
const { vector: queryVec, config: queryEmbeddingConfig } = queryEmbedding;
|
|
1380
1294
|
throwIfAborted(signal);
|
|
1381
1295
|
if (canSearchVectors(kb, queryEmbeddingConfig, queryVec.length)) {
|
|
1382
|
-
const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id),
|
|
1296
|
+
const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), retrievalLimitPerKb, signal);
|
|
1383
1297
|
vecResults = vectorResults.results;
|
|
1384
1298
|
for (const [chunkId, vector] of vectorResults.vectorsByChunkId)
|
|
1385
1299
|
vectorsByChunkId.set(chunkId, vector);
|
|
@@ -1443,8 +1357,15 @@ export class KnowledgeEngine {
|
|
|
1443
1357
|
return true;
|
|
1444
1358
|
});
|
|
1445
1359
|
}
|
|
1446
|
-
|
|
1447
|
-
|
|
1360
|
+
const candidatePool = filtered.slice(0, candidateLimit);
|
|
1361
|
+
const candidatePoolTruncated = candidatePool.length < filtered.length;
|
|
1362
|
+
const candidateIds = new Set(candidatePool.map((candidate) => candidate.chunkId));
|
|
1363
|
+
for (const chunkId of vectorsByChunkId.keys()) {
|
|
1364
|
+
if (!candidateIds.has(chunkId))
|
|
1365
|
+
vectorsByChunkId.delete(chunkId);
|
|
1366
|
+
}
|
|
1367
|
+
if (mode === "deep" && candidatePool.length > 0) {
|
|
1368
|
+
const candidates = candidatePool
|
|
1448
1369
|
.slice(0, tuning.deepRerankCandidates)
|
|
1449
1370
|
.map((r) => {
|
|
1450
1371
|
const chunk = getChunkById(db, r.chunkId);
|
|
@@ -1471,7 +1392,7 @@ export class KnowledgeEngine {
|
|
|
1471
1392
|
sourceChunkIds: [chunk.id],
|
|
1472
1393
|
});
|
|
1473
1394
|
}
|
|
1474
|
-
const diversified = interleaveByFile(diversifyRankedChunks(ranked, diversity, vectorsByChunkId), diversity);
|
|
1395
|
+
const diversified = interleaveByFile(await diversifyRankedChunks(ranked, diversity, vectorsByChunkId, signal), diversity, signal);
|
|
1475
1396
|
const page = diversified.slice(offset, offset + limit);
|
|
1476
1397
|
const results = page.map((r) => {
|
|
1477
1398
|
const kb = kbById.get(r.chunk.kb_id);
|
|
@@ -1503,11 +1424,17 @@ export class KnowledgeEngine {
|
|
|
1503
1424
|
has_more: offset + limit < diversified.length,
|
|
1504
1425
|
warnings: warnings.length > 0 ? warnings : undefined,
|
|
1505
1426
|
mode_used: mode,
|
|
1506
|
-
tuning:
|
|
1427
|
+
tuning: {
|
|
1428
|
+
...tuningSummary,
|
|
1429
|
+
retrieved_candidates: unique.length,
|
|
1430
|
+
filtered_candidates: filtered.length,
|
|
1431
|
+
diversity_candidates: ranked.length,
|
|
1432
|
+
candidate_pool_truncated: candidatePoolTruncated,
|
|
1433
|
+
},
|
|
1507
1434
|
};
|
|
1508
1435
|
}
|
|
1509
1436
|
const ranked = [];
|
|
1510
|
-
for (const r of
|
|
1437
|
+
for (const r of candidatePool) {
|
|
1511
1438
|
throwIfAborted(signal);
|
|
1512
1439
|
const chunk = getChunkById(db, r.chunkId);
|
|
1513
1440
|
if (!chunk)
|
|
@@ -1545,7 +1472,7 @@ export class KnowledgeEngine {
|
|
|
1545
1472
|
});
|
|
1546
1473
|
}
|
|
1547
1474
|
}
|
|
1548
|
-
const diversified = interleaveByFile(diversifyRankedChunks(ranked, diversity, vectorsByChunkId), diversity);
|
|
1475
|
+
const diversified = interleaveByFile(await diversifyRankedChunks(ranked, diversity, vectorsByChunkId, signal), diversity, signal);
|
|
1549
1476
|
const total = diversified.length;
|
|
1550
1477
|
const page = diversified.slice(offset, offset + limit);
|
|
1551
1478
|
const results = page.map((r) => ({
|
|
@@ -1578,7 +1505,13 @@ export class KnowledgeEngine {
|
|
|
1578
1505
|
has_more: offset + limit < total,
|
|
1579
1506
|
warnings: warnings.length > 0 ? warnings : undefined,
|
|
1580
1507
|
mode_used: mode,
|
|
1581
|
-
tuning:
|
|
1508
|
+
tuning: {
|
|
1509
|
+
...tuningSummary,
|
|
1510
|
+
retrieved_candidates: unique.length,
|
|
1511
|
+
filtered_candidates: filtered.length,
|
|
1512
|
+
diversity_candidates: ranked.length,
|
|
1513
|
+
candidate_pool_truncated: candidatePoolTruncated,
|
|
1514
|
+
},
|
|
1582
1515
|
suggestions: results.length === 0
|
|
1583
1516
|
? [
|
|
1584
1517
|
"Try mode 'fast' for exact symbols, mode 'semantic' for pure vector search, or mode 'semantic_hybrid' when weak lexical overlap should not block fused vector results.",
|
|
@@ -1750,8 +1683,48 @@ export class KnowledgeEngine {
|
|
|
1750
1683
|
}
|
|
1751
1684
|
}
|
|
1752
1685
|
if (this.db) {
|
|
1686
|
+
const activeEmbeddingConfig = resolveEmbeddingConfig();
|
|
1753
1687
|
for (const kb of kbs) {
|
|
1754
1688
|
throwIfAborted(signal);
|
|
1689
|
+
if ((kb.status === "ready" || kb.status === "stale") &&
|
|
1690
|
+
kb.chunk_count > 0 &&
|
|
1691
|
+
kb.embedding_dimension !== null &&
|
|
1692
|
+
kb.embedding_signature !== embeddingSignature(activeEmbeddingConfig, kb.embedding_dimension)) {
|
|
1693
|
+
const canUpdate = kb.source_path !== null && kb.source_type !== "text";
|
|
1694
|
+
issues.push({
|
|
1695
|
+
severity: "warning",
|
|
1696
|
+
kb_name: kb.name,
|
|
1697
|
+
message: "This KB's vectors are incompatible with the active embedding provider or normalization policy.",
|
|
1698
|
+
action: canUpdate
|
|
1699
|
+
? "Run knowledge_update to rebuild vectors with the active embedding configuration."
|
|
1700
|
+
: "Re-add inline text or re-import the original source with the active embedding configuration.",
|
|
1701
|
+
action_code: "rebuild_vectors",
|
|
1702
|
+
});
|
|
1703
|
+
}
|
|
1704
|
+
if ((kb.status === "ready" || kb.status === "stale") &&
|
|
1705
|
+
(storedSourcePolicyVersion(kb.source_options, "embedding_input_policy") ?? 0) < CURRENT_EMBEDDING_INPUT_POLICY) {
|
|
1706
|
+
const canUpdate = kb.source_path !== null && kb.source_type !== "text";
|
|
1707
|
+
issues.push({
|
|
1708
|
+
severity: "warning",
|
|
1709
|
+
kb_name: kb.name,
|
|
1710
|
+
message: "This KB predates embedding-safe chunk limits, so oversized chunk tails may be missing from vector retrieval.",
|
|
1711
|
+
action: canUpdate
|
|
1712
|
+
? "Run knowledge_update to re-chunk oversized inputs with the current embedding budget."
|
|
1713
|
+
: "Re-add inline text or re-import the original source to rebuild chunks with the current embedding budget.",
|
|
1714
|
+
action_code: "reindex_for_embedding_budget",
|
|
1715
|
+
});
|
|
1716
|
+
}
|
|
1717
|
+
if (kb.source_type === "url" &&
|
|
1718
|
+
(kb.status === "ready" || kb.status === "stale") &&
|
|
1719
|
+
(storedSourcePolicyVersion(kb.source_options, "url_structure_policy") ?? 0) < CURRENT_URL_STRUCTURE_POLICY) {
|
|
1720
|
+
issues.push({
|
|
1721
|
+
severity: "warning",
|
|
1722
|
+
kb_name: kb.name,
|
|
1723
|
+
message: "This URL KB predates structure-preserving HTML normalization.",
|
|
1724
|
+
action: "Run knowledge_update to rebuild URL headings, sections, lists, tables, and entities.",
|
|
1725
|
+
action_code: "run_update",
|
|
1726
|
+
});
|
|
1727
|
+
}
|
|
1755
1728
|
if (kb.status === "ready" && kb.chunk_count > 0 && getSymbolCount(this.db, kb.id) === 0) {
|
|
1756
1729
|
issues.push({
|
|
1757
1730
|
severity: "info",
|
|
@@ -1879,7 +1852,7 @@ export class KnowledgeEngine {
|
|
|
1879
1852
|
added_chunks: inserted,
|
|
1880
1853
|
});
|
|
1881
1854
|
onProgress?.(message);
|
|
1882
|
-
const vectors = await
|
|
1855
|
+
const vectors = await embedValidatedDocuments(batch.map((chunk) => buildChunkEmbeddingText(chunk)), signal);
|
|
1883
1856
|
throwIfAborted(signal);
|
|
1884
1857
|
assertEmbeddingBatchSize(vectors, batch.length, "import");
|
|
1885
1858
|
persistEmbeddingMetadata(this.db, kb.id, embeddingConfig, vectors);
|
|
@@ -1915,6 +1888,7 @@ export class KnowledgeEngine {
|
|
|
1915
1888
|
name: header.name,
|
|
1916
1889
|
description: header.description,
|
|
1917
1890
|
source_type: "text",
|
|
1891
|
+
source_options: serializeAddOptions(),
|
|
1918
1892
|
embedding_model: embeddingConfigLabel(embeddingConfig),
|
|
1919
1893
|
});
|
|
1920
1894
|
updateKBStatus(this.db, kb.id, "indexing");
|
|
@@ -1934,9 +1908,10 @@ export class KnowledgeEngine {
|
|
|
1934
1908
|
const imported = JSON.parse(line);
|
|
1935
1909
|
const chunk = importedChunkToInsert(imported);
|
|
1936
1910
|
chunk.content_tokenized = preTokenizeForFTS(buildChunkEmbeddingText(chunk));
|
|
1937
|
-
|
|
1911
|
+
const finalizedChunks = await finalizeChunksForEmbedding([chunk], embeddingConfig, signal);
|
|
1912
|
+
pendingChunks.push(...finalizedChunks);
|
|
1938
1913
|
pendingSymbols.push(...extractSymbols(imported.content, imported.file_path, imported.file_type));
|
|
1939
|
-
|
|
1914
|
+
while (pendingChunks.length >= embeddingBatchSize)
|
|
1940
1915
|
await flushPending();
|
|
1941
1916
|
}
|
|
1942
1917
|
if (!header)
|
|
@@ -50,7 +50,7 @@ export declare function chunkIdentityHash(opts: {
|
|
|
50
50
|
metadataJson: string;
|
|
51
51
|
}): string;
|
|
52
52
|
export declare function buildChunkEmbeddingText(chunk: Pick<ChunkInsert, "content" | "file_path" | "file_type" | "metadata_json">): string;
|
|
53
|
-
export declare function chunkMarkdown(content: string, filePath: string): Omit<ChunkInsert, "kb_id">[];
|
|
53
|
+
export declare function chunkMarkdown(content: string, filePath: string, fileType?: string): Omit<ChunkInsert, "kb_id">[];
|
|
54
54
|
export declare function chunkText(content: string, filePath: string): Omit<ChunkInsert, "kb_id">[];
|
|
55
55
|
export declare function isCodeFileType(fileType: string): boolean;
|
|
56
56
|
export declare function analyzeIndexableContent(content: string, filePath: string, fileType?: string): Promise<{
|
|
@@ -397,7 +397,7 @@ function makeChunk(content, filePath, fileType, startLine, endLine, metadata = {
|
|
|
397
397
|
};
|
|
398
398
|
return { ...chunk, content_tokenized: preTokenizeForFTS(buildChunkEmbeddingText(chunk)) };
|
|
399
399
|
}
|
|
400
|
-
export function chunkMarkdown(content, filePath) {
|
|
400
|
+
export function chunkMarkdown(content, filePath, fileType = "markdown") {
|
|
401
401
|
const lines = content.split("\n");
|
|
402
402
|
const chunks = [];
|
|
403
403
|
const headingStack = [];
|
|
@@ -410,7 +410,7 @@ export function chunkMarkdown(content, filePath) {
|
|
|
410
410
|
function pushMarkdownChunk(text, start, end) {
|
|
411
411
|
if (text.trim().length < 50)
|
|
412
412
|
return;
|
|
413
|
-
chunks.push(makeChunk(text.trim(), filePath,
|
|
413
|
+
chunks.push(makeChunk(text.trim(), filePath, fileType, start, end, {
|
|
414
414
|
heading: currentHeading,
|
|
415
415
|
breadcrumb: currentBreadcrumb(),
|
|
416
416
|
}));
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { EmbeddingConfig } from "../embedding/provider.ts";
|
|
2
|
+
import type { ChunkInsert } from "../storage/sqlite.ts";
|
|
3
|
+
type IndexableChunk = Omit<ChunkInsert, "kb_id">;
|
|
4
|
+
export declare function finalizeChunksForEmbedding(chunks: IndexableChunk[], config: EmbeddingConfig, signal?: AbortSignal): Promise<IndexableChunk[]>;
|
|
5
|
+
export {};
|