pi-knowledge 0.12.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/README.md +165 -32
- package/dist/index.js +33 -64
- package/dist/src/embedding/instructions.d.ts +33 -0
- package/dist/src/embedding/instructions.js +146 -0
- package/dist/src/embedding/provider.d.ts +27 -8
- package/dist/src/embedding/provider.js +197 -64
- package/dist/src/engine.d.ts +18 -1
- package/dist/src/engine.js +136 -170
- package/dist/src/indexer/embedding-safe-chunks.js +3 -2
- package/dist/src/model-worker-client.d.ts +4 -0
- package/dist/src/model-worker-client.js +14 -2
- package/dist/src/model-worker.js +34 -3
- package/dist/src/search/contracts.d.ts +2 -0
- package/dist/src/search/contracts.js +2 -0
- package/dist/src/search/diversity.d.ts +16 -0
- package/dist/src/search/diversity.js +183 -0
- package/dist/src/search/tuning.d.ts +10 -0
- package/dist/src/search/tuning.js +20 -0
- package/dist/src/status.d.ts +8 -0
- package/dist/src/status.js +93 -0
- package/docs/configuration.md +14 -11
- package/package.json +1 -1
package/dist/src/engine.js
CHANGED
|
@@ -1,26 +1,27 @@
|
|
|
1
1
|
import { existsSync, readFileSync, renameSync, rmSync, statSync } from "node:fs";
|
|
2
2
|
import { join, resolve } from "node:path";
|
|
3
3
|
import { diagnoseKB } from "./diagnostics/health.js";
|
|
4
|
-
import { dispose as disposeEmbedding, embedDocuments, embeddingConfigLabel, embeddingSignature, embedQueryWithConfig, embedValidatedDocuments, prepareForShutdown as prepareEmbeddingForShutdown, resolveEmbeddingConfig, } from "./embedding/provider.js";
|
|
4
|
+
import { describeEmbeddingCompatibility, describeEmbeddingDecoration, dispose as disposeEmbedding, embedDocuments, embeddingConfigLabel, embeddingSignature, embedQueryWithConfig, embedValidatedDocuments, inspectEmbeddingCompatibility, prepareForShutdown as prepareEmbeddingForShutdown, resolveEmbeddingConfig, } from "./embedding/provider.js";
|
|
5
5
|
import { openVectorReader, openVectorWriter } from "./embedding/vectors.js";
|
|
6
6
|
import { addSkippedScanEntry, analyzeIndexableContent, buildChunkEmbeddingText, chunkFile, chunkIdentityHash, chunkMarkdown, createSkippedScanStats, isReadableTextFile, isSupportedDocumentFile, iterateScannableFiles, preTokenizeForFTS, summarizeSkippedScan, } from "./indexer/chunker.js";
|
|
7
7
|
import { finalizeChunksForEmbedding } from "./indexer/embedding-safe-chunks.js";
|
|
8
8
|
import { htmlToStructuredMarkdown } from "./indexer/html.js";
|
|
9
9
|
import { extractSymbols } from "./indexer/symbols.js";
|
|
10
|
-
import { shutdownModelWorker } from "./model-worker-client.js";
|
|
10
|
+
import { extractDocumentInModelWorker, shutdownModelWorker } from "./model-worker-client.js";
|
|
11
11
|
import { searchBM25 } from "./search/bm25.js";
|
|
12
|
+
import { MAX_SEARCH_WINDOW } from "./search/contracts.js";
|
|
13
|
+
import { diversifyRankedChunks, interleaveByFile } from "./search/diversity.js";
|
|
12
14
|
import { weightedScoreFusion } from "./search/fusion.js";
|
|
13
15
|
import { normalizedQueryText, tokenizeForSearch } from "./search/query.js";
|
|
14
16
|
import { hasAnyLexicalEvidence, hasEnoughLexicalEvidence, normalizeFileTypeFilter, queryCoverage, scoreChunkForQuery, } from "./search/ranking.js";
|
|
15
17
|
import { disposeReranker, prepareRerankerForShutdown, rerank } from "./search/reranker.js";
|
|
16
|
-
import { resolveSearchTuning, summarizeSearchTuning, } from "./search/tuning.js";
|
|
18
|
+
import { resolveSearchTuning, resolveSearchWindow, summarizeSearchTuning, } from "./search/tuning.js";
|
|
17
19
|
import { searchVectorFile } from "./search/vector.js";
|
|
18
20
|
import { countSymbols, createKB, deleteChunksByIds, deleteKB, deleteSymbolsByKB, finishIndexingJob, getChunkById, getChunkCount, getChunksByFile, getFileCount, getKB, getKBByName, getSymbolCount, insertChunks, insertSymbols, iterateChunkIdsByKB, iterateChunksByKB, listKBs, openDatabase, searchSymbols, startIndexingJob, updateIndexingJob, updateKBCounts, updateKBEmbeddingMetadata, updateKBSourceOptions, updateKBStatus, } from "./storage/sqlite.js";
|
|
19
21
|
export { CURRENT_EMBEDDING_MODEL } from "./embedding/provider.js";
|
|
20
22
|
export const CURRENT_URL_STRUCTURE_POLICY = 1;
|
|
21
23
|
export const CURRENT_EMBEDDING_INPUT_POLICY = 2;
|
|
22
24
|
const DEFAULT_INDEX_EMBED_BATCH_SIZE = 64;
|
|
23
|
-
const VECTOR_REDUNDANCY_WEIGHT = 0.35;
|
|
24
25
|
function resolveIndexEmbedBatchSize(env = process.env) {
|
|
25
26
|
const raw = env.PI_KNOWLEDGE_EMBEDDING_BATCH_SIZE?.trim();
|
|
26
27
|
if (!raw)
|
|
@@ -160,116 +161,9 @@ function formatBytes(bytes) {
|
|
|
160
161
|
return `${mb.toFixed(1)} MB`;
|
|
161
162
|
return `${(mb / 1024).toFixed(2)} GB`;
|
|
162
163
|
}
|
|
163
|
-
function cosineSimilarity(a, b) {
|
|
164
|
-
if (!a || !b || a.length !== b.length)
|
|
165
|
-
return 0;
|
|
166
|
-
let dot = 0;
|
|
167
|
-
for (let i = 0; i < a.length; i++)
|
|
168
|
-
dot += a[i] * b[i];
|
|
169
|
-
return dot;
|
|
170
|
-
}
|
|
171
164
|
function tokenizeForSimilarity(text) {
|
|
172
165
|
return tokenizeForSearch(text);
|
|
173
166
|
}
|
|
174
|
-
function jaccardSimilarity(a, b) {
|
|
175
|
-
if (a.size === 0 || b.size === 0)
|
|
176
|
-
return 0;
|
|
177
|
-
let intersection = 0;
|
|
178
|
-
for (const token of a) {
|
|
179
|
-
if (b.has(token))
|
|
180
|
-
intersection++;
|
|
181
|
-
}
|
|
182
|
-
return intersection / (a.size + b.size - intersection);
|
|
183
|
-
}
|
|
184
|
-
function lineProximity(a, b) {
|
|
185
|
-
if (a.chunk.file_path !== b.chunk.file_path || a.chunk.kb_id !== b.chunk.kb_id)
|
|
186
|
-
return 0;
|
|
187
|
-
if (a.sourceChunkIds.some((id) => b.sourceChunkIds.includes(id)))
|
|
188
|
-
return 1;
|
|
189
|
-
const overlap = Math.max(0, Math.min(a.endLine, b.endLine) - Math.max(a.startLine, b.startLine) + 1);
|
|
190
|
-
if (overlap > 0)
|
|
191
|
-
return 1;
|
|
192
|
-
const gap = Math.max(a.startLine, b.startLine) - Math.min(a.endLine, b.endLine);
|
|
193
|
-
if (gap <= 20)
|
|
194
|
-
return 0.8;
|
|
195
|
-
if (gap <= 80)
|
|
196
|
-
return 0.45;
|
|
197
|
-
return 0.2;
|
|
198
|
-
}
|
|
199
|
-
function normalizeScores(candidates) {
|
|
200
|
-
const scores = candidates.map((candidate) => candidate.score);
|
|
201
|
-
const min = Math.min(...scores);
|
|
202
|
-
const max = Math.max(...scores);
|
|
203
|
-
const normalized = new Map();
|
|
204
|
-
for (const candidate of candidates) {
|
|
205
|
-
const value = max === min ? 1 : (candidate.score - min) / (max - min);
|
|
206
|
-
normalized.set(candidate.chunk.id, value);
|
|
207
|
-
}
|
|
208
|
-
return normalized;
|
|
209
|
-
}
|
|
210
|
-
function diversifyRankedChunks(candidates, diversity, vectorsByChunkId = new Map()) {
|
|
211
|
-
if (diversity === "off" || candidates.length <= 2)
|
|
212
|
-
return candidates;
|
|
213
|
-
const lambda = diversity === "strong" ? 0.62 : 0.76;
|
|
214
|
-
const normalized = normalizeScores(candidates);
|
|
215
|
-
const tokenSets = new Map(candidates.map((candidate) => [candidate.chunk.id, tokenizeForSimilarity(candidate.content)]));
|
|
216
|
-
const selected = [];
|
|
217
|
-
const remaining = [...candidates];
|
|
218
|
-
while (remaining.length > 0) {
|
|
219
|
-
let bestIndex = 0;
|
|
220
|
-
let bestScore = Number.NEGATIVE_INFINITY;
|
|
221
|
-
for (let i = 0; i < remaining.length; i++) {
|
|
222
|
-
const candidate = remaining[i];
|
|
223
|
-
let redundancy = 0;
|
|
224
|
-
for (const chosen of selected) {
|
|
225
|
-
const lexical = jaccardSimilarity(tokenSets.get(candidate.chunk.id) ?? new Set(), tokenSets.get(chosen.chunk.id) ?? new Set());
|
|
226
|
-
const vector = Math.max(0, cosineSimilarity(vectorsByChunkId.get(candidate.chunk.id), vectorsByChunkId.get(chosen.chunk.id)));
|
|
227
|
-
redundancy = Math.max(redundancy, Math.max(lexical, lineProximity(candidate, chosen), vector * VECTOR_REDUNDANCY_WEIGHT));
|
|
228
|
-
}
|
|
229
|
-
const relevance = normalized.get(candidate.chunk.id) ?? 0;
|
|
230
|
-
const mmrScore = lambda * relevance - (1 - lambda) * redundancy;
|
|
231
|
-
if (mmrScore > bestScore) {
|
|
232
|
-
bestScore = mmrScore;
|
|
233
|
-
bestIndex = i;
|
|
234
|
-
}
|
|
235
|
-
}
|
|
236
|
-
selected.push(remaining.splice(bestIndex, 1)[0]);
|
|
237
|
-
}
|
|
238
|
-
return selected;
|
|
239
|
-
}
|
|
240
|
-
function interleaveByFile(candidates, diversity) {
|
|
241
|
-
if (diversity === "off" || candidates.length <= 2)
|
|
242
|
-
return candidates;
|
|
243
|
-
const buckets = new Map();
|
|
244
|
-
const fileOrder = [];
|
|
245
|
-
for (const candidate of candidates) {
|
|
246
|
-
const key = `${candidate.chunk.kb_id}:${candidate.chunk.file_path}`;
|
|
247
|
-
if (!buckets.has(key)) {
|
|
248
|
-
buckets.set(key, []);
|
|
249
|
-
fileOrder.push(key);
|
|
250
|
-
}
|
|
251
|
-
buckets.get(key)?.push(candidate);
|
|
252
|
-
}
|
|
253
|
-
if (fileOrder.length <= 1)
|
|
254
|
-
return candidates;
|
|
255
|
-
const result = [];
|
|
256
|
-
let round = 0;
|
|
257
|
-
while (result.length < candidates.length) {
|
|
258
|
-
let added = false;
|
|
259
|
-
for (const key of fileOrder) {
|
|
260
|
-
const bucket = buckets.get(key);
|
|
261
|
-
const item = bucket?.[round];
|
|
262
|
-
if (!item)
|
|
263
|
-
continue;
|
|
264
|
-
result.push(item);
|
|
265
|
-
added = true;
|
|
266
|
-
}
|
|
267
|
-
if (!added)
|
|
268
|
-
break;
|
|
269
|
-
round++;
|
|
270
|
-
}
|
|
271
|
-
return result;
|
|
272
|
-
}
|
|
273
167
|
function buildQuerySnippet(content, query, maxLength = 240) {
|
|
274
168
|
const terms = [...tokenizeForSimilarity(query)].sort((a, b) => b.length - a.length);
|
|
275
169
|
const lower = content.toLowerCase();
|
|
@@ -476,30 +370,19 @@ async function chunkUrl(source, signal) {
|
|
|
476
370
|
const structuredChunks = chunkMarkdown(text, source, "html");
|
|
477
371
|
return structuredChunks.length > 0 ? structuredChunks : chunkFile(text, source);
|
|
478
372
|
}
|
|
479
|
-
function normalizeExtractedText(text) {
|
|
480
|
-
return Array.isArray(text) ? text.join("\n\n") : text;
|
|
481
|
-
}
|
|
482
373
|
async function extractSourceFileContent(filePath, signal) {
|
|
483
374
|
const lowerPath = filePath.toLowerCase();
|
|
484
375
|
if (lowerPath.endsWith(".pdf")) {
|
|
485
376
|
throwIfAborted(signal);
|
|
486
|
-
|
|
487
|
-
const { extractText } = await import("unpdf");
|
|
377
|
+
const extracted = await extractDocumentInModelWorker(filePath, "pdf", signal);
|
|
488
378
|
throwIfAborted(signal);
|
|
489
|
-
|
|
490
|
-
throwIfAborted(signal);
|
|
491
|
-
const { text } = await extractText(new Uint8Array(buf));
|
|
492
|
-
throwIfAborted(signal);
|
|
493
|
-
return { content: normalizeExtractedText(text), fileType: "pdf" };
|
|
379
|
+
return extracted;
|
|
494
380
|
}
|
|
495
381
|
if (lowerPath.endsWith(".docx") || lowerPath.endsWith(".doc")) {
|
|
496
382
|
throwIfAborted(signal);
|
|
497
|
-
|
|
498
|
-
const mammoth = await import("mammoth");
|
|
383
|
+
const extracted = await extractDocumentInModelWorker(filePath, "docx", signal);
|
|
499
384
|
throwIfAborted(signal);
|
|
500
|
-
|
|
501
|
-
throwIfAborted(signal);
|
|
502
|
-
return { content: result.value, fileType: "docx" };
|
|
385
|
+
return extracted;
|
|
503
386
|
}
|
|
504
387
|
if (!isReadableTextFile(filePath)) {
|
|
505
388
|
throw new Error(`File is not readable text and has no supported extractor: ${filePath}`);
|
|
@@ -557,15 +440,8 @@ function persistEmbeddingMetadata(db, kbId, config, vectors) {
|
|
|
557
440
|
updateKBEmbeddingMetadata(db, kbId, embeddingConfigLabel(config), embeddingSignature(config, dimension), dimension);
|
|
558
441
|
return dimension;
|
|
559
442
|
}
|
|
560
|
-
function embeddingMismatchWarning(kb) {
|
|
561
|
-
return `"${kb.name}" has incompatible embedding
|
|
562
|
-
}
|
|
563
|
-
function canSearchVectors(kb, config, queryDimension) {
|
|
564
|
-
if (kb.embedding_signature === null || kb.embedding_dimension === null)
|
|
565
|
-
return false;
|
|
566
|
-
if (kb.embedding_dimension !== queryDimension)
|
|
567
|
-
return false;
|
|
568
|
-
return kb.embedding_signature === embeddingSignature(config, queryDimension);
|
|
443
|
+
function embeddingMismatchWarning(kb, compatibility) {
|
|
444
|
+
return `"${kb.name}" has incompatible embedding vectors (${describeEmbeddingCompatibility(compatibility)}); vector retrieval was skipped and knowledge_update should rebuild them`;
|
|
569
445
|
}
|
|
570
446
|
function throwIfAborted(signal) {
|
|
571
447
|
if (signal?.aborted)
|
|
@@ -950,14 +826,17 @@ export class KnowledgeEngine {
|
|
|
950
826
|
if (!this.db)
|
|
951
827
|
throw new Error("Engine not initialized");
|
|
952
828
|
throwIfAborted(signal);
|
|
953
|
-
updateKBStatus(this.db, kb.id, "indexing");
|
|
954
|
-
startIndexingJob(this.db, kb.id, "update", `Starting update for "${kb.name}"`);
|
|
955
829
|
const scanOptions = toScanOptions(parseAddOptions(kb.source_options));
|
|
956
830
|
const embeddingConfig = resolveEmbeddingConfig();
|
|
957
831
|
const embeddingBatchSize = resolveIndexEmbedBatchSize();
|
|
832
|
+
updateKBStatus(this.db, kb.id, "indexing");
|
|
833
|
+
startIndexingJob(this.db, kb.id, "update", `Starting update for "${kb.name}"`);
|
|
958
834
|
const embeddingModel = embeddingConfigLabel(embeddingConfig);
|
|
959
|
-
const
|
|
960
|
-
let canReuseExistingVectors =
|
|
835
|
+
const compatibility = inspectEmbeddingCompatibility(kb.embedding_signature, kb.embedding_dimension, embeddingConfig, kb.embedding_dimension ?? 0);
|
|
836
|
+
let canReuseExistingVectors = compatibility.compatible;
|
|
837
|
+
let rebuildMessage = compatibility.compatible
|
|
838
|
+
? undefined
|
|
839
|
+
: `Embedding vectors for "${kb.name}" are incompatible (${describeEmbeddingCompatibility(compatibility)}); rebuilding all vectors`;
|
|
961
840
|
let replacementVectorPath;
|
|
962
841
|
let addedVectorPath;
|
|
963
842
|
let addedVectorWriter;
|
|
@@ -986,9 +865,7 @@ export class KnowledgeEngine {
|
|
|
986
865
|
canReuseExistingVectors = false;
|
|
987
866
|
const vectorCount = oldVectorReader?.count ?? 0;
|
|
988
867
|
const vectorDimension = oldVectorReader?.dim ?? 0;
|
|
989
|
-
|
|
990
|
-
updateIndexingJob(this.db, kb.id, { phase: "embedding", message });
|
|
991
|
-
onProgress?.(message);
|
|
868
|
+
rebuildMessage = `Stored vector file for "${kb.name}" is incomplete or incompatible (${vectorCount} vectors/${vectorDimension}d for ${existingIndex} chunks/${kb.embedding_dimension}d); rebuilding all vectors`;
|
|
992
869
|
}
|
|
993
870
|
}
|
|
994
871
|
finally {
|
|
@@ -998,10 +875,9 @@ export class KnowledgeEngine {
|
|
|
998
875
|
const reusableHashes = canReuseExistingVectors
|
|
999
876
|
? existingHashes
|
|
1000
877
|
: new Map();
|
|
1001
|
-
if (!canReuseExistingVectors) {
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
onProgress?.(message);
|
|
878
|
+
if (!canReuseExistingVectors && rebuildMessage) {
|
|
879
|
+
updateIndexingJob(this.db, kb.id, { phase: "embedding", message: rebuildMessage });
|
|
880
|
+
onProgress?.(rebuildMessage);
|
|
1005
881
|
}
|
|
1006
882
|
const oldVectorIndexByHash = new Map();
|
|
1007
883
|
const newVectorIndexByHash = new Map();
|
|
@@ -1326,7 +1202,7 @@ export class KnowledgeEngine {
|
|
|
1326
1202
|
};
|
|
1327
1203
|
}
|
|
1328
1204
|
const db = this.db;
|
|
1329
|
-
const { mode = "hybrid", offset
|
|
1205
|
+
const { mode = "hybrid", offset: requestedOffset, kb_id, filters, diversity = "balanced" } = options;
|
|
1330
1206
|
const resolvedMode = retrievalModeFor(mode);
|
|
1331
1207
|
const retrievalMode = resolvedMode === "adaptive" ? "hybrid" : resolvedMode;
|
|
1332
1208
|
const normalizedQuery = normalizedQueryText(query);
|
|
@@ -1348,10 +1224,9 @@ export class KnowledgeEngine {
|
|
|
1348
1224
|
profile: options.profile,
|
|
1349
1225
|
kbSourceTypes: kbs.map((kb) => kb.source_type),
|
|
1350
1226
|
});
|
|
1351
|
-
const
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
const candidateLimit = Math.max(tuning.candidateMin, offset + limit * tuning.candidateMultiplier);
|
|
1227
|
+
const { limit, offset } = resolveSearchWindow(options.limit, requestedOffset, tuning.defaultLimit);
|
|
1228
|
+
const candidateLimit = Math.min(MAX_SEARCH_WINDOW, Math.max(tuning.candidateMin, offset + limit * tuning.candidateMultiplier));
|
|
1229
|
+
const retrievalLimitPerKb = candidateLimit;
|
|
1355
1230
|
const tuningSummary = summarizeSearchTuning(tuning, limit, candidateLimit);
|
|
1356
1231
|
if (kbs.length === 0) {
|
|
1357
1232
|
return {
|
|
@@ -1365,45 +1240,50 @@ export class KnowledgeEngine {
|
|
|
1365
1240
|
}
|
|
1366
1241
|
const allResults = [];
|
|
1367
1242
|
const vectorsByChunkId = new Map();
|
|
1243
|
+
let queryEmbedding;
|
|
1368
1244
|
for (const kb of kbs) {
|
|
1369
1245
|
throwIfAborted(signal);
|
|
1370
1246
|
if (kb.chunk_count === 0)
|
|
1371
1247
|
continue;
|
|
1372
1248
|
const vectorPath = this.vectorPathFor(kb.id);
|
|
1373
1249
|
if (retrievalMode === "fast") {
|
|
1374
|
-
allResults.push(...searchBM25(db, normalizedQuery || query,
|
|
1250
|
+
allResults.push(...searchBM25(db, normalizedQuery || query, retrievalLimitPerKb, kb.id, { allowOrFallback: false }).map((result) => ({
|
|
1375
1251
|
...result,
|
|
1376
1252
|
score: result.score * kbTrustMultiplier(kb),
|
|
1377
1253
|
})));
|
|
1378
1254
|
}
|
|
1379
1255
|
else if (retrievalMode === "semantic") {
|
|
1380
|
-
|
|
1256
|
+
queryEmbedding ??= await embedQueryWithConfig(query, signal);
|
|
1257
|
+
const { vector: queryVec, config: queryEmbeddingConfig } = queryEmbedding;
|
|
1381
1258
|
throwIfAborted(signal);
|
|
1382
|
-
|
|
1383
|
-
|
|
1259
|
+
const compatibility = inspectEmbeddingCompatibility(kb.embedding_signature, kb.embedding_dimension, queryEmbeddingConfig, queryVec.length);
|
|
1260
|
+
if (!compatibility.compatible) {
|
|
1261
|
+
warnings.push(embeddingMismatchWarning(kb, compatibility));
|
|
1384
1262
|
continue;
|
|
1385
1263
|
}
|
|
1386
|
-
const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id),
|
|
1264
|
+
const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), retrievalLimitPerKb, signal);
|
|
1387
1265
|
allResults.push(...vectorResults.results.map((result) => ({ ...result, score: result.score * kbTrustMultiplier(kb) })));
|
|
1388
1266
|
for (const [chunkId, vector] of vectorResults.vectorsByChunkId)
|
|
1389
1267
|
vectorsByChunkId.set(chunkId, vector);
|
|
1390
1268
|
}
|
|
1391
1269
|
else {
|
|
1392
1270
|
const semanticHybrid = retrievalMode === "semantic_hybrid";
|
|
1393
|
-
const bm25Results = searchBM25(db, normalizedQuery || query,
|
|
1271
|
+
const bm25Results = searchBM25(db, normalizedQuery || query, retrievalLimitPerKb, kb.id);
|
|
1394
1272
|
if (bm25Results.length === 0 && !semanticHybrid)
|
|
1395
1273
|
continue;
|
|
1396
1274
|
let vecResults = [];
|
|
1397
|
-
|
|
1275
|
+
queryEmbedding ??= await embedQueryWithConfig(query, signal);
|
|
1276
|
+
const { vector: queryVec, config: queryEmbeddingConfig } = queryEmbedding;
|
|
1398
1277
|
throwIfAborted(signal);
|
|
1399
|
-
|
|
1400
|
-
|
|
1278
|
+
const compatibility = inspectEmbeddingCompatibility(kb.embedding_signature, kb.embedding_dimension, queryEmbeddingConfig, queryVec.length);
|
|
1279
|
+
if (compatibility.compatible) {
|
|
1280
|
+
const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), retrievalLimitPerKb, signal);
|
|
1401
1281
|
vecResults = vectorResults.results;
|
|
1402
1282
|
for (const [chunkId, vector] of vectorResults.vectorsByChunkId)
|
|
1403
1283
|
vectorsByChunkId.set(chunkId, vector);
|
|
1404
1284
|
}
|
|
1405
1285
|
else {
|
|
1406
|
-
warnings.push(embeddingMismatchWarning(kb));
|
|
1286
|
+
warnings.push(embeddingMismatchWarning(kb, compatibility));
|
|
1407
1287
|
}
|
|
1408
1288
|
const fused = weightedScoreFusion(bm25Results, vecResults);
|
|
1409
1289
|
allResults.push(...fused.map((result) => ({ ...result, score: result.score * kbTrustMultiplier(kb) })));
|
|
@@ -1461,8 +1341,15 @@ export class KnowledgeEngine {
|
|
|
1461
1341
|
return true;
|
|
1462
1342
|
});
|
|
1463
1343
|
}
|
|
1464
|
-
|
|
1465
|
-
|
|
1344
|
+
const candidatePool = filtered.slice(0, candidateLimit);
|
|
1345
|
+
const candidatePoolTruncated = candidatePool.length < filtered.length;
|
|
1346
|
+
const candidateIds = new Set(candidatePool.map((candidate) => candidate.chunkId));
|
|
1347
|
+
for (const chunkId of vectorsByChunkId.keys()) {
|
|
1348
|
+
if (!candidateIds.has(chunkId))
|
|
1349
|
+
vectorsByChunkId.delete(chunkId);
|
|
1350
|
+
}
|
|
1351
|
+
if (mode === "deep" && candidatePool.length > 0) {
|
|
1352
|
+
const candidates = candidatePool
|
|
1466
1353
|
.slice(0, tuning.deepRerankCandidates)
|
|
1467
1354
|
.map((r) => {
|
|
1468
1355
|
const chunk = getChunkById(db, r.chunkId);
|
|
@@ -1489,7 +1376,7 @@ export class KnowledgeEngine {
|
|
|
1489
1376
|
sourceChunkIds: [chunk.id],
|
|
1490
1377
|
});
|
|
1491
1378
|
}
|
|
1492
|
-
const diversified = interleaveByFile(diversifyRankedChunks(ranked, diversity, vectorsByChunkId), diversity);
|
|
1379
|
+
const diversified = interleaveByFile(await diversifyRankedChunks(ranked, diversity, vectorsByChunkId, signal), diversity, signal);
|
|
1493
1380
|
const page = diversified.slice(offset, offset + limit);
|
|
1494
1381
|
const results = page.map((r) => {
|
|
1495
1382
|
const kb = kbById.get(r.chunk.kb_id);
|
|
@@ -1521,11 +1408,17 @@ export class KnowledgeEngine {
|
|
|
1521
1408
|
has_more: offset + limit < diversified.length,
|
|
1522
1409
|
warnings: warnings.length > 0 ? warnings : undefined,
|
|
1523
1410
|
mode_used: mode,
|
|
1524
|
-
tuning:
|
|
1411
|
+
tuning: {
|
|
1412
|
+
...tuningSummary,
|
|
1413
|
+
retrieved_candidates: unique.length,
|
|
1414
|
+
filtered_candidates: filtered.length,
|
|
1415
|
+
diversity_candidates: ranked.length,
|
|
1416
|
+
candidate_pool_truncated: candidatePoolTruncated,
|
|
1417
|
+
},
|
|
1525
1418
|
};
|
|
1526
1419
|
}
|
|
1527
1420
|
const ranked = [];
|
|
1528
|
-
for (const r of
|
|
1421
|
+
for (const r of candidatePool) {
|
|
1529
1422
|
throwIfAborted(signal);
|
|
1530
1423
|
const chunk = getChunkById(db, r.chunkId);
|
|
1531
1424
|
if (!chunk)
|
|
@@ -1563,7 +1456,7 @@ export class KnowledgeEngine {
|
|
|
1563
1456
|
});
|
|
1564
1457
|
}
|
|
1565
1458
|
}
|
|
1566
|
-
const diversified = interleaveByFile(diversifyRankedChunks(ranked, diversity, vectorsByChunkId), diversity);
|
|
1459
|
+
const diversified = interleaveByFile(await diversifyRankedChunks(ranked, diversity, vectorsByChunkId, signal), diversity, signal);
|
|
1567
1460
|
const total = diversified.length;
|
|
1568
1461
|
const page = diversified.slice(offset, offset + limit);
|
|
1569
1462
|
const results = page.map((r) => ({
|
|
@@ -1596,7 +1489,13 @@ export class KnowledgeEngine {
|
|
|
1596
1489
|
has_more: offset + limit < total,
|
|
1597
1490
|
warnings: warnings.length > 0 ? warnings : undefined,
|
|
1598
1491
|
mode_used: mode,
|
|
1599
|
-
tuning:
|
|
1492
|
+
tuning: {
|
|
1493
|
+
...tuningSummary,
|
|
1494
|
+
retrieved_candidates: unique.length,
|
|
1495
|
+
filtered_candidates: filtered.length,
|
|
1496
|
+
diversity_candidates: ranked.length,
|
|
1497
|
+
candidate_pool_truncated: candidatePoolTruncated,
|
|
1498
|
+
},
|
|
1600
1499
|
suggestions: results.length === 0
|
|
1601
1500
|
? [
|
|
1602
1501
|
"Try mode 'fast' for exact symbols, mode 'semantic' for pure vector search, or mode 'semantic_hybrid' when weak lexical overlap should not block fused vector results.",
|
|
@@ -1647,6 +1546,46 @@ export class KnowledgeEngine {
|
|
|
1647
1546
|
return diagnoseKB(db, kb, signal);
|
|
1648
1547
|
});
|
|
1649
1548
|
}
|
|
1549
|
+
statusSnapshot(signal) {
|
|
1550
|
+
if (!this.db)
|
|
1551
|
+
return { knowledge_bases: [], diagnostics: [], embeddings: [] };
|
|
1552
|
+
throwIfAborted(signal);
|
|
1553
|
+
const db = this.db;
|
|
1554
|
+
const knowledgeBases = listKBs(db);
|
|
1555
|
+
const diagnostics = knowledgeBases.map((kb) => {
|
|
1556
|
+
throwIfAborted(signal);
|
|
1557
|
+
return diagnoseKB(db, kb, signal);
|
|
1558
|
+
});
|
|
1559
|
+
try {
|
|
1560
|
+
const config = resolveEmbeddingConfig();
|
|
1561
|
+
const activeModel = embeddingConfigLabel(config);
|
|
1562
|
+
const activeDecoration = describeEmbeddingDecoration(config);
|
|
1563
|
+
return {
|
|
1564
|
+
knowledge_bases: knowledgeBases,
|
|
1565
|
+
diagnostics,
|
|
1566
|
+
embeddings: knowledgeBases.map((kb) => {
|
|
1567
|
+
const compatibility = inspectEmbeddingCompatibility(kb.embedding_signature, kb.embedding_dimension, config, kb.embedding_dimension ?? 0);
|
|
1568
|
+
return {
|
|
1569
|
+
kb_id: kb.id,
|
|
1570
|
+
kb_name: kb.name,
|
|
1571
|
+
indexed_model: kb.embedding_model,
|
|
1572
|
+
active_model: activeModel,
|
|
1573
|
+
active_decoration: activeDecoration,
|
|
1574
|
+
compatibility,
|
|
1575
|
+
detail: describeEmbeddingCompatibility(compatibility),
|
|
1576
|
+
};
|
|
1577
|
+
}),
|
|
1578
|
+
};
|
|
1579
|
+
}
|
|
1580
|
+
catch (error) {
|
|
1581
|
+
return {
|
|
1582
|
+
knowledge_bases: knowledgeBases,
|
|
1583
|
+
diagnostics,
|
|
1584
|
+
embeddings: [],
|
|
1585
|
+
embedding_error: error instanceof Error ? error.message : String(error),
|
|
1586
|
+
};
|
|
1587
|
+
}
|
|
1588
|
+
}
|
|
1650
1589
|
symbolSearch(query, options = {}, signal) {
|
|
1651
1590
|
if (!this.db)
|
|
1652
1591
|
throw new Error("Engine not initialized");
|
|
@@ -1768,8 +1707,35 @@ export class KnowledgeEngine {
|
|
|
1768
1707
|
}
|
|
1769
1708
|
}
|
|
1770
1709
|
if (this.db) {
|
|
1710
|
+
let activeEmbeddingConfig;
|
|
1711
|
+
try {
|
|
1712
|
+
activeEmbeddingConfig = resolveEmbeddingConfig();
|
|
1713
|
+
}
|
|
1714
|
+
catch (error) {
|
|
1715
|
+
issues.push({
|
|
1716
|
+
severity: "blocking",
|
|
1717
|
+
message: `Active embedding configuration is invalid: ${error instanceof Error ? error.message : String(error)}`,
|
|
1718
|
+
action: "Correct the embedding environment variables, then run knowledge_doctor again.",
|
|
1719
|
+
action_code: "fix_embedding_config",
|
|
1720
|
+
});
|
|
1721
|
+
}
|
|
1771
1722
|
for (const kb of kbs) {
|
|
1772
1723
|
throwIfAborted(signal);
|
|
1724
|
+
if (activeEmbeddingConfig && (kb.status === "ready" || kb.status === "stale") && kb.chunk_count > 0) {
|
|
1725
|
+
const compatibility = inspectEmbeddingCompatibility(kb.embedding_signature, kb.embedding_dimension, activeEmbeddingConfig, kb.embedding_dimension ?? 0);
|
|
1726
|
+
if (!compatibility.compatible) {
|
|
1727
|
+
const canUpdate = kb.source_path !== null && kb.source_type !== "text";
|
|
1728
|
+
issues.push({
|
|
1729
|
+
severity: "warning",
|
|
1730
|
+
kb_name: kb.name,
|
|
1731
|
+
message: `This KB's vectors are incompatible with the active embedding configuration: ${describeEmbeddingCompatibility(compatibility)}.`,
|
|
1732
|
+
action: canUpdate
|
|
1733
|
+
? "Run knowledge_update to rebuild vectors with the active embedding configuration."
|
|
1734
|
+
: "Re-add inline text or re-import the original source with the active embedding configuration.",
|
|
1735
|
+
action_code: "rebuild_vectors",
|
|
1736
|
+
});
|
|
1737
|
+
}
|
|
1738
|
+
}
|
|
1773
1739
|
if ((kb.status === "ready" || kb.status === "stale") &&
|
|
1774
1740
|
(storedSourcePolicyVersion(kb.source_options, "embedding_input_policy") ?? 0) < CURRENT_EMBEDDING_INPUT_POLICY) {
|
|
1775
1741
|
const canUpdate = kb.source_path !== null && kb.source_type !== "text";
|
|
@@ -1893,10 +1859,10 @@ export class KnowledgeEngine {
|
|
|
1893
1859
|
throwIfAborted(signal);
|
|
1894
1860
|
const { createReadStream } = await import("node:fs");
|
|
1895
1861
|
const { createInterface } = await import("node:readline");
|
|
1896
|
-
const stream = createReadStream(inputPath, { encoding: "utf-8" });
|
|
1897
|
-
const lines = createInterface({ input: stream, crlfDelay: Infinity });
|
|
1898
1862
|
const embeddingConfig = resolveEmbeddingConfig();
|
|
1899
1863
|
const embeddingBatchSize = resolveIndexEmbedBatchSize();
|
|
1864
|
+
const stream = createReadStream(inputPath, { encoding: "utf-8" });
|
|
1865
|
+
const lines = createInterface({ input: stream, crlfDelay: Infinity });
|
|
1900
1866
|
let header;
|
|
1901
1867
|
let kb;
|
|
1902
1868
|
let vectorWriter;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { embeddingInstructionLength } from "../embedding/instructions.js";
|
|
1
2
|
import { LOCAL_EMBEDDING_PREFERRED_TOKENS, measureEmbeddingInputs } from "../embedding/provider.js";
|
|
2
3
|
import { buildChunkEmbeddingText, chunkIdentityHash, preTokenizeForFTS } from "./chunker.js";
|
|
3
4
|
const SENTENCE_BOUNDARIES = {
|
|
@@ -153,8 +154,8 @@ function withPartMetadata(chunk, part, partCount) {
|
|
|
153
154
|
function isProvablySafeLocalAsciiInput(text, config) {
|
|
154
155
|
if (config.provider !== "local")
|
|
155
156
|
return false;
|
|
156
|
-
const
|
|
157
|
-
if (text.length +
|
|
157
|
+
const instructionLength = embeddingInstructionLength(config.instructions, "passage");
|
|
158
|
+
if (text.length + instructionLength + 2 > LOCAL_EMBEDDING_PREFERRED_TOKENS)
|
|
158
159
|
return false;
|
|
159
160
|
for (let index = 0; index < text.length; index++) {
|
|
160
161
|
if (text.charCodeAt(index) > 0x7f)
|
|
@@ -7,6 +7,10 @@ export declare function measureEmbeddingInputInModelWorker(texts: string[], pref
|
|
|
7
7
|
counts: number[];
|
|
8
8
|
maximum: number;
|
|
9
9
|
}>;
|
|
10
|
+
export declare function extractDocumentInModelWorker(filePath: string, fileType: "pdf" | "docx", signal?: AbortSignal): Promise<{
|
|
11
|
+
content: string;
|
|
12
|
+
fileType: "pdf" | "docx";
|
|
13
|
+
}>;
|
|
10
14
|
export interface RerankWorkerCandidate {
|
|
11
15
|
chunkId: string;
|
|
12
16
|
content: string;
|
|
@@ -39,7 +39,7 @@ function validateNodeExecPath(candidate) {
|
|
|
39
39
|
}).trim();
|
|
40
40
|
const major = parseNodeMajor(version);
|
|
41
41
|
if (major === null || major < MIN_NODE_MAJOR) {
|
|
42
|
-
throw new Error(`Node ${MIN_NODE_MAJOR}+ is required for the
|
|
42
|
+
throw new Error(`Node ${MIN_NODE_MAJOR}+ is required for the isolated runtime worker; ${candidate} reported ${version}`);
|
|
43
43
|
}
|
|
44
44
|
return candidate;
|
|
45
45
|
}
|
|
@@ -149,7 +149,7 @@ function getNodeExecPath() {
|
|
|
149
149
|
// Try the next candidate before surfacing a deterministic error below.
|
|
150
150
|
}
|
|
151
151
|
}
|
|
152
|
-
throw new Error(`Node ${MIN_NODE_MAJOR}+ is required for
|
|
152
|
+
throw new Error(`Node ${MIN_NODE_MAJOR}+ is required for isolated pi-knowledge worker operations such as local models and PDF/DOCX extraction, but no usable node executable was found. Install Node ${MIN_NODE_MAJOR}+ or set PI_KNOWLEDGE_NODE_PATH to the full node.exe path.`);
|
|
153
153
|
}
|
|
154
154
|
export function validateModelWorkerNodePath(nodePath) {
|
|
155
155
|
return validateNodeExecPath(normalizeExecutableCandidate(nodePath));
|
|
@@ -413,6 +413,18 @@ export async function measureEmbeddingInputInModelWorker(texts, prefix, prefixes
|
|
|
413
413
|
}
|
|
414
414
|
return { counts: result.counts, maximum: result.maximum };
|
|
415
415
|
}
|
|
416
|
+
export async function extractDocumentInModelWorker(filePath, fileType, signal) {
|
|
417
|
+
const result = await getWorker().request({ type: "extract_document", filePath, fileType }, signal);
|
|
418
|
+
if (typeof result !== "object" ||
|
|
419
|
+
result === null ||
|
|
420
|
+
!("content" in result) ||
|
|
421
|
+
typeof result.content !== "string" ||
|
|
422
|
+
!("fileType" in result) ||
|
|
423
|
+
result.fileType !== fileType) {
|
|
424
|
+
throw new Error("Invalid document extraction worker response");
|
|
425
|
+
}
|
|
426
|
+
return { content: result.content, fileType };
|
|
427
|
+
}
|
|
416
428
|
export async function rerankInModelWorker(query, candidates, topK, reranker, signal) {
|
|
417
429
|
const result = await getWorker().request({ type: "rerank", query, candidates, topK, reranker }, signal);
|
|
418
430
|
if (!Array.isArray(result))
|
package/dist/src/model-worker.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
|
+
import { readFile } from "node:fs/promises";
|
|
1
2
|
import { join } from "node:path";
|
|
2
3
|
import { createInterface } from "node:readline";
|
|
4
|
+
import { decorateEmbeddingInput, E5_INSTRUCTIONS, NO_INSTRUCTIONS, } from "./embedding/instructions.js";
|
|
3
5
|
import { DEFAULT_RERANKER_REMOTE_HOST, DEFAULT_RERANKER_REMOTE_PATH_TEMPLATE, isHfRerankerConfig, rerankerCacheKey, resolveRerankerConfig, } from "./search/reranker-config.js";
|
|
4
6
|
import { rerankWithSingleRawLogit } from "./search/reranker-logits.js";
|
|
5
7
|
import { getDefaultKnowledgeDir } from "./storage/sqlite.js";
|
|
@@ -76,8 +78,8 @@ async function loadRerankerPipeline(config) {
|
|
|
76
78
|
rerankerPipeline = { key, pipe: loaded };
|
|
77
79
|
return loaded;
|
|
78
80
|
}
|
|
79
|
-
function applyEmbeddingPrefix(text,
|
|
80
|
-
return prefixes === "on" ?
|
|
81
|
+
function applyEmbeddingPrefix(text, role, prefixes) {
|
|
82
|
+
return decorateEmbeddingInput(text, role, prefixes === "on" ? E5_INSTRUCTIONS : NO_INSTRUCTIONS);
|
|
81
83
|
}
|
|
82
84
|
async function handleEmbed(request) {
|
|
83
85
|
const pipe = await loadEmbeddingPipeline();
|
|
@@ -130,6 +132,24 @@ async function handleRerank(request) {
|
|
|
130
132
|
results.sort((a, b) => b.score - a.score);
|
|
131
133
|
return results.slice(0, request.topK);
|
|
132
134
|
}
|
|
135
|
+
const PDFJS_ERRORS_ONLY_VERBOSITY = 0;
|
|
136
|
+
async function handleExtractDocument(request) {
|
|
137
|
+
if (request.fileType === "pdf") {
|
|
138
|
+
const { extractText, getDocumentProxy } = await import("unpdf");
|
|
139
|
+
const bytes = await readFile(request.filePath);
|
|
140
|
+
const pdf = await getDocumentProxy(new Uint8Array(bytes), { verbosity: PDFJS_ERRORS_ONLY_VERBOSITY });
|
|
141
|
+
try {
|
|
142
|
+
const { text } = await extractText(pdf);
|
|
143
|
+
return { content: Array.isArray(text) ? text.join("\n\n") : text, fileType: "pdf" };
|
|
144
|
+
}
|
|
145
|
+
finally {
|
|
146
|
+
await pdf.destroy();
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
const mammoth = await import("mammoth");
|
|
150
|
+
const result = await mammoth.extractRawText({ path: request.filePath });
|
|
151
|
+
return { content: result.value, fileType: "docx" };
|
|
152
|
+
}
|
|
133
153
|
function isStringArray(value) {
|
|
134
154
|
return Array.isArray(value) && value.every((item) => typeof item === "string");
|
|
135
155
|
}
|
|
@@ -166,6 +186,15 @@ function parseWorkerRequest(message) {
|
|
|
166
186
|
prefixes,
|
|
167
187
|
};
|
|
168
188
|
}
|
|
189
|
+
if (message.type === "extract_document") {
|
|
190
|
+
if (!("filePath" in message) || typeof message.filePath !== "string") {
|
|
191
|
+
throw new Error("Invalid document file path");
|
|
192
|
+
}
|
|
193
|
+
if (!("fileType" in message) || (message.fileType !== "pdf" && message.fileType !== "docx")) {
|
|
194
|
+
throw new Error("Invalid document file type");
|
|
195
|
+
}
|
|
196
|
+
return { id: message.id, type: "extract_document", filePath: message.filePath, fileType: message.fileType };
|
|
197
|
+
}
|
|
169
198
|
if (message.type === "rerank") {
|
|
170
199
|
if (!("query" in message) || typeof message.query !== "string")
|
|
171
200
|
throw new Error("Invalid rerank query");
|
|
@@ -199,7 +228,9 @@ async function handleRequestMessage(message) {
|
|
|
199
228
|
? await handleEmbed(request)
|
|
200
229
|
: request.type === "measure_embedding_input"
|
|
201
230
|
? await handleMeasureEmbeddingInput(request)
|
|
202
|
-
:
|
|
231
|
+
: request.type === "extract_document"
|
|
232
|
+
? await handleExtractDocument(request)
|
|
233
|
+
: await handleRerank(request);
|
|
203
234
|
return { id: request.id, result };
|
|
204
235
|
}
|
|
205
236
|
catch (error) {
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { Chunk } from "../storage/sqlite.ts";
|
|
2
|
+
import type { RankingDiagnostics } from "./ranking.ts";
|
|
3
|
+
export type DiversityMode = "off" | "balanced" | "strong" | undefined;
|
|
4
|
+
export interface RankedChunk {
|
|
5
|
+
chunk: Chunk;
|
|
6
|
+
kbName: string;
|
|
7
|
+
ranking?: RankingDiagnostics;
|
|
8
|
+
score: number;
|
|
9
|
+
content: string;
|
|
10
|
+
snippet: string;
|
|
11
|
+
startLine: number;
|
|
12
|
+
endLine: number;
|
|
13
|
+
sourceChunkIds: string[];
|
|
14
|
+
}
|
|
15
|
+
export declare function diversifyRankedChunks(candidates: RankedChunk[], diversity: DiversityMode, vectorsByChunkId?: Map<string, Float32Array>, signal?: AbortSignal): Promise<RankedChunk[]>;
|
|
16
|
+
export declare function interleaveByFile(candidates: RankedChunk[], diversity: DiversityMode, signal?: AbortSignal): RankedChunk[];
|