pi-knowledge 0.5.5 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/README.md +9 -4
- package/dist/index.js +24 -4
- package/dist/src/embedding/provider.d.ts +33 -1
- package/dist/src/embedding/provider.js +103 -37
- package/dist/src/engine.d.ts +4 -1
- package/dist/src/engine.js +98 -34
- package/dist/src/model-worker-client.d.ts +2 -1
- package/dist/src/model-worker-client.js +2 -2
- package/dist/src/model-worker.js +25 -8
- package/dist/src/search/reranker-api.d.ts +7 -0
- package/dist/src/search/reranker-api.js +89 -0
- package/dist/src/search/reranker-config.d.ts +34 -0
- package/dist/src/search/reranker-config.js +151 -0
- package/dist/src/search/reranker.js +7 -1
- package/dist/src/search/tuning.d.ts +41 -0
- package/dist/src/search/tuning.js +190 -0
- package/dist/src/storage/sqlite.d.ts +6 -0
- package/dist/src/storage/sqlite.js +23 -3
- package/docs/configuration.md +41 -6
- package/package.json +1 -1
package/dist/src/engine.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { existsSync, readFileSync, renameSync, rmSync, statSync } from "node:fs";
|
|
2
2
|
import { join, resolve } from "node:path";
|
|
3
3
|
import { diagnoseKB } from "./diagnostics/health.js";
|
|
4
|
-
import { dispose as disposeEmbedding, embedDocuments,
|
|
4
|
+
import { dispose as disposeEmbedding, embedDocuments, embeddingConfigLabel, embeddingSignature, embedQueryWithConfig, prepareForShutdown as prepareEmbeddingForShutdown, resolveEmbeddingConfig, } from "./embedding/provider.js";
|
|
5
5
|
import { openVectorReader, openVectorWriter } from "./embedding/vectors.js";
|
|
6
6
|
import { addSkippedScanEntry, buildChunkEmbeddingText, chunkFile, chunkIdentityHash, createSkippedScanStats, isReadableTextFile, isSupportedDocumentFile, iterateScannableFiles, preTokenizeForFTS, summarizeSkippedScan, } from "./indexer/chunker.js";
|
|
7
7
|
import { extractSymbols } from "./indexer/symbols.js";
|
|
@@ -9,14 +9,12 @@ import { shutdownModelWorker } from "./model-worker-client.js";
|
|
|
9
9
|
import { searchBM25 } from "./search/bm25.js";
|
|
10
10
|
import { weightedScoreFusion } from "./search/fusion.js";
|
|
11
11
|
import { normalizedQueryText, tokenizeForSearch } from "./search/query.js";
|
|
12
|
-
import { hasAnyLexicalEvidence, hasEnoughLexicalEvidence,
|
|
12
|
+
import { hasAnyLexicalEvidence, hasEnoughLexicalEvidence, normalizeFileTypeFilter, queryCoverage, scoreChunkForQuery, } from "./search/ranking.js";
|
|
13
13
|
import { disposeReranker, prepareRerankerForShutdown, rerank } from "./search/reranker.js";
|
|
14
|
+
import { resolveSearchTuning, summarizeSearchTuning, } from "./search/tuning.js";
|
|
14
15
|
import { searchVectorFile } from "./search/vector.js";
|
|
15
|
-
import { countSymbols, createKB, deleteChunksByIds, deleteKB, deleteSymbolsByKB, finishIndexingJob, getChunkById, getChunkCount, getChunksByFile, getFileCount, getKB, getKBByName, getSymbolCount, insertChunks, insertSymbols, iterateChunkIdsByKB, iterateChunksByKB, listKBs, openDatabase, searchSymbols, startIndexingJob, updateIndexingJob, updateKBCounts, updateKBStatus, } from "./storage/sqlite.js";
|
|
16
|
-
export
|
|
17
|
-
const ADAPTIVE_CONTEXT_LINES = 80;
|
|
18
|
-
const ADAPTIVE_MAX_CONTEXT_CHARS = 6_000;
|
|
19
|
-
const ADAPTIVE_NEIGHBOR_TARGET = 5;
|
|
16
|
+
import { countSymbols, createKB, deleteChunksByIds, deleteKB, deleteSymbolsByKB, finishIndexingJob, getChunkById, getChunkCount, getChunksByFile, getFileCount, getKB, getKBByName, getSymbolCount, insertChunks, insertSymbols, iterateChunkIdsByKB, iterateChunksByKB, listKBs, openDatabase, searchSymbols, startIndexingJob, updateIndexingJob, updateKBCounts, updateKBEmbeddingMetadata, updateKBStatus, } from "./storage/sqlite.js";
|
|
17
|
+
export { CURRENT_EMBEDDING_MODEL } from "./embedding/provider.js";
|
|
20
18
|
const INDEX_EMBED_BATCH_SIZE = 64;
|
|
21
19
|
const VECTOR_REDUNDANCY_WEIGHT = 0.35;
|
|
22
20
|
function isCancellationError(error) {
|
|
@@ -256,7 +254,7 @@ function buildQuerySnippet(content, query, maxLength = 240) {
|
|
|
256
254
|
const suffix = end < content.length ? "..." : "";
|
|
257
255
|
return `${prefix}${content.slice(start, end)}${suffix}`;
|
|
258
256
|
}
|
|
259
|
-
function buildAdaptiveContext(seed, chunks, queryTokens) {
|
|
257
|
+
function buildAdaptiveContext(seed, chunks, queryTokens, options) {
|
|
260
258
|
const scored = chunks
|
|
261
259
|
.map((chunk) => {
|
|
262
260
|
const distance = chunk.id === seed.id
|
|
@@ -268,7 +266,7 @@ function buildAdaptiveContext(seed, chunks, queryTokens) {
|
|
|
268
266
|
return { chunk, score: seedBoost + proximity + coverage };
|
|
269
267
|
})
|
|
270
268
|
.sort((a, b) => b.score - a.score)
|
|
271
|
-
.slice(0,
|
|
269
|
+
.slice(0, options.neighborTarget);
|
|
272
270
|
if (!scored.some((item) => item.chunk.id === seed.id))
|
|
273
271
|
scored.push({ chunk: seed, score: Number.MAX_SAFE_INTEGER });
|
|
274
272
|
const selectedIds = new Set(scored.map((item) => item.chunk.id));
|
|
@@ -279,9 +277,9 @@ function buildAdaptiveContext(seed, chunks, queryTokens) {
|
|
|
279
277
|
const sourceChunkIds = [];
|
|
280
278
|
let totalLength = 0;
|
|
281
279
|
for (const chunk of ordered) {
|
|
282
|
-
if (totalLength >=
|
|
280
|
+
if (totalLength >= options.maxContextChars)
|
|
283
281
|
break;
|
|
284
|
-
const remaining =
|
|
282
|
+
const remaining = options.maxContextChars - totalLength;
|
|
285
283
|
const text = chunk.content.slice(0, remaining);
|
|
286
284
|
parts.push(text);
|
|
287
285
|
sourceChunkIds.push(chunk.id);
|
|
@@ -458,6 +456,24 @@ async function extractScannableFileContentOrSkip(file, skipped, signal) {
|
|
|
458
456
|
return undefined;
|
|
459
457
|
}
|
|
460
458
|
}
|
|
459
|
+
function persistEmbeddingMetadata(db, kbId, config, vectors) {
|
|
460
|
+
const firstVector = vectors[0];
|
|
461
|
+
if (!firstVector)
|
|
462
|
+
return undefined;
|
|
463
|
+
const dimension = firstVector.length;
|
|
464
|
+
updateKBEmbeddingMetadata(db, kbId, embeddingConfigLabel(config), embeddingSignature(config, dimension), dimension);
|
|
465
|
+
return dimension;
|
|
466
|
+
}
|
|
467
|
+
function embeddingMismatchWarning(kb) {
|
|
468
|
+
return `"${kb.name}" has incompatible embedding metadata; vector retrieval was skipped and knowledge_update should rebuild it`;
|
|
469
|
+
}
|
|
470
|
+
function canSearchVectors(kb, config, queryDimension) {
|
|
471
|
+
if (kb.embedding_signature === null || kb.embedding_dimension === null)
|
|
472
|
+
return false;
|
|
473
|
+
if (kb.embedding_dimension !== queryDimension)
|
|
474
|
+
return false;
|
|
475
|
+
return kb.embedding_signature === embeddingSignature(config, queryDimension);
|
|
476
|
+
}
|
|
461
477
|
function throwIfAborted(signal) {
|
|
462
478
|
if (signal?.aborted)
|
|
463
479
|
throw new Error("Cancelled");
|
|
@@ -619,11 +635,13 @@ export class KnowledgeEngine {
|
|
|
619
635
|
if (existingKB) {
|
|
620
636
|
throw new Error(`Knowledge base "${name}" already exists. Use knowledge_update to refresh it, or knowledge_remove before adding a replacement.`);
|
|
621
637
|
}
|
|
638
|
+
const embeddingConfig = resolveEmbeddingConfig();
|
|
622
639
|
const kb = createKB(db, {
|
|
623
640
|
name,
|
|
624
641
|
source_path: isDir || isFile ? resolvedSource : isUrl ? source : undefined,
|
|
625
642
|
source_type: sourceType,
|
|
626
643
|
source_options: serializeAddOptions(options),
|
|
644
|
+
embedding_model: embeddingConfigLabel(embeddingConfig),
|
|
627
645
|
});
|
|
628
646
|
updateKBStatus(db, kb.id, "indexing");
|
|
629
647
|
startIndexingJob(db, kb.id, "add", `Starting indexing for "${name}"`);
|
|
@@ -678,6 +696,7 @@ export class KnowledgeEngine {
|
|
|
678
696
|
const vectors = await embedDocuments(batch.map((chunk) => buildChunkEmbeddingText(chunk)), signal);
|
|
679
697
|
if (signal?.aborted)
|
|
680
698
|
throw new Error("Cancelled");
|
|
699
|
+
persistEmbeddingMetadata(db, kb.id, embeddingConfig, vectors);
|
|
681
700
|
insertChunks(db, kb.id, batch);
|
|
682
701
|
writer.append(vectors);
|
|
683
702
|
chunkCount += batch.length;
|
|
@@ -834,6 +853,10 @@ export class KnowledgeEngine {
|
|
|
834
853
|
updateKBStatus(this.db, kb.id, "indexing");
|
|
835
854
|
startIndexingJob(this.db, kb.id, "update", `Starting update for "${kb.name}"`);
|
|
836
855
|
const scanOptions = toScanOptions(parseAddOptions(kb.source_options));
|
|
856
|
+
const embeddingConfig = resolveEmbeddingConfig();
|
|
857
|
+
const embeddingModel = embeddingConfigLabel(embeddingConfig);
|
|
858
|
+
const currentSignature = kb.embedding_dimension === null ? undefined : embeddingSignature(embeddingConfig, kb.embedding_dimension);
|
|
859
|
+
const canReuseExistingVectors = kb.embedding_signature !== null && kb.embedding_dimension !== null && kb.embedding_signature === currentSignature;
|
|
837
860
|
let replacementVectorPath;
|
|
838
861
|
let addedVectorPath;
|
|
839
862
|
let addedVectorWriter;
|
|
@@ -853,6 +876,14 @@ export class KnowledgeEngine {
|
|
|
853
876
|
existingHashes.set(chunk.content_hash, entries);
|
|
854
877
|
existingIndex++;
|
|
855
878
|
}
|
|
879
|
+
const reusableHashes = canReuseExistingVectors
|
|
880
|
+
? existingHashes
|
|
881
|
+
: new Map();
|
|
882
|
+
if (!canReuseExistingVectors) {
|
|
883
|
+
const message = `Embedding metadata changed or missing for "${kb.name}"; rebuilding all vectors`;
|
|
884
|
+
updateIndexingJob(this.db, kb.id, { phase: "embedding", message });
|
|
885
|
+
onProgress?.(message);
|
|
886
|
+
}
|
|
856
887
|
const oldVectorIndexByHash = new Map();
|
|
857
888
|
const newVectorIndexByHash = new Map();
|
|
858
889
|
const pendingChunks = [];
|
|
@@ -862,6 +893,7 @@ export class KnowledgeEngine {
|
|
|
862
893
|
let scannedFiles = 0;
|
|
863
894
|
let scannedChunks = 0;
|
|
864
895
|
let plannedTotalFiles;
|
|
896
|
+
let finalEmbeddingDimension;
|
|
865
897
|
const startedAt = Date.now();
|
|
866
898
|
const flushPending = async () => {
|
|
867
899
|
if (!this.db || !addedVectorWriter || pendingChunks.length === 0)
|
|
@@ -909,7 +941,7 @@ export class KnowledgeEngine {
|
|
|
909
941
|
const processChunks = async (chunks) => {
|
|
910
942
|
for (const chunk of chunks) {
|
|
911
943
|
scannedChunks++;
|
|
912
|
-
const existing =
|
|
944
|
+
const existing = reusableHashes.get(chunk.content_hash);
|
|
913
945
|
if (existing && existing.length > 0) {
|
|
914
946
|
const retained = existing.shift();
|
|
915
947
|
if (retained) {
|
|
@@ -1036,6 +1068,7 @@ export class KnowledgeEngine {
|
|
|
1036
1068
|
: undefined;
|
|
1037
1069
|
if (!vector)
|
|
1038
1070
|
throw new Error(`Missing vector while rebuilding knowledge base: ${chunk.id}`);
|
|
1071
|
+
finalEmbeddingDimension ??= vector.length;
|
|
1039
1072
|
vectorWriter.append([vector]);
|
|
1040
1073
|
finalChunkCount++;
|
|
1041
1074
|
if (finalChunkCount % 1_000 === 0) {
|
|
@@ -1067,6 +1100,7 @@ export class KnowledgeEngine {
|
|
|
1067
1100
|
deleteChunksByIds(this.db, idsToRemove);
|
|
1068
1101
|
deleteSymbolsByKB(this.db, kb.id);
|
|
1069
1102
|
insertSymbols(this.db, kb.id, stagedSymbols);
|
|
1103
|
+
updateKBEmbeddingMetadata(this.db, kb.id, embeddingModel, finalEmbeddingDimension === undefined ? null : embeddingSignature(embeddingConfig, finalEmbeddingDimension), finalEmbeddingDimension ?? null);
|
|
1070
1104
|
updateKBCounts(this.db, kb.id, finalChunkCount, getFileCount(this.db, kb.id));
|
|
1071
1105
|
updateKBStatus(this.db, kb.id, "ready");
|
|
1072
1106
|
const skippedReadySuffix = latestSkippedTotal > 0 ? `, skipped ${latestSkippedTotal} (${latestSkippedSummary})` : "";
|
|
@@ -1109,12 +1143,14 @@ export class KnowledgeEngine {
|
|
|
1109
1143
|
const attempts = [primaryMode, ...fallbackModesFor(primaryMode)];
|
|
1110
1144
|
const tried = [];
|
|
1111
1145
|
const warnings = [];
|
|
1146
|
+
let lastTuning;
|
|
1112
1147
|
for (const mode of attempts) {
|
|
1113
1148
|
if (tried.includes(mode))
|
|
1114
1149
|
continue;
|
|
1115
1150
|
tried.push(mode);
|
|
1116
1151
|
throwIfAborted(signal);
|
|
1117
1152
|
const response = await this.search(query, { ...options, mode }, signal);
|
|
1153
|
+
lastTuning = response.tuning;
|
|
1118
1154
|
if (response.warnings)
|
|
1119
1155
|
warnings.push(...response.warnings);
|
|
1120
1156
|
if (!isWeakAutoResponse(query, response, primaryMode, mode)) {
|
|
@@ -1136,16 +1172,16 @@ export class KnowledgeEngine {
|
|
|
1136
1172
|
warnings: warnings.length > 0 ? [...new Set(warnings)] : undefined,
|
|
1137
1173
|
mode_used: tried.at(-1),
|
|
1138
1174
|
retry_modes: tried.slice(0, -1),
|
|
1175
|
+
tuning: lastTuning,
|
|
1139
1176
|
suggestions: [
|
|
1140
1177
|
"No results after auto mode fallback. Check knowledge_status, try a more exact term, or rebuild the KB if indexing rules changed.",
|
|
1141
1178
|
],
|
|
1142
1179
|
};
|
|
1143
1180
|
}
|
|
1144
1181
|
const db = this.db;
|
|
1145
|
-
const { mode = "hybrid",
|
|
1182
|
+
const { mode = "hybrid", offset = 0, kb_id, filters, diversity = "balanced" } = options;
|
|
1146
1183
|
const resolvedMode = retrievalModeFor(mode);
|
|
1147
1184
|
const retrievalMode = resolvedMode === "adaptive" ? "hybrid" : resolvedMode;
|
|
1148
|
-
const candidateLimit = Math.max(50, offset + limit * 12);
|
|
1149
1185
|
const normalizedQuery = normalizedQueryText(query);
|
|
1150
1186
|
const queryTokens = tokenizeForSimilarity(query);
|
|
1151
1187
|
const normalizedFileType = normalizeFileTypeFilter(filters?.file_type);
|
|
@@ -1159,6 +1195,17 @@ export class KnowledgeEngine {
|
|
|
1159
1195
|
warnings.push(`"${kb.name}" is ${kb.status}; search skipped until indexing is ready`);
|
|
1160
1196
|
}
|
|
1161
1197
|
}
|
|
1198
|
+
const tuning = resolveSearchTuning({
|
|
1199
|
+
query,
|
|
1200
|
+
mode,
|
|
1201
|
+
profile: options.profile,
|
|
1202
|
+
kbSourceTypes: kbs.map((kb) => kb.source_type),
|
|
1203
|
+
});
|
|
1204
|
+
const limit = typeof options.limit === "number" && Number.isFinite(options.limit) && options.limit > 0
|
|
1205
|
+
? Math.trunc(options.limit)
|
|
1206
|
+
: tuning.defaultLimit;
|
|
1207
|
+
const candidateLimit = Math.max(tuning.candidateMin, offset + limit * tuning.candidateMultiplier);
|
|
1208
|
+
const tuningSummary = summarizeSearchTuning(tuning, limit, candidateLimit);
|
|
1162
1209
|
if (kbs.length === 0) {
|
|
1163
1210
|
return {
|
|
1164
1211
|
results: [],
|
|
@@ -1166,15 +1213,13 @@ export class KnowledgeEngine {
|
|
|
1166
1213
|
has_more: false,
|
|
1167
1214
|
warnings: warnings.length > 0 ? warnings : undefined,
|
|
1168
1215
|
mode_used: mode,
|
|
1216
|
+
tuning: tuningSummary,
|
|
1169
1217
|
};
|
|
1170
1218
|
}
|
|
1171
1219
|
const allResults = [];
|
|
1172
1220
|
const vectorsByChunkId = new Map();
|
|
1173
1221
|
for (const kb of kbs) {
|
|
1174
1222
|
throwIfAborted(signal);
|
|
1175
|
-
if (kb.embedding_model !== CURRENT_EMBEDDING_MODEL) {
|
|
1176
|
-
warnings.push(`"${kb.name}" was indexed with ${kb.embedding_model} (current: ${CURRENT_EMBEDDING_MODEL}) — run knowledge_update for best results`);
|
|
1177
|
-
}
|
|
1178
1223
|
if (kb.chunk_count === 0)
|
|
1179
1224
|
continue;
|
|
1180
1225
|
const vectorPath = this.vectorPathFor(kb.id);
|
|
@@ -1185,25 +1230,33 @@ export class KnowledgeEngine {
|
|
|
1185
1230
|
})));
|
|
1186
1231
|
}
|
|
1187
1232
|
else if (retrievalMode === "semantic") {
|
|
1188
|
-
const queryVec = await
|
|
1233
|
+
const { vector: queryVec, config: queryEmbeddingConfig } = await embedQueryWithConfig(query, signal);
|
|
1189
1234
|
throwIfAborted(signal);
|
|
1235
|
+
if (!canSearchVectors(kb, queryEmbeddingConfig, queryVec.length)) {
|
|
1236
|
+
warnings.push(embeddingMismatchWarning(kb));
|
|
1237
|
+
continue;
|
|
1238
|
+
}
|
|
1190
1239
|
const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), candidateLimit, signal);
|
|
1191
1240
|
allResults.push(...vectorResults.results.map((result) => ({ ...result, score: result.score * kbTrustMultiplier(kb) })));
|
|
1192
1241
|
for (const [chunkId, vector] of vectorResults.vectorsByChunkId)
|
|
1193
1242
|
vectorsByChunkId.set(chunkId, vector);
|
|
1194
1243
|
}
|
|
1195
1244
|
else {
|
|
1196
|
-
// hybrid: BM25 + vector weighted fusion (both scoped to this KB)
|
|
1197
1245
|
const bm25Results = searchBM25(db, normalizedQuery || query, candidateLimit, kb.id);
|
|
1198
1246
|
if (bm25Results.length === 0)
|
|
1199
1247
|
continue;
|
|
1200
1248
|
let vecResults = [];
|
|
1201
|
-
const queryVec = await
|
|
1249
|
+
const { vector: queryVec, config: queryEmbeddingConfig } = await embedQueryWithConfig(query, signal);
|
|
1202
1250
|
throwIfAborted(signal);
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1251
|
+
if (canSearchVectors(kb, queryEmbeddingConfig, queryVec.length)) {
|
|
1252
|
+
const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), candidateLimit, signal);
|
|
1253
|
+
vecResults = vectorResults.results;
|
|
1254
|
+
for (const [chunkId, vector] of vectorResults.vectorsByChunkId)
|
|
1255
|
+
vectorsByChunkId.set(chunkId, vector);
|
|
1256
|
+
}
|
|
1257
|
+
else {
|
|
1258
|
+
warnings.push(embeddingMismatchWarning(kb));
|
|
1259
|
+
}
|
|
1207
1260
|
const fused = weightedScoreFusion(bm25Results, vecResults);
|
|
1208
1261
|
allResults.push(...fused.map((result) => ({ ...result, score: result.score * kbTrustMultiplier(kb) })));
|
|
1209
1262
|
}
|
|
@@ -1240,7 +1293,7 @@ export class KnowledgeEngine {
|
|
|
1240
1293
|
const chunk = getChunkById(db, result.chunkId);
|
|
1241
1294
|
if (!chunk)
|
|
1242
1295
|
return false;
|
|
1243
|
-
return result.score >=
|
|
1296
|
+
return result.score >= tuning.minHybridScore && hasEnoughLexicalEvidence(chunk, queryTokens);
|
|
1244
1297
|
});
|
|
1245
1298
|
}
|
|
1246
1299
|
scored.sort((a, b) => b.score - a.score);
|
|
@@ -1260,14 +1313,13 @@ export class KnowledgeEngine {
|
|
|
1260
1313
|
}
|
|
1261
1314
|
if (mode === "deep" && filtered.length > 0) {
|
|
1262
1315
|
const candidates = filtered
|
|
1263
|
-
.slice(0,
|
|
1316
|
+
.slice(0, tuning.deepRerankCandidates)
|
|
1264
1317
|
.map((r) => {
|
|
1265
1318
|
const chunk = getChunkById(db, r.chunkId);
|
|
1266
1319
|
return chunk ? { chunkId: r.chunkId, content: chunk.content } : null;
|
|
1267
1320
|
})
|
|
1268
1321
|
.filter(Boolean);
|
|
1269
|
-
const reranked = await rerank(query, candidates, Math.max(limit *
|
|
1270
|
-
throwIfAborted(signal);
|
|
1322
|
+
const reranked = await rerank(query, candidates, Math.max(limit * tuning.deepRerankTopKMultiplier, limit), signal);
|
|
1271
1323
|
const ranked = [];
|
|
1272
1324
|
for (const r of reranked) {
|
|
1273
1325
|
throwIfAborted(signal);
|
|
@@ -1281,7 +1333,7 @@ export class KnowledgeEngine {
|
|
|
1281
1333
|
ranking: scoreChunkForQuery(r.score, chunk, queryTokens),
|
|
1282
1334
|
score: r.score,
|
|
1283
1335
|
content: chunk.content,
|
|
1284
|
-
snippet: buildQuerySnippet(chunk.content, query),
|
|
1336
|
+
snippet: buildQuerySnippet(chunk.content, query, tuning.snippetMaxLength),
|
|
1285
1337
|
startLine: chunk.start_line,
|
|
1286
1338
|
endLine: chunk.end_line,
|
|
1287
1339
|
sourceChunkIds: [chunk.id],
|
|
@@ -1319,6 +1371,7 @@ export class KnowledgeEngine {
|
|
|
1319
1371
|
has_more: offset + limit < diversified.length,
|
|
1320
1372
|
warnings: warnings.length > 0 ? warnings : undefined,
|
|
1321
1373
|
mode_used: mode,
|
|
1374
|
+
tuning: tuningSummary,
|
|
1322
1375
|
};
|
|
1323
1376
|
}
|
|
1324
1377
|
const ranked = [];
|
|
@@ -1329,15 +1382,18 @@ export class KnowledgeEngine {
|
|
|
1329
1382
|
continue;
|
|
1330
1383
|
const kb = getKB(db, chunk.kb_id);
|
|
1331
1384
|
if (mode === "adaptive") {
|
|
1332
|
-
const contextChunks = getChunksByFile(db, chunk.kb_id, chunk.file_path, Math.max(1, chunk.start_line -
|
|
1333
|
-
const context = buildAdaptiveContext(chunk, contextChunks.length > 0 ? contextChunks : [chunk], queryTokens
|
|
1385
|
+
const contextChunks = getChunksByFile(db, chunk.kb_id, chunk.file_path, Math.max(1, chunk.start_line - tuning.adaptiveContextLines), chunk.end_line + tuning.adaptiveContextLines);
|
|
1386
|
+
const context = buildAdaptiveContext(chunk, contextChunks.length > 0 ? contextChunks : [chunk], queryTokens, {
|
|
1387
|
+
maxContextChars: tuning.adaptiveMaxContextChars,
|
|
1388
|
+
neighborTarget: tuning.adaptiveNeighborTarget,
|
|
1389
|
+
});
|
|
1334
1390
|
pushAdaptiveCandidate(ranked, {
|
|
1335
1391
|
chunk,
|
|
1336
1392
|
kbName: kb?.name ?? "unknown",
|
|
1337
1393
|
ranking: rankingByChunkId.get(r.chunkId),
|
|
1338
1394
|
score: r.score,
|
|
1339
1395
|
content: context.content,
|
|
1340
|
-
snippet: buildQuerySnippet(context.content, query),
|
|
1396
|
+
snippet: buildQuerySnippet(context.content, query, tuning.snippetMaxLength),
|
|
1341
1397
|
startLine: context.startLine,
|
|
1342
1398
|
endLine: context.endLine,
|
|
1343
1399
|
sourceChunkIds: context.sourceChunkIds,
|
|
@@ -1350,7 +1406,7 @@ export class KnowledgeEngine {
|
|
|
1350
1406
|
ranking: rankingByChunkId.get(r.chunkId),
|
|
1351
1407
|
score: r.score,
|
|
1352
1408
|
content: chunk.content,
|
|
1353
|
-
snippet: buildQuerySnippet(chunk.content, query),
|
|
1409
|
+
snippet: buildQuerySnippet(chunk.content, query, tuning.snippetMaxLength),
|
|
1354
1410
|
startLine: chunk.start_line,
|
|
1355
1411
|
endLine: chunk.end_line,
|
|
1356
1412
|
sourceChunkIds: [chunk.id],
|
|
@@ -1390,6 +1446,7 @@ export class KnowledgeEngine {
|
|
|
1390
1446
|
has_more: offset + limit < total,
|
|
1391
1447
|
warnings: warnings.length > 0 ? warnings : undefined,
|
|
1392
1448
|
mode_used: mode,
|
|
1449
|
+
tuning: tuningSummary,
|
|
1393
1450
|
suggestions: results.length === 0
|
|
1394
1451
|
? [
|
|
1395
1452
|
"Try mode 'fast' for exact symbols or mode 'semantic' for conceptual wording.",
|
|
@@ -1663,6 +1720,7 @@ export class KnowledgeEngine {
|
|
|
1663
1720
|
const { createInterface } = await import("node:readline");
|
|
1664
1721
|
const stream = createReadStream(inputPath, { encoding: "utf-8" });
|
|
1665
1722
|
const lines = createInterface({ input: stream, crlfDelay: Infinity });
|
|
1723
|
+
const embeddingConfig = resolveEmbeddingConfig();
|
|
1666
1724
|
let header;
|
|
1667
1725
|
let kb;
|
|
1668
1726
|
let vectorWriter;
|
|
@@ -1689,6 +1747,7 @@ export class KnowledgeEngine {
|
|
|
1689
1747
|
onProgress?.(message);
|
|
1690
1748
|
const vectors = await embedDocuments(batch.map((chunk) => buildChunkEmbeddingText(chunk)), signal);
|
|
1691
1749
|
throwIfAborted(signal);
|
|
1750
|
+
persistEmbeddingMetadata(this.db, kb.id, embeddingConfig, vectors);
|
|
1692
1751
|
insertChunks(this.db, kb.id, batch);
|
|
1693
1752
|
insertSymbols(this.db, kb.id, symbols);
|
|
1694
1753
|
vectorWriter.append(vectors);
|
|
@@ -1717,7 +1776,12 @@ export class KnowledgeEngine {
|
|
|
1717
1776
|
if (existingKB) {
|
|
1718
1777
|
throw new Error(`Knowledge base "${header.name}" already exists. Use knowledge_update to refresh it, or knowledge_remove before importing a replacement.`);
|
|
1719
1778
|
}
|
|
1720
|
-
kb = createKB(this.db, {
|
|
1779
|
+
kb = createKB(this.db, {
|
|
1780
|
+
name: header.name,
|
|
1781
|
+
description: header.description,
|
|
1782
|
+
source_type: "text",
|
|
1783
|
+
embedding_model: embeddingConfigLabel(embeddingConfig),
|
|
1784
|
+
});
|
|
1721
1785
|
updateKBStatus(this.db, kb.id, "indexing");
|
|
1722
1786
|
startIndexingJob(this.db, kb.id, "import", `Starting import for "${header.name}"`);
|
|
1723
1787
|
const importMessage = `Importing ${header.chunk_count ?? "unknown"} chunks...`;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { HfRerankerConfig } from "./search/reranker-config.ts";
|
|
1
2
|
export declare function validateModelWorkerNodePath(nodePath: string): string;
|
|
2
3
|
export declare function resolveModelWorkerNodePath(): string;
|
|
3
4
|
export declare function configureModelWorkerNodePath(nodePath?: string): string;
|
|
@@ -6,7 +7,7 @@ export interface RerankWorkerCandidate {
|
|
|
6
7
|
chunkId: string;
|
|
7
8
|
content: string;
|
|
8
9
|
}
|
|
9
|
-
export declare function rerankInModelWorker(query: string, candidates: RerankWorkerCandidate[], topK: number, signal?: AbortSignal): Promise<Array<{
|
|
10
|
+
export declare function rerankInModelWorker(query: string, candidates: RerankWorkerCandidate[], topK: number, reranker: HfRerankerConfig, signal?: AbortSignal): Promise<Array<{
|
|
10
11
|
chunkId: string;
|
|
11
12
|
score: number;
|
|
12
13
|
}>>;
|
|
@@ -395,8 +395,8 @@ export async function embedInModelWorker(texts, prefix, signal) {
|
|
|
395
395
|
return new Float32Array(vector);
|
|
396
396
|
});
|
|
397
397
|
}
|
|
398
|
-
export async function rerankInModelWorker(query, candidates, topK, signal) {
|
|
399
|
-
const result = await getWorker().request({ type: "rerank", query, candidates, topK }, signal);
|
|
398
|
+
export async function rerankInModelWorker(query, candidates, topK, reranker, signal) {
|
|
399
|
+
const result = await getWorker().request({ type: "rerank", query, candidates, topK, reranker }, signal);
|
|
400
400
|
if (!Array.isArray(result))
|
|
401
401
|
throw new Error("Invalid reranker worker response");
|
|
402
402
|
return result.map((item) => {
|
package/dist/src/model-worker.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { join } from "node:path";
|
|
2
2
|
import { createInterface } from "node:readline";
|
|
3
|
+
import { DEFAULT_RERANKER_REMOTE_HOST, DEFAULT_RERANKER_REMOTE_PATH_TEMPLATE, isHfRerankerConfig, rerankerCacheKey, resolveRerankerConfig, } from "./search/reranker-config.js";
|
|
3
4
|
import { getDefaultKnowledgeDir } from "./storage/sqlite.js";
|
|
4
5
|
let embeddingPipeline = null;
|
|
5
6
|
let rerankerPipeline = null;
|
|
@@ -14,11 +15,16 @@ function configureTransformersEnv(env) {
|
|
|
14
15
|
env.localModelPath = cacheDir;
|
|
15
16
|
}
|
|
16
17
|
}
|
|
18
|
+
function configureEmbeddingTransformersEnv(env) {
|
|
19
|
+
configureTransformersEnv(env);
|
|
20
|
+
env.remoteHost = DEFAULT_RERANKER_REMOTE_HOST;
|
|
21
|
+
env.remotePathTemplate = DEFAULT_RERANKER_REMOTE_PATH_TEMPLATE;
|
|
22
|
+
}
|
|
17
23
|
async function loadEmbeddingPipeline() {
|
|
18
24
|
if (embeddingPipeline)
|
|
19
25
|
return embeddingPipeline;
|
|
20
26
|
const { pipeline, env } = await import("@huggingface/transformers");
|
|
21
|
-
|
|
27
|
+
configureEmbeddingTransformersEnv(env);
|
|
22
28
|
const createPipeline = pipeline;
|
|
23
29
|
const loaded = (await createPipeline("feature-extraction", "Xenova/multilingual-e5-small", {
|
|
24
30
|
quantized: true,
|
|
@@ -27,14 +33,21 @@ async function loadEmbeddingPipeline() {
|
|
|
27
33
|
embeddingPipeline = loaded;
|
|
28
34
|
return loaded;
|
|
29
35
|
}
|
|
30
|
-
async function loadRerankerPipeline() {
|
|
31
|
-
|
|
32
|
-
|
|
36
|
+
async function loadRerankerPipeline(config) {
|
|
37
|
+
const key = rerankerCacheKey(config);
|
|
38
|
+
if (rerankerPipeline?.key === key)
|
|
39
|
+
return rerankerPipeline.pipe;
|
|
33
40
|
const { pipeline, env } = await import("@huggingface/transformers");
|
|
34
|
-
|
|
41
|
+
const transformersEnv = env;
|
|
42
|
+
configureTransformersEnv(transformersEnv);
|
|
43
|
+
transformersEnv.remoteHost = config.remoteHost ?? DEFAULT_RERANKER_REMOTE_HOST;
|
|
44
|
+
transformersEnv.remotePathTemplate = config.remotePathTemplate ?? DEFAULT_RERANKER_REMOTE_PATH_TEMPLATE;
|
|
35
45
|
const createPipeline = pipeline;
|
|
36
|
-
const
|
|
37
|
-
|
|
46
|
+
const options = { revision: config.revision };
|
|
47
|
+
if (config.dtype)
|
|
48
|
+
options.dtype = config.dtype;
|
|
49
|
+
const loaded = (await createPipeline("text-classification", config.model, options));
|
|
50
|
+
rerankerPipeline = { key, pipe: loaded };
|
|
38
51
|
return loaded;
|
|
39
52
|
}
|
|
40
53
|
async function handleEmbed(request) {
|
|
@@ -47,7 +60,7 @@ async function handleEmbed(request) {
|
|
|
47
60
|
return vectors;
|
|
48
61
|
}
|
|
49
62
|
async function handleRerank(request) {
|
|
50
|
-
const pipe = await loadRerankerPipeline();
|
|
63
|
+
const pipe = await loadRerankerPipeline(request.reranker);
|
|
51
64
|
const results = [];
|
|
52
65
|
for (const candidate of request.candidates) {
|
|
53
66
|
const output = await pipe({ text: request.query, text_pair: candidate.content });
|
|
@@ -91,12 +104,16 @@ function parseWorkerRequest(message) {
|
|
|
91
104
|
if (!("candidates" in message) || !isRerankCandidateArray(message.candidates)) {
|
|
92
105
|
throw new Error("Invalid rerank candidates");
|
|
93
106
|
}
|
|
107
|
+
const reranker = "reranker" in message && isHfRerankerConfig(message.reranker) ? message.reranker : resolveRerankerConfig();
|
|
108
|
+
if (reranker.provider !== "hf")
|
|
109
|
+
throw new Error("Reranker API mode is not supported by the model worker");
|
|
94
110
|
return {
|
|
95
111
|
id: message.id,
|
|
96
112
|
type: "rerank",
|
|
97
113
|
query: message.query,
|
|
98
114
|
candidates: message.candidates,
|
|
99
115
|
topK: message.topK,
|
|
116
|
+
reranker,
|
|
100
117
|
};
|
|
101
118
|
}
|
|
102
119
|
throw new Error("Unsupported worker request type");
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { RerankCandidate } from "./reranker.ts";
|
|
2
|
+
import type { ApiRerankerConfig } from "./reranker-config.ts";
|
|
3
|
+
export interface ApiRerankResult {
|
|
4
|
+
chunkId: string;
|
|
5
|
+
score: number;
|
|
6
|
+
}
|
|
7
|
+
export declare function rerankViaApi(query: string, candidates: RerankCandidate[], topK: number, config: ApiRerankerConfig, signal?: AbortSignal): Promise<ApiRerankResult[]>;
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
function valueAtPath(value, path) {
|
|
2
|
+
let current = value;
|
|
3
|
+
for (const segment of path.split(".")) {
|
|
4
|
+
if (segment.length === 0)
|
|
5
|
+
continue;
|
|
6
|
+
if (typeof current !== "object" || current === null)
|
|
7
|
+
return undefined;
|
|
8
|
+
current = current[segment];
|
|
9
|
+
}
|
|
10
|
+
return current;
|
|
11
|
+
}
|
|
12
|
+
function createAbortSignal(timeoutMs, signal) {
|
|
13
|
+
const controller = new AbortController();
|
|
14
|
+
const abortFromParent = () => controller.abort(signal?.reason);
|
|
15
|
+
if (signal?.aborted)
|
|
16
|
+
abortFromParent();
|
|
17
|
+
else
|
|
18
|
+
signal?.addEventListener("abort", abortFromParent, { once: true });
|
|
19
|
+
const timeout = setTimeout(() => controller.abort(new Error(`Reranker API timed out after ${timeoutMs}ms`)), timeoutMs);
|
|
20
|
+
return {
|
|
21
|
+
signal: controller.signal,
|
|
22
|
+
cleanup() {
|
|
23
|
+
clearTimeout(timeout);
|
|
24
|
+
signal?.removeEventListener("abort", abortFromParent);
|
|
25
|
+
},
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
function mapApiResult(result, candidates, config) {
|
|
29
|
+
const index = valueAtPath(result, config.indexField);
|
|
30
|
+
const score = valueAtPath(result, config.scoreField);
|
|
31
|
+
if (typeof index !== "number" || !Number.isInteger(index) || index < 0 || index >= candidates.length) {
|
|
32
|
+
throw new Error("Invalid reranker API response: result index is out of range");
|
|
33
|
+
}
|
|
34
|
+
if (typeof score !== "number" || !Number.isFinite(score)) {
|
|
35
|
+
throw new Error("Invalid reranker API response: result score is not a number");
|
|
36
|
+
}
|
|
37
|
+
return { chunkId: candidates[index].chunkId, score };
|
|
38
|
+
}
|
|
39
|
+
function requestBody(query, candidates, topK, config) {
|
|
40
|
+
const documents = candidates.map((candidate) => {
|
|
41
|
+
const text = candidate.content.length > config.maxDocumentChars
|
|
42
|
+
? candidate.content.slice(0, config.maxDocumentChars)
|
|
43
|
+
: candidate.content;
|
|
44
|
+
return config.format === "custom-json" ? { id: candidate.chunkId, text } : text;
|
|
45
|
+
});
|
|
46
|
+
return JSON.stringify({
|
|
47
|
+
model: config.model,
|
|
48
|
+
query,
|
|
49
|
+
documents,
|
|
50
|
+
top_n: topK,
|
|
51
|
+
return_documents: false,
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
export async function rerankViaApi(query, candidates, topK, config, signal) {
|
|
55
|
+
const abort = createAbortSignal(config.timeoutMs, signal);
|
|
56
|
+
try {
|
|
57
|
+
const headers = { "Content-Type": "application/json" };
|
|
58
|
+
if (config.apiKey)
|
|
59
|
+
headers.Authorization = `Bearer ${config.apiKey}`;
|
|
60
|
+
const response = await fetch(config.endpoint, {
|
|
61
|
+
method: "POST",
|
|
62
|
+
headers,
|
|
63
|
+
body: requestBody(query, candidates, topK, config),
|
|
64
|
+
signal: abort.signal,
|
|
65
|
+
});
|
|
66
|
+
if (!response.ok) {
|
|
67
|
+
const detail = await response.text().catch(() => "");
|
|
68
|
+
throw new Error(`Reranker API error: ${response.status}${detail ? ` ${detail.slice(0, 500)}` : ""}`);
|
|
69
|
+
}
|
|
70
|
+
const data = (await response.json());
|
|
71
|
+
const results = valueAtPath(data, config.resultsPath);
|
|
72
|
+
if (!Array.isArray(results))
|
|
73
|
+
throw new Error("Invalid reranker API response: results field is not an array");
|
|
74
|
+
const ranked = results.map((result) => mapApiResult(result, candidates, config));
|
|
75
|
+
if (config.scoreDirection === "asc")
|
|
76
|
+
ranked.sort((a, b) => a.score - b.score);
|
|
77
|
+
else
|
|
78
|
+
ranked.sort((a, b) => b.score - a.score);
|
|
79
|
+
return ranked.slice(0, topK);
|
|
80
|
+
}
|
|
81
|
+
catch (error) {
|
|
82
|
+
if (signal?.aborted || (error instanceof DOMException && error.name === "AbortError"))
|
|
83
|
+
throw new Error("Cancelled");
|
|
84
|
+
throw error;
|
|
85
|
+
}
|
|
86
|
+
finally {
|
|
87
|
+
abort.cleanup();
|
|
88
|
+
}
|
|
89
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
export type RerankerProvider = "hf" | "api";
|
|
2
|
+
export type ApiRerankerFormat = "cohere" | "jina" | "custom-json";
|
|
3
|
+
export type ApiScoreDirection = "desc" | "asc";
|
|
4
|
+
export interface HfRerankerConfig {
|
|
5
|
+
provider: "hf";
|
|
6
|
+
model: string;
|
|
7
|
+
revision: string;
|
|
8
|
+
dtype?: string;
|
|
9
|
+
remoteHost?: string;
|
|
10
|
+
remotePathTemplate?: string;
|
|
11
|
+
}
|
|
12
|
+
export interface ApiRerankerConfig {
|
|
13
|
+
provider: "api";
|
|
14
|
+
model: string;
|
|
15
|
+
endpoint: string;
|
|
16
|
+
apiKey?: string;
|
|
17
|
+
format: ApiRerankerFormat;
|
|
18
|
+
timeoutMs: number;
|
|
19
|
+
maxDocumentChars: number;
|
|
20
|
+
resultsPath: string;
|
|
21
|
+
indexField: string;
|
|
22
|
+
scoreField: string;
|
|
23
|
+
scoreDirection: ApiScoreDirection;
|
|
24
|
+
}
|
|
25
|
+
export type RerankerConfig = HfRerankerConfig | ApiRerankerConfig;
|
|
26
|
+
export declare const DEFAULT_RERANKER_MODEL = "Xenova/ms-marco-MiniLM-L-4-v2";
|
|
27
|
+
export declare const DEFAULT_RERANKER_REVISION = "main";
|
|
28
|
+
export declare const DEFAULT_RERANKER_REMOTE_HOST = "https://huggingface.co/";
|
|
29
|
+
export declare const DEFAULT_RERANKER_REMOTE_PATH_TEMPLATE = "{model}/resolve/{revision}/";
|
|
30
|
+
export declare const DEFAULT_RERANKER_API_TIMEOUT_MS = 30000;
|
|
31
|
+
export declare const DEFAULT_RERANKER_MAX_DOC_CHARS = 12000;
|
|
32
|
+
export declare function resolveRerankerConfig(env?: NodeJS.ProcessEnv): RerankerConfig;
|
|
33
|
+
export declare function isHfRerankerConfig(value: unknown): value is HfRerankerConfig;
|
|
34
|
+
export declare function rerankerCacheKey(config: HfRerankerConfig): string;
|