pi-knowledge 0.5.5 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,7 +1,7 @@
1
1
  import { existsSync, readFileSync, renameSync, rmSync, statSync } from "node:fs";
2
2
  import { join, resolve } from "node:path";
3
3
  import { diagnoseKB } from "./diagnostics/health.js";
4
- import { dispose as disposeEmbedding, embedDocuments, embedQuery, prepareForShutdown as prepareEmbeddingForShutdown, } from "./embedding/provider.js";
4
+ import { dispose as disposeEmbedding, embedDocuments, embeddingConfigLabel, embeddingSignature, embedQueryWithConfig, prepareForShutdown as prepareEmbeddingForShutdown, resolveEmbeddingConfig, } from "./embedding/provider.js";
5
5
  import { openVectorReader, openVectorWriter } from "./embedding/vectors.js";
6
6
  import { addSkippedScanEntry, buildChunkEmbeddingText, chunkFile, chunkIdentityHash, createSkippedScanStats, isReadableTextFile, isSupportedDocumentFile, iterateScannableFiles, preTokenizeForFTS, summarizeSkippedScan, } from "./indexer/chunker.js";
7
7
  import { extractSymbols } from "./indexer/symbols.js";
@@ -9,14 +9,12 @@ import { shutdownModelWorker } from "./model-worker-client.js";
9
9
  import { searchBM25 } from "./search/bm25.js";
10
10
  import { weightedScoreFusion } from "./search/fusion.js";
11
11
  import { normalizedQueryText, tokenizeForSearch } from "./search/query.js";
12
- import { hasAnyLexicalEvidence, hasEnoughLexicalEvidence, MIN_HYBRID_SCORE, normalizeFileTypeFilter, queryCoverage, scoreChunkForQuery, } from "./search/ranking.js";
12
+ import { hasAnyLexicalEvidence, hasEnoughLexicalEvidence, normalizeFileTypeFilter, queryCoverage, scoreChunkForQuery, } from "./search/ranking.js";
13
13
  import { disposeReranker, prepareRerankerForShutdown, rerank } from "./search/reranker.js";
14
+ import { resolveSearchTuning, summarizeSearchTuning, } from "./search/tuning.js";
14
15
  import { searchVectorFile } from "./search/vector.js";
15
- import { countSymbols, createKB, deleteChunksByIds, deleteKB, deleteSymbolsByKB, finishIndexingJob, getChunkById, getChunkCount, getChunksByFile, getFileCount, getKB, getKBByName, getSymbolCount, insertChunks, insertSymbols, iterateChunkIdsByKB, iterateChunksByKB, listKBs, openDatabase, searchSymbols, startIndexingJob, updateIndexingJob, updateKBCounts, updateKBStatus, } from "./storage/sqlite.js";
16
- export const CURRENT_EMBEDDING_MODEL = "multilingual-e5-small";
17
- const ADAPTIVE_CONTEXT_LINES = 80;
18
- const ADAPTIVE_MAX_CONTEXT_CHARS = 6_000;
19
- const ADAPTIVE_NEIGHBOR_TARGET = 5;
16
+ import { countSymbols, createKB, deleteChunksByIds, deleteKB, deleteSymbolsByKB, finishIndexingJob, getChunkById, getChunkCount, getChunksByFile, getFileCount, getKB, getKBByName, getSymbolCount, insertChunks, insertSymbols, iterateChunkIdsByKB, iterateChunksByKB, listKBs, openDatabase, searchSymbols, startIndexingJob, updateIndexingJob, updateKBCounts, updateKBEmbeddingMetadata, updateKBStatus, } from "./storage/sqlite.js";
17
+ export { CURRENT_EMBEDDING_MODEL } from "./embedding/provider.js";
20
18
  const INDEX_EMBED_BATCH_SIZE = 64;
21
19
  const VECTOR_REDUNDANCY_WEIGHT = 0.35;
22
20
  function isCancellationError(error) {
@@ -256,7 +254,7 @@ function buildQuerySnippet(content, query, maxLength = 240) {
256
254
  const suffix = end < content.length ? "..." : "";
257
255
  return `${prefix}${content.slice(start, end)}${suffix}`;
258
256
  }
259
- function buildAdaptiveContext(seed, chunks, queryTokens) {
257
+ function buildAdaptiveContext(seed, chunks, queryTokens, options) {
260
258
  const scored = chunks
261
259
  .map((chunk) => {
262
260
  const distance = chunk.id === seed.id
@@ -268,7 +266,7 @@ function buildAdaptiveContext(seed, chunks, queryTokens) {
268
266
  return { chunk, score: seedBoost + proximity + coverage };
269
267
  })
270
268
  .sort((a, b) => b.score - a.score)
271
- .slice(0, ADAPTIVE_NEIGHBOR_TARGET);
269
+ .slice(0, options.neighborTarget);
272
270
  if (!scored.some((item) => item.chunk.id === seed.id))
273
271
  scored.push({ chunk: seed, score: Number.MAX_SAFE_INTEGER });
274
272
  const selectedIds = new Set(scored.map((item) => item.chunk.id));
@@ -279,9 +277,9 @@ function buildAdaptiveContext(seed, chunks, queryTokens) {
279
277
  const sourceChunkIds = [];
280
278
  let totalLength = 0;
281
279
  for (const chunk of ordered) {
282
- if (totalLength >= ADAPTIVE_MAX_CONTEXT_CHARS)
280
+ if (totalLength >= options.maxContextChars)
283
281
  break;
284
- const remaining = ADAPTIVE_MAX_CONTEXT_CHARS - totalLength;
282
+ const remaining = options.maxContextChars - totalLength;
285
283
  const text = chunk.content.slice(0, remaining);
286
284
  parts.push(text);
287
285
  sourceChunkIds.push(chunk.id);
@@ -458,6 +456,24 @@ async function extractScannableFileContentOrSkip(file, skipped, signal) {
458
456
  return undefined;
459
457
  }
460
458
  }
459
+ function persistEmbeddingMetadata(db, kbId, config, vectors) {
460
+ const firstVector = vectors[0];
461
+ if (!firstVector)
462
+ return undefined;
463
+ const dimension = firstVector.length;
464
+ updateKBEmbeddingMetadata(db, kbId, embeddingConfigLabel(config), embeddingSignature(config, dimension), dimension);
465
+ return dimension;
466
+ }
467
+ function embeddingMismatchWarning(kb) {
468
+ return `"${kb.name}" has incompatible embedding metadata; vector retrieval was skipped and knowledge_update should rebuild it`;
469
+ }
470
+ function canSearchVectors(kb, config, queryDimension) {
471
+ if (kb.embedding_signature === null || kb.embedding_dimension === null)
472
+ return false;
473
+ if (kb.embedding_dimension !== queryDimension)
474
+ return false;
475
+ return kb.embedding_signature === embeddingSignature(config, queryDimension);
476
+ }
461
477
  function throwIfAborted(signal) {
462
478
  if (signal?.aborted)
463
479
  throw new Error("Cancelled");
@@ -619,11 +635,13 @@ export class KnowledgeEngine {
619
635
  if (existingKB) {
620
636
  throw new Error(`Knowledge base "${name}" already exists. Use knowledge_update to refresh it, or knowledge_remove before adding a replacement.`);
621
637
  }
638
+ const embeddingConfig = resolveEmbeddingConfig();
622
639
  const kb = createKB(db, {
623
640
  name,
624
641
  source_path: isDir || isFile ? resolvedSource : isUrl ? source : undefined,
625
642
  source_type: sourceType,
626
643
  source_options: serializeAddOptions(options),
644
+ embedding_model: embeddingConfigLabel(embeddingConfig),
627
645
  });
628
646
  updateKBStatus(db, kb.id, "indexing");
629
647
  startIndexingJob(db, kb.id, "add", `Starting indexing for "${name}"`);
@@ -678,6 +696,7 @@ export class KnowledgeEngine {
678
696
  const vectors = await embedDocuments(batch.map((chunk) => buildChunkEmbeddingText(chunk)), signal);
679
697
  if (signal?.aborted)
680
698
  throw new Error("Cancelled");
699
+ persistEmbeddingMetadata(db, kb.id, embeddingConfig, vectors);
681
700
  insertChunks(db, kb.id, batch);
682
701
  writer.append(vectors);
683
702
  chunkCount += batch.length;
@@ -834,6 +853,10 @@ export class KnowledgeEngine {
834
853
  updateKBStatus(this.db, kb.id, "indexing");
835
854
  startIndexingJob(this.db, kb.id, "update", `Starting update for "${kb.name}"`);
836
855
  const scanOptions = toScanOptions(parseAddOptions(kb.source_options));
856
+ const embeddingConfig = resolveEmbeddingConfig();
857
+ const embeddingModel = embeddingConfigLabel(embeddingConfig);
858
+ const currentSignature = kb.embedding_dimension === null ? undefined : embeddingSignature(embeddingConfig, kb.embedding_dimension);
859
+ const canReuseExistingVectors = kb.embedding_signature !== null && kb.embedding_dimension !== null && kb.embedding_signature === currentSignature;
837
860
  let replacementVectorPath;
838
861
  let addedVectorPath;
839
862
  let addedVectorWriter;
@@ -853,6 +876,14 @@ export class KnowledgeEngine {
853
876
  existingHashes.set(chunk.content_hash, entries);
854
877
  existingIndex++;
855
878
  }
879
+ const reusableHashes = canReuseExistingVectors
880
+ ? existingHashes
881
+ : new Map();
882
+ if (!canReuseExistingVectors) {
883
+ const message = `Embedding metadata changed or missing for "${kb.name}"; rebuilding all vectors`;
884
+ updateIndexingJob(this.db, kb.id, { phase: "embedding", message });
885
+ onProgress?.(message);
886
+ }
856
887
  const oldVectorIndexByHash = new Map();
857
888
  const newVectorIndexByHash = new Map();
858
889
  const pendingChunks = [];
@@ -862,6 +893,7 @@ export class KnowledgeEngine {
862
893
  let scannedFiles = 0;
863
894
  let scannedChunks = 0;
864
895
  let plannedTotalFiles;
896
+ let finalEmbeddingDimension;
865
897
  const startedAt = Date.now();
866
898
  const flushPending = async () => {
867
899
  if (!this.db || !addedVectorWriter || pendingChunks.length === 0)
@@ -909,7 +941,7 @@ export class KnowledgeEngine {
909
941
  const processChunks = async (chunks) => {
910
942
  for (const chunk of chunks) {
911
943
  scannedChunks++;
912
- const existing = existingHashes.get(chunk.content_hash);
944
+ const existing = reusableHashes.get(chunk.content_hash);
913
945
  if (existing && existing.length > 0) {
914
946
  const retained = existing.shift();
915
947
  if (retained) {
@@ -1036,6 +1068,7 @@ export class KnowledgeEngine {
1036
1068
  : undefined;
1037
1069
  if (!vector)
1038
1070
  throw new Error(`Missing vector while rebuilding knowledge base: ${chunk.id}`);
1071
+ finalEmbeddingDimension ??= vector.length;
1039
1072
  vectorWriter.append([vector]);
1040
1073
  finalChunkCount++;
1041
1074
  if (finalChunkCount % 1_000 === 0) {
@@ -1067,6 +1100,7 @@ export class KnowledgeEngine {
1067
1100
  deleteChunksByIds(this.db, idsToRemove);
1068
1101
  deleteSymbolsByKB(this.db, kb.id);
1069
1102
  insertSymbols(this.db, kb.id, stagedSymbols);
1103
+ updateKBEmbeddingMetadata(this.db, kb.id, embeddingModel, finalEmbeddingDimension === undefined ? null : embeddingSignature(embeddingConfig, finalEmbeddingDimension), finalEmbeddingDimension ?? null);
1070
1104
  updateKBCounts(this.db, kb.id, finalChunkCount, getFileCount(this.db, kb.id));
1071
1105
  updateKBStatus(this.db, kb.id, "ready");
1072
1106
  const skippedReadySuffix = latestSkippedTotal > 0 ? `, skipped ${latestSkippedTotal} (${latestSkippedSummary})` : "";
@@ -1109,12 +1143,14 @@ export class KnowledgeEngine {
1109
1143
  const attempts = [primaryMode, ...fallbackModesFor(primaryMode)];
1110
1144
  const tried = [];
1111
1145
  const warnings = [];
1146
+ let lastTuning;
1112
1147
  for (const mode of attempts) {
1113
1148
  if (tried.includes(mode))
1114
1149
  continue;
1115
1150
  tried.push(mode);
1116
1151
  throwIfAborted(signal);
1117
1152
  const response = await this.search(query, { ...options, mode }, signal);
1153
+ lastTuning = response.tuning;
1118
1154
  if (response.warnings)
1119
1155
  warnings.push(...response.warnings);
1120
1156
  if (!isWeakAutoResponse(query, response, primaryMode, mode)) {
@@ -1136,16 +1172,16 @@ export class KnowledgeEngine {
1136
1172
  warnings: warnings.length > 0 ? [...new Set(warnings)] : undefined,
1137
1173
  mode_used: tried.at(-1),
1138
1174
  retry_modes: tried.slice(0, -1),
1175
+ tuning: lastTuning,
1139
1176
  suggestions: [
1140
1177
  "No results after auto mode fallback. Check knowledge_status, try a more exact term, or rebuild the KB if indexing rules changed.",
1141
1178
  ],
1142
1179
  };
1143
1180
  }
1144
1181
  const db = this.db;
1145
- const { mode = "hybrid", limit = 10, offset = 0, kb_id, filters, diversity = "balanced" } = options;
1182
+ const { mode = "hybrid", offset = 0, kb_id, filters, diversity = "balanced" } = options;
1146
1183
  const resolvedMode = retrievalModeFor(mode);
1147
1184
  const retrievalMode = resolvedMode === "adaptive" ? "hybrid" : resolvedMode;
1148
- const candidateLimit = Math.max(50, offset + limit * 12);
1149
1185
  const normalizedQuery = normalizedQueryText(query);
1150
1186
  const queryTokens = tokenizeForSimilarity(query);
1151
1187
  const normalizedFileType = normalizeFileTypeFilter(filters?.file_type);
@@ -1159,6 +1195,17 @@ export class KnowledgeEngine {
1159
1195
  warnings.push(`"${kb.name}" is ${kb.status}; search skipped until indexing is ready`);
1160
1196
  }
1161
1197
  }
1198
+ const tuning = resolveSearchTuning({
1199
+ query,
1200
+ mode,
1201
+ profile: options.profile,
1202
+ kbSourceTypes: kbs.map((kb) => kb.source_type),
1203
+ });
1204
+ const limit = typeof options.limit === "number" && Number.isFinite(options.limit) && options.limit > 0
1205
+ ? Math.trunc(options.limit)
1206
+ : tuning.defaultLimit;
1207
+ const candidateLimit = Math.max(tuning.candidateMin, offset + limit * tuning.candidateMultiplier);
1208
+ const tuningSummary = summarizeSearchTuning(tuning, limit, candidateLimit);
1162
1209
  if (kbs.length === 0) {
1163
1210
  return {
1164
1211
  results: [],
@@ -1166,15 +1213,13 @@ export class KnowledgeEngine {
1166
1213
  has_more: false,
1167
1214
  warnings: warnings.length > 0 ? warnings : undefined,
1168
1215
  mode_used: mode,
1216
+ tuning: tuningSummary,
1169
1217
  };
1170
1218
  }
1171
1219
  const allResults = [];
1172
1220
  const vectorsByChunkId = new Map();
1173
1221
  for (const kb of kbs) {
1174
1222
  throwIfAborted(signal);
1175
- if (kb.embedding_model !== CURRENT_EMBEDDING_MODEL) {
1176
- warnings.push(`"${kb.name}" was indexed with ${kb.embedding_model} (current: ${CURRENT_EMBEDDING_MODEL}) — run knowledge_update for best results`);
1177
- }
1178
1223
  if (kb.chunk_count === 0)
1179
1224
  continue;
1180
1225
  const vectorPath = this.vectorPathFor(kb.id);
@@ -1185,25 +1230,33 @@ export class KnowledgeEngine {
1185
1230
  })));
1186
1231
  }
1187
1232
  else if (retrievalMode === "semantic") {
1188
- const queryVec = await embedQuery(query, signal);
1233
+ const { vector: queryVec, config: queryEmbeddingConfig } = await embedQueryWithConfig(query, signal);
1189
1234
  throwIfAborted(signal);
1235
+ if (!canSearchVectors(kb, queryEmbeddingConfig, queryVec.length)) {
1236
+ warnings.push(embeddingMismatchWarning(kb));
1237
+ continue;
1238
+ }
1190
1239
  const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), candidateLimit, signal);
1191
1240
  allResults.push(...vectorResults.results.map((result) => ({ ...result, score: result.score * kbTrustMultiplier(kb) })));
1192
1241
  for (const [chunkId, vector] of vectorResults.vectorsByChunkId)
1193
1242
  vectorsByChunkId.set(chunkId, vector);
1194
1243
  }
1195
1244
  else {
1196
- // hybrid: BM25 + vector weighted fusion (both scoped to this KB)
1197
1245
  const bm25Results = searchBM25(db, normalizedQuery || query, candidateLimit, kb.id);
1198
1246
  if (bm25Results.length === 0)
1199
1247
  continue;
1200
1248
  let vecResults = [];
1201
- const queryVec = await embedQuery(query, signal);
1249
+ const { vector: queryVec, config: queryEmbeddingConfig } = await embedQueryWithConfig(query, signal);
1202
1250
  throwIfAborted(signal);
1203
- const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), candidateLimit, signal);
1204
- vecResults = vectorResults.results;
1205
- for (const [chunkId, vector] of vectorResults.vectorsByChunkId)
1206
- vectorsByChunkId.set(chunkId, vector);
1251
+ if (canSearchVectors(kb, queryEmbeddingConfig, queryVec.length)) {
1252
+ const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), candidateLimit, signal);
1253
+ vecResults = vectorResults.results;
1254
+ for (const [chunkId, vector] of vectorResults.vectorsByChunkId)
1255
+ vectorsByChunkId.set(chunkId, vector);
1256
+ }
1257
+ else {
1258
+ warnings.push(embeddingMismatchWarning(kb));
1259
+ }
1207
1260
  const fused = weightedScoreFusion(bm25Results, vecResults);
1208
1261
  allResults.push(...fused.map((result) => ({ ...result, score: result.score * kbTrustMultiplier(kb) })));
1209
1262
  }
@@ -1240,7 +1293,7 @@ export class KnowledgeEngine {
1240
1293
  const chunk = getChunkById(db, result.chunkId);
1241
1294
  if (!chunk)
1242
1295
  return false;
1243
- return result.score >= MIN_HYBRID_SCORE && hasEnoughLexicalEvidence(chunk, queryTokens);
1296
+ return result.score >= tuning.minHybridScore && hasEnoughLexicalEvidence(chunk, queryTokens);
1244
1297
  });
1245
1298
  }
1246
1299
  scored.sort((a, b) => b.score - a.score);
@@ -1260,14 +1313,13 @@ export class KnowledgeEngine {
1260
1313
  }
1261
1314
  if (mode === "deep" && filtered.length > 0) {
1262
1315
  const candidates = filtered
1263
- .slice(0, 30)
1316
+ .slice(0, tuning.deepRerankCandidates)
1264
1317
  .map((r) => {
1265
1318
  const chunk = getChunkById(db, r.chunkId);
1266
1319
  return chunk ? { chunkId: r.chunkId, content: chunk.content } : null;
1267
1320
  })
1268
1321
  .filter(Boolean);
1269
- const reranked = await rerank(query, candidates, Math.max(limit * 3, limit), signal);
1270
- throwIfAborted(signal);
1322
+ const reranked = await rerank(query, candidates, Math.max(limit * tuning.deepRerankTopKMultiplier, limit), signal);
1271
1323
  const ranked = [];
1272
1324
  for (const r of reranked) {
1273
1325
  throwIfAborted(signal);
@@ -1281,7 +1333,7 @@ export class KnowledgeEngine {
1281
1333
  ranking: scoreChunkForQuery(r.score, chunk, queryTokens),
1282
1334
  score: r.score,
1283
1335
  content: chunk.content,
1284
- snippet: buildQuerySnippet(chunk.content, query),
1336
+ snippet: buildQuerySnippet(chunk.content, query, tuning.snippetMaxLength),
1285
1337
  startLine: chunk.start_line,
1286
1338
  endLine: chunk.end_line,
1287
1339
  sourceChunkIds: [chunk.id],
@@ -1319,6 +1371,7 @@ export class KnowledgeEngine {
1319
1371
  has_more: offset + limit < diversified.length,
1320
1372
  warnings: warnings.length > 0 ? warnings : undefined,
1321
1373
  mode_used: mode,
1374
+ tuning: tuningSummary,
1322
1375
  };
1323
1376
  }
1324
1377
  const ranked = [];
@@ -1329,15 +1382,18 @@ export class KnowledgeEngine {
1329
1382
  continue;
1330
1383
  const kb = getKB(db, chunk.kb_id);
1331
1384
  if (mode === "adaptive") {
1332
- const contextChunks = getChunksByFile(db, chunk.kb_id, chunk.file_path, Math.max(1, chunk.start_line - ADAPTIVE_CONTEXT_LINES), chunk.end_line + ADAPTIVE_CONTEXT_LINES);
1333
- const context = buildAdaptiveContext(chunk, contextChunks.length > 0 ? contextChunks : [chunk], queryTokens);
1385
+ const contextChunks = getChunksByFile(db, chunk.kb_id, chunk.file_path, Math.max(1, chunk.start_line - tuning.adaptiveContextLines), chunk.end_line + tuning.adaptiveContextLines);
1386
+ const context = buildAdaptiveContext(chunk, contextChunks.length > 0 ? contextChunks : [chunk], queryTokens, {
1387
+ maxContextChars: tuning.adaptiveMaxContextChars,
1388
+ neighborTarget: tuning.adaptiveNeighborTarget,
1389
+ });
1334
1390
  pushAdaptiveCandidate(ranked, {
1335
1391
  chunk,
1336
1392
  kbName: kb?.name ?? "unknown",
1337
1393
  ranking: rankingByChunkId.get(r.chunkId),
1338
1394
  score: r.score,
1339
1395
  content: context.content,
1340
- snippet: buildQuerySnippet(context.content, query),
1396
+ snippet: buildQuerySnippet(context.content, query, tuning.snippetMaxLength),
1341
1397
  startLine: context.startLine,
1342
1398
  endLine: context.endLine,
1343
1399
  sourceChunkIds: context.sourceChunkIds,
@@ -1350,7 +1406,7 @@ export class KnowledgeEngine {
1350
1406
  ranking: rankingByChunkId.get(r.chunkId),
1351
1407
  score: r.score,
1352
1408
  content: chunk.content,
1353
- snippet: buildQuerySnippet(chunk.content, query),
1409
+ snippet: buildQuerySnippet(chunk.content, query, tuning.snippetMaxLength),
1354
1410
  startLine: chunk.start_line,
1355
1411
  endLine: chunk.end_line,
1356
1412
  sourceChunkIds: [chunk.id],
@@ -1390,6 +1446,7 @@ export class KnowledgeEngine {
1390
1446
  has_more: offset + limit < total,
1391
1447
  warnings: warnings.length > 0 ? warnings : undefined,
1392
1448
  mode_used: mode,
1449
+ tuning: tuningSummary,
1393
1450
  suggestions: results.length === 0
1394
1451
  ? [
1395
1452
  "Try mode 'fast' for exact symbols or mode 'semantic' for conceptual wording.",
@@ -1663,6 +1720,7 @@ export class KnowledgeEngine {
1663
1720
  const { createInterface } = await import("node:readline");
1664
1721
  const stream = createReadStream(inputPath, { encoding: "utf-8" });
1665
1722
  const lines = createInterface({ input: stream, crlfDelay: Infinity });
1723
+ const embeddingConfig = resolveEmbeddingConfig();
1666
1724
  let header;
1667
1725
  let kb;
1668
1726
  let vectorWriter;
@@ -1689,6 +1747,7 @@ export class KnowledgeEngine {
1689
1747
  onProgress?.(message);
1690
1748
  const vectors = await embedDocuments(batch.map((chunk) => buildChunkEmbeddingText(chunk)), signal);
1691
1749
  throwIfAborted(signal);
1750
+ persistEmbeddingMetadata(this.db, kb.id, embeddingConfig, vectors);
1692
1751
  insertChunks(this.db, kb.id, batch);
1693
1752
  insertSymbols(this.db, kb.id, symbols);
1694
1753
  vectorWriter.append(vectors);
@@ -1717,7 +1776,12 @@ export class KnowledgeEngine {
1717
1776
  if (existingKB) {
1718
1777
  throw new Error(`Knowledge base "${header.name}" already exists. Use knowledge_update to refresh it, or knowledge_remove before importing a replacement.`);
1719
1778
  }
1720
- kb = createKB(this.db, { name: header.name, description: header.description, source_type: "text" });
1779
+ kb = createKB(this.db, {
1780
+ name: header.name,
1781
+ description: header.description,
1782
+ source_type: "text",
1783
+ embedding_model: embeddingConfigLabel(embeddingConfig),
1784
+ });
1721
1785
  updateKBStatus(this.db, kb.id, "indexing");
1722
1786
  startIndexingJob(this.db, kb.id, "import", `Starting import for "${header.name}"`);
1723
1787
  const importMessage = `Importing ${header.chunk_count ?? "unknown"} chunks...`;
@@ -1,3 +1,4 @@
1
+ import type { HfRerankerConfig } from "./search/reranker-config.ts";
1
2
  export declare function validateModelWorkerNodePath(nodePath: string): string;
2
3
  export declare function resolveModelWorkerNodePath(): string;
3
4
  export declare function configureModelWorkerNodePath(nodePath?: string): string;
@@ -6,7 +7,7 @@ export interface RerankWorkerCandidate {
6
7
  chunkId: string;
7
8
  content: string;
8
9
  }
9
- export declare function rerankInModelWorker(query: string, candidates: RerankWorkerCandidate[], topK: number, signal?: AbortSignal): Promise<Array<{
10
+ export declare function rerankInModelWorker(query: string, candidates: RerankWorkerCandidate[], topK: number, reranker: HfRerankerConfig, signal?: AbortSignal): Promise<Array<{
10
11
  chunkId: string;
11
12
  score: number;
12
13
  }>>;
@@ -395,8 +395,8 @@ export async function embedInModelWorker(texts, prefix, signal) {
395
395
  return new Float32Array(vector);
396
396
  });
397
397
  }
398
- export async function rerankInModelWorker(query, candidates, topK, signal) {
399
- const result = await getWorker().request({ type: "rerank", query, candidates, topK }, signal);
398
+ export async function rerankInModelWorker(query, candidates, topK, reranker, signal) {
399
+ const result = await getWorker().request({ type: "rerank", query, candidates, topK, reranker }, signal);
400
400
  if (!Array.isArray(result))
401
401
  throw new Error("Invalid reranker worker response");
402
402
  return result.map((item) => {
@@ -1,5 +1,6 @@
1
1
  import { join } from "node:path";
2
2
  import { createInterface } from "node:readline";
3
+ import { DEFAULT_RERANKER_REMOTE_HOST, DEFAULT_RERANKER_REMOTE_PATH_TEMPLATE, isHfRerankerConfig, rerankerCacheKey, resolveRerankerConfig, } from "./search/reranker-config.js";
3
4
  import { getDefaultKnowledgeDir } from "./storage/sqlite.js";
4
5
  let embeddingPipeline = null;
5
6
  let rerankerPipeline = null;
@@ -14,11 +15,16 @@ function configureTransformersEnv(env) {
14
15
  env.localModelPath = cacheDir;
15
16
  }
16
17
  }
18
+ function configureEmbeddingTransformersEnv(env) {
19
+ configureTransformersEnv(env);
20
+ env.remoteHost = DEFAULT_RERANKER_REMOTE_HOST;
21
+ env.remotePathTemplate = DEFAULT_RERANKER_REMOTE_PATH_TEMPLATE;
22
+ }
17
23
  async function loadEmbeddingPipeline() {
18
24
  if (embeddingPipeline)
19
25
  return embeddingPipeline;
20
26
  const { pipeline, env } = await import("@huggingface/transformers");
21
- configureTransformersEnv(env);
27
+ configureEmbeddingTransformersEnv(env);
22
28
  const createPipeline = pipeline;
23
29
  const loaded = (await createPipeline("feature-extraction", "Xenova/multilingual-e5-small", {
24
30
  quantized: true,
@@ -27,14 +33,21 @@ async function loadEmbeddingPipeline() {
27
33
  embeddingPipeline = loaded;
28
34
  return loaded;
29
35
  }
30
- async function loadRerankerPipeline() {
31
- if (rerankerPipeline)
32
- return rerankerPipeline;
36
+ async function loadRerankerPipeline(config) {
37
+ const key = rerankerCacheKey(config);
38
+ if (rerankerPipeline?.key === key)
39
+ return rerankerPipeline.pipe;
33
40
  const { pipeline, env } = await import("@huggingface/transformers");
34
- configureTransformersEnv(env);
41
+ const transformersEnv = env;
42
+ configureTransformersEnv(transformersEnv);
43
+ transformersEnv.remoteHost = config.remoteHost ?? DEFAULT_RERANKER_REMOTE_HOST;
44
+ transformersEnv.remotePathTemplate = config.remotePathTemplate ?? DEFAULT_RERANKER_REMOTE_PATH_TEMPLATE;
35
45
  const createPipeline = pipeline;
36
- const loaded = (await createPipeline("text-classification", "Xenova/ms-marco-MiniLM-L-4-v2"));
37
- rerankerPipeline = loaded;
46
+ const options = { revision: config.revision };
47
+ if (config.dtype)
48
+ options.dtype = config.dtype;
49
+ const loaded = (await createPipeline("text-classification", config.model, options));
50
+ rerankerPipeline = { key, pipe: loaded };
38
51
  return loaded;
39
52
  }
40
53
  async function handleEmbed(request) {
@@ -47,7 +60,7 @@ async function handleEmbed(request) {
47
60
  return vectors;
48
61
  }
49
62
  async function handleRerank(request) {
50
- const pipe = await loadRerankerPipeline();
63
+ const pipe = await loadRerankerPipeline(request.reranker);
51
64
  const results = [];
52
65
  for (const candidate of request.candidates) {
53
66
  const output = await pipe({ text: request.query, text_pair: candidate.content });
@@ -91,12 +104,16 @@ function parseWorkerRequest(message) {
91
104
  if (!("candidates" in message) || !isRerankCandidateArray(message.candidates)) {
92
105
  throw new Error("Invalid rerank candidates");
93
106
  }
107
+ const reranker = "reranker" in message && isHfRerankerConfig(message.reranker) ? message.reranker : resolveRerankerConfig();
108
+ if (reranker.provider !== "hf")
109
+ throw new Error("Reranker API mode is not supported by the model worker");
94
110
  return {
95
111
  id: message.id,
96
112
  type: "rerank",
97
113
  query: message.query,
98
114
  candidates: message.candidates,
99
115
  topK: message.topK,
116
+ reranker,
100
117
  };
101
118
  }
102
119
  throw new Error("Unsupported worker request type");
@@ -0,0 +1,7 @@
1
+ import type { RerankCandidate } from "./reranker.ts";
2
+ import type { ApiRerankerConfig } from "./reranker-config.ts";
3
+ export interface ApiRerankResult {
4
+ chunkId: string;
5
+ score: number;
6
+ }
7
+ export declare function rerankViaApi(query: string, candidates: RerankCandidate[], topK: number, config: ApiRerankerConfig, signal?: AbortSignal): Promise<ApiRerankResult[]>;
@@ -0,0 +1,89 @@
1
+ function valueAtPath(value, path) {
2
+ let current = value;
3
+ for (const segment of path.split(".")) {
4
+ if (segment.length === 0)
5
+ continue;
6
+ if (typeof current !== "object" || current === null)
7
+ return undefined;
8
+ current = current[segment];
9
+ }
10
+ return current;
11
+ }
12
+ function createAbortSignal(timeoutMs, signal) {
13
+ const controller = new AbortController();
14
+ const abortFromParent = () => controller.abort(signal?.reason);
15
+ if (signal?.aborted)
16
+ abortFromParent();
17
+ else
18
+ signal?.addEventListener("abort", abortFromParent, { once: true });
19
+ const timeout = setTimeout(() => controller.abort(new Error(`Reranker API timed out after ${timeoutMs}ms`)), timeoutMs);
20
+ return {
21
+ signal: controller.signal,
22
+ cleanup() {
23
+ clearTimeout(timeout);
24
+ signal?.removeEventListener("abort", abortFromParent);
25
+ },
26
+ };
27
+ }
28
+ function mapApiResult(result, candidates, config) {
29
+ const index = valueAtPath(result, config.indexField);
30
+ const score = valueAtPath(result, config.scoreField);
31
+ if (typeof index !== "number" || !Number.isInteger(index) || index < 0 || index >= candidates.length) {
32
+ throw new Error("Invalid reranker API response: result index is out of range");
33
+ }
34
+ if (typeof score !== "number" || !Number.isFinite(score)) {
35
+ throw new Error("Invalid reranker API response: result score is not a number");
36
+ }
37
+ return { chunkId: candidates[index].chunkId, score };
38
+ }
39
+ function requestBody(query, candidates, topK, config) {
40
+ const documents = candidates.map((candidate) => {
41
+ const text = candidate.content.length > config.maxDocumentChars
42
+ ? candidate.content.slice(0, config.maxDocumentChars)
43
+ : candidate.content;
44
+ return config.format === "custom-json" ? { id: candidate.chunkId, text } : text;
45
+ });
46
+ return JSON.stringify({
47
+ model: config.model,
48
+ query,
49
+ documents,
50
+ top_n: topK,
51
+ return_documents: false,
52
+ });
53
+ }
54
+ export async function rerankViaApi(query, candidates, topK, config, signal) {
55
+ const abort = createAbortSignal(config.timeoutMs, signal);
56
+ try {
57
+ const headers = { "Content-Type": "application/json" };
58
+ if (config.apiKey)
59
+ headers.Authorization = `Bearer ${config.apiKey}`;
60
+ const response = await fetch(config.endpoint, {
61
+ method: "POST",
62
+ headers,
63
+ body: requestBody(query, candidates, topK, config),
64
+ signal: abort.signal,
65
+ });
66
+ if (!response.ok) {
67
+ const detail = await response.text().catch(() => "");
68
+ throw new Error(`Reranker API error: ${response.status}${detail ? ` ${detail.slice(0, 500)}` : ""}`);
69
+ }
70
+ const data = (await response.json());
71
+ const results = valueAtPath(data, config.resultsPath);
72
+ if (!Array.isArray(results))
73
+ throw new Error("Invalid reranker API response: results field is not an array");
74
+ const ranked = results.map((result) => mapApiResult(result, candidates, config));
75
+ if (config.scoreDirection === "asc")
76
+ ranked.sort((a, b) => a.score - b.score);
77
+ else
78
+ ranked.sort((a, b) => b.score - a.score);
79
+ return ranked.slice(0, topK);
80
+ }
81
+ catch (error) {
82
+ if (signal?.aborted || (error instanceof DOMException && error.name === "AbortError"))
83
+ throw new Error("Cancelled");
84
+ throw error;
85
+ }
86
+ finally {
87
+ abort.cleanup();
88
+ }
89
+ }
@@ -0,0 +1,34 @@
1
+ export type RerankerProvider = "hf" | "api";
2
+ export type ApiRerankerFormat = "cohere" | "jina" | "custom-json";
3
+ export type ApiScoreDirection = "desc" | "asc";
4
+ export interface HfRerankerConfig {
5
+ provider: "hf";
6
+ model: string;
7
+ revision: string;
8
+ dtype?: string;
9
+ remoteHost?: string;
10
+ remotePathTemplate?: string;
11
+ }
12
+ export interface ApiRerankerConfig {
13
+ provider: "api";
14
+ model: string;
15
+ endpoint: string;
16
+ apiKey?: string;
17
+ format: ApiRerankerFormat;
18
+ timeoutMs: number;
19
+ maxDocumentChars: number;
20
+ resultsPath: string;
21
+ indexField: string;
22
+ scoreField: string;
23
+ scoreDirection: ApiScoreDirection;
24
+ }
25
+ export type RerankerConfig = HfRerankerConfig | ApiRerankerConfig;
26
+ export declare const DEFAULT_RERANKER_MODEL = "Xenova/ms-marco-MiniLM-L-4-v2";
27
+ export declare const DEFAULT_RERANKER_REVISION = "main";
28
+ export declare const DEFAULT_RERANKER_REMOTE_HOST = "https://huggingface.co/";
29
+ export declare const DEFAULT_RERANKER_REMOTE_PATH_TEMPLATE = "{model}/resolve/{revision}/";
30
+ export declare const DEFAULT_RERANKER_API_TIMEOUT_MS = 30000;
31
+ export declare const DEFAULT_RERANKER_MAX_DOC_CHARS = 12000;
32
+ export declare function resolveRerankerConfig(env?: NodeJS.ProcessEnv): RerankerConfig;
33
+ export declare function isHfRerankerConfig(value: unknown): value is HfRerankerConfig;
34
+ export declare function rerankerCacheKey(config: HfRerankerConfig): string;