pi-knowledge 0.3.3 → 0.3.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/engine.ts CHANGED
@@ -2,20 +2,36 @@ import { existsSync, statSync } from "node:fs";
2
2
  import { join, resolve } from "node:path";
3
3
  import type Database from "better-sqlite3";
4
4
  import { type DiagnosticResult, diagnoseKB } from "./diagnostics/health.ts";
5
- import { dispose as disposeEmbedding, embedDocuments, embedQuery } from "./embedding/provider.ts";
5
+ import {
6
+ dispose as disposeEmbedding,
7
+ embedDocuments,
8
+ embedQuery,
9
+ prepareForShutdown as prepareEmbeddingForShutdown,
10
+ } from "./embedding/provider.ts";
6
11
  import { loadVectors, saveVectors } from "./embedding/vectors.ts";
7
- import { chunkFile, contentHash, preTokenizeForFTS, walkDir } from "./indexer/chunker.ts";
12
+ import { buildChunkEmbeddingText, chunkFile, contentHash, preTokenizeForFTS, walkDir } from "./indexer/chunker.ts";
8
13
  import { searchBM25 } from "./search/bm25.ts";
9
- import { reciprocalRankFusion } from "./search/fusion.ts";
10
- import { disposeReranker, rerank } from "./search/reranker.ts";
14
+ import { weightedScoreFusion } from "./search/fusion.ts";
15
+ import { normalizedQueryText, tokenizeForSearch } from "./search/query.ts";
16
+ import {
17
+ hasEnoughLexicalEvidence,
18
+ MIN_HYBRID_SCORE,
19
+ normalizeFileTypeFilter,
20
+ queryCoverage,
21
+ type RankingDiagnostics,
22
+ scoreChunkForQuery,
23
+ } from "./search/ranking.ts";
24
+ import { disposeReranker, prepareRerankerForShutdown, rerank } from "./search/reranker.ts";
11
25
  import { searchVector } from "./search/vector.ts";
12
26
  import {
27
+ type Chunk,
13
28
  createKB,
14
29
  deleteChunksByIds,
15
30
  deleteKB,
16
31
  getChunkById,
17
32
  getChunkHashesByKB,
18
33
  getChunkIdsByKB,
34
+ getChunksByFile,
19
35
  getChunksByKB,
20
36
  getFileCount,
21
37
  getKB,
@@ -29,11 +45,12 @@ import {
29
45
  } from "./storage/sqlite.ts";
30
46
 
31
47
  export interface SearchOptions {
32
- mode?: "fast" | "semantic" | "hybrid" | "deep";
48
+ mode?: "fast" | "semantic" | "hybrid" | "deep" | "adaptive";
33
49
  limit?: number;
34
50
  offset?: number;
35
51
  kb_id?: string;
36
52
  filters?: { file_type?: string; path_pattern?: string };
53
+ diversity?: "off" | "balanced" | "strong";
37
54
  }
38
55
 
39
56
  export interface SearchResult {
@@ -45,6 +62,7 @@ export interface SearchResult {
45
62
  snippet: string;
46
63
  start_line: number;
47
64
  end_line: number;
65
+ ranking?: RankingDiagnostics;
48
66
  }
49
67
 
50
68
  export const CURRENT_EMBEDDING_MODEL = "multilingual-e5-small";
@@ -67,6 +85,226 @@ interface ImportedChunk {
67
85
  metadata_json?: string;
68
86
  }
69
87
 
88
+ interface RankedChunk {
89
+ chunk: Chunk;
90
+ kbName: string;
91
+ ranking?: RankingDiagnostics;
92
+ score: number;
93
+ content: string;
94
+ snippet: string;
95
+ startLine: number;
96
+ endLine: number;
97
+ sourceChunkIds: string[];
98
+ }
99
+
100
+ const ADAPTIVE_CONTEXT_LINES = 80;
101
+ const ADAPTIVE_MAX_CONTEXT_CHARS = 6_000;
102
+ const ADAPTIVE_NEIGHBOR_TARGET = 5;
103
+ const VECTOR_REDUNDANCY_WEIGHT = 0.35;
104
+
105
+ function cosineSimilarity(a: Float32Array | undefined, b: Float32Array | undefined): number {
106
+ if (!a || !b || a.length !== b.length) return 0;
107
+ let dot = 0;
108
+ for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
109
+ return dot;
110
+ }
111
+
112
+ function tokenizeForSimilarity(text: string): Set<string> {
113
+ return tokenizeForSearch(text);
114
+ }
115
+
116
+ function jaccardSimilarity(a: Set<string>, b: Set<string>): number {
117
+ if (a.size === 0 || b.size === 0) return 0;
118
+ let intersection = 0;
119
+ for (const token of a) {
120
+ if (b.has(token)) intersection++;
121
+ }
122
+ return intersection / (a.size + b.size - intersection);
123
+ }
124
+
125
+ function lineProximity(a: RankedChunk, b: RankedChunk): number {
126
+ if (a.chunk.file_path !== b.chunk.file_path || a.chunk.kb_id !== b.chunk.kb_id) return 0;
127
+ if (a.sourceChunkIds.some((id) => b.sourceChunkIds.includes(id))) return 1;
128
+ const overlap = Math.max(0, Math.min(a.endLine, b.endLine) - Math.max(a.startLine, b.startLine) + 1);
129
+ if (overlap > 0) return 1;
130
+ const gap = Math.max(a.startLine, b.startLine) - Math.min(a.endLine, b.endLine);
131
+ if (gap <= 20) return 0.8;
132
+ if (gap <= 80) return 0.45;
133
+ return 0.2;
134
+ }
135
+
136
+ function normalizeScores(candidates: RankedChunk[]): Map<string, number> {
137
+ const scores = candidates.map((candidate) => candidate.score);
138
+ const min = Math.min(...scores);
139
+ const max = Math.max(...scores);
140
+ const normalized = new Map<string, number>();
141
+ for (const candidate of candidates) {
142
+ const value = max === min ? 1 : (candidate.score - min) / (max - min);
143
+ normalized.set(candidate.chunk.id, value);
144
+ }
145
+ return normalized;
146
+ }
147
+
148
+ function diversifyRankedChunks(
149
+ candidates: RankedChunk[],
150
+ diversity: SearchOptions["diversity"],
151
+ vectorsByChunkId: Map<string, Float32Array> = new Map(),
152
+ ): RankedChunk[] {
153
+ if (diversity === "off" || candidates.length <= 2) return candidates;
154
+ const lambda = diversity === "strong" ? 0.62 : 0.76;
155
+ const normalized = normalizeScores(candidates);
156
+ const tokenSets = new Map(
157
+ candidates.map((candidate) => [candidate.chunk.id, tokenizeForSimilarity(candidate.content)]),
158
+ );
159
+ const selected: RankedChunk[] = [];
160
+ const remaining = [...candidates];
161
+
162
+ while (remaining.length > 0) {
163
+ let bestIndex = 0;
164
+ let bestScore = Number.NEGATIVE_INFINITY;
165
+ for (let i = 0; i < remaining.length; i++) {
166
+ const candidate = remaining[i];
167
+ let redundancy = 0;
168
+ for (const chosen of selected) {
169
+ const lexical = jaccardSimilarity(
170
+ tokenSets.get(candidate.chunk.id) ?? new Set<string>(),
171
+ tokenSets.get(chosen.chunk.id) ?? new Set<string>(),
172
+ );
173
+ const vector = Math.max(
174
+ 0,
175
+ cosineSimilarity(vectorsByChunkId.get(candidate.chunk.id), vectorsByChunkId.get(chosen.chunk.id)),
176
+ );
177
+ redundancy = Math.max(
178
+ redundancy,
179
+ Math.max(lexical, lineProximity(candidate, chosen), vector * VECTOR_REDUNDANCY_WEIGHT),
180
+ );
181
+ }
182
+ const relevance = normalized.get(candidate.chunk.id) ?? 0;
183
+ const mmrScore = lambda * relevance - (1 - lambda) * redundancy;
184
+ if (mmrScore > bestScore) {
185
+ bestScore = mmrScore;
186
+ bestIndex = i;
187
+ }
188
+ }
189
+ selected.push(remaining.splice(bestIndex, 1)[0]);
190
+ }
191
+
192
+ return selected;
193
+ }
194
+
195
+ function interleaveByFile(candidates: RankedChunk[], diversity: SearchOptions["diversity"]): RankedChunk[] {
196
+ if (diversity === "off" || candidates.length <= 2) return candidates;
197
+ const buckets = new Map<string, RankedChunk[]>();
198
+ const fileOrder: string[] = [];
199
+ for (const candidate of candidates) {
200
+ const key = `${candidate.chunk.kb_id}:${candidate.chunk.file_path}`;
201
+ if (!buckets.has(key)) {
202
+ buckets.set(key, []);
203
+ fileOrder.push(key);
204
+ }
205
+ buckets.get(key)?.push(candidate);
206
+ }
207
+ if (fileOrder.length <= 1) return candidates;
208
+
209
+ const result: RankedChunk[] = [];
210
+ let round = 0;
211
+ while (result.length < candidates.length) {
212
+ let added = false;
213
+ for (const key of fileOrder) {
214
+ const bucket = buckets.get(key);
215
+ const item = bucket?.[round];
216
+ if (!item) continue;
217
+ result.push(item);
218
+ added = true;
219
+ }
220
+ if (!added) break;
221
+ round++;
222
+ }
223
+ return result;
224
+ }
225
+
226
+ function buildQuerySnippet(content: string, query: string, maxLength = 240): string {
227
+ const terms = [...tokenizeForSimilarity(query)].sort((a, b) => b.length - a.length);
228
+ const lower = content.toLowerCase();
229
+ let matchIndex = -1;
230
+ for (const term of terms) {
231
+ matchIndex = lower.indexOf(term.toLowerCase());
232
+ if (matchIndex >= 0) break;
233
+ }
234
+ if (matchIndex < 0) return content.slice(0, maxLength);
235
+ const half = Math.floor(maxLength / 2);
236
+ const start = Math.max(0, matchIndex - half);
237
+ const end = Math.min(content.length, start + maxLength);
238
+ const prefix = start > 0 ? "..." : "";
239
+ const suffix = end < content.length ? "..." : "";
240
+ return `${prefix}${content.slice(start, end)}${suffix}`;
241
+ }
242
+
243
+ function buildAdaptiveContext(
244
+ seed: Chunk,
245
+ chunks: Chunk[],
246
+ queryTokens: Set<string>,
247
+ ): {
248
+ content: string;
249
+ startLine: number;
250
+ endLine: number;
251
+ sourceChunkIds: string[];
252
+ } {
253
+ const scored = chunks
254
+ .map((chunk) => {
255
+ const distance =
256
+ chunk.id === seed.id
257
+ ? 0
258
+ : Math.min(Math.abs(chunk.start_line - seed.start_line), Math.abs(chunk.end_line - seed.end_line));
259
+ const proximity = 1 / (1 + distance / 20);
260
+ const coverage = queryCoverage(chunk.content, queryTokens);
261
+ const seedBoost = chunk.id === seed.id ? 2 : 0;
262
+ return { chunk, score: seedBoost + proximity + coverage };
263
+ })
264
+ .sort((a, b) => b.score - a.score)
265
+ .slice(0, ADAPTIVE_NEIGHBOR_TARGET);
266
+ if (!scored.some((item) => item.chunk.id === seed.id)) scored.push({ chunk: seed, score: Number.MAX_SAFE_INTEGER });
267
+
268
+ const selectedIds = new Set(scored.map((item) => item.chunk.id));
269
+ const ordered = chunks
270
+ .filter((chunk) => selectedIds.has(chunk.id))
271
+ .sort((a, b) => a.start_line - b.start_line || a.end_line - b.end_line);
272
+ const parts: string[] = [];
273
+ const sourceChunkIds: string[] = [];
274
+ let totalLength = 0;
275
+ for (const chunk of ordered) {
276
+ if (totalLength >= ADAPTIVE_MAX_CONTEXT_CHARS) break;
277
+ const remaining = ADAPTIVE_MAX_CONTEXT_CHARS - totalLength;
278
+ const text = chunk.content.slice(0, remaining);
279
+ parts.push(text);
280
+ sourceChunkIds.push(chunk.id);
281
+ totalLength += text.length + 2;
282
+ }
283
+ return {
284
+ content: parts.join("\n\n"),
285
+ startLine: Math.min(...ordered.map((chunk) => chunk.start_line)),
286
+ endLine: Math.max(...ordered.map((chunk) => chunk.end_line)),
287
+ sourceChunkIds,
288
+ };
289
+ }
290
+
291
+ function pushAdaptiveCandidate(candidates: RankedChunk[], candidate: RankedChunk): void {
292
+ const overlappingIndex = candidates.findIndex(
293
+ (existing) =>
294
+ existing.chunk.kb_id === candidate.chunk.kb_id &&
295
+ existing.chunk.file_path === candidate.chunk.file_path &&
296
+ existing.sourceChunkIds.some((id) => candidate.sourceChunkIds.includes(id)),
297
+ );
298
+ if (overlappingIndex < 0) {
299
+ candidates.push(candidate);
300
+ return;
301
+ }
302
+ const existing = candidates[overlappingIndex];
303
+ if (candidate.score > existing.score || candidate.sourceChunkIds.length > existing.sourceChunkIds.length) {
304
+ candidates[overlappingIndex] = candidate;
305
+ }
306
+ }
307
+
70
308
  async function chunkUrl(source: string, signal?: AbortSignal): Promise<Awaited<ReturnType<typeof chunkFile>>> {
71
309
  const res = await fetch(source, { signal });
72
310
  if (!res.ok) throw new Error(`Fetch failed: ${res.status}`);
@@ -109,6 +347,15 @@ export class KnowledgeEngine {
109
347
  this.vectorCache.delete(kbId);
110
348
  }
111
349
 
350
+ private getVectorsByChunkId(kbId: string, chunkIds: string[]): Map<string, Float32Array> {
351
+ const vectors = this.getVectors(kbId);
352
+ const mapped = new Map<string, Float32Array>();
353
+ for (let i = 0; i < chunkIds.length; i++) {
354
+ if (vectors[i]) mapped.set(chunkIds[i], vectors[i]);
355
+ }
356
+ return mapped;
357
+ }
358
+
112
359
  async add(
113
360
  source: string,
114
361
  name: string,
@@ -122,6 +369,13 @@ export class KnowledgeEngine {
122
369
  const isFile = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isFile();
123
370
  const sourceType = isUrl ? "url" : isDir ? "directory" : isFile ? "file" : "text";
124
371
 
372
+ const existingKB = getKBByName(this.db, name);
373
+ if (existingKB) {
374
+ throw new Error(
375
+ `Knowledge base "${name}" already exists. Use knowledge_update to refresh it, or knowledge_remove before adding a replacement.`,
376
+ );
377
+ }
378
+
125
379
  const kb = createKB(this.db, {
126
380
  name,
127
381
  source_path: isDir || isFile ? resolvedSource : isUrl ? source : undefined,
@@ -162,7 +416,7 @@ export class KnowledgeEngine {
162
416
  }
163
417
 
164
418
  onProgress?.(`${allChunks.length} chunks, embedding...`);
165
- const texts = allChunks.map((c) => c.content);
419
+ const texts = allChunks.map((c) => buildChunkEmbeddingText(c));
166
420
  const vectors = await embedDocuments(texts, signal);
167
421
 
168
422
  onProgress?.(`Storing...`);
@@ -243,7 +497,7 @@ export class KnowledgeEngine {
243
497
  if (chunksToAdd.length > 0) {
244
498
  onProgress?.(`Embedding ${chunksToAdd.length} new chunks...`);
245
499
  newVectors = await embedDocuments(
246
- chunksToAdd.map((c) => c.content),
500
+ chunksToAdd.map((c) => buildChunkEmbeddingText(c)),
247
501
  signal,
248
502
  );
249
503
  insertChunks(this.db, kb.id, chunksToAdd);
@@ -278,13 +532,20 @@ export class KnowledgeEngine {
278
532
  async search(query: string, options: SearchOptions = {}): Promise<SearchResponse> {
279
533
  if (!this.db) throw new Error("Engine not initialized");
280
534
  const db = this.db;
281
- const { mode = "hybrid", limit = 10, offset = 0, kb_id, filters } = options;
282
-
283
- const kbs = kb_id ? ([getKB(db, kb_id)].filter(Boolean) as KnowledgeBase[]) : listKBs(db);
535
+ const { mode = "hybrid", limit = 10, offset = 0, kb_id, filters, diversity = "balanced" } = options;
536
+ const retrievalMode = mode === "adaptive" ? "hybrid" : mode;
537
+ const candidateLimit = Math.max(50, offset + limit * 12);
538
+ const normalizedQuery = normalizedQueryText(query);
539
+ const queryTokens = tokenizeForSimilarity(query);
540
+ const normalizedFileType = normalizeFileTypeFilter(filters?.file_type);
541
+
542
+ const selectedKB = kb_id ? (getKB(db, kb_id) ?? getKBByName(db, kb_id)) : undefined;
543
+ const kbs = kb_id ? ([selectedKB].filter(Boolean) as KnowledgeBase[]) : listKBs(db);
284
544
  if (kbs.length === 0) return { results: [], total_count: 0, has_more: false };
285
545
 
286
546
  const warnings: string[] = [];
287
547
  const allResults: { chunkId: string; score: number }[] = [];
548
+ const vectorsByChunkId = new Map<string, Float32Array>();
288
549
 
289
550
  for (const kb of kbs) {
290
551
  if (kb.embedding_model !== CURRENT_EMBEDDING_MODEL) {
@@ -294,26 +555,30 @@ export class KnowledgeEngine {
294
555
  }
295
556
  const chunkIds = getChunkIdsByKB(db, kb.id);
296
557
  if (chunkIds.length === 0) continue;
558
+ for (const [chunkId, vector] of this.getVectorsByChunkId(kb.id, chunkIds)) {
559
+ vectorsByChunkId.set(chunkId, vector);
560
+ }
297
561
 
298
- if (mode === "fast") {
299
- allResults.push(...searchBM25(db, query, 50, kb.id));
300
- } else if (mode === "semantic") {
301
- const vectors = this.getVectors(kb.id);
562
+ if (retrievalMode === "fast") {
563
+ allResults.push(...searchBM25(db, normalizedQuery || query, candidateLimit, kb.id));
564
+ } else if (retrievalMode === "semantic") {
565
+ const vectors = [...this.getVectorsByChunkId(kb.id, chunkIds).values()];
302
566
  if (vectors.length > 0) {
303
567
  const queryVec = await embedQuery(query);
304
- allResults.push(...searchVector(queryVec, vectors, chunkIds));
568
+ allResults.push(...searchVector(queryVec, vectors, chunkIds, candidateLimit));
305
569
  }
306
570
  } else {
307
- // hybrid: BM25 + vector + RRF (both scoped to this KB)
308
- const bm25Results = searchBM25(db, query, 50, kb.id);
571
+ // hybrid: BM25 + vector weighted fusion (both scoped to this KB)
572
+ const bm25Results = searchBM25(db, normalizedQuery || query, candidateLimit, kb.id);
309
573
 
310
- const vectors = this.getVectors(kb.id);
574
+ const vectors = [...this.getVectorsByChunkId(kb.id, chunkIds).values()];
311
575
  let vecResults: { chunkId: string; score: number }[] = [];
312
576
  if (vectors.length > 0) {
313
577
  const queryVec = await embedQuery(query);
314
- vecResults = searchVector(queryVec, vectors, chunkIds);
578
+ vecResults = searchVector(queryVec, vectors, chunkIds, candidateLimit);
315
579
  }
316
- const fused = reciprocalRankFusion([bm25Results, vecResults]);
580
+ if (bm25Results.length === 0) continue;
581
+ const fused = weightedScoreFusion(bm25Results, vecResults);
317
582
  allResults.push(...fused);
318
583
  }
319
584
  }
@@ -325,74 +590,139 @@ export class KnowledgeEngine {
325
590
  seen.add(r.chunkId);
326
591
  return true;
327
592
  });
328
- unique.sort((a, b) => b.score - a.score);
593
+ const rankingByChunkId = new Map<string, RankingDiagnostics>();
594
+ for (const result of unique) {
595
+ const chunk = getChunkById(db, result.chunkId);
596
+ if (chunk) {
597
+ const ranking = scoreChunkForQuery(result.score, chunk, queryTokens);
598
+ rankingByChunkId.set(result.chunkId, ranking);
599
+ result.score = ranking.adjusted_score;
600
+ }
601
+ }
602
+ let scored = unique;
603
+ if (retrievalMode !== "fast" && retrievalMode !== "semantic") {
604
+ scored = unique.filter((result) => {
605
+ const chunk = getChunkById(db, result.chunkId);
606
+ if (!chunk) return false;
607
+ return result.score >= MIN_HYBRID_SCORE && hasEnoughLexicalEvidence(chunk, queryTokens);
608
+ });
609
+ }
610
+ scored.sort((a, b) => b.score - a.score);
329
611
 
330
612
  // Apply metadata filters post-retrieval
331
- let filtered = unique;
332
- if (filters?.file_type || filters?.path_pattern) {
333
- filtered = unique.filter((r) => {
613
+ let filtered = scored;
614
+ if (normalizedFileType || filters?.path_pattern) {
615
+ filtered = scored.filter((r) => {
334
616
  const chunk = getChunkById(db, r.chunkId);
335
617
  if (!chunk) return false;
336
- if (filters.file_type && chunk.file_type !== filters.file_type) return false;
337
- if (filters.path_pattern && !chunk.file_path.includes(filters.path_pattern)) return false;
618
+ if (normalizedFileType && chunk.file_type !== normalizedFileType) return false;
619
+ if (filters?.path_pattern && !chunk.file_path.includes(filters.path_pattern)) return false;
338
620
  return true;
339
621
  });
340
622
  }
341
623
 
342
- // Deep mode: rerank top-20 with cross-encoder
343
624
  if (mode === "deep" && filtered.length > 0) {
344
625
  const candidates = filtered
345
- .slice(0, 20)
626
+ .slice(0, 30)
346
627
  .map((r) => {
347
628
  const chunk = getChunkById(db, r.chunkId);
348
629
  return chunk ? { chunkId: r.chunkId, content: chunk.content } : null;
349
630
  })
350
631
  .filter(Boolean) as Array<{ chunkId: string; content: string }>;
351
- const reranked = await rerank(query, candidates, limit);
352
- const results: SearchResult[] = [];
632
+ const reranked = await rerank(query, candidates, Math.max(limit * 3, limit));
633
+ const ranked: RankedChunk[] = [];
353
634
  for (const r of reranked) {
354
635
  const chunk = getChunkById(db, r.chunkId);
355
636
  if (!chunk) continue;
356
637
  const kbObj = getKB(db, chunk.kb_id);
357
- results.push({
358
- content: chunk.content,
359
- file_path: chunk.file_path,
360
- file_type: chunk.file_type,
361
- kb_name: kbObj?.name ?? "unknown",
638
+ ranked.push({
639
+ chunk,
640
+ kbName: kbObj?.name ?? "unknown",
641
+ ranking: scoreChunkForQuery(r.score, chunk, queryTokens),
362
642
  score: r.score,
363
- snippet: chunk.content.slice(0, 200),
364
- start_line: chunk.start_line,
365
- end_line: chunk.end_line,
643
+ content: chunk.content,
644
+ snippet: buildQuerySnippet(chunk.content, query),
645
+ startLine: chunk.start_line,
646
+ endLine: chunk.end_line,
647
+ sourceChunkIds: [chunk.id],
366
648
  });
367
649
  }
650
+ const diversified = interleaveByFile(diversifyRankedChunks(ranked, diversity, vectorsByChunkId), diversity);
651
+ const page = diversified.slice(offset, offset + limit);
652
+ const results = page.map((r) => ({
653
+ content: r.content,
654
+ file_path: r.chunk.file_path,
655
+ file_type: r.chunk.file_type,
656
+ kb_name: r.kbName,
657
+ score: r.score,
658
+ ranking: r.ranking,
659
+ snippet: r.snippet,
660
+ start_line: r.startLine,
661
+ end_line: r.endLine,
662
+ }));
368
663
  return {
369
664
  results,
370
- total_count: results.length,
371
- has_more: false,
665
+ total_count: diversified.length,
666
+ has_more: offset + limit < diversified.length,
372
667
  warnings: warnings.length > 0 ? warnings : undefined,
373
668
  };
374
669
  }
375
670
 
376
- const total = filtered.length;
377
- const page = filtered.slice(offset, offset + limit);
378
-
379
- const results: SearchResult[] = page
380
- .map((r) => {
381
- const chunk = getChunkById(db, r.chunkId);
382
- if (!chunk) return null;
383
- const kb = getKB(db, chunk.kb_id);
384
- return {
385
- content: chunk.content,
386
- file_path: chunk.file_path,
387
- file_type: chunk.file_type,
388
- kb_name: kb?.name ?? "unknown",
671
+ const ranked: RankedChunk[] = [];
672
+ for (const r of filtered) {
673
+ const chunk = getChunkById(db, r.chunkId);
674
+ if (!chunk) continue;
675
+ const kb = getKB(db, chunk.kb_id);
676
+
677
+ if (mode === "adaptive") {
678
+ const contextChunks = getChunksByFile(
679
+ db,
680
+ chunk.kb_id,
681
+ chunk.file_path,
682
+ Math.max(1, chunk.start_line - ADAPTIVE_CONTEXT_LINES),
683
+ chunk.end_line + ADAPTIVE_CONTEXT_LINES,
684
+ );
685
+ const context = buildAdaptiveContext(chunk, contextChunks.length > 0 ? contextChunks : [chunk], queryTokens);
686
+ pushAdaptiveCandidate(ranked, {
687
+ chunk,
688
+ kbName: kb?.name ?? "unknown",
689
+ ranking: rankingByChunkId.get(r.chunkId),
389
690
  score: r.score,
390
- snippet: chunk.content.slice(0, 200),
391
- start_line: chunk.start_line,
392
- end_line: chunk.end_line,
393
- };
394
- })
395
- .filter(Boolean) as SearchResult[];
691
+ content: context.content,
692
+ snippet: buildQuerySnippet(context.content, query),
693
+ startLine: context.startLine,
694
+ endLine: context.endLine,
695
+ sourceChunkIds: context.sourceChunkIds,
696
+ });
697
+ } else {
698
+ ranked.push({
699
+ chunk,
700
+ kbName: kb?.name ?? "unknown",
701
+ ranking: rankingByChunkId.get(r.chunkId),
702
+ score: r.score,
703
+ content: chunk.content,
704
+ snippet: buildQuerySnippet(chunk.content, query),
705
+ startLine: chunk.start_line,
706
+ endLine: chunk.end_line,
707
+ sourceChunkIds: [chunk.id],
708
+ });
709
+ }
710
+ }
711
+
712
+ const diversified = interleaveByFile(diversifyRankedChunks(ranked, diversity, vectorsByChunkId), diversity);
713
+ const total = diversified.length;
714
+ const page = diversified.slice(offset, offset + limit);
715
+ const results: SearchResult[] = page.map((r) => ({
716
+ content: r.content,
717
+ file_path: r.chunk.file_path,
718
+ file_type: r.chunk.file_type,
719
+ kb_name: r.kbName,
720
+ score: r.score,
721
+ ranking: r.ranking,
722
+ snippet: r.snippet,
723
+ start_line: r.startLine,
724
+ end_line: r.endLine,
725
+ }));
396
726
 
397
727
  return {
398
728
  results,
@@ -472,23 +802,29 @@ export class KnowledgeEngine {
472
802
  try {
473
803
  const chunkData = lines.slice(1).map((l) => JSON.parse(l) as ImportedChunk);
474
804
  onProgress?.(`Importing ${chunkData.length} chunks, embedding...`);
475
- const texts = chunkData.map((c) => c.content);
476
- const vectors = await embedDocuments(texts, signal);
477
805
  const chunks = chunkData.map((c) => ({
478
806
  content_hash: contentHash(c.content),
479
807
  content: c.content,
480
- content_tokenized: preTokenizeForFTS(c.content),
808
+ content_tokenized: "",
481
809
  file_path: c.file_path,
482
810
  file_type: c.file_type,
483
811
  start_line: c.start_line,
484
812
  end_line: c.end_line,
485
813
  metadata_json: c.metadata_json || "{}",
486
814
  }));
487
- insertChunks(this.db, kb.id, chunks);
815
+ const indexedChunks = chunks.map((chunk) => ({
816
+ ...chunk,
817
+ content_tokenized: preTokenizeForFTS(buildChunkEmbeddingText(chunk)),
818
+ }));
819
+ const vectors = await embedDocuments(
820
+ indexedChunks.map((c) => buildChunkEmbeddingText(c)),
821
+ signal,
822
+ );
823
+ insertChunks(this.db, kb.id, indexedChunks);
488
824
  const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
489
825
  saveVectors(vectorPath, vectors);
490
826
  this.vectorCache.set(kb.id, vectors);
491
- updateKBCounts(this.db, kb.id, chunks.length, new Set(chunks.map((c) => c.file_path)).size);
827
+ updateKBCounts(this.db, kb.id, indexedChunks.length, new Set(indexedChunks.map((c) => c.file_path)).size);
492
828
  updateKBStatus(this.db, kb.id, "ready");
493
829
  const savedKB = getKB(this.db, kb.id);
494
830
  if (!savedKB) throw new Error(`Knowledge base disappeared after import: ${kb.id}`);
@@ -499,9 +835,15 @@ export class KnowledgeEngine {
499
835
  }
500
836
  }
501
837
 
502
- async dispose(): Promise<void> {
503
- await disposeEmbedding();
504
- await disposeReranker();
838
+ async dispose(options: { disposeModels?: boolean } = {}): Promise<void> {
839
+ const disposeModels = options.disposeModels ?? true;
840
+ if (disposeModels) {
841
+ await disposeEmbedding();
842
+ await disposeReranker();
843
+ } else {
844
+ await prepareEmbeddingForShutdown();
845
+ await prepareRerankerForShutdown();
846
+ }
505
847
  this.db?.close();
506
848
  this.db = null;
507
849
  }