pi-knowledge 0.3.4 → 0.3.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/engine.ts CHANGED
@@ -2,20 +2,36 @@ import { existsSync, statSync } from "node:fs";
2
2
  import { join, resolve } from "node:path";
3
3
  import type Database from "better-sqlite3";
4
4
  import { type DiagnosticResult, diagnoseKB } from "./diagnostics/health.ts";
5
- import { dispose as disposeEmbedding, embedDocuments, embedQuery } from "./embedding/provider.ts";
5
+ import {
6
+ dispose as disposeEmbedding,
7
+ embedDocuments,
8
+ embedQuery,
9
+ prepareForShutdown as prepareEmbeddingForShutdown,
10
+ } from "./embedding/provider.ts";
6
11
  import { loadVectors, saveVectors } from "./embedding/vectors.ts";
7
- import { chunkFile, contentHash, preTokenizeForFTS, walkDir } from "./indexer/chunker.ts";
12
+ import { buildChunkEmbeddingText, chunkFile, contentHash, preTokenizeForFTS, walkDir } from "./indexer/chunker.ts";
8
13
  import { searchBM25 } from "./search/bm25.ts";
9
- import { reciprocalRankFusion } from "./search/fusion.ts";
10
- import { disposeReranker, rerank } from "./search/reranker.ts";
14
+ import { weightedScoreFusion } from "./search/fusion.ts";
15
+ import { normalizedQueryText, tokenizeForSearch } from "./search/query.ts";
16
+ import {
17
+ hasEnoughLexicalEvidence,
18
+ MIN_HYBRID_SCORE,
19
+ normalizeFileTypeFilter,
20
+ queryCoverage,
21
+ type RankingDiagnostics,
22
+ scoreChunkForQuery,
23
+ } from "./search/ranking.ts";
24
+ import { disposeReranker, prepareRerankerForShutdown, rerank } from "./search/reranker.ts";
11
25
  import { searchVector } from "./search/vector.ts";
12
26
  import {
27
+ type Chunk,
13
28
  createKB,
14
29
  deleteChunksByIds,
15
30
  deleteKB,
16
31
  getChunkById,
17
32
  getChunkHashesByKB,
18
33
  getChunkIdsByKB,
34
+ getChunksByFile,
19
35
  getChunksByKB,
20
36
  getFileCount,
21
37
  getKB,
@@ -29,11 +45,12 @@ import {
29
45
  } from "./storage/sqlite.ts";
30
46
 
31
47
  export interface SearchOptions {
32
- mode?: "fast" | "semantic" | "hybrid" | "deep";
48
+ mode?: "fast" | "semantic" | "hybrid" | "deep" | "adaptive";
33
49
  limit?: number;
34
50
  offset?: number;
35
51
  kb_id?: string;
36
52
  filters?: { file_type?: string; path_pattern?: string };
53
+ diversity?: "off" | "balanced" | "strong";
37
54
  }
38
55
 
39
56
  export interface SearchResult {
@@ -45,6 +62,7 @@ export interface SearchResult {
45
62
  snippet: string;
46
63
  start_line: number;
47
64
  end_line: number;
65
+ ranking?: RankingDiagnostics;
48
66
  }
49
67
 
50
68
  export const CURRENT_EMBEDDING_MODEL = "multilingual-e5-small";
@@ -67,6 +85,226 @@ interface ImportedChunk {
67
85
  metadata_json?: string;
68
86
  }
69
87
 
88
+ interface RankedChunk {
89
+ chunk: Chunk;
90
+ kbName: string;
91
+ ranking?: RankingDiagnostics;
92
+ score: number;
93
+ content: string;
94
+ snippet: string;
95
+ startLine: number;
96
+ endLine: number;
97
+ sourceChunkIds: string[];
98
+ }
99
+
100
+ const ADAPTIVE_CONTEXT_LINES = 80;
101
+ const ADAPTIVE_MAX_CONTEXT_CHARS = 6_000;
102
+ const ADAPTIVE_NEIGHBOR_TARGET = 5;
103
+ const VECTOR_REDUNDANCY_WEIGHT = 0.35;
104
+
105
+ function cosineSimilarity(a: Float32Array | undefined, b: Float32Array | undefined): number {
106
+ if (!a || !b || a.length !== b.length) return 0;
107
+ let dot = 0;
108
+ for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
109
+ return dot;
110
+ }
111
+
112
+ function tokenizeForSimilarity(text: string): Set<string> {
113
+ return tokenizeForSearch(text);
114
+ }
115
+
116
+ function jaccardSimilarity(a: Set<string>, b: Set<string>): number {
117
+ if (a.size === 0 || b.size === 0) return 0;
118
+ let intersection = 0;
119
+ for (const token of a) {
120
+ if (b.has(token)) intersection++;
121
+ }
122
+ return intersection / (a.size + b.size - intersection);
123
+ }
124
+
125
+ function lineProximity(a: RankedChunk, b: RankedChunk): number {
126
+ if (a.chunk.file_path !== b.chunk.file_path || a.chunk.kb_id !== b.chunk.kb_id) return 0;
127
+ if (a.sourceChunkIds.some((id) => b.sourceChunkIds.includes(id))) return 1;
128
+ const overlap = Math.max(0, Math.min(a.endLine, b.endLine) - Math.max(a.startLine, b.startLine) + 1);
129
+ if (overlap > 0) return 1;
130
+ const gap = Math.max(a.startLine, b.startLine) - Math.min(a.endLine, b.endLine);
131
+ if (gap <= 20) return 0.8;
132
+ if (gap <= 80) return 0.45;
133
+ return 0.2;
134
+ }
135
+
136
+ function normalizeScores(candidates: RankedChunk[]): Map<string, number> {
137
+ const scores = candidates.map((candidate) => candidate.score);
138
+ const min = Math.min(...scores);
139
+ const max = Math.max(...scores);
140
+ const normalized = new Map<string, number>();
141
+ for (const candidate of candidates) {
142
+ const value = max === min ? 1 : (candidate.score - min) / (max - min);
143
+ normalized.set(candidate.chunk.id, value);
144
+ }
145
+ return normalized;
146
+ }
147
+
148
+ function diversifyRankedChunks(
149
+ candidates: RankedChunk[],
150
+ diversity: SearchOptions["diversity"],
151
+ vectorsByChunkId: Map<string, Float32Array> = new Map(),
152
+ ): RankedChunk[] {
153
+ if (diversity === "off" || candidates.length <= 2) return candidates;
154
+ const lambda = diversity === "strong" ? 0.62 : 0.76;
155
+ const normalized = normalizeScores(candidates);
156
+ const tokenSets = new Map(
157
+ candidates.map((candidate) => [candidate.chunk.id, tokenizeForSimilarity(candidate.content)]),
158
+ );
159
+ const selected: RankedChunk[] = [];
160
+ const remaining = [...candidates];
161
+
162
+ while (remaining.length > 0) {
163
+ let bestIndex = 0;
164
+ let bestScore = Number.NEGATIVE_INFINITY;
165
+ for (let i = 0; i < remaining.length; i++) {
166
+ const candidate = remaining[i];
167
+ let redundancy = 0;
168
+ for (const chosen of selected) {
169
+ const lexical = jaccardSimilarity(
170
+ tokenSets.get(candidate.chunk.id) ?? new Set<string>(),
171
+ tokenSets.get(chosen.chunk.id) ?? new Set<string>(),
172
+ );
173
+ const vector = Math.max(
174
+ 0,
175
+ cosineSimilarity(vectorsByChunkId.get(candidate.chunk.id), vectorsByChunkId.get(chosen.chunk.id)),
176
+ );
177
+ redundancy = Math.max(
178
+ redundancy,
179
+ Math.max(lexical, lineProximity(candidate, chosen), vector * VECTOR_REDUNDANCY_WEIGHT),
180
+ );
181
+ }
182
+ const relevance = normalized.get(candidate.chunk.id) ?? 0;
183
+ const mmrScore = lambda * relevance - (1 - lambda) * redundancy;
184
+ if (mmrScore > bestScore) {
185
+ bestScore = mmrScore;
186
+ bestIndex = i;
187
+ }
188
+ }
189
+ selected.push(remaining.splice(bestIndex, 1)[0]);
190
+ }
191
+
192
+ return selected;
193
+ }
194
+
195
+ function interleaveByFile(candidates: RankedChunk[], diversity: SearchOptions["diversity"]): RankedChunk[] {
196
+ if (diversity === "off" || candidates.length <= 2) return candidates;
197
+ const buckets = new Map<string, RankedChunk[]>();
198
+ const fileOrder: string[] = [];
199
+ for (const candidate of candidates) {
200
+ const key = `${candidate.chunk.kb_id}:${candidate.chunk.file_path}`;
201
+ if (!buckets.has(key)) {
202
+ buckets.set(key, []);
203
+ fileOrder.push(key);
204
+ }
205
+ buckets.get(key)?.push(candidate);
206
+ }
207
+ if (fileOrder.length <= 1) return candidates;
208
+
209
+ const result: RankedChunk[] = [];
210
+ let round = 0;
211
+ while (result.length < candidates.length) {
212
+ let added = false;
213
+ for (const key of fileOrder) {
214
+ const bucket = buckets.get(key);
215
+ const item = bucket?.[round];
216
+ if (!item) continue;
217
+ result.push(item);
218
+ added = true;
219
+ }
220
+ if (!added) break;
221
+ round++;
222
+ }
223
+ return result;
224
+ }
225
+
226
+ function buildQuerySnippet(content: string, query: string, maxLength = 240): string {
227
+ const terms = [...tokenizeForSimilarity(query)].sort((a, b) => b.length - a.length);
228
+ const lower = content.toLowerCase();
229
+ let matchIndex = -1;
230
+ for (const term of terms) {
231
+ matchIndex = lower.indexOf(term.toLowerCase());
232
+ if (matchIndex >= 0) break;
233
+ }
234
+ if (matchIndex < 0) return content.slice(0, maxLength);
235
+ const half = Math.floor(maxLength / 2);
236
+ const start = Math.max(0, matchIndex - half);
237
+ const end = Math.min(content.length, start + maxLength);
238
+ const prefix = start > 0 ? "..." : "";
239
+ const suffix = end < content.length ? "..." : "";
240
+ return `${prefix}${content.slice(start, end)}${suffix}`;
241
+ }
242
+
243
+ function buildAdaptiveContext(
244
+ seed: Chunk,
245
+ chunks: Chunk[],
246
+ queryTokens: Set<string>,
247
+ ): {
248
+ content: string;
249
+ startLine: number;
250
+ endLine: number;
251
+ sourceChunkIds: string[];
252
+ } {
253
+ const scored = chunks
254
+ .map((chunk) => {
255
+ const distance =
256
+ chunk.id === seed.id
257
+ ? 0
258
+ : Math.min(Math.abs(chunk.start_line - seed.start_line), Math.abs(chunk.end_line - seed.end_line));
259
+ const proximity = 1 / (1 + distance / 20);
260
+ const coverage = queryCoverage(chunk.content, queryTokens);
261
+ const seedBoost = chunk.id === seed.id ? 2 : 0;
262
+ return { chunk, score: seedBoost + proximity + coverage };
263
+ })
264
+ .sort((a, b) => b.score - a.score)
265
+ .slice(0, ADAPTIVE_NEIGHBOR_TARGET);
266
+ if (!scored.some((item) => item.chunk.id === seed.id)) scored.push({ chunk: seed, score: Number.MAX_SAFE_INTEGER });
267
+
268
+ const selectedIds = new Set(scored.map((item) => item.chunk.id));
269
+ const ordered = chunks
270
+ .filter((chunk) => selectedIds.has(chunk.id))
271
+ .sort((a, b) => a.start_line - b.start_line || a.end_line - b.end_line);
272
+ const parts: string[] = [];
273
+ const sourceChunkIds: string[] = [];
274
+ let totalLength = 0;
275
+ for (const chunk of ordered) {
276
+ if (totalLength >= ADAPTIVE_MAX_CONTEXT_CHARS) break;
277
+ const remaining = ADAPTIVE_MAX_CONTEXT_CHARS - totalLength;
278
+ const text = chunk.content.slice(0, remaining);
279
+ parts.push(text);
280
+ sourceChunkIds.push(chunk.id);
281
+ totalLength += text.length + 2;
282
+ }
283
+ return {
284
+ content: parts.join("\n\n"),
285
+ startLine: Math.min(...ordered.map((chunk) => chunk.start_line)),
286
+ endLine: Math.max(...ordered.map((chunk) => chunk.end_line)),
287
+ sourceChunkIds,
288
+ };
289
+ }
290
+
291
+ function pushAdaptiveCandidate(candidates: RankedChunk[], candidate: RankedChunk): void {
292
+ const overlappingIndex = candidates.findIndex(
293
+ (existing) =>
294
+ existing.chunk.kb_id === candidate.chunk.kb_id &&
295
+ existing.chunk.file_path === candidate.chunk.file_path &&
296
+ existing.sourceChunkIds.some((id) => candidate.sourceChunkIds.includes(id)),
297
+ );
298
+ if (overlappingIndex < 0) {
299
+ candidates.push(candidate);
300
+ return;
301
+ }
302
+ const existing = candidates[overlappingIndex];
303
+ if (candidate.score > existing.score || candidate.sourceChunkIds.length > existing.sourceChunkIds.length) {
304
+ candidates[overlappingIndex] = candidate;
305
+ }
306
+ }
307
+
70
308
  async function chunkUrl(source: string, signal?: AbortSignal): Promise<Awaited<ReturnType<typeof chunkFile>>> {
71
309
  const res = await fetch(source, { signal });
72
310
  if (!res.ok) throw new Error(`Fetch failed: ${res.status}`);
@@ -109,6 +347,15 @@ export class KnowledgeEngine {
109
347
  this.vectorCache.delete(kbId);
110
348
  }
111
349
 
350
+ private getVectorsByChunkId(kbId: string, chunkIds: string[]): Map<string, Float32Array> {
351
+ const vectors = this.getVectors(kbId);
352
+ const mapped = new Map<string, Float32Array>();
353
+ for (let i = 0; i < chunkIds.length; i++) {
354
+ if (vectors[i]) mapped.set(chunkIds[i], vectors[i]);
355
+ }
356
+ return mapped;
357
+ }
358
+
112
359
  async add(
113
360
  source: string,
114
361
  name: string,
@@ -169,7 +416,7 @@ export class KnowledgeEngine {
169
416
  }
170
417
 
171
418
  onProgress?.(`${allChunks.length} chunks, embedding...`);
172
- const texts = allChunks.map((c) => c.content);
419
+ const texts = allChunks.map((c) => buildChunkEmbeddingText(c));
173
420
  const vectors = await embedDocuments(texts, signal);
174
421
 
175
422
  onProgress?.(`Storing...`);
@@ -250,7 +497,7 @@ export class KnowledgeEngine {
250
497
  if (chunksToAdd.length > 0) {
251
498
  onProgress?.(`Embedding ${chunksToAdd.length} new chunks...`);
252
499
  newVectors = await embedDocuments(
253
- chunksToAdd.map((c) => c.content),
500
+ chunksToAdd.map((c) => buildChunkEmbeddingText(c)),
254
501
  signal,
255
502
  );
256
503
  insertChunks(this.db, kb.id, chunksToAdd);
@@ -285,13 +532,20 @@ export class KnowledgeEngine {
285
532
  async search(query: string, options: SearchOptions = {}): Promise<SearchResponse> {
286
533
  if (!this.db) throw new Error("Engine not initialized");
287
534
  const db = this.db;
288
- const { mode = "hybrid", limit = 10, offset = 0, kb_id, filters } = options;
289
-
290
- const kbs = kb_id ? ([getKB(db, kb_id)].filter(Boolean) as KnowledgeBase[]) : listKBs(db);
535
+ const { mode = "hybrid", limit = 10, offset = 0, kb_id, filters, diversity = "balanced" } = options;
536
+ const retrievalMode = mode === "adaptive" ? "hybrid" : mode;
537
+ const candidateLimit = Math.max(50, offset + limit * 12);
538
+ const normalizedQuery = normalizedQueryText(query);
539
+ const queryTokens = tokenizeForSimilarity(query);
540
+ const normalizedFileType = normalizeFileTypeFilter(filters?.file_type);
541
+
542
+ const selectedKB = kb_id ? (getKB(db, kb_id) ?? getKBByName(db, kb_id)) : undefined;
543
+ const kbs = kb_id ? ([selectedKB].filter(Boolean) as KnowledgeBase[]) : listKBs(db);
291
544
  if (kbs.length === 0) return { results: [], total_count: 0, has_more: false };
292
545
 
293
546
  const warnings: string[] = [];
294
547
  const allResults: { chunkId: string; score: number }[] = [];
548
+ const vectorsByChunkId = new Map<string, Float32Array>();
295
549
 
296
550
  for (const kb of kbs) {
297
551
  if (kb.embedding_model !== CURRENT_EMBEDDING_MODEL) {
@@ -301,26 +555,30 @@ export class KnowledgeEngine {
301
555
  }
302
556
  const chunkIds = getChunkIdsByKB(db, kb.id);
303
557
  if (chunkIds.length === 0) continue;
558
+ for (const [chunkId, vector] of this.getVectorsByChunkId(kb.id, chunkIds)) {
559
+ vectorsByChunkId.set(chunkId, vector);
560
+ }
304
561
 
305
- if (mode === "fast") {
306
- allResults.push(...searchBM25(db, query, 50, kb.id));
307
- } else if (mode === "semantic") {
308
- const vectors = this.getVectors(kb.id);
562
+ if (retrievalMode === "fast") {
563
+ allResults.push(...searchBM25(db, normalizedQuery || query, candidateLimit, kb.id));
564
+ } else if (retrievalMode === "semantic") {
565
+ const vectors = [...this.getVectorsByChunkId(kb.id, chunkIds).values()];
309
566
  if (vectors.length > 0) {
310
567
  const queryVec = await embedQuery(query);
311
- allResults.push(...searchVector(queryVec, vectors, chunkIds));
568
+ allResults.push(...searchVector(queryVec, vectors, chunkIds, candidateLimit));
312
569
  }
313
570
  } else {
314
- // hybrid: BM25 + vector + RRF (both scoped to this KB)
315
- const bm25Results = searchBM25(db, query, 50, kb.id);
571
+ // hybrid: BM25 + vector weighted fusion (both scoped to this KB)
572
+ const bm25Results = searchBM25(db, normalizedQuery || query, candidateLimit, kb.id);
316
573
 
317
- const vectors = this.getVectors(kb.id);
574
+ const vectors = [...this.getVectorsByChunkId(kb.id, chunkIds).values()];
318
575
  let vecResults: { chunkId: string; score: number }[] = [];
319
576
  if (vectors.length > 0) {
320
577
  const queryVec = await embedQuery(query);
321
- vecResults = searchVector(queryVec, vectors, chunkIds);
578
+ vecResults = searchVector(queryVec, vectors, chunkIds, candidateLimit);
322
579
  }
323
- const fused = reciprocalRankFusion([bm25Results, vecResults]);
580
+ if (bm25Results.length === 0) continue;
581
+ const fused = weightedScoreFusion(bm25Results, vecResults);
324
582
  allResults.push(...fused);
325
583
  }
326
584
  }
@@ -332,74 +590,139 @@ export class KnowledgeEngine {
332
590
  seen.add(r.chunkId);
333
591
  return true;
334
592
  });
335
- unique.sort((a, b) => b.score - a.score);
593
+ const rankingByChunkId = new Map<string, RankingDiagnostics>();
594
+ for (const result of unique) {
595
+ const chunk = getChunkById(db, result.chunkId);
596
+ if (chunk) {
597
+ const ranking = scoreChunkForQuery(result.score, chunk, queryTokens);
598
+ rankingByChunkId.set(result.chunkId, ranking);
599
+ result.score = ranking.adjusted_score;
600
+ }
601
+ }
602
+ let scored = unique;
603
+ if (retrievalMode !== "fast" && retrievalMode !== "semantic") {
604
+ scored = unique.filter((result) => {
605
+ const chunk = getChunkById(db, result.chunkId);
606
+ if (!chunk) return false;
607
+ return result.score >= MIN_HYBRID_SCORE && hasEnoughLexicalEvidence(chunk, queryTokens);
608
+ });
609
+ }
610
+ scored.sort((a, b) => b.score - a.score);
336
611
 
337
612
  // Apply metadata filters post-retrieval
338
- let filtered = unique;
339
- if (filters?.file_type || filters?.path_pattern) {
340
- filtered = unique.filter((r) => {
613
+ let filtered = scored;
614
+ if (normalizedFileType || filters?.path_pattern) {
615
+ filtered = scored.filter((r) => {
341
616
  const chunk = getChunkById(db, r.chunkId);
342
617
  if (!chunk) return false;
343
- if (filters.file_type && chunk.file_type !== filters.file_type) return false;
344
- if (filters.path_pattern && !chunk.file_path.includes(filters.path_pattern)) return false;
618
+ if (normalizedFileType && chunk.file_type !== normalizedFileType) return false;
619
+ if (filters?.path_pattern && !chunk.file_path.includes(filters.path_pattern)) return false;
345
620
  return true;
346
621
  });
347
622
  }
348
623
 
349
- // Deep mode: rerank top-20 with cross-encoder
350
624
  if (mode === "deep" && filtered.length > 0) {
351
625
  const candidates = filtered
352
- .slice(0, 20)
626
+ .slice(0, 30)
353
627
  .map((r) => {
354
628
  const chunk = getChunkById(db, r.chunkId);
355
629
  return chunk ? { chunkId: r.chunkId, content: chunk.content } : null;
356
630
  })
357
631
  .filter(Boolean) as Array<{ chunkId: string; content: string }>;
358
- const reranked = await rerank(query, candidates, limit);
359
- const results: SearchResult[] = [];
632
+ const reranked = await rerank(query, candidates, Math.max(limit * 3, limit));
633
+ const ranked: RankedChunk[] = [];
360
634
  for (const r of reranked) {
361
635
  const chunk = getChunkById(db, r.chunkId);
362
636
  if (!chunk) continue;
363
637
  const kbObj = getKB(db, chunk.kb_id);
364
- results.push({
365
- content: chunk.content,
366
- file_path: chunk.file_path,
367
- file_type: chunk.file_type,
368
- kb_name: kbObj?.name ?? "unknown",
638
+ ranked.push({
639
+ chunk,
640
+ kbName: kbObj?.name ?? "unknown",
641
+ ranking: scoreChunkForQuery(r.score, chunk, queryTokens),
369
642
  score: r.score,
370
- snippet: chunk.content.slice(0, 200),
371
- start_line: chunk.start_line,
372
- end_line: chunk.end_line,
643
+ content: chunk.content,
644
+ snippet: buildQuerySnippet(chunk.content, query),
645
+ startLine: chunk.start_line,
646
+ endLine: chunk.end_line,
647
+ sourceChunkIds: [chunk.id],
373
648
  });
374
649
  }
650
+ const diversified = interleaveByFile(diversifyRankedChunks(ranked, diversity, vectorsByChunkId), diversity);
651
+ const page = diversified.slice(offset, offset + limit);
652
+ const results = page.map((r) => ({
653
+ content: r.content,
654
+ file_path: r.chunk.file_path,
655
+ file_type: r.chunk.file_type,
656
+ kb_name: r.kbName,
657
+ score: r.score,
658
+ ranking: r.ranking,
659
+ snippet: r.snippet,
660
+ start_line: r.startLine,
661
+ end_line: r.endLine,
662
+ }));
375
663
  return {
376
664
  results,
377
- total_count: results.length,
378
- has_more: false,
665
+ total_count: diversified.length,
666
+ has_more: offset + limit < diversified.length,
379
667
  warnings: warnings.length > 0 ? warnings : undefined,
380
668
  };
381
669
  }
382
670
 
383
- const total = filtered.length;
384
- const page = filtered.slice(offset, offset + limit);
385
-
386
- const results: SearchResult[] = page
387
- .map((r) => {
388
- const chunk = getChunkById(db, r.chunkId);
389
- if (!chunk) return null;
390
- const kb = getKB(db, chunk.kb_id);
391
- return {
392
- content: chunk.content,
393
- file_path: chunk.file_path,
394
- file_type: chunk.file_type,
395
- kb_name: kb?.name ?? "unknown",
671
+ const ranked: RankedChunk[] = [];
672
+ for (const r of filtered) {
673
+ const chunk = getChunkById(db, r.chunkId);
674
+ if (!chunk) continue;
675
+ const kb = getKB(db, chunk.kb_id);
676
+
677
+ if (mode === "adaptive") {
678
+ const contextChunks = getChunksByFile(
679
+ db,
680
+ chunk.kb_id,
681
+ chunk.file_path,
682
+ Math.max(1, chunk.start_line - ADAPTIVE_CONTEXT_LINES),
683
+ chunk.end_line + ADAPTIVE_CONTEXT_LINES,
684
+ );
685
+ const context = buildAdaptiveContext(chunk, contextChunks.length > 0 ? contextChunks : [chunk], queryTokens);
686
+ pushAdaptiveCandidate(ranked, {
687
+ chunk,
688
+ kbName: kb?.name ?? "unknown",
689
+ ranking: rankingByChunkId.get(r.chunkId),
690
+ score: r.score,
691
+ content: context.content,
692
+ snippet: buildQuerySnippet(context.content, query),
693
+ startLine: context.startLine,
694
+ endLine: context.endLine,
695
+ sourceChunkIds: context.sourceChunkIds,
696
+ });
697
+ } else {
698
+ ranked.push({
699
+ chunk,
700
+ kbName: kb?.name ?? "unknown",
701
+ ranking: rankingByChunkId.get(r.chunkId),
396
702
  score: r.score,
397
- snippet: chunk.content.slice(0, 200),
398
- start_line: chunk.start_line,
399
- end_line: chunk.end_line,
400
- };
401
- })
402
- .filter(Boolean) as SearchResult[];
703
+ content: chunk.content,
704
+ snippet: buildQuerySnippet(chunk.content, query),
705
+ startLine: chunk.start_line,
706
+ endLine: chunk.end_line,
707
+ sourceChunkIds: [chunk.id],
708
+ });
709
+ }
710
+ }
711
+
712
+ const diversified = interleaveByFile(diversifyRankedChunks(ranked, diversity, vectorsByChunkId), diversity);
713
+ const total = diversified.length;
714
+ const page = diversified.slice(offset, offset + limit);
715
+ const results: SearchResult[] = page.map((r) => ({
716
+ content: r.content,
717
+ file_path: r.chunk.file_path,
718
+ file_type: r.chunk.file_type,
719
+ kb_name: r.kbName,
720
+ score: r.score,
721
+ ranking: r.ranking,
722
+ snippet: r.snippet,
723
+ start_line: r.startLine,
724
+ end_line: r.endLine,
725
+ }));
403
726
 
404
727
  return {
405
728
  results,
@@ -479,23 +802,29 @@ export class KnowledgeEngine {
479
802
  try {
480
803
  const chunkData = lines.slice(1).map((l) => JSON.parse(l) as ImportedChunk);
481
804
  onProgress?.(`Importing ${chunkData.length} chunks, embedding...`);
482
- const texts = chunkData.map((c) => c.content);
483
- const vectors = await embedDocuments(texts, signal);
484
805
  const chunks = chunkData.map((c) => ({
485
806
  content_hash: contentHash(c.content),
486
807
  content: c.content,
487
- content_tokenized: preTokenizeForFTS(c.content),
808
+ content_tokenized: "",
488
809
  file_path: c.file_path,
489
810
  file_type: c.file_type,
490
811
  start_line: c.start_line,
491
812
  end_line: c.end_line,
492
813
  metadata_json: c.metadata_json || "{}",
493
814
  }));
494
- insertChunks(this.db, kb.id, chunks);
815
+ const indexedChunks = chunks.map((chunk) => ({
816
+ ...chunk,
817
+ content_tokenized: preTokenizeForFTS(buildChunkEmbeddingText(chunk)),
818
+ }));
819
+ const vectors = await embedDocuments(
820
+ indexedChunks.map((c) => buildChunkEmbeddingText(c)),
821
+ signal,
822
+ );
823
+ insertChunks(this.db, kb.id, indexedChunks);
495
824
  const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
496
825
  saveVectors(vectorPath, vectors);
497
826
  this.vectorCache.set(kb.id, vectors);
498
- updateKBCounts(this.db, kb.id, chunks.length, new Set(chunks.map((c) => c.file_path)).size);
827
+ updateKBCounts(this.db, kb.id, indexedChunks.length, new Set(indexedChunks.map((c) => c.file_path)).size);
499
828
  updateKBStatus(this.db, kb.id, "ready");
500
829
  const savedKB = getKB(this.db, kb.id);
501
830
  if (!savedKB) throw new Error(`Knowledge base disappeared after import: ${kb.id}`);
@@ -506,9 +835,15 @@ export class KnowledgeEngine {
506
835
  }
507
836
  }
508
837
 
509
- async dispose(): Promise<void> {
510
- await disposeEmbedding();
511
- await disposeReranker();
838
+ async dispose(options: { disposeModels?: boolean } = {}): Promise<void> {
839
+ const disposeModels = options.disposeModels ?? true;
840
+ if (disposeModels) {
841
+ await disposeEmbedding();
842
+ await disposeReranker();
843
+ } else {
844
+ await prepareEmbeddingForShutdown();
845
+ await prepareRerankerForShutdown();
846
+ }
512
847
  this.db?.close();
513
848
  this.db = null;
514
849
  }