pi-knowledge 0.3.3 → 0.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +36 -0
- package/README.md +16 -2
- package/index.ts +67 -7
- package/package.json +1 -1
- package/src/embedding/provider.ts +50 -12
- package/src/engine.ts +409 -67
- package/src/indexer/chunker.ts +121 -77
- package/src/indexer/chunkers/code-ast.ts +14 -11
- package/src/search/bm25.ts +26 -22
- package/src/search/fusion.ts +33 -0
- package/src/search/query.ts +59 -0
- package/src/search/ranking.ts +197 -0
- package/src/search/reranker.ts +49 -11
- package/src/storage/sqlite.ts +16 -0
package/src/engine.ts
CHANGED
|
@@ -2,20 +2,36 @@ import { existsSync, statSync } from "node:fs";
|
|
|
2
2
|
import { join, resolve } from "node:path";
|
|
3
3
|
import type Database from "better-sqlite3";
|
|
4
4
|
import { type DiagnosticResult, diagnoseKB } from "./diagnostics/health.ts";
|
|
5
|
-
import {
|
|
5
|
+
import {
|
|
6
|
+
dispose as disposeEmbedding,
|
|
7
|
+
embedDocuments,
|
|
8
|
+
embedQuery,
|
|
9
|
+
prepareForShutdown as prepareEmbeddingForShutdown,
|
|
10
|
+
} from "./embedding/provider.ts";
|
|
6
11
|
import { loadVectors, saveVectors } from "./embedding/vectors.ts";
|
|
7
|
-
import { chunkFile, contentHash, preTokenizeForFTS, walkDir } from "./indexer/chunker.ts";
|
|
12
|
+
import { buildChunkEmbeddingText, chunkFile, contentHash, preTokenizeForFTS, walkDir } from "./indexer/chunker.ts";
|
|
8
13
|
import { searchBM25 } from "./search/bm25.ts";
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
14
|
+
import { weightedScoreFusion } from "./search/fusion.ts";
|
|
15
|
+
import { normalizedQueryText, tokenizeForSearch } from "./search/query.ts";
|
|
16
|
+
import {
|
|
17
|
+
hasEnoughLexicalEvidence,
|
|
18
|
+
MIN_HYBRID_SCORE,
|
|
19
|
+
normalizeFileTypeFilter,
|
|
20
|
+
queryCoverage,
|
|
21
|
+
type RankingDiagnostics,
|
|
22
|
+
scoreChunkForQuery,
|
|
23
|
+
} from "./search/ranking.ts";
|
|
24
|
+
import { disposeReranker, prepareRerankerForShutdown, rerank } from "./search/reranker.ts";
|
|
11
25
|
import { searchVector } from "./search/vector.ts";
|
|
12
26
|
import {
|
|
27
|
+
type Chunk,
|
|
13
28
|
createKB,
|
|
14
29
|
deleteChunksByIds,
|
|
15
30
|
deleteKB,
|
|
16
31
|
getChunkById,
|
|
17
32
|
getChunkHashesByKB,
|
|
18
33
|
getChunkIdsByKB,
|
|
34
|
+
getChunksByFile,
|
|
19
35
|
getChunksByKB,
|
|
20
36
|
getFileCount,
|
|
21
37
|
getKB,
|
|
@@ -29,11 +45,12 @@ import {
|
|
|
29
45
|
} from "./storage/sqlite.ts";
|
|
30
46
|
|
|
31
47
|
export interface SearchOptions {
|
|
32
|
-
mode?: "fast" | "semantic" | "hybrid" | "deep";
|
|
48
|
+
mode?: "fast" | "semantic" | "hybrid" | "deep" | "adaptive";
|
|
33
49
|
limit?: number;
|
|
34
50
|
offset?: number;
|
|
35
51
|
kb_id?: string;
|
|
36
52
|
filters?: { file_type?: string; path_pattern?: string };
|
|
53
|
+
diversity?: "off" | "balanced" | "strong";
|
|
37
54
|
}
|
|
38
55
|
|
|
39
56
|
export interface SearchResult {
|
|
@@ -45,6 +62,7 @@ export interface SearchResult {
|
|
|
45
62
|
snippet: string;
|
|
46
63
|
start_line: number;
|
|
47
64
|
end_line: number;
|
|
65
|
+
ranking?: RankingDiagnostics;
|
|
48
66
|
}
|
|
49
67
|
|
|
50
68
|
export const CURRENT_EMBEDDING_MODEL = "multilingual-e5-small";
|
|
@@ -67,6 +85,226 @@ interface ImportedChunk {
|
|
|
67
85
|
metadata_json?: string;
|
|
68
86
|
}
|
|
69
87
|
|
|
88
|
+
interface RankedChunk {
|
|
89
|
+
chunk: Chunk;
|
|
90
|
+
kbName: string;
|
|
91
|
+
ranking?: RankingDiagnostics;
|
|
92
|
+
score: number;
|
|
93
|
+
content: string;
|
|
94
|
+
snippet: string;
|
|
95
|
+
startLine: number;
|
|
96
|
+
endLine: number;
|
|
97
|
+
sourceChunkIds: string[];
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
const ADAPTIVE_CONTEXT_LINES = 80;
|
|
101
|
+
const ADAPTIVE_MAX_CONTEXT_CHARS = 6_000;
|
|
102
|
+
const ADAPTIVE_NEIGHBOR_TARGET = 5;
|
|
103
|
+
const VECTOR_REDUNDANCY_WEIGHT = 0.35;
|
|
104
|
+
|
|
105
|
+
function cosineSimilarity(a: Float32Array | undefined, b: Float32Array | undefined): number {
|
|
106
|
+
if (!a || !b || a.length !== b.length) return 0;
|
|
107
|
+
let dot = 0;
|
|
108
|
+
for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
|
|
109
|
+
return dot;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
function tokenizeForSimilarity(text: string): Set<string> {
|
|
113
|
+
return tokenizeForSearch(text);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function jaccardSimilarity(a: Set<string>, b: Set<string>): number {
|
|
117
|
+
if (a.size === 0 || b.size === 0) return 0;
|
|
118
|
+
let intersection = 0;
|
|
119
|
+
for (const token of a) {
|
|
120
|
+
if (b.has(token)) intersection++;
|
|
121
|
+
}
|
|
122
|
+
return intersection / (a.size + b.size - intersection);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function lineProximity(a: RankedChunk, b: RankedChunk): number {
|
|
126
|
+
if (a.chunk.file_path !== b.chunk.file_path || a.chunk.kb_id !== b.chunk.kb_id) return 0;
|
|
127
|
+
if (a.sourceChunkIds.some((id) => b.sourceChunkIds.includes(id))) return 1;
|
|
128
|
+
const overlap = Math.max(0, Math.min(a.endLine, b.endLine) - Math.max(a.startLine, b.startLine) + 1);
|
|
129
|
+
if (overlap > 0) return 1;
|
|
130
|
+
const gap = Math.max(a.startLine, b.startLine) - Math.min(a.endLine, b.endLine);
|
|
131
|
+
if (gap <= 20) return 0.8;
|
|
132
|
+
if (gap <= 80) return 0.45;
|
|
133
|
+
return 0.2;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function normalizeScores(candidates: RankedChunk[]): Map<string, number> {
|
|
137
|
+
const scores = candidates.map((candidate) => candidate.score);
|
|
138
|
+
const min = Math.min(...scores);
|
|
139
|
+
const max = Math.max(...scores);
|
|
140
|
+
const normalized = new Map<string, number>();
|
|
141
|
+
for (const candidate of candidates) {
|
|
142
|
+
const value = max === min ? 1 : (candidate.score - min) / (max - min);
|
|
143
|
+
normalized.set(candidate.chunk.id, value);
|
|
144
|
+
}
|
|
145
|
+
return normalized;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
function diversifyRankedChunks(
|
|
149
|
+
candidates: RankedChunk[],
|
|
150
|
+
diversity: SearchOptions["diversity"],
|
|
151
|
+
vectorsByChunkId: Map<string, Float32Array> = new Map(),
|
|
152
|
+
): RankedChunk[] {
|
|
153
|
+
if (diversity === "off" || candidates.length <= 2) return candidates;
|
|
154
|
+
const lambda = diversity === "strong" ? 0.62 : 0.76;
|
|
155
|
+
const normalized = normalizeScores(candidates);
|
|
156
|
+
const tokenSets = new Map(
|
|
157
|
+
candidates.map((candidate) => [candidate.chunk.id, tokenizeForSimilarity(candidate.content)]),
|
|
158
|
+
);
|
|
159
|
+
const selected: RankedChunk[] = [];
|
|
160
|
+
const remaining = [...candidates];
|
|
161
|
+
|
|
162
|
+
while (remaining.length > 0) {
|
|
163
|
+
let bestIndex = 0;
|
|
164
|
+
let bestScore = Number.NEGATIVE_INFINITY;
|
|
165
|
+
for (let i = 0; i < remaining.length; i++) {
|
|
166
|
+
const candidate = remaining[i];
|
|
167
|
+
let redundancy = 0;
|
|
168
|
+
for (const chosen of selected) {
|
|
169
|
+
const lexical = jaccardSimilarity(
|
|
170
|
+
tokenSets.get(candidate.chunk.id) ?? new Set<string>(),
|
|
171
|
+
tokenSets.get(chosen.chunk.id) ?? new Set<string>(),
|
|
172
|
+
);
|
|
173
|
+
const vector = Math.max(
|
|
174
|
+
0,
|
|
175
|
+
cosineSimilarity(vectorsByChunkId.get(candidate.chunk.id), vectorsByChunkId.get(chosen.chunk.id)),
|
|
176
|
+
);
|
|
177
|
+
redundancy = Math.max(
|
|
178
|
+
redundancy,
|
|
179
|
+
Math.max(lexical, lineProximity(candidate, chosen), vector * VECTOR_REDUNDANCY_WEIGHT),
|
|
180
|
+
);
|
|
181
|
+
}
|
|
182
|
+
const relevance = normalized.get(candidate.chunk.id) ?? 0;
|
|
183
|
+
const mmrScore = lambda * relevance - (1 - lambda) * redundancy;
|
|
184
|
+
if (mmrScore > bestScore) {
|
|
185
|
+
bestScore = mmrScore;
|
|
186
|
+
bestIndex = i;
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
selected.push(remaining.splice(bestIndex, 1)[0]);
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
return selected;
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
function interleaveByFile(candidates: RankedChunk[], diversity: SearchOptions["diversity"]): RankedChunk[] {
|
|
196
|
+
if (diversity === "off" || candidates.length <= 2) return candidates;
|
|
197
|
+
const buckets = new Map<string, RankedChunk[]>();
|
|
198
|
+
const fileOrder: string[] = [];
|
|
199
|
+
for (const candidate of candidates) {
|
|
200
|
+
const key = `${candidate.chunk.kb_id}:${candidate.chunk.file_path}`;
|
|
201
|
+
if (!buckets.has(key)) {
|
|
202
|
+
buckets.set(key, []);
|
|
203
|
+
fileOrder.push(key);
|
|
204
|
+
}
|
|
205
|
+
buckets.get(key)?.push(candidate);
|
|
206
|
+
}
|
|
207
|
+
if (fileOrder.length <= 1) return candidates;
|
|
208
|
+
|
|
209
|
+
const result: RankedChunk[] = [];
|
|
210
|
+
let round = 0;
|
|
211
|
+
while (result.length < candidates.length) {
|
|
212
|
+
let added = false;
|
|
213
|
+
for (const key of fileOrder) {
|
|
214
|
+
const bucket = buckets.get(key);
|
|
215
|
+
const item = bucket?.[round];
|
|
216
|
+
if (!item) continue;
|
|
217
|
+
result.push(item);
|
|
218
|
+
added = true;
|
|
219
|
+
}
|
|
220
|
+
if (!added) break;
|
|
221
|
+
round++;
|
|
222
|
+
}
|
|
223
|
+
return result;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
function buildQuerySnippet(content: string, query: string, maxLength = 240): string {
|
|
227
|
+
const terms = [...tokenizeForSimilarity(query)].sort((a, b) => b.length - a.length);
|
|
228
|
+
const lower = content.toLowerCase();
|
|
229
|
+
let matchIndex = -1;
|
|
230
|
+
for (const term of terms) {
|
|
231
|
+
matchIndex = lower.indexOf(term.toLowerCase());
|
|
232
|
+
if (matchIndex >= 0) break;
|
|
233
|
+
}
|
|
234
|
+
if (matchIndex < 0) return content.slice(0, maxLength);
|
|
235
|
+
const half = Math.floor(maxLength / 2);
|
|
236
|
+
const start = Math.max(0, matchIndex - half);
|
|
237
|
+
const end = Math.min(content.length, start + maxLength);
|
|
238
|
+
const prefix = start > 0 ? "..." : "";
|
|
239
|
+
const suffix = end < content.length ? "..." : "";
|
|
240
|
+
return `${prefix}${content.slice(start, end)}${suffix}`;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
function buildAdaptiveContext(
|
|
244
|
+
seed: Chunk,
|
|
245
|
+
chunks: Chunk[],
|
|
246
|
+
queryTokens: Set<string>,
|
|
247
|
+
): {
|
|
248
|
+
content: string;
|
|
249
|
+
startLine: number;
|
|
250
|
+
endLine: number;
|
|
251
|
+
sourceChunkIds: string[];
|
|
252
|
+
} {
|
|
253
|
+
const scored = chunks
|
|
254
|
+
.map((chunk) => {
|
|
255
|
+
const distance =
|
|
256
|
+
chunk.id === seed.id
|
|
257
|
+
? 0
|
|
258
|
+
: Math.min(Math.abs(chunk.start_line - seed.start_line), Math.abs(chunk.end_line - seed.end_line));
|
|
259
|
+
const proximity = 1 / (1 + distance / 20);
|
|
260
|
+
const coverage = queryCoverage(chunk.content, queryTokens);
|
|
261
|
+
const seedBoost = chunk.id === seed.id ? 2 : 0;
|
|
262
|
+
return { chunk, score: seedBoost + proximity + coverage };
|
|
263
|
+
})
|
|
264
|
+
.sort((a, b) => b.score - a.score)
|
|
265
|
+
.slice(0, ADAPTIVE_NEIGHBOR_TARGET);
|
|
266
|
+
if (!scored.some((item) => item.chunk.id === seed.id)) scored.push({ chunk: seed, score: Number.MAX_SAFE_INTEGER });
|
|
267
|
+
|
|
268
|
+
const selectedIds = new Set(scored.map((item) => item.chunk.id));
|
|
269
|
+
const ordered = chunks
|
|
270
|
+
.filter((chunk) => selectedIds.has(chunk.id))
|
|
271
|
+
.sort((a, b) => a.start_line - b.start_line || a.end_line - b.end_line);
|
|
272
|
+
const parts: string[] = [];
|
|
273
|
+
const sourceChunkIds: string[] = [];
|
|
274
|
+
let totalLength = 0;
|
|
275
|
+
for (const chunk of ordered) {
|
|
276
|
+
if (totalLength >= ADAPTIVE_MAX_CONTEXT_CHARS) break;
|
|
277
|
+
const remaining = ADAPTIVE_MAX_CONTEXT_CHARS - totalLength;
|
|
278
|
+
const text = chunk.content.slice(0, remaining);
|
|
279
|
+
parts.push(text);
|
|
280
|
+
sourceChunkIds.push(chunk.id);
|
|
281
|
+
totalLength += text.length + 2;
|
|
282
|
+
}
|
|
283
|
+
return {
|
|
284
|
+
content: parts.join("\n\n"),
|
|
285
|
+
startLine: Math.min(...ordered.map((chunk) => chunk.start_line)),
|
|
286
|
+
endLine: Math.max(...ordered.map((chunk) => chunk.end_line)),
|
|
287
|
+
sourceChunkIds,
|
|
288
|
+
};
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
function pushAdaptiveCandidate(candidates: RankedChunk[], candidate: RankedChunk): void {
|
|
292
|
+
const overlappingIndex = candidates.findIndex(
|
|
293
|
+
(existing) =>
|
|
294
|
+
existing.chunk.kb_id === candidate.chunk.kb_id &&
|
|
295
|
+
existing.chunk.file_path === candidate.chunk.file_path &&
|
|
296
|
+
existing.sourceChunkIds.some((id) => candidate.sourceChunkIds.includes(id)),
|
|
297
|
+
);
|
|
298
|
+
if (overlappingIndex < 0) {
|
|
299
|
+
candidates.push(candidate);
|
|
300
|
+
return;
|
|
301
|
+
}
|
|
302
|
+
const existing = candidates[overlappingIndex];
|
|
303
|
+
if (candidate.score > existing.score || candidate.sourceChunkIds.length > existing.sourceChunkIds.length) {
|
|
304
|
+
candidates[overlappingIndex] = candidate;
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
|
|
70
308
|
async function chunkUrl(source: string, signal?: AbortSignal): Promise<Awaited<ReturnType<typeof chunkFile>>> {
|
|
71
309
|
const res = await fetch(source, { signal });
|
|
72
310
|
if (!res.ok) throw new Error(`Fetch failed: ${res.status}`);
|
|
@@ -109,6 +347,15 @@ export class KnowledgeEngine {
|
|
|
109
347
|
this.vectorCache.delete(kbId);
|
|
110
348
|
}
|
|
111
349
|
|
|
350
|
+
private getVectorsByChunkId(kbId: string, chunkIds: string[]): Map<string, Float32Array> {
|
|
351
|
+
const vectors = this.getVectors(kbId);
|
|
352
|
+
const mapped = new Map<string, Float32Array>();
|
|
353
|
+
for (let i = 0; i < chunkIds.length; i++) {
|
|
354
|
+
if (vectors[i]) mapped.set(chunkIds[i], vectors[i]);
|
|
355
|
+
}
|
|
356
|
+
return mapped;
|
|
357
|
+
}
|
|
358
|
+
|
|
112
359
|
async add(
|
|
113
360
|
source: string,
|
|
114
361
|
name: string,
|
|
@@ -122,6 +369,13 @@ export class KnowledgeEngine {
|
|
|
122
369
|
const isFile = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isFile();
|
|
123
370
|
const sourceType = isUrl ? "url" : isDir ? "directory" : isFile ? "file" : "text";
|
|
124
371
|
|
|
372
|
+
const existingKB = getKBByName(this.db, name);
|
|
373
|
+
if (existingKB) {
|
|
374
|
+
throw new Error(
|
|
375
|
+
`Knowledge base "${name}" already exists. Use knowledge_update to refresh it, or knowledge_remove before adding a replacement.`,
|
|
376
|
+
);
|
|
377
|
+
}
|
|
378
|
+
|
|
125
379
|
const kb = createKB(this.db, {
|
|
126
380
|
name,
|
|
127
381
|
source_path: isDir || isFile ? resolvedSource : isUrl ? source : undefined,
|
|
@@ -162,7 +416,7 @@ export class KnowledgeEngine {
|
|
|
162
416
|
}
|
|
163
417
|
|
|
164
418
|
onProgress?.(`${allChunks.length} chunks, embedding...`);
|
|
165
|
-
const texts = allChunks.map((c) => c
|
|
419
|
+
const texts = allChunks.map((c) => buildChunkEmbeddingText(c));
|
|
166
420
|
const vectors = await embedDocuments(texts, signal);
|
|
167
421
|
|
|
168
422
|
onProgress?.(`Storing...`);
|
|
@@ -243,7 +497,7 @@ export class KnowledgeEngine {
|
|
|
243
497
|
if (chunksToAdd.length > 0) {
|
|
244
498
|
onProgress?.(`Embedding ${chunksToAdd.length} new chunks...`);
|
|
245
499
|
newVectors = await embedDocuments(
|
|
246
|
-
chunksToAdd.map((c) => c
|
|
500
|
+
chunksToAdd.map((c) => buildChunkEmbeddingText(c)),
|
|
247
501
|
signal,
|
|
248
502
|
);
|
|
249
503
|
insertChunks(this.db, kb.id, chunksToAdd);
|
|
@@ -278,13 +532,20 @@ export class KnowledgeEngine {
|
|
|
278
532
|
async search(query: string, options: SearchOptions = {}): Promise<SearchResponse> {
|
|
279
533
|
if (!this.db) throw new Error("Engine not initialized");
|
|
280
534
|
const db = this.db;
|
|
281
|
-
const { mode = "hybrid", limit = 10, offset = 0, kb_id, filters } = options;
|
|
282
|
-
|
|
283
|
-
const
|
|
535
|
+
const { mode = "hybrid", limit = 10, offset = 0, kb_id, filters, diversity = "balanced" } = options;
|
|
536
|
+
const retrievalMode = mode === "adaptive" ? "hybrid" : mode;
|
|
537
|
+
const candidateLimit = Math.max(50, offset + limit * 12);
|
|
538
|
+
const normalizedQuery = normalizedQueryText(query);
|
|
539
|
+
const queryTokens = tokenizeForSimilarity(query);
|
|
540
|
+
const normalizedFileType = normalizeFileTypeFilter(filters?.file_type);
|
|
541
|
+
|
|
542
|
+
const selectedKB = kb_id ? (getKB(db, kb_id) ?? getKBByName(db, kb_id)) : undefined;
|
|
543
|
+
const kbs = kb_id ? ([selectedKB].filter(Boolean) as KnowledgeBase[]) : listKBs(db);
|
|
284
544
|
if (kbs.length === 0) return { results: [], total_count: 0, has_more: false };
|
|
285
545
|
|
|
286
546
|
const warnings: string[] = [];
|
|
287
547
|
const allResults: { chunkId: string; score: number }[] = [];
|
|
548
|
+
const vectorsByChunkId = new Map<string, Float32Array>();
|
|
288
549
|
|
|
289
550
|
for (const kb of kbs) {
|
|
290
551
|
if (kb.embedding_model !== CURRENT_EMBEDDING_MODEL) {
|
|
@@ -294,26 +555,30 @@ export class KnowledgeEngine {
|
|
|
294
555
|
}
|
|
295
556
|
const chunkIds = getChunkIdsByKB(db, kb.id);
|
|
296
557
|
if (chunkIds.length === 0) continue;
|
|
558
|
+
for (const [chunkId, vector] of this.getVectorsByChunkId(kb.id, chunkIds)) {
|
|
559
|
+
vectorsByChunkId.set(chunkId, vector);
|
|
560
|
+
}
|
|
297
561
|
|
|
298
|
-
if (
|
|
299
|
-
allResults.push(...searchBM25(db, query,
|
|
300
|
-
} else if (
|
|
301
|
-
const vectors = this.
|
|
562
|
+
if (retrievalMode === "fast") {
|
|
563
|
+
allResults.push(...searchBM25(db, normalizedQuery || query, candidateLimit, kb.id));
|
|
564
|
+
} else if (retrievalMode === "semantic") {
|
|
565
|
+
const vectors = [...this.getVectorsByChunkId(kb.id, chunkIds).values()];
|
|
302
566
|
if (vectors.length > 0) {
|
|
303
567
|
const queryVec = await embedQuery(query);
|
|
304
|
-
allResults.push(...searchVector(queryVec, vectors, chunkIds));
|
|
568
|
+
allResults.push(...searchVector(queryVec, vectors, chunkIds, candidateLimit));
|
|
305
569
|
}
|
|
306
570
|
} else {
|
|
307
|
-
// hybrid: BM25 + vector
|
|
308
|
-
const bm25Results = searchBM25(db, query,
|
|
571
|
+
// hybrid: BM25 + vector weighted fusion (both scoped to this KB)
|
|
572
|
+
const bm25Results = searchBM25(db, normalizedQuery || query, candidateLimit, kb.id);
|
|
309
573
|
|
|
310
|
-
const vectors = this.
|
|
574
|
+
const vectors = [...this.getVectorsByChunkId(kb.id, chunkIds).values()];
|
|
311
575
|
let vecResults: { chunkId: string; score: number }[] = [];
|
|
312
576
|
if (vectors.length > 0) {
|
|
313
577
|
const queryVec = await embedQuery(query);
|
|
314
|
-
vecResults = searchVector(queryVec, vectors, chunkIds);
|
|
578
|
+
vecResults = searchVector(queryVec, vectors, chunkIds, candidateLimit);
|
|
315
579
|
}
|
|
316
|
-
|
|
580
|
+
if (bm25Results.length === 0) continue;
|
|
581
|
+
const fused = weightedScoreFusion(bm25Results, vecResults);
|
|
317
582
|
allResults.push(...fused);
|
|
318
583
|
}
|
|
319
584
|
}
|
|
@@ -325,74 +590,139 @@ export class KnowledgeEngine {
|
|
|
325
590
|
seen.add(r.chunkId);
|
|
326
591
|
return true;
|
|
327
592
|
});
|
|
328
|
-
|
|
593
|
+
const rankingByChunkId = new Map<string, RankingDiagnostics>();
|
|
594
|
+
for (const result of unique) {
|
|
595
|
+
const chunk = getChunkById(db, result.chunkId);
|
|
596
|
+
if (chunk) {
|
|
597
|
+
const ranking = scoreChunkForQuery(result.score, chunk, queryTokens);
|
|
598
|
+
rankingByChunkId.set(result.chunkId, ranking);
|
|
599
|
+
result.score = ranking.adjusted_score;
|
|
600
|
+
}
|
|
601
|
+
}
|
|
602
|
+
let scored = unique;
|
|
603
|
+
if (retrievalMode !== "fast" && retrievalMode !== "semantic") {
|
|
604
|
+
scored = unique.filter((result) => {
|
|
605
|
+
const chunk = getChunkById(db, result.chunkId);
|
|
606
|
+
if (!chunk) return false;
|
|
607
|
+
return result.score >= MIN_HYBRID_SCORE && hasEnoughLexicalEvidence(chunk, queryTokens);
|
|
608
|
+
});
|
|
609
|
+
}
|
|
610
|
+
scored.sort((a, b) => b.score - a.score);
|
|
329
611
|
|
|
330
612
|
// Apply metadata filters post-retrieval
|
|
331
|
-
let filtered =
|
|
332
|
-
if (
|
|
333
|
-
filtered =
|
|
613
|
+
let filtered = scored;
|
|
614
|
+
if (normalizedFileType || filters?.path_pattern) {
|
|
615
|
+
filtered = scored.filter((r) => {
|
|
334
616
|
const chunk = getChunkById(db, r.chunkId);
|
|
335
617
|
if (!chunk) return false;
|
|
336
|
-
if (
|
|
337
|
-
if (filters
|
|
618
|
+
if (normalizedFileType && chunk.file_type !== normalizedFileType) return false;
|
|
619
|
+
if (filters?.path_pattern && !chunk.file_path.includes(filters.path_pattern)) return false;
|
|
338
620
|
return true;
|
|
339
621
|
});
|
|
340
622
|
}
|
|
341
623
|
|
|
342
|
-
// Deep mode: rerank top-20 with cross-encoder
|
|
343
624
|
if (mode === "deep" && filtered.length > 0) {
|
|
344
625
|
const candidates = filtered
|
|
345
|
-
.slice(0,
|
|
626
|
+
.slice(0, 30)
|
|
346
627
|
.map((r) => {
|
|
347
628
|
const chunk = getChunkById(db, r.chunkId);
|
|
348
629
|
return chunk ? { chunkId: r.chunkId, content: chunk.content } : null;
|
|
349
630
|
})
|
|
350
631
|
.filter(Boolean) as Array<{ chunkId: string; content: string }>;
|
|
351
|
-
const reranked = await rerank(query, candidates, limit);
|
|
352
|
-
const
|
|
632
|
+
const reranked = await rerank(query, candidates, Math.max(limit * 3, limit));
|
|
633
|
+
const ranked: RankedChunk[] = [];
|
|
353
634
|
for (const r of reranked) {
|
|
354
635
|
const chunk = getChunkById(db, r.chunkId);
|
|
355
636
|
if (!chunk) continue;
|
|
356
637
|
const kbObj = getKB(db, chunk.kb_id);
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
kb_name: kbObj?.name ?? "unknown",
|
|
638
|
+
ranked.push({
|
|
639
|
+
chunk,
|
|
640
|
+
kbName: kbObj?.name ?? "unknown",
|
|
641
|
+
ranking: scoreChunkForQuery(r.score, chunk, queryTokens),
|
|
362
642
|
score: r.score,
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
643
|
+
content: chunk.content,
|
|
644
|
+
snippet: buildQuerySnippet(chunk.content, query),
|
|
645
|
+
startLine: chunk.start_line,
|
|
646
|
+
endLine: chunk.end_line,
|
|
647
|
+
sourceChunkIds: [chunk.id],
|
|
366
648
|
});
|
|
367
649
|
}
|
|
650
|
+
const diversified = interleaveByFile(diversifyRankedChunks(ranked, diversity, vectorsByChunkId), diversity);
|
|
651
|
+
const page = diversified.slice(offset, offset + limit);
|
|
652
|
+
const results = page.map((r) => ({
|
|
653
|
+
content: r.content,
|
|
654
|
+
file_path: r.chunk.file_path,
|
|
655
|
+
file_type: r.chunk.file_type,
|
|
656
|
+
kb_name: r.kbName,
|
|
657
|
+
score: r.score,
|
|
658
|
+
ranking: r.ranking,
|
|
659
|
+
snippet: r.snippet,
|
|
660
|
+
start_line: r.startLine,
|
|
661
|
+
end_line: r.endLine,
|
|
662
|
+
}));
|
|
368
663
|
return {
|
|
369
664
|
results,
|
|
370
|
-
total_count:
|
|
371
|
-
has_more:
|
|
665
|
+
total_count: diversified.length,
|
|
666
|
+
has_more: offset + limit < diversified.length,
|
|
372
667
|
warnings: warnings.length > 0 ? warnings : undefined,
|
|
373
668
|
};
|
|
374
669
|
}
|
|
375
670
|
|
|
376
|
-
const
|
|
377
|
-
const
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
.
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
const
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
671
|
+
const ranked: RankedChunk[] = [];
|
|
672
|
+
for (const r of filtered) {
|
|
673
|
+
const chunk = getChunkById(db, r.chunkId);
|
|
674
|
+
if (!chunk) continue;
|
|
675
|
+
const kb = getKB(db, chunk.kb_id);
|
|
676
|
+
|
|
677
|
+
if (mode === "adaptive") {
|
|
678
|
+
const contextChunks = getChunksByFile(
|
|
679
|
+
db,
|
|
680
|
+
chunk.kb_id,
|
|
681
|
+
chunk.file_path,
|
|
682
|
+
Math.max(1, chunk.start_line - ADAPTIVE_CONTEXT_LINES),
|
|
683
|
+
chunk.end_line + ADAPTIVE_CONTEXT_LINES,
|
|
684
|
+
);
|
|
685
|
+
const context = buildAdaptiveContext(chunk, contextChunks.length > 0 ? contextChunks : [chunk], queryTokens);
|
|
686
|
+
pushAdaptiveCandidate(ranked, {
|
|
687
|
+
chunk,
|
|
688
|
+
kbName: kb?.name ?? "unknown",
|
|
689
|
+
ranking: rankingByChunkId.get(r.chunkId),
|
|
389
690
|
score: r.score,
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
691
|
+
content: context.content,
|
|
692
|
+
snippet: buildQuerySnippet(context.content, query),
|
|
693
|
+
startLine: context.startLine,
|
|
694
|
+
endLine: context.endLine,
|
|
695
|
+
sourceChunkIds: context.sourceChunkIds,
|
|
696
|
+
});
|
|
697
|
+
} else {
|
|
698
|
+
ranked.push({
|
|
699
|
+
chunk,
|
|
700
|
+
kbName: kb?.name ?? "unknown",
|
|
701
|
+
ranking: rankingByChunkId.get(r.chunkId),
|
|
702
|
+
score: r.score,
|
|
703
|
+
content: chunk.content,
|
|
704
|
+
snippet: buildQuerySnippet(chunk.content, query),
|
|
705
|
+
startLine: chunk.start_line,
|
|
706
|
+
endLine: chunk.end_line,
|
|
707
|
+
sourceChunkIds: [chunk.id],
|
|
708
|
+
});
|
|
709
|
+
}
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
const diversified = interleaveByFile(diversifyRankedChunks(ranked, diversity, vectorsByChunkId), diversity);
|
|
713
|
+
const total = diversified.length;
|
|
714
|
+
const page = diversified.slice(offset, offset + limit);
|
|
715
|
+
const results: SearchResult[] = page.map((r) => ({
|
|
716
|
+
content: r.content,
|
|
717
|
+
file_path: r.chunk.file_path,
|
|
718
|
+
file_type: r.chunk.file_type,
|
|
719
|
+
kb_name: r.kbName,
|
|
720
|
+
score: r.score,
|
|
721
|
+
ranking: r.ranking,
|
|
722
|
+
snippet: r.snippet,
|
|
723
|
+
start_line: r.startLine,
|
|
724
|
+
end_line: r.endLine,
|
|
725
|
+
}));
|
|
396
726
|
|
|
397
727
|
return {
|
|
398
728
|
results,
|
|
@@ -472,23 +802,29 @@ export class KnowledgeEngine {
|
|
|
472
802
|
try {
|
|
473
803
|
const chunkData = lines.slice(1).map((l) => JSON.parse(l) as ImportedChunk);
|
|
474
804
|
onProgress?.(`Importing ${chunkData.length} chunks, embedding...`);
|
|
475
|
-
const texts = chunkData.map((c) => c.content);
|
|
476
|
-
const vectors = await embedDocuments(texts, signal);
|
|
477
805
|
const chunks = chunkData.map((c) => ({
|
|
478
806
|
content_hash: contentHash(c.content),
|
|
479
807
|
content: c.content,
|
|
480
|
-
content_tokenized:
|
|
808
|
+
content_tokenized: "",
|
|
481
809
|
file_path: c.file_path,
|
|
482
810
|
file_type: c.file_type,
|
|
483
811
|
start_line: c.start_line,
|
|
484
812
|
end_line: c.end_line,
|
|
485
813
|
metadata_json: c.metadata_json || "{}",
|
|
486
814
|
}));
|
|
487
|
-
|
|
815
|
+
const indexedChunks = chunks.map((chunk) => ({
|
|
816
|
+
...chunk,
|
|
817
|
+
content_tokenized: preTokenizeForFTS(buildChunkEmbeddingText(chunk)),
|
|
818
|
+
}));
|
|
819
|
+
const vectors = await embedDocuments(
|
|
820
|
+
indexedChunks.map((c) => buildChunkEmbeddingText(c)),
|
|
821
|
+
signal,
|
|
822
|
+
);
|
|
823
|
+
insertChunks(this.db, kb.id, indexedChunks);
|
|
488
824
|
const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
|
|
489
825
|
saveVectors(vectorPath, vectors);
|
|
490
826
|
this.vectorCache.set(kb.id, vectors);
|
|
491
|
-
updateKBCounts(this.db, kb.id,
|
|
827
|
+
updateKBCounts(this.db, kb.id, indexedChunks.length, new Set(indexedChunks.map((c) => c.file_path)).size);
|
|
492
828
|
updateKBStatus(this.db, kb.id, "ready");
|
|
493
829
|
const savedKB = getKB(this.db, kb.id);
|
|
494
830
|
if (!savedKB) throw new Error(`Knowledge base disappeared after import: ${kb.id}`);
|
|
@@ -499,9 +835,15 @@ export class KnowledgeEngine {
|
|
|
499
835
|
}
|
|
500
836
|
}
|
|
501
837
|
|
|
502
|
-
async dispose(): Promise<void> {
|
|
503
|
-
|
|
504
|
-
|
|
838
|
+
async dispose(options: { disposeModels?: boolean } = {}): Promise<void> {
|
|
839
|
+
const disposeModels = options.disposeModels ?? true;
|
|
840
|
+
if (disposeModels) {
|
|
841
|
+
await disposeEmbedding();
|
|
842
|
+
await disposeReranker();
|
|
843
|
+
} else {
|
|
844
|
+
await prepareEmbeddingForShutdown();
|
|
845
|
+
await prepareRerankerForShutdown();
|
|
846
|
+
}
|
|
505
847
|
this.db?.close();
|
|
506
848
|
this.db = null;
|
|
507
849
|
}
|