pi-knowledge 0.3.4 → 0.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/README.md +16 -2
- package/index.ts +63 -6
- package/package.json +1 -1
- package/src/embedding/provider.ts +50 -12
- package/src/engine.ts +402 -67
- package/src/indexer/chunker.ts +100 -77
- package/src/indexer/chunkers/code-ast.ts +14 -11
- package/src/search/bm25.ts +26 -22
- package/src/search/fusion.ts +33 -0
- package/src/search/query.ts +59 -0
- package/src/search/ranking.ts +197 -0
- package/src/search/reranker.ts +49 -11
- package/src/storage/sqlite.ts +16 -0
package/src/engine.ts
CHANGED
|
@@ -2,20 +2,36 @@ import { existsSync, statSync } from "node:fs";
|
|
|
2
2
|
import { join, resolve } from "node:path";
|
|
3
3
|
import type Database from "better-sqlite3";
|
|
4
4
|
import { type DiagnosticResult, diagnoseKB } from "./diagnostics/health.ts";
|
|
5
|
-
import {
|
|
5
|
+
import {
|
|
6
|
+
dispose as disposeEmbedding,
|
|
7
|
+
embedDocuments,
|
|
8
|
+
embedQuery,
|
|
9
|
+
prepareForShutdown as prepareEmbeddingForShutdown,
|
|
10
|
+
} from "./embedding/provider.ts";
|
|
6
11
|
import { loadVectors, saveVectors } from "./embedding/vectors.ts";
|
|
7
|
-
import { chunkFile, contentHash, preTokenizeForFTS, walkDir } from "./indexer/chunker.ts";
|
|
12
|
+
import { buildChunkEmbeddingText, chunkFile, contentHash, preTokenizeForFTS, walkDir } from "./indexer/chunker.ts";
|
|
8
13
|
import { searchBM25 } from "./search/bm25.ts";
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
14
|
+
import { weightedScoreFusion } from "./search/fusion.ts";
|
|
15
|
+
import { normalizedQueryText, tokenizeForSearch } from "./search/query.ts";
|
|
16
|
+
import {
|
|
17
|
+
hasEnoughLexicalEvidence,
|
|
18
|
+
MIN_HYBRID_SCORE,
|
|
19
|
+
normalizeFileTypeFilter,
|
|
20
|
+
queryCoverage,
|
|
21
|
+
type RankingDiagnostics,
|
|
22
|
+
scoreChunkForQuery,
|
|
23
|
+
} from "./search/ranking.ts";
|
|
24
|
+
import { disposeReranker, prepareRerankerForShutdown, rerank } from "./search/reranker.ts";
|
|
11
25
|
import { searchVector } from "./search/vector.ts";
|
|
12
26
|
import {
|
|
27
|
+
type Chunk,
|
|
13
28
|
createKB,
|
|
14
29
|
deleteChunksByIds,
|
|
15
30
|
deleteKB,
|
|
16
31
|
getChunkById,
|
|
17
32
|
getChunkHashesByKB,
|
|
18
33
|
getChunkIdsByKB,
|
|
34
|
+
getChunksByFile,
|
|
19
35
|
getChunksByKB,
|
|
20
36
|
getFileCount,
|
|
21
37
|
getKB,
|
|
@@ -29,11 +45,12 @@ import {
|
|
|
29
45
|
} from "./storage/sqlite.ts";
|
|
30
46
|
|
|
31
47
|
export interface SearchOptions {
|
|
32
|
-
mode?: "fast" | "semantic" | "hybrid" | "deep";
|
|
48
|
+
mode?: "fast" | "semantic" | "hybrid" | "deep" | "adaptive";
|
|
33
49
|
limit?: number;
|
|
34
50
|
offset?: number;
|
|
35
51
|
kb_id?: string;
|
|
36
52
|
filters?: { file_type?: string; path_pattern?: string };
|
|
53
|
+
diversity?: "off" | "balanced" | "strong";
|
|
37
54
|
}
|
|
38
55
|
|
|
39
56
|
export interface SearchResult {
|
|
@@ -45,6 +62,7 @@ export interface SearchResult {
|
|
|
45
62
|
snippet: string;
|
|
46
63
|
start_line: number;
|
|
47
64
|
end_line: number;
|
|
65
|
+
ranking?: RankingDiagnostics;
|
|
48
66
|
}
|
|
49
67
|
|
|
50
68
|
export const CURRENT_EMBEDDING_MODEL = "multilingual-e5-small";
|
|
@@ -67,6 +85,226 @@ interface ImportedChunk {
|
|
|
67
85
|
metadata_json?: string;
|
|
68
86
|
}
|
|
69
87
|
|
|
88
|
+
interface RankedChunk {
|
|
89
|
+
chunk: Chunk;
|
|
90
|
+
kbName: string;
|
|
91
|
+
ranking?: RankingDiagnostics;
|
|
92
|
+
score: number;
|
|
93
|
+
content: string;
|
|
94
|
+
snippet: string;
|
|
95
|
+
startLine: number;
|
|
96
|
+
endLine: number;
|
|
97
|
+
sourceChunkIds: string[];
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
const ADAPTIVE_CONTEXT_LINES = 80;
|
|
101
|
+
const ADAPTIVE_MAX_CONTEXT_CHARS = 6_000;
|
|
102
|
+
const ADAPTIVE_NEIGHBOR_TARGET = 5;
|
|
103
|
+
const VECTOR_REDUNDANCY_WEIGHT = 0.35;
|
|
104
|
+
|
|
105
|
+
function cosineSimilarity(a: Float32Array | undefined, b: Float32Array | undefined): number {
|
|
106
|
+
if (!a || !b || a.length !== b.length) return 0;
|
|
107
|
+
let dot = 0;
|
|
108
|
+
for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
|
|
109
|
+
return dot;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
function tokenizeForSimilarity(text: string): Set<string> {
|
|
113
|
+
return tokenizeForSearch(text);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function jaccardSimilarity(a: Set<string>, b: Set<string>): number {
|
|
117
|
+
if (a.size === 0 || b.size === 0) return 0;
|
|
118
|
+
let intersection = 0;
|
|
119
|
+
for (const token of a) {
|
|
120
|
+
if (b.has(token)) intersection++;
|
|
121
|
+
}
|
|
122
|
+
return intersection / (a.size + b.size - intersection);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function lineProximity(a: RankedChunk, b: RankedChunk): number {
|
|
126
|
+
if (a.chunk.file_path !== b.chunk.file_path || a.chunk.kb_id !== b.chunk.kb_id) return 0;
|
|
127
|
+
if (a.sourceChunkIds.some((id) => b.sourceChunkIds.includes(id))) return 1;
|
|
128
|
+
const overlap = Math.max(0, Math.min(a.endLine, b.endLine) - Math.max(a.startLine, b.startLine) + 1);
|
|
129
|
+
if (overlap > 0) return 1;
|
|
130
|
+
const gap = Math.max(a.startLine, b.startLine) - Math.min(a.endLine, b.endLine);
|
|
131
|
+
if (gap <= 20) return 0.8;
|
|
132
|
+
if (gap <= 80) return 0.45;
|
|
133
|
+
return 0.2;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function normalizeScores(candidates: RankedChunk[]): Map<string, number> {
|
|
137
|
+
const scores = candidates.map((candidate) => candidate.score);
|
|
138
|
+
const min = Math.min(...scores);
|
|
139
|
+
const max = Math.max(...scores);
|
|
140
|
+
const normalized = new Map<string, number>();
|
|
141
|
+
for (const candidate of candidates) {
|
|
142
|
+
const value = max === min ? 1 : (candidate.score - min) / (max - min);
|
|
143
|
+
normalized.set(candidate.chunk.id, value);
|
|
144
|
+
}
|
|
145
|
+
return normalized;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
function diversifyRankedChunks(
|
|
149
|
+
candidates: RankedChunk[],
|
|
150
|
+
diversity: SearchOptions["diversity"],
|
|
151
|
+
vectorsByChunkId: Map<string, Float32Array> = new Map(),
|
|
152
|
+
): RankedChunk[] {
|
|
153
|
+
if (diversity === "off" || candidates.length <= 2) return candidates;
|
|
154
|
+
const lambda = diversity === "strong" ? 0.62 : 0.76;
|
|
155
|
+
const normalized = normalizeScores(candidates);
|
|
156
|
+
const tokenSets = new Map(
|
|
157
|
+
candidates.map((candidate) => [candidate.chunk.id, tokenizeForSimilarity(candidate.content)]),
|
|
158
|
+
);
|
|
159
|
+
const selected: RankedChunk[] = [];
|
|
160
|
+
const remaining = [...candidates];
|
|
161
|
+
|
|
162
|
+
while (remaining.length > 0) {
|
|
163
|
+
let bestIndex = 0;
|
|
164
|
+
let bestScore = Number.NEGATIVE_INFINITY;
|
|
165
|
+
for (let i = 0; i < remaining.length; i++) {
|
|
166
|
+
const candidate = remaining[i];
|
|
167
|
+
let redundancy = 0;
|
|
168
|
+
for (const chosen of selected) {
|
|
169
|
+
const lexical = jaccardSimilarity(
|
|
170
|
+
tokenSets.get(candidate.chunk.id) ?? new Set<string>(),
|
|
171
|
+
tokenSets.get(chosen.chunk.id) ?? new Set<string>(),
|
|
172
|
+
);
|
|
173
|
+
const vector = Math.max(
|
|
174
|
+
0,
|
|
175
|
+
cosineSimilarity(vectorsByChunkId.get(candidate.chunk.id), vectorsByChunkId.get(chosen.chunk.id)),
|
|
176
|
+
);
|
|
177
|
+
redundancy = Math.max(
|
|
178
|
+
redundancy,
|
|
179
|
+
Math.max(lexical, lineProximity(candidate, chosen), vector * VECTOR_REDUNDANCY_WEIGHT),
|
|
180
|
+
);
|
|
181
|
+
}
|
|
182
|
+
const relevance = normalized.get(candidate.chunk.id) ?? 0;
|
|
183
|
+
const mmrScore = lambda * relevance - (1 - lambda) * redundancy;
|
|
184
|
+
if (mmrScore > bestScore) {
|
|
185
|
+
bestScore = mmrScore;
|
|
186
|
+
bestIndex = i;
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
selected.push(remaining.splice(bestIndex, 1)[0]);
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
return selected;
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
function interleaveByFile(candidates: RankedChunk[], diversity: SearchOptions["diversity"]): RankedChunk[] {
|
|
196
|
+
if (diversity === "off" || candidates.length <= 2) return candidates;
|
|
197
|
+
const buckets = new Map<string, RankedChunk[]>();
|
|
198
|
+
const fileOrder: string[] = [];
|
|
199
|
+
for (const candidate of candidates) {
|
|
200
|
+
const key = `${candidate.chunk.kb_id}:${candidate.chunk.file_path}`;
|
|
201
|
+
if (!buckets.has(key)) {
|
|
202
|
+
buckets.set(key, []);
|
|
203
|
+
fileOrder.push(key);
|
|
204
|
+
}
|
|
205
|
+
buckets.get(key)?.push(candidate);
|
|
206
|
+
}
|
|
207
|
+
if (fileOrder.length <= 1) return candidates;
|
|
208
|
+
|
|
209
|
+
const result: RankedChunk[] = [];
|
|
210
|
+
let round = 0;
|
|
211
|
+
while (result.length < candidates.length) {
|
|
212
|
+
let added = false;
|
|
213
|
+
for (const key of fileOrder) {
|
|
214
|
+
const bucket = buckets.get(key);
|
|
215
|
+
const item = bucket?.[round];
|
|
216
|
+
if (!item) continue;
|
|
217
|
+
result.push(item);
|
|
218
|
+
added = true;
|
|
219
|
+
}
|
|
220
|
+
if (!added) break;
|
|
221
|
+
round++;
|
|
222
|
+
}
|
|
223
|
+
return result;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
function buildQuerySnippet(content: string, query: string, maxLength = 240): string {
|
|
227
|
+
const terms = [...tokenizeForSimilarity(query)].sort((a, b) => b.length - a.length);
|
|
228
|
+
const lower = content.toLowerCase();
|
|
229
|
+
let matchIndex = -1;
|
|
230
|
+
for (const term of terms) {
|
|
231
|
+
matchIndex = lower.indexOf(term.toLowerCase());
|
|
232
|
+
if (matchIndex >= 0) break;
|
|
233
|
+
}
|
|
234
|
+
if (matchIndex < 0) return content.slice(0, maxLength);
|
|
235
|
+
const half = Math.floor(maxLength / 2);
|
|
236
|
+
const start = Math.max(0, matchIndex - half);
|
|
237
|
+
const end = Math.min(content.length, start + maxLength);
|
|
238
|
+
const prefix = start > 0 ? "..." : "";
|
|
239
|
+
const suffix = end < content.length ? "..." : "";
|
|
240
|
+
return `${prefix}${content.slice(start, end)}${suffix}`;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
function buildAdaptiveContext(
|
|
244
|
+
seed: Chunk,
|
|
245
|
+
chunks: Chunk[],
|
|
246
|
+
queryTokens: Set<string>,
|
|
247
|
+
): {
|
|
248
|
+
content: string;
|
|
249
|
+
startLine: number;
|
|
250
|
+
endLine: number;
|
|
251
|
+
sourceChunkIds: string[];
|
|
252
|
+
} {
|
|
253
|
+
const scored = chunks
|
|
254
|
+
.map((chunk) => {
|
|
255
|
+
const distance =
|
|
256
|
+
chunk.id === seed.id
|
|
257
|
+
? 0
|
|
258
|
+
: Math.min(Math.abs(chunk.start_line - seed.start_line), Math.abs(chunk.end_line - seed.end_line));
|
|
259
|
+
const proximity = 1 / (1 + distance / 20);
|
|
260
|
+
const coverage = queryCoverage(chunk.content, queryTokens);
|
|
261
|
+
const seedBoost = chunk.id === seed.id ? 2 : 0;
|
|
262
|
+
return { chunk, score: seedBoost + proximity + coverage };
|
|
263
|
+
})
|
|
264
|
+
.sort((a, b) => b.score - a.score)
|
|
265
|
+
.slice(0, ADAPTIVE_NEIGHBOR_TARGET);
|
|
266
|
+
if (!scored.some((item) => item.chunk.id === seed.id)) scored.push({ chunk: seed, score: Number.MAX_SAFE_INTEGER });
|
|
267
|
+
|
|
268
|
+
const selectedIds = new Set(scored.map((item) => item.chunk.id));
|
|
269
|
+
const ordered = chunks
|
|
270
|
+
.filter((chunk) => selectedIds.has(chunk.id))
|
|
271
|
+
.sort((a, b) => a.start_line - b.start_line || a.end_line - b.end_line);
|
|
272
|
+
const parts: string[] = [];
|
|
273
|
+
const sourceChunkIds: string[] = [];
|
|
274
|
+
let totalLength = 0;
|
|
275
|
+
for (const chunk of ordered) {
|
|
276
|
+
if (totalLength >= ADAPTIVE_MAX_CONTEXT_CHARS) break;
|
|
277
|
+
const remaining = ADAPTIVE_MAX_CONTEXT_CHARS - totalLength;
|
|
278
|
+
const text = chunk.content.slice(0, remaining);
|
|
279
|
+
parts.push(text);
|
|
280
|
+
sourceChunkIds.push(chunk.id);
|
|
281
|
+
totalLength += text.length + 2;
|
|
282
|
+
}
|
|
283
|
+
return {
|
|
284
|
+
content: parts.join("\n\n"),
|
|
285
|
+
startLine: Math.min(...ordered.map((chunk) => chunk.start_line)),
|
|
286
|
+
endLine: Math.max(...ordered.map((chunk) => chunk.end_line)),
|
|
287
|
+
sourceChunkIds,
|
|
288
|
+
};
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
function pushAdaptiveCandidate(candidates: RankedChunk[], candidate: RankedChunk): void {
|
|
292
|
+
const overlappingIndex = candidates.findIndex(
|
|
293
|
+
(existing) =>
|
|
294
|
+
existing.chunk.kb_id === candidate.chunk.kb_id &&
|
|
295
|
+
existing.chunk.file_path === candidate.chunk.file_path &&
|
|
296
|
+
existing.sourceChunkIds.some((id) => candidate.sourceChunkIds.includes(id)),
|
|
297
|
+
);
|
|
298
|
+
if (overlappingIndex < 0) {
|
|
299
|
+
candidates.push(candidate);
|
|
300
|
+
return;
|
|
301
|
+
}
|
|
302
|
+
const existing = candidates[overlappingIndex];
|
|
303
|
+
if (candidate.score > existing.score || candidate.sourceChunkIds.length > existing.sourceChunkIds.length) {
|
|
304
|
+
candidates[overlappingIndex] = candidate;
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
|
|
70
308
|
async function chunkUrl(source: string, signal?: AbortSignal): Promise<Awaited<ReturnType<typeof chunkFile>>> {
|
|
71
309
|
const res = await fetch(source, { signal });
|
|
72
310
|
if (!res.ok) throw new Error(`Fetch failed: ${res.status}`);
|
|
@@ -109,6 +347,15 @@ export class KnowledgeEngine {
|
|
|
109
347
|
this.vectorCache.delete(kbId);
|
|
110
348
|
}
|
|
111
349
|
|
|
350
|
+
private getVectorsByChunkId(kbId: string, chunkIds: string[]): Map<string, Float32Array> {
|
|
351
|
+
const vectors = this.getVectors(kbId);
|
|
352
|
+
const mapped = new Map<string, Float32Array>();
|
|
353
|
+
for (let i = 0; i < chunkIds.length; i++) {
|
|
354
|
+
if (vectors[i]) mapped.set(chunkIds[i], vectors[i]);
|
|
355
|
+
}
|
|
356
|
+
return mapped;
|
|
357
|
+
}
|
|
358
|
+
|
|
112
359
|
async add(
|
|
113
360
|
source: string,
|
|
114
361
|
name: string,
|
|
@@ -169,7 +416,7 @@ export class KnowledgeEngine {
|
|
|
169
416
|
}
|
|
170
417
|
|
|
171
418
|
onProgress?.(`${allChunks.length} chunks, embedding...`);
|
|
172
|
-
const texts = allChunks.map((c) => c
|
|
419
|
+
const texts = allChunks.map((c) => buildChunkEmbeddingText(c));
|
|
173
420
|
const vectors = await embedDocuments(texts, signal);
|
|
174
421
|
|
|
175
422
|
onProgress?.(`Storing...`);
|
|
@@ -250,7 +497,7 @@ export class KnowledgeEngine {
|
|
|
250
497
|
if (chunksToAdd.length > 0) {
|
|
251
498
|
onProgress?.(`Embedding ${chunksToAdd.length} new chunks...`);
|
|
252
499
|
newVectors = await embedDocuments(
|
|
253
|
-
chunksToAdd.map((c) => c
|
|
500
|
+
chunksToAdd.map((c) => buildChunkEmbeddingText(c)),
|
|
254
501
|
signal,
|
|
255
502
|
);
|
|
256
503
|
insertChunks(this.db, kb.id, chunksToAdd);
|
|
@@ -285,13 +532,20 @@ export class KnowledgeEngine {
|
|
|
285
532
|
async search(query: string, options: SearchOptions = {}): Promise<SearchResponse> {
|
|
286
533
|
if (!this.db) throw new Error("Engine not initialized");
|
|
287
534
|
const db = this.db;
|
|
288
|
-
const { mode = "hybrid", limit = 10, offset = 0, kb_id, filters } = options;
|
|
289
|
-
|
|
290
|
-
const
|
|
535
|
+
const { mode = "hybrid", limit = 10, offset = 0, kb_id, filters, diversity = "balanced" } = options;
|
|
536
|
+
const retrievalMode = mode === "adaptive" ? "hybrid" : mode;
|
|
537
|
+
const candidateLimit = Math.max(50, offset + limit * 12);
|
|
538
|
+
const normalizedQuery = normalizedQueryText(query);
|
|
539
|
+
const queryTokens = tokenizeForSimilarity(query);
|
|
540
|
+
const normalizedFileType = normalizeFileTypeFilter(filters?.file_type);
|
|
541
|
+
|
|
542
|
+
const selectedKB = kb_id ? (getKB(db, kb_id) ?? getKBByName(db, kb_id)) : undefined;
|
|
543
|
+
const kbs = kb_id ? ([selectedKB].filter(Boolean) as KnowledgeBase[]) : listKBs(db);
|
|
291
544
|
if (kbs.length === 0) return { results: [], total_count: 0, has_more: false };
|
|
292
545
|
|
|
293
546
|
const warnings: string[] = [];
|
|
294
547
|
const allResults: { chunkId: string; score: number }[] = [];
|
|
548
|
+
const vectorsByChunkId = new Map<string, Float32Array>();
|
|
295
549
|
|
|
296
550
|
for (const kb of kbs) {
|
|
297
551
|
if (kb.embedding_model !== CURRENT_EMBEDDING_MODEL) {
|
|
@@ -301,26 +555,30 @@ export class KnowledgeEngine {
|
|
|
301
555
|
}
|
|
302
556
|
const chunkIds = getChunkIdsByKB(db, kb.id);
|
|
303
557
|
if (chunkIds.length === 0) continue;
|
|
558
|
+
for (const [chunkId, vector] of this.getVectorsByChunkId(kb.id, chunkIds)) {
|
|
559
|
+
vectorsByChunkId.set(chunkId, vector);
|
|
560
|
+
}
|
|
304
561
|
|
|
305
|
-
if (
|
|
306
|
-
allResults.push(...searchBM25(db, query,
|
|
307
|
-
} else if (
|
|
308
|
-
const vectors = this.
|
|
562
|
+
if (retrievalMode === "fast") {
|
|
563
|
+
allResults.push(...searchBM25(db, normalizedQuery || query, candidateLimit, kb.id));
|
|
564
|
+
} else if (retrievalMode === "semantic") {
|
|
565
|
+
const vectors = [...this.getVectorsByChunkId(kb.id, chunkIds).values()];
|
|
309
566
|
if (vectors.length > 0) {
|
|
310
567
|
const queryVec = await embedQuery(query);
|
|
311
|
-
allResults.push(...searchVector(queryVec, vectors, chunkIds));
|
|
568
|
+
allResults.push(...searchVector(queryVec, vectors, chunkIds, candidateLimit));
|
|
312
569
|
}
|
|
313
570
|
} else {
|
|
314
|
-
// hybrid: BM25 + vector
|
|
315
|
-
const bm25Results = searchBM25(db, query,
|
|
571
|
+
// hybrid: BM25 + vector weighted fusion (both scoped to this KB)
|
|
572
|
+
const bm25Results = searchBM25(db, normalizedQuery || query, candidateLimit, kb.id);
|
|
316
573
|
|
|
317
|
-
const vectors = this.
|
|
574
|
+
const vectors = [...this.getVectorsByChunkId(kb.id, chunkIds).values()];
|
|
318
575
|
let vecResults: { chunkId: string; score: number }[] = [];
|
|
319
576
|
if (vectors.length > 0) {
|
|
320
577
|
const queryVec = await embedQuery(query);
|
|
321
|
-
vecResults = searchVector(queryVec, vectors, chunkIds);
|
|
578
|
+
vecResults = searchVector(queryVec, vectors, chunkIds, candidateLimit);
|
|
322
579
|
}
|
|
323
|
-
|
|
580
|
+
if (bm25Results.length === 0) continue;
|
|
581
|
+
const fused = weightedScoreFusion(bm25Results, vecResults);
|
|
324
582
|
allResults.push(...fused);
|
|
325
583
|
}
|
|
326
584
|
}
|
|
@@ -332,74 +590,139 @@ export class KnowledgeEngine {
|
|
|
332
590
|
seen.add(r.chunkId);
|
|
333
591
|
return true;
|
|
334
592
|
});
|
|
335
|
-
|
|
593
|
+
const rankingByChunkId = new Map<string, RankingDiagnostics>();
|
|
594
|
+
for (const result of unique) {
|
|
595
|
+
const chunk = getChunkById(db, result.chunkId);
|
|
596
|
+
if (chunk) {
|
|
597
|
+
const ranking = scoreChunkForQuery(result.score, chunk, queryTokens);
|
|
598
|
+
rankingByChunkId.set(result.chunkId, ranking);
|
|
599
|
+
result.score = ranking.adjusted_score;
|
|
600
|
+
}
|
|
601
|
+
}
|
|
602
|
+
let scored = unique;
|
|
603
|
+
if (retrievalMode !== "fast" && retrievalMode !== "semantic") {
|
|
604
|
+
scored = unique.filter((result) => {
|
|
605
|
+
const chunk = getChunkById(db, result.chunkId);
|
|
606
|
+
if (!chunk) return false;
|
|
607
|
+
return result.score >= MIN_HYBRID_SCORE && hasEnoughLexicalEvidence(chunk, queryTokens);
|
|
608
|
+
});
|
|
609
|
+
}
|
|
610
|
+
scored.sort((a, b) => b.score - a.score);
|
|
336
611
|
|
|
337
612
|
// Apply metadata filters post-retrieval
|
|
338
|
-
let filtered =
|
|
339
|
-
if (
|
|
340
|
-
filtered =
|
|
613
|
+
let filtered = scored;
|
|
614
|
+
if (normalizedFileType || filters?.path_pattern) {
|
|
615
|
+
filtered = scored.filter((r) => {
|
|
341
616
|
const chunk = getChunkById(db, r.chunkId);
|
|
342
617
|
if (!chunk) return false;
|
|
343
|
-
if (
|
|
344
|
-
if (filters
|
|
618
|
+
if (normalizedFileType && chunk.file_type !== normalizedFileType) return false;
|
|
619
|
+
if (filters?.path_pattern && !chunk.file_path.includes(filters.path_pattern)) return false;
|
|
345
620
|
return true;
|
|
346
621
|
});
|
|
347
622
|
}
|
|
348
623
|
|
|
349
|
-
// Deep mode: rerank top-20 with cross-encoder
|
|
350
624
|
if (mode === "deep" && filtered.length > 0) {
|
|
351
625
|
const candidates = filtered
|
|
352
|
-
.slice(0,
|
|
626
|
+
.slice(0, 30)
|
|
353
627
|
.map((r) => {
|
|
354
628
|
const chunk = getChunkById(db, r.chunkId);
|
|
355
629
|
return chunk ? { chunkId: r.chunkId, content: chunk.content } : null;
|
|
356
630
|
})
|
|
357
631
|
.filter(Boolean) as Array<{ chunkId: string; content: string }>;
|
|
358
|
-
const reranked = await rerank(query, candidates, limit);
|
|
359
|
-
const
|
|
632
|
+
const reranked = await rerank(query, candidates, Math.max(limit * 3, limit));
|
|
633
|
+
const ranked: RankedChunk[] = [];
|
|
360
634
|
for (const r of reranked) {
|
|
361
635
|
const chunk = getChunkById(db, r.chunkId);
|
|
362
636
|
if (!chunk) continue;
|
|
363
637
|
const kbObj = getKB(db, chunk.kb_id);
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
kb_name: kbObj?.name ?? "unknown",
|
|
638
|
+
ranked.push({
|
|
639
|
+
chunk,
|
|
640
|
+
kbName: kbObj?.name ?? "unknown",
|
|
641
|
+
ranking: scoreChunkForQuery(r.score, chunk, queryTokens),
|
|
369
642
|
score: r.score,
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
643
|
+
content: chunk.content,
|
|
644
|
+
snippet: buildQuerySnippet(chunk.content, query),
|
|
645
|
+
startLine: chunk.start_line,
|
|
646
|
+
endLine: chunk.end_line,
|
|
647
|
+
sourceChunkIds: [chunk.id],
|
|
373
648
|
});
|
|
374
649
|
}
|
|
650
|
+
const diversified = interleaveByFile(diversifyRankedChunks(ranked, diversity, vectorsByChunkId), diversity);
|
|
651
|
+
const page = diversified.slice(offset, offset + limit);
|
|
652
|
+
const results = page.map((r) => ({
|
|
653
|
+
content: r.content,
|
|
654
|
+
file_path: r.chunk.file_path,
|
|
655
|
+
file_type: r.chunk.file_type,
|
|
656
|
+
kb_name: r.kbName,
|
|
657
|
+
score: r.score,
|
|
658
|
+
ranking: r.ranking,
|
|
659
|
+
snippet: r.snippet,
|
|
660
|
+
start_line: r.startLine,
|
|
661
|
+
end_line: r.endLine,
|
|
662
|
+
}));
|
|
375
663
|
return {
|
|
376
664
|
results,
|
|
377
|
-
total_count:
|
|
378
|
-
has_more:
|
|
665
|
+
total_count: diversified.length,
|
|
666
|
+
has_more: offset + limit < diversified.length,
|
|
379
667
|
warnings: warnings.length > 0 ? warnings : undefined,
|
|
380
668
|
};
|
|
381
669
|
}
|
|
382
670
|
|
|
383
|
-
const
|
|
384
|
-
const
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
.
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
const
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
671
|
+
const ranked: RankedChunk[] = [];
|
|
672
|
+
for (const r of filtered) {
|
|
673
|
+
const chunk = getChunkById(db, r.chunkId);
|
|
674
|
+
if (!chunk) continue;
|
|
675
|
+
const kb = getKB(db, chunk.kb_id);
|
|
676
|
+
|
|
677
|
+
if (mode === "adaptive") {
|
|
678
|
+
const contextChunks = getChunksByFile(
|
|
679
|
+
db,
|
|
680
|
+
chunk.kb_id,
|
|
681
|
+
chunk.file_path,
|
|
682
|
+
Math.max(1, chunk.start_line - ADAPTIVE_CONTEXT_LINES),
|
|
683
|
+
chunk.end_line + ADAPTIVE_CONTEXT_LINES,
|
|
684
|
+
);
|
|
685
|
+
const context = buildAdaptiveContext(chunk, contextChunks.length > 0 ? contextChunks : [chunk], queryTokens);
|
|
686
|
+
pushAdaptiveCandidate(ranked, {
|
|
687
|
+
chunk,
|
|
688
|
+
kbName: kb?.name ?? "unknown",
|
|
689
|
+
ranking: rankingByChunkId.get(r.chunkId),
|
|
690
|
+
score: r.score,
|
|
691
|
+
content: context.content,
|
|
692
|
+
snippet: buildQuerySnippet(context.content, query),
|
|
693
|
+
startLine: context.startLine,
|
|
694
|
+
endLine: context.endLine,
|
|
695
|
+
sourceChunkIds: context.sourceChunkIds,
|
|
696
|
+
});
|
|
697
|
+
} else {
|
|
698
|
+
ranked.push({
|
|
699
|
+
chunk,
|
|
700
|
+
kbName: kb?.name ?? "unknown",
|
|
701
|
+
ranking: rankingByChunkId.get(r.chunkId),
|
|
396
702
|
score: r.score,
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
703
|
+
content: chunk.content,
|
|
704
|
+
snippet: buildQuerySnippet(chunk.content, query),
|
|
705
|
+
startLine: chunk.start_line,
|
|
706
|
+
endLine: chunk.end_line,
|
|
707
|
+
sourceChunkIds: [chunk.id],
|
|
708
|
+
});
|
|
709
|
+
}
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
const diversified = interleaveByFile(diversifyRankedChunks(ranked, diversity, vectorsByChunkId), diversity);
|
|
713
|
+
const total = diversified.length;
|
|
714
|
+
const page = diversified.slice(offset, offset + limit);
|
|
715
|
+
const results: SearchResult[] = page.map((r) => ({
|
|
716
|
+
content: r.content,
|
|
717
|
+
file_path: r.chunk.file_path,
|
|
718
|
+
file_type: r.chunk.file_type,
|
|
719
|
+
kb_name: r.kbName,
|
|
720
|
+
score: r.score,
|
|
721
|
+
ranking: r.ranking,
|
|
722
|
+
snippet: r.snippet,
|
|
723
|
+
start_line: r.startLine,
|
|
724
|
+
end_line: r.endLine,
|
|
725
|
+
}));
|
|
403
726
|
|
|
404
727
|
return {
|
|
405
728
|
results,
|
|
@@ -479,23 +802,29 @@ export class KnowledgeEngine {
|
|
|
479
802
|
try {
|
|
480
803
|
const chunkData = lines.slice(1).map((l) => JSON.parse(l) as ImportedChunk);
|
|
481
804
|
onProgress?.(`Importing ${chunkData.length} chunks, embedding...`);
|
|
482
|
-
const texts = chunkData.map((c) => c.content);
|
|
483
|
-
const vectors = await embedDocuments(texts, signal);
|
|
484
805
|
const chunks = chunkData.map((c) => ({
|
|
485
806
|
content_hash: contentHash(c.content),
|
|
486
807
|
content: c.content,
|
|
487
|
-
content_tokenized:
|
|
808
|
+
content_tokenized: "",
|
|
488
809
|
file_path: c.file_path,
|
|
489
810
|
file_type: c.file_type,
|
|
490
811
|
start_line: c.start_line,
|
|
491
812
|
end_line: c.end_line,
|
|
492
813
|
metadata_json: c.metadata_json || "{}",
|
|
493
814
|
}));
|
|
494
|
-
|
|
815
|
+
const indexedChunks = chunks.map((chunk) => ({
|
|
816
|
+
...chunk,
|
|
817
|
+
content_tokenized: preTokenizeForFTS(buildChunkEmbeddingText(chunk)),
|
|
818
|
+
}));
|
|
819
|
+
const vectors = await embedDocuments(
|
|
820
|
+
indexedChunks.map((c) => buildChunkEmbeddingText(c)),
|
|
821
|
+
signal,
|
|
822
|
+
);
|
|
823
|
+
insertChunks(this.db, kb.id, indexedChunks);
|
|
495
824
|
const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
|
|
496
825
|
saveVectors(vectorPath, vectors);
|
|
497
826
|
this.vectorCache.set(kb.id, vectors);
|
|
498
|
-
updateKBCounts(this.db, kb.id,
|
|
827
|
+
updateKBCounts(this.db, kb.id, indexedChunks.length, new Set(indexedChunks.map((c) => c.file_path)).size);
|
|
499
828
|
updateKBStatus(this.db, kb.id, "ready");
|
|
500
829
|
const savedKB = getKB(this.db, kb.id);
|
|
501
830
|
if (!savedKB) throw new Error(`Knowledge base disappeared after import: ${kb.id}`);
|
|
@@ -506,9 +835,15 @@ export class KnowledgeEngine {
|
|
|
506
835
|
}
|
|
507
836
|
}
|
|
508
837
|
|
|
509
|
-
async dispose(): Promise<void> {
|
|
510
|
-
|
|
511
|
-
|
|
838
|
+
async dispose(options: { disposeModels?: boolean } = {}): Promise<void> {
|
|
839
|
+
const disposeModels = options.disposeModels ?? true;
|
|
840
|
+
if (disposeModels) {
|
|
841
|
+
await disposeEmbedding();
|
|
842
|
+
await disposeReranker();
|
|
843
|
+
} else {
|
|
844
|
+
await prepareEmbeddingForShutdown();
|
|
845
|
+
await prepareRerankerForShutdown();
|
|
846
|
+
}
|
|
512
847
|
this.db?.close();
|
|
513
848
|
this.db = null;
|
|
514
849
|
}
|