pi-knowledge 0.3.4 → 0.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/README.md +16 -2
- package/index.ts +63 -6
- package/package.json +1 -1
- package/src/embedding/provider.ts +50 -12
- package/src/engine.ts +402 -67
- package/src/indexer/chunker.ts +100 -77
- package/src/indexer/chunkers/code-ast.ts +14 -11
- package/src/search/bm25.ts +26 -22
- package/src/search/fusion.ts +33 -0
- package/src/search/query.ts +59 -0
- package/src/search/ranking.ts +197 -0
- package/src/search/reranker.ts +49 -11
- package/src/storage/sqlite.ts +16 -0
package/src/search/reranker.ts
CHANGED
|
@@ -9,6 +9,9 @@ type RerankerPipeline = {
|
|
|
9
9
|
let rerankerPipeline: RerankerPipeline | null = null;
|
|
10
10
|
let disposeTimer: ReturnType<typeof setTimeout> | null = null;
|
|
11
11
|
let disposePromise: Promise<void> | null = null;
|
|
12
|
+
let activeRuns = 0;
|
|
13
|
+
let disposeRequested = false;
|
|
14
|
+
const idleWaiters: Array<() => void> = [];
|
|
12
15
|
const IDLE_MS = 30_000;
|
|
13
16
|
|
|
14
17
|
async function load(): Promise<RerankerPipeline> {
|
|
@@ -20,26 +23,57 @@ async function load(): Promise<RerankerPipeline> {
|
|
|
20
23
|
return rerankerPipeline;
|
|
21
24
|
}
|
|
22
25
|
|
|
23
|
-
function
|
|
26
|
+
function clearTimer(): void {
|
|
24
27
|
if (disposeTimer) clearTimeout(disposeTimer);
|
|
28
|
+
disposeTimer = null;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function scheduleDispose(): void {
|
|
32
|
+
if (activeRuns > 0 || disposeRequested) return;
|
|
33
|
+
clearTimer();
|
|
25
34
|
disposeTimer = setTimeout(() => disposeReranker(), IDLE_MS);
|
|
26
35
|
}
|
|
27
36
|
|
|
37
|
+
function beginRun(): void {
|
|
38
|
+
activeRuns++;
|
|
39
|
+
clearTimer();
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function endRun(): void {
|
|
43
|
+
activeRuns--;
|
|
44
|
+
if (activeRuns > 0) return;
|
|
45
|
+
for (const resolve of idleWaiters.splice(0)) resolve();
|
|
46
|
+
if (!disposeRequested) scheduleDispose();
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function waitForNoActiveRuns(): Promise<void> {
|
|
50
|
+
if (activeRuns === 0) return Promise.resolve();
|
|
51
|
+
return new Promise((resolve) => idleWaiters.push(resolve));
|
|
52
|
+
}
|
|
53
|
+
|
|
28
54
|
export async function disposeReranker(): Promise<void> {
|
|
29
|
-
|
|
30
|
-
clearTimeout(disposeTimer);
|
|
31
|
-
disposeTimer = null;
|
|
32
|
-
}
|
|
55
|
+
clearTimer();
|
|
33
56
|
if (disposePromise) return disposePromise;
|
|
57
|
+
disposeRequested = true;
|
|
58
|
+
await waitForNoActiveRuns();
|
|
34
59
|
const instance = rerankerPipeline;
|
|
35
60
|
rerankerPipeline = null;
|
|
36
|
-
if (!instance)
|
|
61
|
+
if (!instance) {
|
|
62
|
+
disposeRequested = false;
|
|
63
|
+
return;
|
|
64
|
+
}
|
|
37
65
|
disposePromise = Promise.resolve(instance.dispose()).finally(() => {
|
|
38
66
|
disposePromise = null;
|
|
67
|
+
disposeRequested = false;
|
|
39
68
|
});
|
|
40
69
|
return disposePromise;
|
|
41
70
|
}
|
|
42
71
|
|
|
72
|
+
export async function prepareRerankerForShutdown(): Promise<void> {
|
|
73
|
+
clearTimer();
|
|
74
|
+
await waitForNoActiveRuns();
|
|
75
|
+
}
|
|
76
|
+
|
|
43
77
|
export interface RerankCandidate {
|
|
44
78
|
chunkId: string;
|
|
45
79
|
content: string;
|
|
@@ -52,13 +86,17 @@ export async function rerank(
|
|
|
52
86
|
): Promise<Array<{ chunkId: string; score: number }>> {
|
|
53
87
|
if (candidates.length === 0) return [];
|
|
54
88
|
const pipe = await load();
|
|
55
|
-
|
|
89
|
+
beginRun();
|
|
56
90
|
|
|
57
91
|
const results: Array<{ chunkId: string; score: number }> = [];
|
|
58
|
-
|
|
59
|
-
const
|
|
60
|
-
|
|
61
|
-
|
|
92
|
+
try {
|
|
93
|
+
for (const c of candidates) {
|
|
94
|
+
const output = await pipe({ text: query, text_pair: c.content });
|
|
95
|
+
const score = Array.isArray(output) ? (output[0]?.score ?? 0) : (output?.score ?? 0);
|
|
96
|
+
results.push({ chunkId: c.chunkId, score });
|
|
97
|
+
}
|
|
98
|
+
} finally {
|
|
99
|
+
endRun();
|
|
62
100
|
}
|
|
63
101
|
|
|
64
102
|
results.sort((a, b) => b.score - a.score);
|
package/src/storage/sqlite.ts
CHANGED
|
@@ -223,6 +223,22 @@ export function getChunkById(db: Database.Database, id: string): Chunk | undefin
|
|
|
223
223
|
return db.prepare("SELECT * FROM chunks WHERE id = ?").get(id) as Chunk | undefined;
|
|
224
224
|
}
|
|
225
225
|
|
|
226
|
+
export function getChunksByFile(
|
|
227
|
+
db: Database.Database,
|
|
228
|
+
kbId: string,
|
|
229
|
+
filePath: string,
|
|
230
|
+
startLine: number,
|
|
231
|
+
endLine: number,
|
|
232
|
+
): Chunk[] {
|
|
233
|
+
return db
|
|
234
|
+
.prepare(
|
|
235
|
+
`SELECT * FROM chunks
|
|
236
|
+
WHERE kb_id = ? AND file_path = ? AND end_line >= ? AND start_line <= ?
|
|
237
|
+
ORDER BY start_line, end_line`,
|
|
238
|
+
)
|
|
239
|
+
.all(kbId, filePath, startLine, endLine) as Chunk[];
|
|
240
|
+
}
|
|
241
|
+
|
|
226
242
|
export function getChunkByRowid(db: Database.Database, rowid: number): Chunk | undefined {
|
|
227
243
|
return db.prepare("SELECT * FROM chunks WHERE rowid = ?").get(rowid) as Chunk | undefined;
|
|
228
244
|
}
|