pi-knowledge 0.3.6 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.pi/skills/search-docs.md +24 -4
- package/CHANGELOG.md +19 -2
- package/README.md +42 -3
- package/index.ts +201 -124
- package/package.json +1 -1
- package/src/diagnostics/health.ts +84 -14
- package/src/embedding/provider.ts +5 -39
- package/src/embedding/vectors.ts +95 -12
- package/src/engine.ts +882 -140
- package/src/indexer/chunker.ts +232 -24
- package/src/model-worker-client.ts +146 -0
- package/src/model-worker.ts +88 -0
- package/src/search/bm25.ts +15 -4
- package/src/search/ranking.ts +4 -0
- package/src/search/reranker.ts +5 -35
- package/src/search/vector.ts +60 -0
- package/src/storage/sqlite.ts +224 -9
package/src/indexer/chunker.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
-
import { type Dirent, existsSync, readdirSync, readFileSync, statSync } from "node:fs";
|
|
2
|
+
import { closeSync, type Dirent, existsSync, openSync, readdirSync, readFileSync, readSync, statSync } from "node:fs";
|
|
3
3
|
import { extname, join, relative, sep } from "node:path";
|
|
4
4
|
import ignore from "ignore";
|
|
5
5
|
import type { ChunkInsert } from "../storage/sqlite.ts";
|
|
@@ -54,7 +54,7 @@ const BINARY_EXTENSIONS = new Set([
|
|
|
54
54
|
".lib",
|
|
55
55
|
]);
|
|
56
56
|
|
|
57
|
-
const
|
|
57
|
+
const DEFAULT_SUGGESTED_EXCLUDE = [
|
|
58
58
|
"node_modules",
|
|
59
59
|
".git",
|
|
60
60
|
"dist",
|
|
@@ -67,7 +67,13 @@ const DEFAULT_IGNORE = [
|
|
|
67
67
|
".next",
|
|
68
68
|
".cache",
|
|
69
69
|
".playwright",
|
|
70
|
+
".browser",
|
|
71
|
+
".browsers",
|
|
70
72
|
"__pycache__",
|
|
73
|
+
"*.app",
|
|
74
|
+
"*.asar",
|
|
75
|
+
"*.asar.unpacked",
|
|
76
|
+
"*.pak",
|
|
71
77
|
".env",
|
|
72
78
|
".env.*",
|
|
73
79
|
"*.pem",
|
|
@@ -84,19 +90,28 @@ const DEFAULT_IGNORE = [
|
|
|
84
90
|
"docs/*evaluation-report*.md",
|
|
85
91
|
"docs/*knowledge-base*report*.md",
|
|
86
92
|
"*knowledge-base-evaluation-report*.md",
|
|
87
|
-
"
|
|
88
|
-
"appsettings*.json",
|
|
93
|
+
"*knowledge*.jsonl",
|
|
89
94
|
"*.min.js",
|
|
90
95
|
"*.min.css",
|
|
91
96
|
"*.map",
|
|
92
97
|
"*.lock",
|
|
93
98
|
"package-lock.json",
|
|
99
|
+
"playwright-report",
|
|
100
|
+
"test-results",
|
|
101
|
+
"browser-cache",
|
|
102
|
+
"ms-playwright",
|
|
94
103
|
".DS_Store",
|
|
95
104
|
"Thumbs.db",
|
|
96
105
|
"*.pyc",
|
|
97
106
|
"*.class",
|
|
98
107
|
];
|
|
99
108
|
|
|
109
|
+
export interface ScanOptions {
|
|
110
|
+
includeSuggestedText?: boolean;
|
|
111
|
+
includePaths?: string[];
|
|
112
|
+
excludePaths?: string[];
|
|
113
|
+
}
|
|
114
|
+
|
|
100
115
|
export interface ScannedFile {
|
|
101
116
|
path: string; // absolute path
|
|
102
117
|
relPath: string; // relative to root
|
|
@@ -104,6 +119,59 @@ export interface ScannedFile {
|
|
|
104
119
|
fileType: string;
|
|
105
120
|
}
|
|
106
121
|
|
|
122
|
+
export interface ScannableFile {
|
|
123
|
+
path: string; // absolute path
|
|
124
|
+
relPath: string; // relative to root
|
|
125
|
+
fileType: string;
|
|
126
|
+
size: number;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
export interface SkippedScanEntry {
|
|
130
|
+
path: string;
|
|
131
|
+
reason: "suggested_excluded" | "oversized" | "binary" | "unreadable" | "inaccessible";
|
|
132
|
+
size?: number;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
export interface ScanResult {
|
|
136
|
+
files: ScannedFile[];
|
|
137
|
+
skipped: {
|
|
138
|
+
total: number;
|
|
139
|
+
by_reason: Record<SkippedScanEntry["reason"], number>;
|
|
140
|
+
samples: SkippedScanEntry[];
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
const MAX_SKIPPED_SAMPLES = 25;
|
|
145
|
+
|
|
146
|
+
export function createSkippedScanStats(): ScanResult["skipped"] {
|
|
147
|
+
return {
|
|
148
|
+
total: 0,
|
|
149
|
+
by_reason: {
|
|
150
|
+
suggested_excluded: 0,
|
|
151
|
+
oversized: 0,
|
|
152
|
+
binary: 0,
|
|
153
|
+
unreadable: 0,
|
|
154
|
+
inaccessible: 0,
|
|
155
|
+
},
|
|
156
|
+
samples: [],
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
function addSkipped(skipped: ScanResult["skipped"], entry: SkippedScanEntry): void {
|
|
161
|
+
skipped.total++;
|
|
162
|
+
skipped.by_reason[entry.reason]++;
|
|
163
|
+
if (skipped.samples.length < MAX_SKIPPED_SAMPLES) skipped.samples.push(entry);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
export function summarizeSkippedScan(skipped: ScanResult["skipped"]): string {
|
|
167
|
+
return (
|
|
168
|
+
Object.entries(skipped.by_reason)
|
|
169
|
+
.filter(([, count]) => count > 0)
|
|
170
|
+
.map(([reason, count]) => `${reason}: ${count}`)
|
|
171
|
+
.join(", ") || "none"
|
|
172
|
+
);
|
|
173
|
+
}
|
|
174
|
+
|
|
107
175
|
function detectFileType(filePath: string): string {
|
|
108
176
|
const ext = extname(filePath).toLowerCase();
|
|
109
177
|
const map: Record<string, string> = {
|
|
@@ -143,59 +211,162 @@ function detectFileType(filePath: string): string {
|
|
|
143
211
|
|
|
144
212
|
function isBinaryFile(filePath: string): boolean {
|
|
145
213
|
if (BINARY_EXTENSIONS.has(extname(filePath).toLowerCase())) return true;
|
|
214
|
+
let fd: number | undefined;
|
|
146
215
|
try {
|
|
147
|
-
|
|
148
|
-
const sample =
|
|
149
|
-
|
|
216
|
+
fd = openSync(filePath, "r");
|
|
217
|
+
const sample = Buffer.alloc(512);
|
|
218
|
+
const bytesRead = readSync(fd, sample, 0, sample.length, 0);
|
|
219
|
+
if (bytesRead === 0) return false;
|
|
220
|
+
return sample.subarray(0, bytesRead).includes(0x00);
|
|
150
221
|
} catch {
|
|
151
222
|
return true;
|
|
223
|
+
} finally {
|
|
224
|
+
if (fd !== undefined) closeSync(fd);
|
|
152
225
|
}
|
|
153
226
|
}
|
|
154
227
|
|
|
155
|
-
|
|
228
|
+
function buildIgnoreMatcher(dirPath: string): ReturnType<typeof ignore> {
|
|
156
229
|
const ig = ignore();
|
|
157
|
-
ig.add(
|
|
230
|
+
ig.add(DEFAULT_SUGGESTED_EXCLUDE);
|
|
158
231
|
|
|
159
232
|
const gitignorePath = join(dirPath, ".gitignore");
|
|
160
233
|
if (existsSync(gitignorePath)) {
|
|
161
234
|
ig.add(readFileSync(gitignorePath, "utf-8"));
|
|
162
235
|
}
|
|
163
236
|
|
|
164
|
-
|
|
237
|
+
return ig;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
function normalizeScanPath(path: string): string {
|
|
241
|
+
return path.replaceAll("\\", "/").replace(/^\/+/, "").replace(/\/+$/, "");
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
function normalizeScanPaths(paths: string[] | undefined): string[] {
|
|
245
|
+
return (paths ?? []).map(normalizeScanPath).filter(Boolean);
|
|
246
|
+
}
|
|
165
247
|
|
|
166
|
-
|
|
248
|
+
function pathMatches(relPath: string, patterns: string[]): boolean {
|
|
249
|
+
const normalized = normalizeScanPath(relPath);
|
|
250
|
+
return patterns.some((pattern) => normalized === pattern || normalized.startsWith(`${pattern}/`));
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
function directoryCouldContainIncludedPath(relPath: string, includePaths: string[]): boolean {
|
|
254
|
+
const normalized = normalizeScanPath(relPath);
|
|
255
|
+
return includePaths.some((includePath) => includePath.startsWith(`${normalized}/`));
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
export function* iterateScannableFiles(
|
|
259
|
+
dirPath: string,
|
|
260
|
+
skipped: ScanResult["skipped"] = createSkippedScanStats(),
|
|
261
|
+
options: ScanOptions = {},
|
|
262
|
+
): Generator<ScannableFile> {
|
|
263
|
+
const ig = buildIgnoreMatcher(dirPath);
|
|
264
|
+
const includePaths = normalizeScanPaths(options.includePaths);
|
|
265
|
+
const excludePaths = normalizeScanPaths(options.excludePaths);
|
|
266
|
+
const includeSuggestedText = options.includeSuggestedText === true;
|
|
267
|
+
|
|
268
|
+
function* walk(dir: string): Generator<ScannableFile> {
|
|
167
269
|
let entries: Dirent[];
|
|
168
270
|
try {
|
|
169
271
|
entries = readdirSync(dir, { withFileTypes: true });
|
|
170
272
|
} catch {
|
|
273
|
+
addSkipped(skipped, { path: relative(dirPath, dir).split(sep).join("/") || ".", reason: "inaccessible" });
|
|
171
274
|
return;
|
|
172
275
|
}
|
|
173
276
|
for (const entry of entries) {
|
|
174
277
|
const fullPath = join(dir, entry.name);
|
|
175
278
|
const relPath = relative(dirPath, fullPath).split(sep).join("/");
|
|
279
|
+
const explicitlyIncluded = pathMatches(relPath, includePaths);
|
|
176
280
|
|
|
177
|
-
if (
|
|
281
|
+
if (pathMatches(relPath, excludePaths)) {
|
|
282
|
+
addSkipped(skipped, { path: relPath, reason: "suggested_excluded" });
|
|
283
|
+
continue;
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
if (ig.ignores(relPath) && !includeSuggestedText && !explicitlyIncluded) {
|
|
287
|
+
if (entry.isDirectory() && directoryCouldContainIncludedPath(relPath, includePaths)) {
|
|
288
|
+
yield* walk(fullPath);
|
|
289
|
+
continue;
|
|
290
|
+
}
|
|
291
|
+
addSkipped(skipped, { path: relPath, reason: "suggested_excluded" });
|
|
292
|
+
continue;
|
|
293
|
+
}
|
|
178
294
|
|
|
179
295
|
if (entry.isDirectory()) {
|
|
180
|
-
if (ig.ignores(`${relPath}/`))
|
|
181
|
-
|
|
296
|
+
if (ig.ignores(`${relPath}/`) && !includeSuggestedText && !explicitlyIncluded) {
|
|
297
|
+
if (directoryCouldContainIncludedPath(relPath, includePaths)) {
|
|
298
|
+
yield* walk(fullPath);
|
|
299
|
+
continue;
|
|
300
|
+
}
|
|
301
|
+
addSkipped(skipped, { path: `${relPath}/`, reason: "suggested_excluded" });
|
|
302
|
+
continue;
|
|
303
|
+
}
|
|
304
|
+
yield* walk(fullPath);
|
|
182
305
|
} else if (entry.isFile()) {
|
|
183
|
-
|
|
184
|
-
if (stat.size > MAX_FILE_SIZE) continue;
|
|
185
|
-
if (isBinaryFile(fullPath)) continue;
|
|
186
|
-
|
|
306
|
+
let size = 0;
|
|
187
307
|
try {
|
|
188
|
-
|
|
189
|
-
results.push({ path: fullPath, relPath, content, fileType: detectFileType(fullPath) });
|
|
308
|
+
size = statSync(fullPath).size;
|
|
190
309
|
} catch {
|
|
191
|
-
|
|
310
|
+
addSkipped(skipped, { path: relPath, reason: "unreadable" });
|
|
311
|
+
continue;
|
|
312
|
+
}
|
|
313
|
+
if (size > MAX_FILE_SIZE) {
|
|
314
|
+
addSkipped(skipped, { path: relPath, reason: "oversized", size });
|
|
315
|
+
continue;
|
|
316
|
+
}
|
|
317
|
+
if (isBinaryFile(fullPath)) {
|
|
318
|
+
addSkipped(skipped, { path: relPath, reason: "binary", size });
|
|
319
|
+
continue;
|
|
192
320
|
}
|
|
321
|
+
|
|
322
|
+
yield { path: fullPath, relPath, fileType: detectFileType(fullPath), size };
|
|
193
323
|
}
|
|
194
324
|
}
|
|
195
325
|
}
|
|
196
326
|
|
|
197
|
-
walk(dirPath);
|
|
198
|
-
|
|
327
|
+
yield* walk(dirPath);
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
export function* iterateScannedFiles(
|
|
331
|
+
dirPath: string,
|
|
332
|
+
skipped: ScanResult["skipped"] = createSkippedScanStats(),
|
|
333
|
+
options: ScanOptions = {},
|
|
334
|
+
): Generator<ScannedFile> {
|
|
335
|
+
for (const file of iterateScannableFiles(dirPath, skipped, options)) {
|
|
336
|
+
try {
|
|
337
|
+
const content = readFileSync(file.path, "utf-8");
|
|
338
|
+
yield { ...file, content };
|
|
339
|
+
} catch {
|
|
340
|
+
addSkipped(skipped, { path: file.relPath, reason: "unreadable", size: file.size });
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
export async function iterateScannedFilesAsync(
|
|
346
|
+
dirPath: string,
|
|
347
|
+
onFile: (file: ScannedFile) => Promise<void> | void,
|
|
348
|
+
skipped: ScanResult["skipped"] = createSkippedScanStats(),
|
|
349
|
+
options: ScanOptions = {},
|
|
350
|
+
): Promise<ScanResult["skipped"]> {
|
|
351
|
+
for (const file of iterateScannedFiles(dirPath, skipped, options)) {
|
|
352
|
+
await onFile(file);
|
|
353
|
+
}
|
|
354
|
+
return skipped;
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
export function walkDir(dirPath: string, options: ScanOptions = {}): ScannedFile[] {
|
|
358
|
+
return walkDirDetailed(dirPath, options).files;
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
export function walkDirDetailed(dirPath: string, options: ScanOptions = {}): ScanResult {
|
|
362
|
+
const results: ScannedFile[] = [];
|
|
363
|
+
const skipped = createSkippedScanStats();
|
|
364
|
+
for (const file of iterateScannedFiles(dirPath, skipped, options)) results.push(file);
|
|
365
|
+
return { files: results, skipped };
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
export function isReadableTextFile(filePath: string): boolean {
|
|
369
|
+
return !isBinaryFile(filePath);
|
|
199
370
|
}
|
|
200
371
|
|
|
201
372
|
// --- Chunking ---
|
|
@@ -206,6 +377,7 @@ function estimateTokens(text: string): number {
|
|
|
206
377
|
|
|
207
378
|
const MARKDOWN_TARGET_TOKENS = 450;
|
|
208
379
|
const TEXT_TARGET_TOKENS = 550;
|
|
380
|
+
const MAX_TEXT_CHUNK_CHARS = 6_000;
|
|
209
381
|
|
|
210
382
|
export function preTokenizeForFTS(content: string): string {
|
|
211
383
|
return content
|
|
@@ -222,6 +394,21 @@ export function contentHash(content: string): string {
|
|
|
222
394
|
return createHash("sha256").update(content).digest("hex");
|
|
223
395
|
}
|
|
224
396
|
|
|
397
|
+
export function chunkIdentityHash(opts: {
|
|
398
|
+
content: string;
|
|
399
|
+
filePath: string;
|
|
400
|
+
fileType: string;
|
|
401
|
+
startLine: number;
|
|
402
|
+
endLine: number;
|
|
403
|
+
metadataJson: string;
|
|
404
|
+
}): string {
|
|
405
|
+
return contentHash(
|
|
406
|
+
[opts.filePath, opts.fileType, String(opts.startLine), String(opts.endLine), opts.metadataJson, opts.content].join(
|
|
407
|
+
"\0",
|
|
408
|
+
),
|
|
409
|
+
);
|
|
410
|
+
}
|
|
411
|
+
|
|
225
412
|
function normalizeHeading(heading: string): string {
|
|
226
413
|
return heading.replace(/^#{1,6}\s+/, "").trim();
|
|
227
414
|
}
|
|
@@ -263,7 +450,7 @@ function makeChunk(
|
|
|
263
450
|
): Omit<ChunkInsert, "kb_id"> {
|
|
264
451
|
const metadata_json = JSON.stringify(metadata);
|
|
265
452
|
const chunk = {
|
|
266
|
-
content_hash:
|
|
453
|
+
content_hash: chunkIdentityHash({ content, filePath, fileType, startLine, endLine, metadataJson: metadata_json }),
|
|
267
454
|
content,
|
|
268
455
|
content_tokenized: "",
|
|
269
456
|
file_path: filePath,
|
|
@@ -363,8 +550,29 @@ export function chunkText(content: string, filePath: string): Omit<ChunkInsert,
|
|
|
363
550
|
lineOffset += text.split("\n").length + 1;
|
|
364
551
|
}
|
|
365
552
|
|
|
553
|
+
function pushOversizedParagraph(para: string): void {
|
|
554
|
+
let offset = 0;
|
|
555
|
+
while (offset < para.length) {
|
|
556
|
+
const slice = para.slice(offset, offset + MAX_TEXT_CHUNK_CHARS).trim();
|
|
557
|
+
if (slice.length >= 50) {
|
|
558
|
+
chunks.push(makeChunk(slice, filePath, fileType, lineOffset, lineOffset + slice.split("\n").length));
|
|
559
|
+
}
|
|
560
|
+
lineOffset += slice.split("\n").length;
|
|
561
|
+
offset += MAX_TEXT_CHUNK_CHARS;
|
|
562
|
+
}
|
|
563
|
+
}
|
|
564
|
+
|
|
366
565
|
for (const para of paragraphs) {
|
|
367
566
|
const paraTokens = estimateTokens(para);
|
|
567
|
+
if (para.length > MAX_TEXT_CHUNK_CHARS) {
|
|
568
|
+
if (buffer.length > 0) {
|
|
569
|
+
flush();
|
|
570
|
+
buffer = [];
|
|
571
|
+
bufferTokens = 0;
|
|
572
|
+
}
|
|
573
|
+
pushOversizedParagraph(para);
|
|
574
|
+
continue;
|
|
575
|
+
}
|
|
368
576
|
if (bufferTokens + paraTokens > TEXT_TARGET_TOKENS && buffer.length > 0) {
|
|
369
577
|
flush();
|
|
370
578
|
buffer = [];
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
import { type ChildProcess, fork } from "node:child_process";
|
|
2
|
+
import { fileURLToPath } from "node:url";
|
|
3
|
+
|
|
4
|
+
type WorkerResponse = {
|
|
5
|
+
id: number;
|
|
6
|
+
result?: unknown;
|
|
7
|
+
error?: string;
|
|
8
|
+
};
|
|
9
|
+
|
|
10
|
+
type PendingRequest = {
|
|
11
|
+
resolve: (value: unknown) => void;
|
|
12
|
+
reject: (reason: Error) => void;
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
let worker: ChildProcess | null = null;
|
|
16
|
+
let nextRequestId = 1;
|
|
17
|
+
const pending = new Map<number, PendingRequest>();
|
|
18
|
+
|
|
19
|
+
function rejectPending(error: Error): void {
|
|
20
|
+
for (const request of pending.values()) {
|
|
21
|
+
request.reject(error);
|
|
22
|
+
}
|
|
23
|
+
pending.clear();
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function getWorker(): ChildProcess {
|
|
27
|
+
if (worker?.connected) return worker;
|
|
28
|
+
const workerPath = fileURLToPath(new URL("./model-worker.ts", import.meta.url));
|
|
29
|
+
worker = fork(workerPath, {
|
|
30
|
+
execArgv: ["--experimental-strip-types"],
|
|
31
|
+
stdio: ["ignore", "ignore", "ignore", "ipc"],
|
|
32
|
+
env: process.env,
|
|
33
|
+
});
|
|
34
|
+
const child = worker;
|
|
35
|
+
worker.on("message", (message: WorkerResponse) => {
|
|
36
|
+
const request = pending.get(message.id);
|
|
37
|
+
if (!request) return;
|
|
38
|
+
pending.delete(message.id);
|
|
39
|
+
if (message.error) {
|
|
40
|
+
request.reject(new Error(message.error));
|
|
41
|
+
} else {
|
|
42
|
+
request.resolve(message.result);
|
|
43
|
+
}
|
|
44
|
+
});
|
|
45
|
+
worker.on("exit", (code, signal) => {
|
|
46
|
+
if (worker !== child) return;
|
|
47
|
+
worker = null;
|
|
48
|
+
if (pending.size > 0) {
|
|
49
|
+
rejectPending(
|
|
50
|
+
new Error(`Model worker exited before responding (code ${code ?? "null"}, signal ${signal ?? "null"})`),
|
|
51
|
+
);
|
|
52
|
+
}
|
|
53
|
+
});
|
|
54
|
+
worker.on("error", (error) => {
|
|
55
|
+
if (worker !== child) return;
|
|
56
|
+
worker = null;
|
|
57
|
+
rejectPending(error);
|
|
58
|
+
});
|
|
59
|
+
return worker;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
async function requestModelWorker(message: Record<string, unknown>, signal?: AbortSignal): Promise<unknown> {
|
|
63
|
+
if (signal?.aborted) throw new Error("Cancelled");
|
|
64
|
+
const child = getWorker();
|
|
65
|
+
const id = nextRequestId++;
|
|
66
|
+
return new Promise((resolve, reject) => {
|
|
67
|
+
let abortHandler: (() => void) | undefined;
|
|
68
|
+
pending.set(id, { resolve, reject });
|
|
69
|
+
if (signal) {
|
|
70
|
+
abortHandler = () => {
|
|
71
|
+
pending.delete(id);
|
|
72
|
+
reject(new Error("Cancelled"));
|
|
73
|
+
shutdownModelWorker();
|
|
74
|
+
};
|
|
75
|
+
signal.addEventListener("abort", abortHandler, { once: true });
|
|
76
|
+
}
|
|
77
|
+
const cleanup = (): void => {
|
|
78
|
+
if (abortHandler) signal?.removeEventListener("abort", abortHandler);
|
|
79
|
+
};
|
|
80
|
+
const originalResolve = resolve;
|
|
81
|
+
const originalReject = reject;
|
|
82
|
+
pending.set(id, {
|
|
83
|
+
resolve(value) {
|
|
84
|
+
cleanup();
|
|
85
|
+
originalResolve(value);
|
|
86
|
+
},
|
|
87
|
+
reject(error) {
|
|
88
|
+
cleanup();
|
|
89
|
+
originalReject(error);
|
|
90
|
+
},
|
|
91
|
+
});
|
|
92
|
+
child.send({ id, ...message }, (error) => {
|
|
93
|
+
if (!error) return;
|
|
94
|
+
pending.delete(id);
|
|
95
|
+
cleanup();
|
|
96
|
+
reject(error);
|
|
97
|
+
});
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
export async function embedInModelWorker(
|
|
102
|
+
texts: string[],
|
|
103
|
+
prefix: "query" | "passage",
|
|
104
|
+
signal?: AbortSignal,
|
|
105
|
+
): Promise<Float32Array[]> {
|
|
106
|
+
const result = await requestModelWorker({ type: "embed", texts, prefix }, signal);
|
|
107
|
+
if (!Array.isArray(result)) throw new Error("Invalid embedding worker response");
|
|
108
|
+
return result.map((vector) => {
|
|
109
|
+
if (!Array.isArray(vector)) throw new Error("Invalid embedding vector from worker");
|
|
110
|
+
return new Float32Array(vector);
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
export interface RerankWorkerCandidate {
|
|
115
|
+
chunkId: string;
|
|
116
|
+
content: string;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
export async function rerankInModelWorker(
|
|
120
|
+
query: string,
|
|
121
|
+
candidates: RerankWorkerCandidate[],
|
|
122
|
+
topK: number,
|
|
123
|
+
): Promise<Array<{ chunkId: string; score: number }>> {
|
|
124
|
+
const result = await requestModelWorker({ type: "rerank", query, candidates, topK });
|
|
125
|
+
if (!Array.isArray(result)) throw new Error("Invalid reranker worker response");
|
|
126
|
+
return result.map((item) => {
|
|
127
|
+
if (
|
|
128
|
+
typeof item !== "object" ||
|
|
129
|
+
item === null ||
|
|
130
|
+
typeof (item as { chunkId?: unknown }).chunkId !== "string" ||
|
|
131
|
+
typeof (item as { score?: unknown }).score !== "number"
|
|
132
|
+
) {
|
|
133
|
+
throw new Error("Invalid reranker result from worker");
|
|
134
|
+
}
|
|
135
|
+
return item as { chunkId: string; score: number };
|
|
136
|
+
});
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
export function shutdownModelWorker(): void {
|
|
140
|
+
const child = worker;
|
|
141
|
+
worker = null;
|
|
142
|
+
rejectPending(new Error("Model worker shut down"));
|
|
143
|
+
if (child && !child.killed) {
|
|
144
|
+
child.kill("SIGKILL");
|
|
145
|
+
}
|
|
146
|
+
}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import { join } from "node:path";
|
|
2
|
+
import { getDefaultKnowledgeDir } from "./storage/sqlite.ts";
|
|
3
|
+
|
|
4
|
+
type FeatureExtractionPipeline = (
|
|
5
|
+
input: string,
|
|
6
|
+
options: { pooling: "mean"; normalize: true },
|
|
7
|
+
) => Promise<{ data: ArrayLike<number> }>;
|
|
8
|
+
|
|
9
|
+
type RerankerPipeline = (input: {
|
|
10
|
+
text: string;
|
|
11
|
+
text_pair: string;
|
|
12
|
+
}) => Promise<{ score?: number } | Array<{ score?: number }>>;
|
|
13
|
+
|
|
14
|
+
type EmbedRequest = {
|
|
15
|
+
id: number;
|
|
16
|
+
type: "embed";
|
|
17
|
+
texts: string[];
|
|
18
|
+
prefix: "query" | "passage";
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
type RerankRequest = {
|
|
22
|
+
id: number;
|
|
23
|
+
type: "rerank";
|
|
24
|
+
query: string;
|
|
25
|
+
candidates: Array<{ chunkId: string; content: string }>;
|
|
26
|
+
topK: number;
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
type WorkerRequest = EmbedRequest | RerankRequest;
|
|
30
|
+
|
|
31
|
+
let embeddingPipeline: FeatureExtractionPipeline | null = null;
|
|
32
|
+
let rerankerPipeline: RerankerPipeline | null = null;
|
|
33
|
+
|
|
34
|
+
function getModelCacheDir(): string {
|
|
35
|
+
return process.env.PI_KNOWLEDGE_MODEL_CACHE_DIR ?? join(getDefaultKnowledgeDir(), "models");
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
async function loadEmbeddingPipeline(): Promise<FeatureExtractionPipeline> {
|
|
39
|
+
if (embeddingPipeline) return embeddingPipeline;
|
|
40
|
+
const { pipeline, env } = await import("@huggingface/transformers");
|
|
41
|
+
env.cacheDir = getModelCacheDir();
|
|
42
|
+
embeddingPipeline = await pipeline("feature-extraction", "Xenova/multilingual-e5-small", {
|
|
43
|
+
quantized: true,
|
|
44
|
+
dtype: "fp32",
|
|
45
|
+
});
|
|
46
|
+
return embeddingPipeline;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
async function loadRerankerPipeline(): Promise<RerankerPipeline> {
|
|
50
|
+
if (rerankerPipeline) return rerankerPipeline;
|
|
51
|
+
const { pipeline, env } = await import("@huggingface/transformers");
|
|
52
|
+
env.cacheDir = getModelCacheDir();
|
|
53
|
+
rerankerPipeline = await pipeline("text-classification", "Xenova/ms-marco-MiniLM-L-4-v2");
|
|
54
|
+
return rerankerPipeline;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
async function handleEmbed(request: EmbedRequest): Promise<number[][]> {
|
|
58
|
+
const pipe = await loadEmbeddingPipeline();
|
|
59
|
+
const vectors: number[][] = [];
|
|
60
|
+
for (const text of request.texts) {
|
|
61
|
+
const output = await pipe(`${request.prefix}: ${text}`, { pooling: "mean", normalize: true });
|
|
62
|
+
vectors.push(Array.from(output.data));
|
|
63
|
+
}
|
|
64
|
+
return vectors;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
async function handleRerank(request: RerankRequest): Promise<Array<{ chunkId: string; score: number }>> {
|
|
68
|
+
const pipe = await loadRerankerPipeline();
|
|
69
|
+
const results: Array<{ chunkId: string; score: number }> = [];
|
|
70
|
+
for (const candidate of request.candidates) {
|
|
71
|
+
const output = await pipe({ text: request.query, text_pair: candidate.content });
|
|
72
|
+
const score = Array.isArray(output) ? (output[0]?.score ?? 0) : (output?.score ?? 0);
|
|
73
|
+
results.push({ chunkId: candidate.chunkId, score });
|
|
74
|
+
}
|
|
75
|
+
results.sort((a, b) => b.score - a.score);
|
|
76
|
+
return results.slice(0, request.topK);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
process.on("message", (request: WorkerRequest) => {
|
|
80
|
+
void (async () => {
|
|
81
|
+
try {
|
|
82
|
+
const result = request.type === "embed" ? await handleEmbed(request) : await handleRerank(request);
|
|
83
|
+
process.send?.({ id: request.id, result });
|
|
84
|
+
} catch (error) {
|
|
85
|
+
process.send?.({ id: request.id, error: error instanceof Error ? error.message : String(error) });
|
|
86
|
+
}
|
|
87
|
+
})();
|
|
88
|
+
});
|
package/src/search/bm25.ts
CHANGED
|
@@ -12,6 +12,10 @@ function prepareFtsTerms(query: string): string[] {
|
|
|
12
12
|
.filter((t) => t.length > 0);
|
|
13
13
|
}
|
|
14
14
|
|
|
15
|
+
function quoteFtsTerm(term: string): string {
|
|
16
|
+
return `"${term.replace(/"/g, '""')}"`;
|
|
17
|
+
}
|
|
18
|
+
|
|
15
19
|
function runSearch(db: Database.Database, ftsQuery: string, limit: number, kbId?: string): BM25Result[] {
|
|
16
20
|
if (kbId) {
|
|
17
21
|
return db
|
|
@@ -31,13 +35,20 @@ function runSearch(db: Database.Database, ftsQuery: string, limit: number, kbId?
|
|
|
31
35
|
.all(ftsQuery, limit) as BM25Result[];
|
|
32
36
|
}
|
|
33
37
|
|
|
34
|
-
export function searchBM25(
|
|
38
|
+
export function searchBM25(
|
|
39
|
+
db: Database.Database,
|
|
40
|
+
query: string,
|
|
41
|
+
limit = 50,
|
|
42
|
+
kbId?: string,
|
|
43
|
+
options: { allowOrFallback?: boolean } = {},
|
|
44
|
+
): BM25Result[] {
|
|
35
45
|
const terms = prepareFtsTerms(query);
|
|
36
46
|
if (terms.length === 0) return [];
|
|
37
47
|
try {
|
|
38
|
-
const
|
|
39
|
-
|
|
40
|
-
|
|
48
|
+
const quotedTerms = terms.map(quoteFtsTerm);
|
|
49
|
+
const strict = runSearch(db, quotedTerms.join(" AND "), limit, kbId);
|
|
50
|
+
if (strict.length > 0 || terms.length === 1 || options.allowOrFallback === false) return strict;
|
|
51
|
+
return runSearch(db, quotedTerms.join(" OR "), limit, kbId);
|
|
41
52
|
} catch {
|
|
42
53
|
return [];
|
|
43
54
|
}
|
package/src/search/ranking.ts
CHANGED
|
@@ -108,6 +108,10 @@ export function queryCoverage(text: string, queryTokens: Set<string>): number {
|
|
|
108
108
|
return matched / signals.size;
|
|
109
109
|
}
|
|
110
110
|
|
|
111
|
+
export function hasAnyLexicalEvidence(text: string, queryTokens: Set<string>): boolean {
|
|
112
|
+
return queryCoverage(text, queryTokens) > 0;
|
|
113
|
+
}
|
|
114
|
+
|
|
111
115
|
export function pathTokenBoost(filePath: string, queryTokens: Set<string>): number {
|
|
112
116
|
const pathTokens = signalTokens(tokenizeForSearch(filePath.replace(/[./_-]/g, " ")));
|
|
113
117
|
const signals = signalTokens(queryTokens);
|