pi-knowledge 0.3.6 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.pi/skills/search-docs.md +24 -4
- package/CHANGELOG.md +9 -2
- package/README.md +40 -2
- package/index.ts +200 -14
- package/package.json +1 -1
- package/src/diagnostics/health.ts +84 -14
- package/src/embedding/vectors.ts +95 -12
- package/src/engine.ts +880 -140
- package/src/indexer/chunker.ts +232 -24
- package/src/search/bm25.ts +15 -4
- package/src/search/ranking.ts +4 -0
- package/src/search/vector.ts +60 -0
- package/src/storage/sqlite.ts +224 -9
package/src/indexer/chunker.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
-
import { type Dirent, existsSync, readdirSync, readFileSync, statSync } from "node:fs";
|
|
2
|
+
import { closeSync, type Dirent, existsSync, openSync, readdirSync, readFileSync, readSync, statSync } from "node:fs";
|
|
3
3
|
import { extname, join, relative, sep } from "node:path";
|
|
4
4
|
import ignore from "ignore";
|
|
5
5
|
import type { ChunkInsert } from "../storage/sqlite.ts";
|
|
@@ -54,7 +54,7 @@ const BINARY_EXTENSIONS = new Set([
|
|
|
54
54
|
".lib",
|
|
55
55
|
]);
|
|
56
56
|
|
|
57
|
-
const
|
|
57
|
+
const DEFAULT_SUGGESTED_EXCLUDE = [
|
|
58
58
|
"node_modules",
|
|
59
59
|
".git",
|
|
60
60
|
"dist",
|
|
@@ -67,7 +67,13 @@ const DEFAULT_IGNORE = [
|
|
|
67
67
|
".next",
|
|
68
68
|
".cache",
|
|
69
69
|
".playwright",
|
|
70
|
+
".browser",
|
|
71
|
+
".browsers",
|
|
70
72
|
"__pycache__",
|
|
73
|
+
"*.app",
|
|
74
|
+
"*.asar",
|
|
75
|
+
"*.asar.unpacked",
|
|
76
|
+
"*.pak",
|
|
71
77
|
".env",
|
|
72
78
|
".env.*",
|
|
73
79
|
"*.pem",
|
|
@@ -84,19 +90,28 @@ const DEFAULT_IGNORE = [
|
|
|
84
90
|
"docs/*evaluation-report*.md",
|
|
85
91
|
"docs/*knowledge-base*report*.md",
|
|
86
92
|
"*knowledge-base-evaluation-report*.md",
|
|
87
|
-
"
|
|
88
|
-
"appsettings*.json",
|
|
93
|
+
"*knowledge*.jsonl",
|
|
89
94
|
"*.min.js",
|
|
90
95
|
"*.min.css",
|
|
91
96
|
"*.map",
|
|
92
97
|
"*.lock",
|
|
93
98
|
"package-lock.json",
|
|
99
|
+
"playwright-report",
|
|
100
|
+
"test-results",
|
|
101
|
+
"browser-cache",
|
|
102
|
+
"ms-playwright",
|
|
94
103
|
".DS_Store",
|
|
95
104
|
"Thumbs.db",
|
|
96
105
|
"*.pyc",
|
|
97
106
|
"*.class",
|
|
98
107
|
];
|
|
99
108
|
|
|
109
|
+
export interface ScanOptions {
|
|
110
|
+
includeSuggestedText?: boolean;
|
|
111
|
+
includePaths?: string[];
|
|
112
|
+
excludePaths?: string[];
|
|
113
|
+
}
|
|
114
|
+
|
|
100
115
|
export interface ScannedFile {
|
|
101
116
|
path: string; // absolute path
|
|
102
117
|
relPath: string; // relative to root
|
|
@@ -104,6 +119,59 @@ export interface ScannedFile {
|
|
|
104
119
|
fileType: string;
|
|
105
120
|
}
|
|
106
121
|
|
|
122
|
+
export interface ScannableFile {
|
|
123
|
+
path: string; // absolute path
|
|
124
|
+
relPath: string; // relative to root
|
|
125
|
+
fileType: string;
|
|
126
|
+
size: number;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
export interface SkippedScanEntry {
|
|
130
|
+
path: string;
|
|
131
|
+
reason: "suggested_excluded" | "oversized" | "binary" | "unreadable" | "inaccessible";
|
|
132
|
+
size?: number;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
export interface ScanResult {
|
|
136
|
+
files: ScannedFile[];
|
|
137
|
+
skipped: {
|
|
138
|
+
total: number;
|
|
139
|
+
by_reason: Record<SkippedScanEntry["reason"], number>;
|
|
140
|
+
samples: SkippedScanEntry[];
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
const MAX_SKIPPED_SAMPLES = 25;
|
|
145
|
+
|
|
146
|
+
export function createSkippedScanStats(): ScanResult["skipped"] {
|
|
147
|
+
return {
|
|
148
|
+
total: 0,
|
|
149
|
+
by_reason: {
|
|
150
|
+
suggested_excluded: 0,
|
|
151
|
+
oversized: 0,
|
|
152
|
+
binary: 0,
|
|
153
|
+
unreadable: 0,
|
|
154
|
+
inaccessible: 0,
|
|
155
|
+
},
|
|
156
|
+
samples: [],
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
function addSkipped(skipped: ScanResult["skipped"], entry: SkippedScanEntry): void {
|
|
161
|
+
skipped.total++;
|
|
162
|
+
skipped.by_reason[entry.reason]++;
|
|
163
|
+
if (skipped.samples.length < MAX_SKIPPED_SAMPLES) skipped.samples.push(entry);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
export function summarizeSkippedScan(skipped: ScanResult["skipped"]): string {
|
|
167
|
+
return (
|
|
168
|
+
Object.entries(skipped.by_reason)
|
|
169
|
+
.filter(([, count]) => count > 0)
|
|
170
|
+
.map(([reason, count]) => `${reason}: ${count}`)
|
|
171
|
+
.join(", ") || "none"
|
|
172
|
+
);
|
|
173
|
+
}
|
|
174
|
+
|
|
107
175
|
function detectFileType(filePath: string): string {
|
|
108
176
|
const ext = extname(filePath).toLowerCase();
|
|
109
177
|
const map: Record<string, string> = {
|
|
@@ -143,59 +211,162 @@ function detectFileType(filePath: string): string {
|
|
|
143
211
|
|
|
144
212
|
function isBinaryFile(filePath: string): boolean {
|
|
145
213
|
if (BINARY_EXTENSIONS.has(extname(filePath).toLowerCase())) return true;
|
|
214
|
+
let fd: number | undefined;
|
|
146
215
|
try {
|
|
147
|
-
|
|
148
|
-
const sample =
|
|
149
|
-
|
|
216
|
+
fd = openSync(filePath, "r");
|
|
217
|
+
const sample = Buffer.alloc(512);
|
|
218
|
+
const bytesRead = readSync(fd, sample, 0, sample.length, 0);
|
|
219
|
+
if (bytesRead === 0) return false;
|
|
220
|
+
return sample.subarray(0, bytesRead).includes(0x00);
|
|
150
221
|
} catch {
|
|
151
222
|
return true;
|
|
223
|
+
} finally {
|
|
224
|
+
if (fd !== undefined) closeSync(fd);
|
|
152
225
|
}
|
|
153
226
|
}
|
|
154
227
|
|
|
155
|
-
|
|
228
|
+
function buildIgnoreMatcher(dirPath: string): ReturnType<typeof ignore> {
|
|
156
229
|
const ig = ignore();
|
|
157
|
-
ig.add(
|
|
230
|
+
ig.add(DEFAULT_SUGGESTED_EXCLUDE);
|
|
158
231
|
|
|
159
232
|
const gitignorePath = join(dirPath, ".gitignore");
|
|
160
233
|
if (existsSync(gitignorePath)) {
|
|
161
234
|
ig.add(readFileSync(gitignorePath, "utf-8"));
|
|
162
235
|
}
|
|
163
236
|
|
|
164
|
-
|
|
237
|
+
return ig;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
function normalizeScanPath(path: string): string {
|
|
241
|
+
return path.replaceAll("\\", "/").replace(/^\/+/, "").replace(/\/+$/, "");
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
function normalizeScanPaths(paths: string[] | undefined): string[] {
|
|
245
|
+
return (paths ?? []).map(normalizeScanPath).filter(Boolean);
|
|
246
|
+
}
|
|
165
247
|
|
|
166
|
-
|
|
248
|
+
function pathMatches(relPath: string, patterns: string[]): boolean {
|
|
249
|
+
const normalized = normalizeScanPath(relPath);
|
|
250
|
+
return patterns.some((pattern) => normalized === pattern || normalized.startsWith(`${pattern}/`));
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
function directoryCouldContainIncludedPath(relPath: string, includePaths: string[]): boolean {
|
|
254
|
+
const normalized = normalizeScanPath(relPath);
|
|
255
|
+
return includePaths.some((includePath) => includePath.startsWith(`${normalized}/`));
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
export function* iterateScannableFiles(
|
|
259
|
+
dirPath: string,
|
|
260
|
+
skipped: ScanResult["skipped"] = createSkippedScanStats(),
|
|
261
|
+
options: ScanOptions = {},
|
|
262
|
+
): Generator<ScannableFile> {
|
|
263
|
+
const ig = buildIgnoreMatcher(dirPath);
|
|
264
|
+
const includePaths = normalizeScanPaths(options.includePaths);
|
|
265
|
+
const excludePaths = normalizeScanPaths(options.excludePaths);
|
|
266
|
+
const includeSuggestedText = options.includeSuggestedText === true;
|
|
267
|
+
|
|
268
|
+
function* walk(dir: string): Generator<ScannableFile> {
|
|
167
269
|
let entries: Dirent[];
|
|
168
270
|
try {
|
|
169
271
|
entries = readdirSync(dir, { withFileTypes: true });
|
|
170
272
|
} catch {
|
|
273
|
+
addSkipped(skipped, { path: relative(dirPath, dir).split(sep).join("/") || ".", reason: "inaccessible" });
|
|
171
274
|
return;
|
|
172
275
|
}
|
|
173
276
|
for (const entry of entries) {
|
|
174
277
|
const fullPath = join(dir, entry.name);
|
|
175
278
|
const relPath = relative(dirPath, fullPath).split(sep).join("/");
|
|
279
|
+
const explicitlyIncluded = pathMatches(relPath, includePaths);
|
|
176
280
|
|
|
177
|
-
if (
|
|
281
|
+
if (pathMatches(relPath, excludePaths)) {
|
|
282
|
+
addSkipped(skipped, { path: relPath, reason: "suggested_excluded" });
|
|
283
|
+
continue;
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
if (ig.ignores(relPath) && !includeSuggestedText && !explicitlyIncluded) {
|
|
287
|
+
if (entry.isDirectory() && directoryCouldContainIncludedPath(relPath, includePaths)) {
|
|
288
|
+
yield* walk(fullPath);
|
|
289
|
+
continue;
|
|
290
|
+
}
|
|
291
|
+
addSkipped(skipped, { path: relPath, reason: "suggested_excluded" });
|
|
292
|
+
continue;
|
|
293
|
+
}
|
|
178
294
|
|
|
179
295
|
if (entry.isDirectory()) {
|
|
180
|
-
if (ig.ignores(`${relPath}/`))
|
|
181
|
-
|
|
296
|
+
if (ig.ignores(`${relPath}/`) && !includeSuggestedText && !explicitlyIncluded) {
|
|
297
|
+
if (directoryCouldContainIncludedPath(relPath, includePaths)) {
|
|
298
|
+
yield* walk(fullPath);
|
|
299
|
+
continue;
|
|
300
|
+
}
|
|
301
|
+
addSkipped(skipped, { path: `${relPath}/`, reason: "suggested_excluded" });
|
|
302
|
+
continue;
|
|
303
|
+
}
|
|
304
|
+
yield* walk(fullPath);
|
|
182
305
|
} else if (entry.isFile()) {
|
|
183
|
-
|
|
184
|
-
if (stat.size > MAX_FILE_SIZE) continue;
|
|
185
|
-
if (isBinaryFile(fullPath)) continue;
|
|
186
|
-
|
|
306
|
+
let size = 0;
|
|
187
307
|
try {
|
|
188
|
-
|
|
189
|
-
results.push({ path: fullPath, relPath, content, fileType: detectFileType(fullPath) });
|
|
308
|
+
size = statSync(fullPath).size;
|
|
190
309
|
} catch {
|
|
191
|
-
|
|
310
|
+
addSkipped(skipped, { path: relPath, reason: "unreadable" });
|
|
311
|
+
continue;
|
|
312
|
+
}
|
|
313
|
+
if (size > MAX_FILE_SIZE) {
|
|
314
|
+
addSkipped(skipped, { path: relPath, reason: "oversized", size });
|
|
315
|
+
continue;
|
|
316
|
+
}
|
|
317
|
+
if (isBinaryFile(fullPath)) {
|
|
318
|
+
addSkipped(skipped, { path: relPath, reason: "binary", size });
|
|
319
|
+
continue;
|
|
192
320
|
}
|
|
321
|
+
|
|
322
|
+
yield { path: fullPath, relPath, fileType: detectFileType(fullPath), size };
|
|
193
323
|
}
|
|
194
324
|
}
|
|
195
325
|
}
|
|
196
326
|
|
|
197
|
-
walk(dirPath);
|
|
198
|
-
|
|
327
|
+
yield* walk(dirPath);
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
export function* iterateScannedFiles(
|
|
331
|
+
dirPath: string,
|
|
332
|
+
skipped: ScanResult["skipped"] = createSkippedScanStats(),
|
|
333
|
+
options: ScanOptions = {},
|
|
334
|
+
): Generator<ScannedFile> {
|
|
335
|
+
for (const file of iterateScannableFiles(dirPath, skipped, options)) {
|
|
336
|
+
try {
|
|
337
|
+
const content = readFileSync(file.path, "utf-8");
|
|
338
|
+
yield { ...file, content };
|
|
339
|
+
} catch {
|
|
340
|
+
addSkipped(skipped, { path: file.relPath, reason: "unreadable", size: file.size });
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
export async function iterateScannedFilesAsync(
|
|
346
|
+
dirPath: string,
|
|
347
|
+
onFile: (file: ScannedFile) => Promise<void> | void,
|
|
348
|
+
skipped: ScanResult["skipped"] = createSkippedScanStats(),
|
|
349
|
+
options: ScanOptions = {},
|
|
350
|
+
): Promise<ScanResult["skipped"]> {
|
|
351
|
+
for (const file of iterateScannedFiles(dirPath, skipped, options)) {
|
|
352
|
+
await onFile(file);
|
|
353
|
+
}
|
|
354
|
+
return skipped;
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
export function walkDir(dirPath: string, options: ScanOptions = {}): ScannedFile[] {
|
|
358
|
+
return walkDirDetailed(dirPath, options).files;
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
export function walkDirDetailed(dirPath: string, options: ScanOptions = {}): ScanResult {
|
|
362
|
+
const results: ScannedFile[] = [];
|
|
363
|
+
const skipped = createSkippedScanStats();
|
|
364
|
+
for (const file of iterateScannedFiles(dirPath, skipped, options)) results.push(file);
|
|
365
|
+
return { files: results, skipped };
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
export function isReadableTextFile(filePath: string): boolean {
|
|
369
|
+
return !isBinaryFile(filePath);
|
|
199
370
|
}
|
|
200
371
|
|
|
201
372
|
// --- Chunking ---
|
|
@@ -206,6 +377,7 @@ function estimateTokens(text: string): number {
|
|
|
206
377
|
|
|
207
378
|
const MARKDOWN_TARGET_TOKENS = 450;
|
|
208
379
|
const TEXT_TARGET_TOKENS = 550;
|
|
380
|
+
const MAX_TEXT_CHUNK_CHARS = 6_000;
|
|
209
381
|
|
|
210
382
|
export function preTokenizeForFTS(content: string): string {
|
|
211
383
|
return content
|
|
@@ -222,6 +394,21 @@ export function contentHash(content: string): string {
|
|
|
222
394
|
return createHash("sha256").update(content).digest("hex");
|
|
223
395
|
}
|
|
224
396
|
|
|
397
|
+
export function chunkIdentityHash(opts: {
|
|
398
|
+
content: string;
|
|
399
|
+
filePath: string;
|
|
400
|
+
fileType: string;
|
|
401
|
+
startLine: number;
|
|
402
|
+
endLine: number;
|
|
403
|
+
metadataJson: string;
|
|
404
|
+
}): string {
|
|
405
|
+
return contentHash(
|
|
406
|
+
[opts.filePath, opts.fileType, String(opts.startLine), String(opts.endLine), opts.metadataJson, opts.content].join(
|
|
407
|
+
"\0",
|
|
408
|
+
),
|
|
409
|
+
);
|
|
410
|
+
}
|
|
411
|
+
|
|
225
412
|
function normalizeHeading(heading: string): string {
|
|
226
413
|
return heading.replace(/^#{1,6}\s+/, "").trim();
|
|
227
414
|
}
|
|
@@ -263,7 +450,7 @@ function makeChunk(
|
|
|
263
450
|
): Omit<ChunkInsert, "kb_id"> {
|
|
264
451
|
const metadata_json = JSON.stringify(metadata);
|
|
265
452
|
const chunk = {
|
|
266
|
-
content_hash:
|
|
453
|
+
content_hash: chunkIdentityHash({ content, filePath, fileType, startLine, endLine, metadataJson: metadata_json }),
|
|
267
454
|
content,
|
|
268
455
|
content_tokenized: "",
|
|
269
456
|
file_path: filePath,
|
|
@@ -363,8 +550,29 @@ export function chunkText(content: string, filePath: string): Omit<ChunkInsert,
|
|
|
363
550
|
lineOffset += text.split("\n").length + 1;
|
|
364
551
|
}
|
|
365
552
|
|
|
553
|
+
function pushOversizedParagraph(para: string): void {
|
|
554
|
+
let offset = 0;
|
|
555
|
+
while (offset < para.length) {
|
|
556
|
+
const slice = para.slice(offset, offset + MAX_TEXT_CHUNK_CHARS).trim();
|
|
557
|
+
if (slice.length >= 50) {
|
|
558
|
+
chunks.push(makeChunk(slice, filePath, fileType, lineOffset, lineOffset + slice.split("\n").length));
|
|
559
|
+
}
|
|
560
|
+
lineOffset += slice.split("\n").length;
|
|
561
|
+
offset += MAX_TEXT_CHUNK_CHARS;
|
|
562
|
+
}
|
|
563
|
+
}
|
|
564
|
+
|
|
366
565
|
for (const para of paragraphs) {
|
|
367
566
|
const paraTokens = estimateTokens(para);
|
|
567
|
+
if (para.length > MAX_TEXT_CHUNK_CHARS) {
|
|
568
|
+
if (buffer.length > 0) {
|
|
569
|
+
flush();
|
|
570
|
+
buffer = [];
|
|
571
|
+
bufferTokens = 0;
|
|
572
|
+
}
|
|
573
|
+
pushOversizedParagraph(para);
|
|
574
|
+
continue;
|
|
575
|
+
}
|
|
368
576
|
if (bufferTokens + paraTokens > TEXT_TARGET_TOKENS && buffer.length > 0) {
|
|
369
577
|
flush();
|
|
370
578
|
buffer = [];
|
package/src/search/bm25.ts
CHANGED
|
@@ -12,6 +12,10 @@ function prepareFtsTerms(query: string): string[] {
|
|
|
12
12
|
.filter((t) => t.length > 0);
|
|
13
13
|
}
|
|
14
14
|
|
|
15
|
+
function quoteFtsTerm(term: string): string {
|
|
16
|
+
return `"${term.replace(/"/g, '""')}"`;
|
|
17
|
+
}
|
|
18
|
+
|
|
15
19
|
function runSearch(db: Database.Database, ftsQuery: string, limit: number, kbId?: string): BM25Result[] {
|
|
16
20
|
if (kbId) {
|
|
17
21
|
return db
|
|
@@ -31,13 +35,20 @@ function runSearch(db: Database.Database, ftsQuery: string, limit: number, kbId?
|
|
|
31
35
|
.all(ftsQuery, limit) as BM25Result[];
|
|
32
36
|
}
|
|
33
37
|
|
|
34
|
-
export function searchBM25(
|
|
38
|
+
export function searchBM25(
|
|
39
|
+
db: Database.Database,
|
|
40
|
+
query: string,
|
|
41
|
+
limit = 50,
|
|
42
|
+
kbId?: string,
|
|
43
|
+
options: { allowOrFallback?: boolean } = {},
|
|
44
|
+
): BM25Result[] {
|
|
35
45
|
const terms = prepareFtsTerms(query);
|
|
36
46
|
if (terms.length === 0) return [];
|
|
37
47
|
try {
|
|
38
|
-
const
|
|
39
|
-
|
|
40
|
-
|
|
48
|
+
const quotedTerms = terms.map(quoteFtsTerm);
|
|
49
|
+
const strict = runSearch(db, quotedTerms.join(" AND "), limit, kbId);
|
|
50
|
+
if (strict.length > 0 || terms.length === 1 || options.allowOrFallback === false) return strict;
|
|
51
|
+
return runSearch(db, quotedTerms.join(" OR "), limit, kbId);
|
|
41
52
|
} catch {
|
|
42
53
|
return [];
|
|
43
54
|
}
|
package/src/search/ranking.ts
CHANGED
|
@@ -108,6 +108,10 @@ export function queryCoverage(text: string, queryTokens: Set<string>): number {
|
|
|
108
108
|
return matched / signals.size;
|
|
109
109
|
}
|
|
110
110
|
|
|
111
|
+
export function hasAnyLexicalEvidence(text: string, queryTokens: Set<string>): boolean {
|
|
112
|
+
return queryCoverage(text, queryTokens) > 0;
|
|
113
|
+
}
|
|
114
|
+
|
|
111
115
|
export function pathTokenBoost(filePath: string, queryTokens: Set<string>): number {
|
|
112
116
|
const pathTokens = signalTokens(tokenizeForSearch(filePath.replace(/[./_-]/g, " ")));
|
|
113
117
|
const signals = signalTokens(queryTokens);
|
package/src/search/vector.ts
CHANGED
|
@@ -1,14 +1,42 @@
|
|
|
1
|
+
import { openVectorReader } from "../embedding/vectors.ts";
|
|
2
|
+
|
|
1
3
|
export interface VectorResult {
|
|
2
4
|
chunkId: string;
|
|
3
5
|
score: number;
|
|
4
6
|
}
|
|
5
7
|
|
|
8
|
+
export interface VectorFileSearchResult {
|
|
9
|
+
results: VectorResult[];
|
|
10
|
+
vectorsByChunkId: Map<string, Float32Array>;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
type ChunkIdSource = Iterable<string | { id: string }>;
|
|
14
|
+
|
|
6
15
|
function cosine(a: Float32Array, b: Float32Array): number {
|
|
7
16
|
let dot = 0;
|
|
8
17
|
for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
|
|
9
18
|
return dot;
|
|
10
19
|
}
|
|
11
20
|
|
|
21
|
+
function keepTopK(
|
|
22
|
+
top: Array<VectorResult & { vector: Float32Array }>,
|
|
23
|
+
item: VectorResult,
|
|
24
|
+
vector: Float32Array,
|
|
25
|
+
limit: number,
|
|
26
|
+
): void {
|
|
27
|
+
if (limit <= 0) return;
|
|
28
|
+
if (top.length < limit) {
|
|
29
|
+
const stored = { ...item, vector: new Float32Array(vector) };
|
|
30
|
+
top.push(stored);
|
|
31
|
+
top.sort((a, b) => a.score - b.score);
|
|
32
|
+
return;
|
|
33
|
+
}
|
|
34
|
+
if (item.score <= top[0].score) return;
|
|
35
|
+
const stored = { ...item, vector: new Float32Array(vector) };
|
|
36
|
+
top[0] = stored;
|
|
37
|
+
top.sort((a, b) => a.score - b.score);
|
|
38
|
+
}
|
|
39
|
+
|
|
12
40
|
export function searchVector(
|
|
13
41
|
queryVec: Float32Array,
|
|
14
42
|
vectors: Float32Array[],
|
|
@@ -19,3 +47,35 @@ export function searchVector(
|
|
|
19
47
|
scores.sort((a, b) => b.score - a.score);
|
|
20
48
|
return scores.slice(0, limit);
|
|
21
49
|
}
|
|
50
|
+
|
|
51
|
+
export function searchVectorFile(
|
|
52
|
+
queryVec: Float32Array,
|
|
53
|
+
vectorPath: string,
|
|
54
|
+
chunkIds: ChunkIdSource,
|
|
55
|
+
limit = 50,
|
|
56
|
+
): VectorFileSearchResult {
|
|
57
|
+
const reader = openVectorReader(vectorPath);
|
|
58
|
+
if (!reader) return { results: [], vectorsByChunkId: new Map() };
|
|
59
|
+
try {
|
|
60
|
+
if (reader.dim !== queryVec.length) return { results: [], vectorsByChunkId: new Map() };
|
|
61
|
+
const scratch = new Float32Array(reader.dim);
|
|
62
|
+
const top: Array<VectorResult & { vector: Float32Array }> = [];
|
|
63
|
+
let i = 0;
|
|
64
|
+
for (const chunkIdRef of chunkIds) {
|
|
65
|
+
if (i >= reader.count) break;
|
|
66
|
+
const vectorIndex = i;
|
|
67
|
+
i++;
|
|
68
|
+
if (!reader.readInto(vectorIndex, scratch)) continue;
|
|
69
|
+
const chunkId = typeof chunkIdRef === "string" ? chunkIdRef : chunkIdRef.id;
|
|
70
|
+
keepTopK(top, { chunkId, score: cosine(queryVec, scratch) }, scratch, limit);
|
|
71
|
+
}
|
|
72
|
+
const sorted = top.sort((a, b) => b.score - a.score);
|
|
73
|
+
const vectorsByChunkId = new Map(sorted.map((item) => [item.chunkId, item.vector]));
|
|
74
|
+
return {
|
|
75
|
+
results: sorted.map(({ chunkId, score }) => ({ chunkId, score })),
|
|
76
|
+
vectorsByChunkId,
|
|
77
|
+
};
|
|
78
|
+
} finally {
|
|
79
|
+
reader.close();
|
|
80
|
+
}
|
|
81
|
+
}
|