pi-knowledge 0.3.4 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.pi/skills/search-docs.md +24 -4
- package/CHANGELOG.md +34 -0
- package/README.md +55 -3
- package/index.ts +263 -20
- package/package.json +1 -1
- package/src/diagnostics/health.ts +84 -14
- package/src/embedding/provider.ts +50 -12
- package/src/embedding/vectors.ts +95 -12
- package/src/engine.ts +1252 -177
- package/src/indexer/chunker.ts +331 -100
- package/src/indexer/chunkers/code-ast.ts +14 -11
- package/src/search/bm25.ts +37 -22
- package/src/search/fusion.ts +33 -0
- package/src/search/query.ts +59 -0
- package/src/search/ranking.ts +201 -0
- package/src/search/reranker.ts +49 -11
- package/src/search/vector.ts +60 -0
- package/src/storage/sqlite.ts +240 -9
package/src/indexer/chunker.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
-
import { type Dirent, existsSync, readdirSync, readFileSync, statSync } from "node:fs";
|
|
2
|
+
import { closeSync, type Dirent, existsSync, openSync, readdirSync, readFileSync, readSync, statSync } from "node:fs";
|
|
3
3
|
import { extname, join, relative, sep } from "node:path";
|
|
4
4
|
import ignore from "ignore";
|
|
5
5
|
import type { ChunkInsert } from "../storage/sqlite.ts";
|
|
@@ -54,7 +54,7 @@ const BINARY_EXTENSIONS = new Set([
|
|
|
54
54
|
".lib",
|
|
55
55
|
]);
|
|
56
56
|
|
|
57
|
-
const
|
|
57
|
+
const DEFAULT_SUGGESTED_EXCLUDE = [
|
|
58
58
|
"node_modules",
|
|
59
59
|
".git",
|
|
60
60
|
"dist",
|
|
@@ -67,7 +67,13 @@ const DEFAULT_IGNORE = [
|
|
|
67
67
|
".next",
|
|
68
68
|
".cache",
|
|
69
69
|
".playwright",
|
|
70
|
+
".browser",
|
|
71
|
+
".browsers",
|
|
70
72
|
"__pycache__",
|
|
73
|
+
"*.app",
|
|
74
|
+
"*.asar",
|
|
75
|
+
"*.asar.unpacked",
|
|
76
|
+
"*.pak",
|
|
71
77
|
".env",
|
|
72
78
|
".env.*",
|
|
73
79
|
"*.pem",
|
|
@@ -80,19 +86,32 @@ const DEFAULT_IGNORE = [
|
|
|
80
86
|
"*secrets*",
|
|
81
87
|
"*credential*",
|
|
82
88
|
"*credentials*",
|
|
83
|
-
"
|
|
84
|
-
"
|
|
89
|
+
"docs/*eval-report*.md",
|
|
90
|
+
"docs/*evaluation-report*.md",
|
|
91
|
+
"docs/*knowledge-base*report*.md",
|
|
92
|
+
"*knowledge-base-evaluation-report*.md",
|
|
93
|
+
"*knowledge*.jsonl",
|
|
85
94
|
"*.min.js",
|
|
86
95
|
"*.min.css",
|
|
87
96
|
"*.map",
|
|
88
97
|
"*.lock",
|
|
89
98
|
"package-lock.json",
|
|
99
|
+
"playwright-report",
|
|
100
|
+
"test-results",
|
|
101
|
+
"browser-cache",
|
|
102
|
+
"ms-playwright",
|
|
90
103
|
".DS_Store",
|
|
91
104
|
"Thumbs.db",
|
|
92
105
|
"*.pyc",
|
|
93
106
|
"*.class",
|
|
94
107
|
];
|
|
95
108
|
|
|
109
|
+
export interface ScanOptions {
|
|
110
|
+
includeSuggestedText?: boolean;
|
|
111
|
+
includePaths?: string[];
|
|
112
|
+
excludePaths?: string[];
|
|
113
|
+
}
|
|
114
|
+
|
|
96
115
|
export interface ScannedFile {
|
|
97
116
|
path: string; // absolute path
|
|
98
117
|
relPath: string; // relative to root
|
|
@@ -100,6 +119,59 @@ export interface ScannedFile {
|
|
|
100
119
|
fileType: string;
|
|
101
120
|
}
|
|
102
121
|
|
|
122
|
+
export interface ScannableFile {
|
|
123
|
+
path: string; // absolute path
|
|
124
|
+
relPath: string; // relative to root
|
|
125
|
+
fileType: string;
|
|
126
|
+
size: number;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
export interface SkippedScanEntry {
|
|
130
|
+
path: string;
|
|
131
|
+
reason: "suggested_excluded" | "oversized" | "binary" | "unreadable" | "inaccessible";
|
|
132
|
+
size?: number;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
export interface ScanResult {
|
|
136
|
+
files: ScannedFile[];
|
|
137
|
+
skipped: {
|
|
138
|
+
total: number;
|
|
139
|
+
by_reason: Record<SkippedScanEntry["reason"], number>;
|
|
140
|
+
samples: SkippedScanEntry[];
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
const MAX_SKIPPED_SAMPLES = 25;
|
|
145
|
+
|
|
146
|
+
export function createSkippedScanStats(): ScanResult["skipped"] {
|
|
147
|
+
return {
|
|
148
|
+
total: 0,
|
|
149
|
+
by_reason: {
|
|
150
|
+
suggested_excluded: 0,
|
|
151
|
+
oversized: 0,
|
|
152
|
+
binary: 0,
|
|
153
|
+
unreadable: 0,
|
|
154
|
+
inaccessible: 0,
|
|
155
|
+
},
|
|
156
|
+
samples: [],
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
function addSkipped(skipped: ScanResult["skipped"], entry: SkippedScanEntry): void {
|
|
161
|
+
skipped.total++;
|
|
162
|
+
skipped.by_reason[entry.reason]++;
|
|
163
|
+
if (skipped.samples.length < MAX_SKIPPED_SAMPLES) skipped.samples.push(entry);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
export function summarizeSkippedScan(skipped: ScanResult["skipped"]): string {
|
|
167
|
+
return (
|
|
168
|
+
Object.entries(skipped.by_reason)
|
|
169
|
+
.filter(([, count]) => count > 0)
|
|
170
|
+
.map(([reason, count]) => `${reason}: ${count}`)
|
|
171
|
+
.join(", ") || "none"
|
|
172
|
+
);
|
|
173
|
+
}
|
|
174
|
+
|
|
103
175
|
function detectFileType(filePath: string): string {
|
|
104
176
|
const ext = extname(filePath).toLowerCase();
|
|
105
177
|
const map: Record<string, string> = {
|
|
@@ -139,59 +211,162 @@ function detectFileType(filePath: string): string {
|
|
|
139
211
|
|
|
140
212
|
function isBinaryFile(filePath: string): boolean {
|
|
141
213
|
if (BINARY_EXTENSIONS.has(extname(filePath).toLowerCase())) return true;
|
|
214
|
+
let fd: number | undefined;
|
|
142
215
|
try {
|
|
143
|
-
|
|
144
|
-
const sample =
|
|
145
|
-
|
|
216
|
+
fd = openSync(filePath, "r");
|
|
217
|
+
const sample = Buffer.alloc(512);
|
|
218
|
+
const bytesRead = readSync(fd, sample, 0, sample.length, 0);
|
|
219
|
+
if (bytesRead === 0) return false;
|
|
220
|
+
return sample.subarray(0, bytesRead).includes(0x00);
|
|
146
221
|
} catch {
|
|
147
222
|
return true;
|
|
223
|
+
} finally {
|
|
224
|
+
if (fd !== undefined) closeSync(fd);
|
|
148
225
|
}
|
|
149
226
|
}
|
|
150
227
|
|
|
151
|
-
|
|
228
|
+
function buildIgnoreMatcher(dirPath: string): ReturnType<typeof ignore> {
|
|
152
229
|
const ig = ignore();
|
|
153
|
-
ig.add(
|
|
230
|
+
ig.add(DEFAULT_SUGGESTED_EXCLUDE);
|
|
154
231
|
|
|
155
232
|
const gitignorePath = join(dirPath, ".gitignore");
|
|
156
233
|
if (existsSync(gitignorePath)) {
|
|
157
234
|
ig.add(readFileSync(gitignorePath, "utf-8"));
|
|
158
235
|
}
|
|
159
236
|
|
|
160
|
-
|
|
237
|
+
return ig;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
function normalizeScanPath(path: string): string {
|
|
241
|
+
return path.replaceAll("\\", "/").replace(/^\/+/, "").replace(/\/+$/, "");
|
|
242
|
+
}
|
|
161
243
|
|
|
162
|
-
|
|
244
|
+
function normalizeScanPaths(paths: string[] | undefined): string[] {
|
|
245
|
+
return (paths ?? []).map(normalizeScanPath).filter(Boolean);
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
function pathMatches(relPath: string, patterns: string[]): boolean {
|
|
249
|
+
const normalized = normalizeScanPath(relPath);
|
|
250
|
+
return patterns.some((pattern) => normalized === pattern || normalized.startsWith(`${pattern}/`));
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
function directoryCouldContainIncludedPath(relPath: string, includePaths: string[]): boolean {
|
|
254
|
+
const normalized = normalizeScanPath(relPath);
|
|
255
|
+
return includePaths.some((includePath) => includePath.startsWith(`${normalized}/`));
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
export function* iterateScannableFiles(
|
|
259
|
+
dirPath: string,
|
|
260
|
+
skipped: ScanResult["skipped"] = createSkippedScanStats(),
|
|
261
|
+
options: ScanOptions = {},
|
|
262
|
+
): Generator<ScannableFile> {
|
|
263
|
+
const ig = buildIgnoreMatcher(dirPath);
|
|
264
|
+
const includePaths = normalizeScanPaths(options.includePaths);
|
|
265
|
+
const excludePaths = normalizeScanPaths(options.excludePaths);
|
|
266
|
+
const includeSuggestedText = options.includeSuggestedText === true;
|
|
267
|
+
|
|
268
|
+
function* walk(dir: string): Generator<ScannableFile> {
|
|
163
269
|
let entries: Dirent[];
|
|
164
270
|
try {
|
|
165
271
|
entries = readdirSync(dir, { withFileTypes: true });
|
|
166
272
|
} catch {
|
|
273
|
+
addSkipped(skipped, { path: relative(dirPath, dir).split(sep).join("/") || ".", reason: "inaccessible" });
|
|
167
274
|
return;
|
|
168
275
|
}
|
|
169
276
|
for (const entry of entries) {
|
|
170
277
|
const fullPath = join(dir, entry.name);
|
|
171
278
|
const relPath = relative(dirPath, fullPath).split(sep).join("/");
|
|
279
|
+
const explicitlyIncluded = pathMatches(relPath, includePaths);
|
|
172
280
|
|
|
173
|
-
if (
|
|
281
|
+
if (pathMatches(relPath, excludePaths)) {
|
|
282
|
+
addSkipped(skipped, { path: relPath, reason: "suggested_excluded" });
|
|
283
|
+
continue;
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
if (ig.ignores(relPath) && !includeSuggestedText && !explicitlyIncluded) {
|
|
287
|
+
if (entry.isDirectory() && directoryCouldContainIncludedPath(relPath, includePaths)) {
|
|
288
|
+
yield* walk(fullPath);
|
|
289
|
+
continue;
|
|
290
|
+
}
|
|
291
|
+
addSkipped(skipped, { path: relPath, reason: "suggested_excluded" });
|
|
292
|
+
continue;
|
|
293
|
+
}
|
|
174
294
|
|
|
175
295
|
if (entry.isDirectory()) {
|
|
176
|
-
if (ig.ignores(`${relPath}/`))
|
|
177
|
-
|
|
296
|
+
if (ig.ignores(`${relPath}/`) && !includeSuggestedText && !explicitlyIncluded) {
|
|
297
|
+
if (directoryCouldContainIncludedPath(relPath, includePaths)) {
|
|
298
|
+
yield* walk(fullPath);
|
|
299
|
+
continue;
|
|
300
|
+
}
|
|
301
|
+
addSkipped(skipped, { path: `${relPath}/`, reason: "suggested_excluded" });
|
|
302
|
+
continue;
|
|
303
|
+
}
|
|
304
|
+
yield* walk(fullPath);
|
|
178
305
|
} else if (entry.isFile()) {
|
|
179
|
-
|
|
180
|
-
if (stat.size > MAX_FILE_SIZE) continue;
|
|
181
|
-
if (isBinaryFile(fullPath)) continue;
|
|
182
|
-
|
|
306
|
+
let size = 0;
|
|
183
307
|
try {
|
|
184
|
-
|
|
185
|
-
results.push({ path: fullPath, relPath, content, fileType: detectFileType(fullPath) });
|
|
308
|
+
size = statSync(fullPath).size;
|
|
186
309
|
} catch {
|
|
187
|
-
|
|
310
|
+
addSkipped(skipped, { path: relPath, reason: "unreadable" });
|
|
311
|
+
continue;
|
|
312
|
+
}
|
|
313
|
+
if (size > MAX_FILE_SIZE) {
|
|
314
|
+
addSkipped(skipped, { path: relPath, reason: "oversized", size });
|
|
315
|
+
continue;
|
|
316
|
+
}
|
|
317
|
+
if (isBinaryFile(fullPath)) {
|
|
318
|
+
addSkipped(skipped, { path: relPath, reason: "binary", size });
|
|
319
|
+
continue;
|
|
188
320
|
}
|
|
321
|
+
|
|
322
|
+
yield { path: fullPath, relPath, fileType: detectFileType(fullPath), size };
|
|
189
323
|
}
|
|
190
324
|
}
|
|
191
325
|
}
|
|
192
326
|
|
|
193
|
-
walk(dirPath);
|
|
194
|
-
|
|
327
|
+
yield* walk(dirPath);
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
export function* iterateScannedFiles(
|
|
331
|
+
dirPath: string,
|
|
332
|
+
skipped: ScanResult["skipped"] = createSkippedScanStats(),
|
|
333
|
+
options: ScanOptions = {},
|
|
334
|
+
): Generator<ScannedFile> {
|
|
335
|
+
for (const file of iterateScannableFiles(dirPath, skipped, options)) {
|
|
336
|
+
try {
|
|
337
|
+
const content = readFileSync(file.path, "utf-8");
|
|
338
|
+
yield { ...file, content };
|
|
339
|
+
} catch {
|
|
340
|
+
addSkipped(skipped, { path: file.relPath, reason: "unreadable", size: file.size });
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
export async function iterateScannedFilesAsync(
|
|
346
|
+
dirPath: string,
|
|
347
|
+
onFile: (file: ScannedFile) => Promise<void> | void,
|
|
348
|
+
skipped: ScanResult["skipped"] = createSkippedScanStats(),
|
|
349
|
+
options: ScanOptions = {},
|
|
350
|
+
): Promise<ScanResult["skipped"]> {
|
|
351
|
+
for (const file of iterateScannedFiles(dirPath, skipped, options)) {
|
|
352
|
+
await onFile(file);
|
|
353
|
+
}
|
|
354
|
+
return skipped;
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
export function walkDir(dirPath: string, options: ScanOptions = {}): ScannedFile[] {
|
|
358
|
+
return walkDirDetailed(dirPath, options).files;
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
export function walkDirDetailed(dirPath: string, options: ScanOptions = {}): ScanResult {
|
|
362
|
+
const results: ScannedFile[] = [];
|
|
363
|
+
const skipped = createSkippedScanStats();
|
|
364
|
+
for (const file of iterateScannedFiles(dirPath, skipped, options)) results.push(file);
|
|
365
|
+
return { files: results, skipped };
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
export function isReadableTextFile(filePath: string): boolean {
|
|
369
|
+
return !isBinaryFile(filePath);
|
|
195
370
|
}
|
|
196
371
|
|
|
197
372
|
// --- Chunking ---
|
|
@@ -200,6 +375,10 @@ function estimateTokens(text: string): number {
|
|
|
200
375
|
return Math.ceil(text.length / 3);
|
|
201
376
|
}
|
|
202
377
|
|
|
378
|
+
const MARKDOWN_TARGET_TOKENS = 450;
|
|
379
|
+
const TEXT_TARGET_TOKENS = 550;
|
|
380
|
+
const MAX_TEXT_CHUNK_CHARS = 6_000;
|
|
381
|
+
|
|
203
382
|
export function preTokenizeForFTS(content: string): string {
|
|
204
383
|
return content
|
|
205
384
|
.replace(/([a-z])([A-Z])/g, "$1 $2")
|
|
@@ -215,74 +394,123 @@ export function contentHash(content: string): string {
|
|
|
215
394
|
return createHash("sha256").update(content).digest("hex");
|
|
216
395
|
}
|
|
217
396
|
|
|
397
|
+
export function chunkIdentityHash(opts: {
|
|
398
|
+
content: string;
|
|
399
|
+
filePath: string;
|
|
400
|
+
fileType: string;
|
|
401
|
+
startLine: number;
|
|
402
|
+
endLine: number;
|
|
403
|
+
metadataJson: string;
|
|
404
|
+
}): string {
|
|
405
|
+
return contentHash(
|
|
406
|
+
[opts.filePath, opts.fileType, String(opts.startLine), String(opts.endLine), opts.metadataJson, opts.content].join(
|
|
407
|
+
"\0",
|
|
408
|
+
),
|
|
409
|
+
);
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
function normalizeHeading(heading: string): string {
|
|
413
|
+
return heading.replace(/^#{1,6}\s+/, "").trim();
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
function buildContextPrefix(
|
|
417
|
+
filePath: string,
|
|
418
|
+
fileType: string,
|
|
419
|
+
metadata: Record<string, string | number | null | undefined>,
|
|
420
|
+
): string {
|
|
421
|
+
const parts = [`File: ${filePath}`, `Type: ${fileType}`];
|
|
422
|
+
const heading = typeof metadata.heading === "string" ? metadata.heading : "";
|
|
423
|
+
const breadcrumb = typeof metadata.breadcrumb === "string" ? metadata.breadcrumb : "";
|
|
424
|
+
const symbol = typeof metadata.function_name === "string" ? metadata.function_name : "";
|
|
425
|
+
if (breadcrumb) parts.push(`Section: ${breadcrumb}`);
|
|
426
|
+
else if (heading) parts.push(`Section: ${normalizeHeading(heading)}`);
|
|
427
|
+
if (symbol) parts.push(`Symbol: ${symbol}`);
|
|
428
|
+
return parts.join("\n");
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
export function buildChunkEmbeddingText(
|
|
432
|
+
chunk: Pick<ChunkInsert, "content" | "file_path" | "file_type" | "metadata_json">,
|
|
433
|
+
): string {
|
|
434
|
+
let metadata: Record<string, string | number | null | undefined> = {};
|
|
435
|
+
try {
|
|
436
|
+
metadata = JSON.parse(chunk.metadata_json) as Record<string, string | number | null | undefined>;
|
|
437
|
+
} catch {
|
|
438
|
+
metadata = {};
|
|
439
|
+
}
|
|
440
|
+
return `${buildContextPrefix(chunk.file_path, chunk.file_type, metadata)}\n\n${chunk.content}`;
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
function makeChunk(
|
|
444
|
+
content: string,
|
|
445
|
+
filePath: string,
|
|
446
|
+
fileType: string,
|
|
447
|
+
startLine: number,
|
|
448
|
+
endLine: number,
|
|
449
|
+
metadata: Record<string, string | number | null | undefined> = {},
|
|
450
|
+
): Omit<ChunkInsert, "kb_id"> {
|
|
451
|
+
const metadata_json = JSON.stringify(metadata);
|
|
452
|
+
const chunk = {
|
|
453
|
+
content_hash: chunkIdentityHash({ content, filePath, fileType, startLine, endLine, metadataJson: metadata_json }),
|
|
454
|
+
content,
|
|
455
|
+
content_tokenized: "",
|
|
456
|
+
file_path: filePath,
|
|
457
|
+
file_type: fileType,
|
|
458
|
+
start_line: startLine,
|
|
459
|
+
end_line: endLine,
|
|
460
|
+
metadata_json,
|
|
461
|
+
};
|
|
462
|
+
return { ...chunk, content_tokenized: preTokenizeForFTS(buildChunkEmbeddingText(chunk)) };
|
|
463
|
+
}
|
|
464
|
+
|
|
218
465
|
export function chunkMarkdown(content: string, filePath: string): Omit<ChunkInsert, "kb_id">[] {
|
|
219
466
|
const lines = content.split("\n");
|
|
220
467
|
const chunks: Omit<ChunkInsert, "kb_id">[] = [];
|
|
221
|
-
|
|
468
|
+
const headingStack: string[] = [];
|
|
222
469
|
let sectionLines: string[] = [];
|
|
223
470
|
let startLine = 1;
|
|
471
|
+
let currentHeading = "";
|
|
472
|
+
|
|
473
|
+
function currentBreadcrumb(): string {
|
|
474
|
+
return headingStack.map(normalizeHeading).filter(Boolean).join(" > ");
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
function pushMarkdownChunk(text: string, start: number, end: number): void {
|
|
478
|
+
if (text.trim().length < 50) return;
|
|
479
|
+
chunks.push(
|
|
480
|
+
makeChunk(text.trim(), filePath, "markdown", start, end, {
|
|
481
|
+
heading: currentHeading,
|
|
482
|
+
breadcrumb: currentBreadcrumb(),
|
|
483
|
+
}),
|
|
484
|
+
);
|
|
485
|
+
}
|
|
224
486
|
|
|
225
487
|
function flush(endLine: number): void {
|
|
226
488
|
const text = sectionLines.join("\n").trim();
|
|
227
489
|
if (text.length < 50) return;
|
|
228
490
|
|
|
229
491
|
const fullText = currentHeading ? `${currentHeading}\n\n${text}` : text;
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
chunks.push({
|
|
241
|
-
content_hash: contentHash(chunkText),
|
|
242
|
-
content: chunkText,
|
|
243
|
-
content_tokenized: preTokenizeForFTS(chunkText),
|
|
244
|
-
file_path: filePath,
|
|
245
|
-
file_type: "markdown",
|
|
246
|
-
start_line: bufStart,
|
|
247
|
-
end_line: endLine,
|
|
248
|
-
metadata_json: JSON.stringify({ heading: currentHeading }),
|
|
249
|
-
});
|
|
250
|
-
buf = [];
|
|
251
|
-
bufStart = endLine;
|
|
252
|
-
}
|
|
492
|
+
const paragraphs = fullText.split(/\n\n+/);
|
|
493
|
+
let buffer: string[] = [];
|
|
494
|
+
let bufferStart = startLine;
|
|
495
|
+
|
|
496
|
+
for (const para of paragraphs) {
|
|
497
|
+
const next = [...buffer, para].join("\n\n");
|
|
498
|
+
if (estimateTokens(next) > MARKDOWN_TARGET_TOKENS && buffer.length > 0) {
|
|
499
|
+
pushMarkdownChunk(buffer.join("\n\n"), bufferStart, endLine);
|
|
500
|
+
buffer = [];
|
|
501
|
+
bufferStart = endLine;
|
|
253
502
|
}
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
content: chunkText,
|
|
260
|
-
content_tokenized: preTokenizeForFTS(chunkText),
|
|
261
|
-
file_path: filePath,
|
|
262
|
-
file_type: "markdown",
|
|
263
|
-
start_line: bufStart,
|
|
264
|
-
end_line: endLine,
|
|
265
|
-
metadata_json: JSON.stringify({ heading: currentHeading }),
|
|
266
|
-
});
|
|
267
|
-
}
|
|
268
|
-
}
|
|
269
|
-
} else {
|
|
270
|
-
chunks.push({
|
|
271
|
-
content_hash: contentHash(fullText),
|
|
272
|
-
content: fullText,
|
|
273
|
-
content_tokenized: preTokenizeForFTS(fullText),
|
|
274
|
-
file_path: filePath,
|
|
275
|
-
file_type: "markdown",
|
|
276
|
-
start_line: startLine,
|
|
277
|
-
end_line: endLine,
|
|
278
|
-
metadata_json: JSON.stringify({ heading: currentHeading }),
|
|
279
|
-
});
|
|
503
|
+
buffer.push(para);
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
if (buffer.length > 0) {
|
|
507
|
+
pushMarkdownChunk(buffer.join("\n\n"), bufferStart, endLine);
|
|
280
508
|
}
|
|
281
509
|
}
|
|
282
510
|
|
|
283
511
|
for (let i = 0; i < lines.length; i++) {
|
|
284
512
|
const line = lines[i];
|
|
285
|
-
const headingMatch = line.match(/^(#{1,
|
|
513
|
+
const headingMatch = line.match(/^(#{1,6})\s+(.+)$/);
|
|
286
514
|
|
|
287
515
|
if (headingMatch && sectionLines.length > 0) {
|
|
288
516
|
flush(i);
|
|
@@ -291,6 +519,9 @@ export function chunkMarkdown(content: string, filePath: string): Omit<ChunkInse
|
|
|
291
519
|
}
|
|
292
520
|
|
|
293
521
|
if (headingMatch) {
|
|
522
|
+
const level = headingMatch[1].length;
|
|
523
|
+
headingStack.splice(level - 1);
|
|
524
|
+
headingStack[level - 1] = line;
|
|
294
525
|
currentHeading = line;
|
|
295
526
|
} else {
|
|
296
527
|
sectionLines.push(line);
|
|
@@ -315,26 +546,37 @@ export function chunkText(content: string, filePath: string): Omit<ChunkInsert,
|
|
|
315
546
|
function flush(): void {
|
|
316
547
|
const text = buffer.join("\n\n").trim();
|
|
317
548
|
if (text.length < 50) return;
|
|
318
|
-
chunks.push(
|
|
319
|
-
content_hash: contentHash(text),
|
|
320
|
-
content: text,
|
|
321
|
-
content_tokenized: preTokenizeForFTS(text),
|
|
322
|
-
file_path: filePath,
|
|
323
|
-
file_type: fileType,
|
|
324
|
-
start_line: lineOffset,
|
|
325
|
-
end_line: lineOffset + text.split("\n").length,
|
|
326
|
-
metadata_json: "{}",
|
|
327
|
-
});
|
|
549
|
+
chunks.push(makeChunk(text, filePath, fileType, lineOffset, lineOffset + text.split("\n").length));
|
|
328
550
|
lineOffset += text.split("\n").length + 1;
|
|
329
551
|
}
|
|
330
552
|
|
|
553
|
+
function pushOversizedParagraph(para: string): void {
|
|
554
|
+
let offset = 0;
|
|
555
|
+
while (offset < para.length) {
|
|
556
|
+
const slice = para.slice(offset, offset + MAX_TEXT_CHUNK_CHARS).trim();
|
|
557
|
+
if (slice.length >= 50) {
|
|
558
|
+
chunks.push(makeChunk(slice, filePath, fileType, lineOffset, lineOffset + slice.split("\n").length));
|
|
559
|
+
}
|
|
560
|
+
lineOffset += slice.split("\n").length;
|
|
561
|
+
offset += MAX_TEXT_CHUNK_CHARS;
|
|
562
|
+
}
|
|
563
|
+
}
|
|
564
|
+
|
|
331
565
|
for (const para of paragraphs) {
|
|
332
566
|
const paraTokens = estimateTokens(para);
|
|
333
|
-
if (
|
|
567
|
+
if (para.length > MAX_TEXT_CHUNK_CHARS) {
|
|
568
|
+
if (buffer.length > 0) {
|
|
569
|
+
flush();
|
|
570
|
+
buffer = [];
|
|
571
|
+
bufferTokens = 0;
|
|
572
|
+
}
|
|
573
|
+
pushOversizedParagraph(para);
|
|
574
|
+
continue;
|
|
575
|
+
}
|
|
576
|
+
if (bufferTokens + paraTokens > TEXT_TARGET_TOKENS && buffer.length > 0) {
|
|
334
577
|
flush();
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
bufferTokens = buffer.length > 0 ? estimateTokens(buffer[0]) : 0;
|
|
578
|
+
buffer = [];
|
|
579
|
+
bufferTokens = 0;
|
|
338
580
|
}
|
|
339
581
|
buffer.push(para);
|
|
340
582
|
bufferTokens += paraTokens;
|
|
@@ -363,18 +605,7 @@ export async function chunkFile(content: string, filePath: string): Promise<Omit
|
|
|
363
605
|
|
|
364
606
|
// Fallback: if file has content but no chunks (too short for splitting), keep as single chunk
|
|
365
607
|
if (chunks.length === 0 && content.trim().length > 10) {
|
|
366
|
-
chunks = [
|
|
367
|
-
{
|
|
368
|
-
content_hash: contentHash(content),
|
|
369
|
-
content: content.trim(),
|
|
370
|
-
content_tokenized: preTokenizeForFTS(content),
|
|
371
|
-
file_path: filePath,
|
|
372
|
-
file_type: fileType,
|
|
373
|
-
start_line: 1,
|
|
374
|
-
end_line: content.split("\n").length,
|
|
375
|
-
metadata_json: "{}",
|
|
376
|
-
},
|
|
377
|
-
];
|
|
608
|
+
chunks = [makeChunk(content.trim(), filePath, fileType, 1, content.split("\n").length)];
|
|
378
609
|
}
|
|
379
610
|
|
|
380
611
|
return chunks;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { ChunkInsert } from "../../storage/sqlite.ts";
|
|
2
|
-
import { contentHash, preTokenizeForFTS } from "../chunker.ts";
|
|
2
|
+
import { buildChunkEmbeddingText, contentHash, preTokenizeForFTS } from "../chunker.ts";
|
|
3
3
|
|
|
4
4
|
interface ASTNode {
|
|
5
5
|
type: string;
|
|
@@ -191,16 +191,19 @@ export async function chunkWithAST(
|
|
|
191
191
|
const tree = parser.parse(content);
|
|
192
192
|
const fns = collectChunks(tree.rootNode as unknown as ASTNode, config);
|
|
193
193
|
if (fns.length === 0) return [];
|
|
194
|
-
return fns.map((fn) =>
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
194
|
+
return fns.map((fn) => {
|
|
195
|
+
const chunk = {
|
|
196
|
+
content_hash: contentHash(fn.text),
|
|
197
|
+
content: fn.text,
|
|
198
|
+
content_tokenized: "",
|
|
199
|
+
file_path: filePath,
|
|
200
|
+
file_type: config.fileType,
|
|
201
|
+
start_line: fn.start,
|
|
202
|
+
end_line: fn.end,
|
|
203
|
+
metadata_json: JSON.stringify({ function_name: fn.name }),
|
|
204
|
+
};
|
|
205
|
+
return { ...chunk, content_tokenized: preTokenizeForFTS(buildChunkEmbeddingText(chunk)) };
|
|
206
|
+
});
|
|
204
207
|
}
|
|
205
208
|
|
|
206
209
|
export const SUPPORTED_LANGUAGES = new Set(Object.keys(LANGS));
|
package/src/search/bm25.ts
CHANGED
|
@@ -1,39 +1,54 @@
|
|
|
1
1
|
import type Database from "better-sqlite3";
|
|
2
|
-
import {
|
|
2
|
+
import { normalizedQueryText } from "./query.ts";
|
|
3
3
|
|
|
4
4
|
export interface BM25Result {
|
|
5
5
|
chunkId: string;
|
|
6
6
|
score: number;
|
|
7
7
|
}
|
|
8
8
|
|
|
9
|
-
function
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
if (terms.length === 0) return "";
|
|
14
|
-
return terms.join(" AND ");
|
|
9
|
+
function prepareFtsTerms(query: string): string[] {
|
|
10
|
+
return normalizedQueryText(query)
|
|
11
|
+
.split(/\s+/)
|
|
12
|
+
.filter((t) => t.length > 0);
|
|
15
13
|
}
|
|
16
14
|
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
.prepare(
|
|
24
|
-
`SELECT c.id as chunkId, -bm25(chunks_fts) as score
|
|
25
|
-
FROM chunks_fts JOIN chunks c ON chunks_fts.rowid = c.rowid
|
|
26
|
-
WHERE chunks_fts MATCH ? AND c.kb_id = ? ORDER BY bm25(chunks_fts) LIMIT ?`,
|
|
27
|
-
)
|
|
28
|
-
.all(ftsQuery, kbId, limit) as BM25Result[];
|
|
29
|
-
}
|
|
15
|
+
function quoteFtsTerm(term: string): string {
|
|
16
|
+
return `"${term.replace(/"/g, '""')}"`;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function runSearch(db: Database.Database, ftsQuery: string, limit: number, kbId?: string): BM25Result[] {
|
|
20
|
+
if (kbId) {
|
|
30
21
|
return db
|
|
31
22
|
.prepare(
|
|
32
23
|
`SELECT c.id as chunkId, -bm25(chunks_fts) as score
|
|
33
24
|
FROM chunks_fts JOIN chunks c ON chunks_fts.rowid = c.rowid
|
|
34
|
-
WHERE chunks_fts MATCH ? ORDER BY bm25(chunks_fts) LIMIT ?`,
|
|
25
|
+
WHERE chunks_fts MATCH ? AND c.kb_id = ? ORDER BY bm25(chunks_fts) LIMIT ?`,
|
|
35
26
|
)
|
|
36
|
-
.all(ftsQuery, limit) as BM25Result[];
|
|
27
|
+
.all(ftsQuery, kbId, limit) as BM25Result[];
|
|
28
|
+
}
|
|
29
|
+
return db
|
|
30
|
+
.prepare(
|
|
31
|
+
`SELECT c.id as chunkId, -bm25(chunks_fts) as score
|
|
32
|
+
FROM chunks_fts JOIN chunks c ON chunks_fts.rowid = c.rowid
|
|
33
|
+
WHERE chunks_fts MATCH ? ORDER BY bm25(chunks_fts) LIMIT ?`,
|
|
34
|
+
)
|
|
35
|
+
.all(ftsQuery, limit) as BM25Result[];
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export function searchBM25(
|
|
39
|
+
db: Database.Database,
|
|
40
|
+
query: string,
|
|
41
|
+
limit = 50,
|
|
42
|
+
kbId?: string,
|
|
43
|
+
options: { allowOrFallback?: boolean } = {},
|
|
44
|
+
): BM25Result[] {
|
|
45
|
+
const terms = prepareFtsTerms(query);
|
|
46
|
+
if (terms.length === 0) return [];
|
|
47
|
+
try {
|
|
48
|
+
const quotedTerms = terms.map(quoteFtsTerm);
|
|
49
|
+
const strict = runSearch(db, quotedTerms.join(" AND "), limit, kbId);
|
|
50
|
+
if (strict.length > 0 || terms.length === 1 || options.allowOrFallback === false) return strict;
|
|
51
|
+
return runSearch(db, quotedTerms.join(" OR "), limit, kbId);
|
|
37
52
|
} catch {
|
|
38
53
|
return [];
|
|
39
54
|
}
|