pi-knowledge 0.3.4 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,5 @@
1
1
  import { createHash } from "node:crypto";
2
- import { type Dirent, existsSync, readdirSync, readFileSync, statSync } from "node:fs";
2
+ import { closeSync, type Dirent, existsSync, openSync, readdirSync, readFileSync, readSync, statSync } from "node:fs";
3
3
  import { extname, join, relative, sep } from "node:path";
4
4
  import ignore from "ignore";
5
5
  import type { ChunkInsert } from "../storage/sqlite.ts";
@@ -54,7 +54,7 @@ const BINARY_EXTENSIONS = new Set([
54
54
  ".lib",
55
55
  ]);
56
56
 
57
- const DEFAULT_IGNORE = [
57
+ const DEFAULT_SUGGESTED_EXCLUDE = [
58
58
  "node_modules",
59
59
  ".git",
60
60
  "dist",
@@ -67,7 +67,13 @@ const DEFAULT_IGNORE = [
67
67
  ".next",
68
68
  ".cache",
69
69
  ".playwright",
70
+ ".browser",
71
+ ".browsers",
70
72
  "__pycache__",
73
+ "*.app",
74
+ "*.asar",
75
+ "*.asar.unpacked",
76
+ "*.pak",
71
77
  ".env",
72
78
  ".env.*",
73
79
  "*.pem",
@@ -80,19 +86,32 @@ const DEFAULT_IGNORE = [
80
86
  "*secrets*",
81
87
  "*credential*",
82
88
  "*credentials*",
83
- "setting*.json",
84
- "appsettings*.json",
89
+ "docs/*eval-report*.md",
90
+ "docs/*evaluation-report*.md",
91
+ "docs/*knowledge-base*report*.md",
92
+ "*knowledge-base-evaluation-report*.md",
93
+ "*knowledge*.jsonl",
85
94
  "*.min.js",
86
95
  "*.min.css",
87
96
  "*.map",
88
97
  "*.lock",
89
98
  "package-lock.json",
99
+ "playwright-report",
100
+ "test-results",
101
+ "browser-cache",
102
+ "ms-playwright",
90
103
  ".DS_Store",
91
104
  "Thumbs.db",
92
105
  "*.pyc",
93
106
  "*.class",
94
107
  ];
95
108
 
109
+ export interface ScanOptions {
110
+ includeSuggestedText?: boolean;
111
+ includePaths?: string[];
112
+ excludePaths?: string[];
113
+ }
114
+
96
115
  export interface ScannedFile {
97
116
  path: string; // absolute path
98
117
  relPath: string; // relative to root
@@ -100,6 +119,59 @@ export interface ScannedFile {
100
119
  fileType: string;
101
120
  }
102
121
 
122
+ export interface ScannableFile {
123
+ path: string; // absolute path
124
+ relPath: string; // relative to root
125
+ fileType: string;
126
+ size: number;
127
+ }
128
+
129
+ export interface SkippedScanEntry {
130
+ path: string;
131
+ reason: "suggested_excluded" | "oversized" | "binary" | "unreadable" | "inaccessible";
132
+ size?: number;
133
+ }
134
+
135
+ export interface ScanResult {
136
+ files: ScannedFile[];
137
+ skipped: {
138
+ total: number;
139
+ by_reason: Record<SkippedScanEntry["reason"], number>;
140
+ samples: SkippedScanEntry[];
141
+ };
142
+ }
143
+
144
+ const MAX_SKIPPED_SAMPLES = 25;
145
+
146
+ export function createSkippedScanStats(): ScanResult["skipped"] {
147
+ return {
148
+ total: 0,
149
+ by_reason: {
150
+ suggested_excluded: 0,
151
+ oversized: 0,
152
+ binary: 0,
153
+ unreadable: 0,
154
+ inaccessible: 0,
155
+ },
156
+ samples: [],
157
+ };
158
+ }
159
+
160
+ function addSkipped(skipped: ScanResult["skipped"], entry: SkippedScanEntry): void {
161
+ skipped.total++;
162
+ skipped.by_reason[entry.reason]++;
163
+ if (skipped.samples.length < MAX_SKIPPED_SAMPLES) skipped.samples.push(entry);
164
+ }
165
+
166
+ export function summarizeSkippedScan(skipped: ScanResult["skipped"]): string {
167
+ return (
168
+ Object.entries(skipped.by_reason)
169
+ .filter(([, count]) => count > 0)
170
+ .map(([reason, count]) => `${reason}: ${count}`)
171
+ .join(", ") || "none"
172
+ );
173
+ }
174
+
103
175
  function detectFileType(filePath: string): string {
104
176
  const ext = extname(filePath).toLowerCase();
105
177
  const map: Record<string, string> = {
@@ -139,59 +211,162 @@ function detectFileType(filePath: string): string {
139
211
 
140
212
  function isBinaryFile(filePath: string): boolean {
141
213
  if (BINARY_EXTENSIONS.has(extname(filePath).toLowerCase())) return true;
214
+ let fd: number | undefined;
142
215
  try {
143
- const fd = readFileSync(filePath, { flag: "r" });
144
- const sample = fd.subarray(0, 512);
145
- return sample.includes(0x00);
216
+ fd = openSync(filePath, "r");
217
+ const sample = Buffer.alloc(512);
218
+ const bytesRead = readSync(fd, sample, 0, sample.length, 0);
219
+ if (bytesRead === 0) return false;
220
+ return sample.subarray(0, bytesRead).includes(0x00);
146
221
  } catch {
147
222
  return true;
223
+ } finally {
224
+ if (fd !== undefined) closeSync(fd);
148
225
  }
149
226
  }
150
227
 
151
- export function walkDir(dirPath: string): ScannedFile[] {
228
+ function buildIgnoreMatcher(dirPath: string): ReturnType<typeof ignore> {
152
229
  const ig = ignore();
153
- ig.add(DEFAULT_IGNORE);
230
+ ig.add(DEFAULT_SUGGESTED_EXCLUDE);
154
231
 
155
232
  const gitignorePath = join(dirPath, ".gitignore");
156
233
  if (existsSync(gitignorePath)) {
157
234
  ig.add(readFileSync(gitignorePath, "utf-8"));
158
235
  }
159
236
 
160
- const results: ScannedFile[] = [];
237
+ return ig;
238
+ }
239
+
240
+ function normalizeScanPath(path: string): string {
241
+ return path.replaceAll("\\", "/").replace(/^\/+/, "").replace(/\/+$/, "");
242
+ }
161
243
 
162
- function walk(dir: string): void {
244
+ function normalizeScanPaths(paths: string[] | undefined): string[] {
245
+ return (paths ?? []).map(normalizeScanPath).filter(Boolean);
246
+ }
247
+
248
+ function pathMatches(relPath: string, patterns: string[]): boolean {
249
+ const normalized = normalizeScanPath(relPath);
250
+ return patterns.some((pattern) => normalized === pattern || normalized.startsWith(`${pattern}/`));
251
+ }
252
+
253
+ function directoryCouldContainIncludedPath(relPath: string, includePaths: string[]): boolean {
254
+ const normalized = normalizeScanPath(relPath);
255
+ return includePaths.some((includePath) => includePath.startsWith(`${normalized}/`));
256
+ }
257
+
258
+ export function* iterateScannableFiles(
259
+ dirPath: string,
260
+ skipped: ScanResult["skipped"] = createSkippedScanStats(),
261
+ options: ScanOptions = {},
262
+ ): Generator<ScannableFile> {
263
+ const ig = buildIgnoreMatcher(dirPath);
264
+ const includePaths = normalizeScanPaths(options.includePaths);
265
+ const excludePaths = normalizeScanPaths(options.excludePaths);
266
+ const includeSuggestedText = options.includeSuggestedText === true;
267
+
268
+ function* walk(dir: string): Generator<ScannableFile> {
163
269
  let entries: Dirent[];
164
270
  try {
165
271
  entries = readdirSync(dir, { withFileTypes: true });
166
272
  } catch {
273
+ addSkipped(skipped, { path: relative(dirPath, dir).split(sep).join("/") || ".", reason: "inaccessible" });
167
274
  return;
168
275
  }
169
276
  for (const entry of entries) {
170
277
  const fullPath = join(dir, entry.name);
171
278
  const relPath = relative(dirPath, fullPath).split(sep).join("/");
279
+ const explicitlyIncluded = pathMatches(relPath, includePaths);
172
280
 
173
- if (ig.ignores(relPath)) continue;
281
+ if (pathMatches(relPath, excludePaths)) {
282
+ addSkipped(skipped, { path: relPath, reason: "suggested_excluded" });
283
+ continue;
284
+ }
285
+
286
+ if (ig.ignores(relPath) && !includeSuggestedText && !explicitlyIncluded) {
287
+ if (entry.isDirectory() && directoryCouldContainIncludedPath(relPath, includePaths)) {
288
+ yield* walk(fullPath);
289
+ continue;
290
+ }
291
+ addSkipped(skipped, { path: relPath, reason: "suggested_excluded" });
292
+ continue;
293
+ }
174
294
 
175
295
  if (entry.isDirectory()) {
176
- if (ig.ignores(`${relPath}/`)) continue;
177
- walk(fullPath);
296
+ if (ig.ignores(`${relPath}/`) && !includeSuggestedText && !explicitlyIncluded) {
297
+ if (directoryCouldContainIncludedPath(relPath, includePaths)) {
298
+ yield* walk(fullPath);
299
+ continue;
300
+ }
301
+ addSkipped(skipped, { path: `${relPath}/`, reason: "suggested_excluded" });
302
+ continue;
303
+ }
304
+ yield* walk(fullPath);
178
305
  } else if (entry.isFile()) {
179
- const stat = statSync(fullPath);
180
- if (stat.size > MAX_FILE_SIZE) continue;
181
- if (isBinaryFile(fullPath)) continue;
182
-
306
+ let size = 0;
183
307
  try {
184
- const content = readFileSync(fullPath, "utf-8");
185
- results.push({ path: fullPath, relPath, content, fileType: detectFileType(fullPath) });
308
+ size = statSync(fullPath).size;
186
309
  } catch {
187
- // skip unreadable files
310
+ addSkipped(skipped, { path: relPath, reason: "unreadable" });
311
+ continue;
312
+ }
313
+ if (size > MAX_FILE_SIZE) {
314
+ addSkipped(skipped, { path: relPath, reason: "oversized", size });
315
+ continue;
316
+ }
317
+ if (isBinaryFile(fullPath)) {
318
+ addSkipped(skipped, { path: relPath, reason: "binary", size });
319
+ continue;
188
320
  }
321
+
322
+ yield { path: fullPath, relPath, fileType: detectFileType(fullPath), size };
189
323
  }
190
324
  }
191
325
  }
192
326
 
193
- walk(dirPath);
194
- return results;
327
+ yield* walk(dirPath);
328
+ }
329
+
330
+ export function* iterateScannedFiles(
331
+ dirPath: string,
332
+ skipped: ScanResult["skipped"] = createSkippedScanStats(),
333
+ options: ScanOptions = {},
334
+ ): Generator<ScannedFile> {
335
+ for (const file of iterateScannableFiles(dirPath, skipped, options)) {
336
+ try {
337
+ const content = readFileSync(file.path, "utf-8");
338
+ yield { ...file, content };
339
+ } catch {
340
+ addSkipped(skipped, { path: file.relPath, reason: "unreadable", size: file.size });
341
+ }
342
+ }
343
+ }
344
+
345
+ export async function iterateScannedFilesAsync(
346
+ dirPath: string,
347
+ onFile: (file: ScannedFile) => Promise<void> | void,
348
+ skipped: ScanResult["skipped"] = createSkippedScanStats(),
349
+ options: ScanOptions = {},
350
+ ): Promise<ScanResult["skipped"]> {
351
+ for (const file of iterateScannedFiles(dirPath, skipped, options)) {
352
+ await onFile(file);
353
+ }
354
+ return skipped;
355
+ }
356
+
357
+ export function walkDir(dirPath: string, options: ScanOptions = {}): ScannedFile[] {
358
+ return walkDirDetailed(dirPath, options).files;
359
+ }
360
+
361
+ export function walkDirDetailed(dirPath: string, options: ScanOptions = {}): ScanResult {
362
+ const results: ScannedFile[] = [];
363
+ const skipped = createSkippedScanStats();
364
+ for (const file of iterateScannedFiles(dirPath, skipped, options)) results.push(file);
365
+ return { files: results, skipped };
366
+ }
367
+
368
+ export function isReadableTextFile(filePath: string): boolean {
369
+ return !isBinaryFile(filePath);
195
370
  }
196
371
 
197
372
  // --- Chunking ---
@@ -200,6 +375,10 @@ function estimateTokens(text: string): number {
200
375
  return Math.ceil(text.length / 3);
201
376
  }
202
377
 
378
+ const MARKDOWN_TARGET_TOKENS = 450;
379
+ const TEXT_TARGET_TOKENS = 550;
380
+ const MAX_TEXT_CHUNK_CHARS = 6_000;
381
+
203
382
  export function preTokenizeForFTS(content: string): string {
204
383
  return content
205
384
  .replace(/([a-z])([A-Z])/g, "$1 $2")
@@ -215,74 +394,123 @@ export function contentHash(content: string): string {
215
394
  return createHash("sha256").update(content).digest("hex");
216
395
  }
217
396
 
397
+ export function chunkIdentityHash(opts: {
398
+ content: string;
399
+ filePath: string;
400
+ fileType: string;
401
+ startLine: number;
402
+ endLine: number;
403
+ metadataJson: string;
404
+ }): string {
405
+ return contentHash(
406
+ [opts.filePath, opts.fileType, String(opts.startLine), String(opts.endLine), opts.metadataJson, opts.content].join(
407
+ "\0",
408
+ ),
409
+ );
410
+ }
411
+
412
+ function normalizeHeading(heading: string): string {
413
+ return heading.replace(/^#{1,6}\s+/, "").trim();
414
+ }
415
+
416
+ function buildContextPrefix(
417
+ filePath: string,
418
+ fileType: string,
419
+ metadata: Record<string, string | number | null | undefined>,
420
+ ): string {
421
+ const parts = [`File: ${filePath}`, `Type: ${fileType}`];
422
+ const heading = typeof metadata.heading === "string" ? metadata.heading : "";
423
+ const breadcrumb = typeof metadata.breadcrumb === "string" ? metadata.breadcrumb : "";
424
+ const symbol = typeof metadata.function_name === "string" ? metadata.function_name : "";
425
+ if (breadcrumb) parts.push(`Section: ${breadcrumb}`);
426
+ else if (heading) parts.push(`Section: ${normalizeHeading(heading)}`);
427
+ if (symbol) parts.push(`Symbol: ${symbol}`);
428
+ return parts.join("\n");
429
+ }
430
+
431
+ export function buildChunkEmbeddingText(
432
+ chunk: Pick<ChunkInsert, "content" | "file_path" | "file_type" | "metadata_json">,
433
+ ): string {
434
+ let metadata: Record<string, string | number | null | undefined> = {};
435
+ try {
436
+ metadata = JSON.parse(chunk.metadata_json) as Record<string, string | number | null | undefined>;
437
+ } catch {
438
+ metadata = {};
439
+ }
440
+ return `${buildContextPrefix(chunk.file_path, chunk.file_type, metadata)}\n\n${chunk.content}`;
441
+ }
442
+
443
+ function makeChunk(
444
+ content: string,
445
+ filePath: string,
446
+ fileType: string,
447
+ startLine: number,
448
+ endLine: number,
449
+ metadata: Record<string, string | number | null | undefined> = {},
450
+ ): Omit<ChunkInsert, "kb_id"> {
451
+ const metadata_json = JSON.stringify(metadata);
452
+ const chunk = {
453
+ content_hash: chunkIdentityHash({ content, filePath, fileType, startLine, endLine, metadataJson: metadata_json }),
454
+ content,
455
+ content_tokenized: "",
456
+ file_path: filePath,
457
+ file_type: fileType,
458
+ start_line: startLine,
459
+ end_line: endLine,
460
+ metadata_json,
461
+ };
462
+ return { ...chunk, content_tokenized: preTokenizeForFTS(buildChunkEmbeddingText(chunk)) };
463
+ }
464
+
218
465
  export function chunkMarkdown(content: string, filePath: string): Omit<ChunkInsert, "kb_id">[] {
219
466
  const lines = content.split("\n");
220
467
  const chunks: Omit<ChunkInsert, "kb_id">[] = [];
221
- let currentHeading = "";
468
+ const headingStack: string[] = [];
222
469
  let sectionLines: string[] = [];
223
470
  let startLine = 1;
471
+ let currentHeading = "";
472
+
473
+ function currentBreadcrumb(): string {
474
+ return headingStack.map(normalizeHeading).filter(Boolean).join(" > ");
475
+ }
476
+
477
+ function pushMarkdownChunk(text: string, start: number, end: number): void {
478
+ if (text.trim().length < 50) return;
479
+ chunks.push(
480
+ makeChunk(text.trim(), filePath, "markdown", start, end, {
481
+ heading: currentHeading,
482
+ breadcrumb: currentBreadcrumb(),
483
+ }),
484
+ );
485
+ }
224
486
 
225
487
  function flush(endLine: number): void {
226
488
  const text = sectionLines.join("\n").trim();
227
489
  if (text.length < 50) return;
228
490
 
229
491
  const fullText = currentHeading ? `${currentHeading}\n\n${text}` : text;
230
-
231
- if (estimateTokens(fullText) > 2000) {
232
- // Split large sections by paragraph
233
- const paragraphs = fullText.split(/\n\n+/);
234
- let buf: string[] = [];
235
- let bufStart = startLine;
236
- for (const para of paragraphs) {
237
- buf.push(para);
238
- if (estimateTokens(buf.join("\n\n")) > 1000) {
239
- const chunkText = buf.join("\n\n");
240
- chunks.push({
241
- content_hash: contentHash(chunkText),
242
- content: chunkText,
243
- content_tokenized: preTokenizeForFTS(chunkText),
244
- file_path: filePath,
245
- file_type: "markdown",
246
- start_line: bufStart,
247
- end_line: endLine,
248
- metadata_json: JSON.stringify({ heading: currentHeading }),
249
- });
250
- buf = [];
251
- bufStart = endLine;
252
- }
492
+ const paragraphs = fullText.split(/\n\n+/);
493
+ let buffer: string[] = [];
494
+ let bufferStart = startLine;
495
+
496
+ for (const para of paragraphs) {
497
+ const next = [...buffer, para].join("\n\n");
498
+ if (estimateTokens(next) > MARKDOWN_TARGET_TOKENS && buffer.length > 0) {
499
+ pushMarkdownChunk(buffer.join("\n\n"), bufferStart, endLine);
500
+ buffer = [];
501
+ bufferStart = endLine;
253
502
  }
254
- if (buf.length > 0) {
255
- const chunkText = buf.join("\n\n");
256
- if (chunkText.length >= 50) {
257
- chunks.push({
258
- content_hash: contentHash(chunkText),
259
- content: chunkText,
260
- content_tokenized: preTokenizeForFTS(chunkText),
261
- file_path: filePath,
262
- file_type: "markdown",
263
- start_line: bufStart,
264
- end_line: endLine,
265
- metadata_json: JSON.stringify({ heading: currentHeading }),
266
- });
267
- }
268
- }
269
- } else {
270
- chunks.push({
271
- content_hash: contentHash(fullText),
272
- content: fullText,
273
- content_tokenized: preTokenizeForFTS(fullText),
274
- file_path: filePath,
275
- file_type: "markdown",
276
- start_line: startLine,
277
- end_line: endLine,
278
- metadata_json: JSON.stringify({ heading: currentHeading }),
279
- });
503
+ buffer.push(para);
504
+ }
505
+
506
+ if (buffer.length > 0) {
507
+ pushMarkdownChunk(buffer.join("\n\n"), bufferStart, endLine);
280
508
  }
281
509
  }
282
510
 
283
511
  for (let i = 0; i < lines.length; i++) {
284
512
  const line = lines[i];
285
- const headingMatch = line.match(/^(#{1,4})\s+(.+)$/);
513
+ const headingMatch = line.match(/^(#{1,6})\s+(.+)$/);
286
514
 
287
515
  if (headingMatch && sectionLines.length > 0) {
288
516
  flush(i);
@@ -291,6 +519,9 @@ export function chunkMarkdown(content: string, filePath: string): Omit<ChunkInse
291
519
  }
292
520
 
293
521
  if (headingMatch) {
522
+ const level = headingMatch[1].length;
523
+ headingStack.splice(level - 1);
524
+ headingStack[level - 1] = line;
294
525
  currentHeading = line;
295
526
  } else {
296
527
  sectionLines.push(line);
@@ -315,26 +546,37 @@ export function chunkText(content: string, filePath: string): Omit<ChunkInsert,
315
546
  function flush(): void {
316
547
  const text = buffer.join("\n\n").trim();
317
548
  if (text.length < 50) return;
318
- chunks.push({
319
- content_hash: contentHash(text),
320
- content: text,
321
- content_tokenized: preTokenizeForFTS(text),
322
- file_path: filePath,
323
- file_type: fileType,
324
- start_line: lineOffset,
325
- end_line: lineOffset + text.split("\n").length,
326
- metadata_json: "{}",
327
- });
549
+ chunks.push(makeChunk(text, filePath, fileType, lineOffset, lineOffset + text.split("\n").length));
328
550
  lineOffset += text.split("\n").length + 1;
329
551
  }
330
552
 
553
+ function pushOversizedParagraph(para: string): void {
554
+ let offset = 0;
555
+ while (offset < para.length) {
556
+ const slice = para.slice(offset, offset + MAX_TEXT_CHUNK_CHARS).trim();
557
+ if (slice.length >= 50) {
558
+ chunks.push(makeChunk(slice, filePath, fileType, lineOffset, lineOffset + slice.split("\n").length));
559
+ }
560
+ lineOffset += slice.split("\n").length;
561
+ offset += MAX_TEXT_CHUNK_CHARS;
562
+ }
563
+ }
564
+
331
565
  for (const para of paragraphs) {
332
566
  const paraTokens = estimateTokens(para);
333
- if (bufferTokens + paraTokens > 1000 && buffer.length > 0) {
567
+ if (para.length > MAX_TEXT_CHUNK_CHARS) {
568
+ if (buffer.length > 0) {
569
+ flush();
570
+ buffer = [];
571
+ bufferTokens = 0;
572
+ }
573
+ pushOversizedParagraph(para);
574
+ continue;
575
+ }
576
+ if (bufferTokens + paraTokens > TEXT_TARGET_TOKENS && buffer.length > 0) {
334
577
  flush();
335
- // overlap: keep last paragraph
336
- buffer = buffer.length > 0 ? [buffer[buffer.length - 1]] : [];
337
- bufferTokens = buffer.length > 0 ? estimateTokens(buffer[0]) : 0;
578
+ buffer = [];
579
+ bufferTokens = 0;
338
580
  }
339
581
  buffer.push(para);
340
582
  bufferTokens += paraTokens;
@@ -363,18 +605,7 @@ export async function chunkFile(content: string, filePath: string): Promise<Omit
363
605
 
364
606
  // Fallback: if file has content but no chunks (too short for splitting), keep as single chunk
365
607
  if (chunks.length === 0 && content.trim().length > 10) {
366
- chunks = [
367
- {
368
- content_hash: contentHash(content),
369
- content: content.trim(),
370
- content_tokenized: preTokenizeForFTS(content),
371
- file_path: filePath,
372
- file_type: fileType,
373
- start_line: 1,
374
- end_line: content.split("\n").length,
375
- metadata_json: "{}",
376
- },
377
- ];
608
+ chunks = [makeChunk(content.trim(), filePath, fileType, 1, content.split("\n").length)];
378
609
  }
379
610
 
380
611
  return chunks;
@@ -1,5 +1,5 @@
1
1
  import type { ChunkInsert } from "../../storage/sqlite.ts";
2
- import { contentHash, preTokenizeForFTS } from "../chunker.ts";
2
+ import { buildChunkEmbeddingText, contentHash, preTokenizeForFTS } from "../chunker.ts";
3
3
 
4
4
  interface ASTNode {
5
5
  type: string;
@@ -191,16 +191,19 @@ export async function chunkWithAST(
191
191
  const tree = parser.parse(content);
192
192
  const fns = collectChunks(tree.rootNode as unknown as ASTNode, config);
193
193
  if (fns.length === 0) return [];
194
- return fns.map((fn) => ({
195
- content_hash: contentHash(fn.text),
196
- content: fn.text,
197
- content_tokenized: preTokenizeForFTS(fn.text),
198
- file_path: filePath,
199
- file_type: config.fileType,
200
- start_line: fn.start,
201
- end_line: fn.end,
202
- metadata_json: JSON.stringify({ function_name: fn.name }),
203
- }));
194
+ return fns.map((fn) => {
195
+ const chunk = {
196
+ content_hash: contentHash(fn.text),
197
+ content: fn.text,
198
+ content_tokenized: "",
199
+ file_path: filePath,
200
+ file_type: config.fileType,
201
+ start_line: fn.start,
202
+ end_line: fn.end,
203
+ metadata_json: JSON.stringify({ function_name: fn.name }),
204
+ };
205
+ return { ...chunk, content_tokenized: preTokenizeForFTS(buildChunkEmbeddingText(chunk)) };
206
+ });
204
207
  }
205
208
 
206
209
  export const SUPPORTED_LANGUAGES = new Set(Object.keys(LANGS));
@@ -1,39 +1,54 @@
1
1
  import type Database from "better-sqlite3";
2
- import { preTokenizeForFTS } from "../indexer/chunker.ts";
2
+ import { normalizedQueryText } from "./query.ts";
3
3
 
4
4
  export interface BM25Result {
5
5
  chunkId: string;
6
6
  score: number;
7
7
  }
8
8
 
9
- function prepareFtsQuery(query: string): string {
10
- let q = preTokenizeForFTS(query);
11
- q = q.replace(/[*"(){}[\]^~:+.#@!\\/<>|&$%]/g, " ");
12
- const terms = q.split(/\s+/).filter((t) => t.length > 0);
13
- if (terms.length === 0) return "";
14
- return terms.join(" AND ");
9
+ function prepareFtsTerms(query: string): string[] {
10
+ return normalizedQueryText(query)
11
+ .split(/\s+/)
12
+ .filter((t) => t.length > 0);
15
13
  }
16
14
 
17
- export function searchBM25(db: Database.Database, query: string, limit = 50, kbId?: string): BM25Result[] {
18
- const ftsQuery = prepareFtsQuery(query);
19
- if (!ftsQuery) return [];
20
- try {
21
- if (kbId) {
22
- return db
23
- .prepare(
24
- `SELECT c.id as chunkId, -bm25(chunks_fts) as score
25
- FROM chunks_fts JOIN chunks c ON chunks_fts.rowid = c.rowid
26
- WHERE chunks_fts MATCH ? AND c.kb_id = ? ORDER BY bm25(chunks_fts) LIMIT ?`,
27
- )
28
- .all(ftsQuery, kbId, limit) as BM25Result[];
29
- }
15
+ function quoteFtsTerm(term: string): string {
16
+ return `"${term.replace(/"/g, '""')}"`;
17
+ }
18
+
19
+ function runSearch(db: Database.Database, ftsQuery: string, limit: number, kbId?: string): BM25Result[] {
20
+ if (kbId) {
30
21
  return db
31
22
  .prepare(
32
23
  `SELECT c.id as chunkId, -bm25(chunks_fts) as score
33
24
  FROM chunks_fts JOIN chunks c ON chunks_fts.rowid = c.rowid
34
- WHERE chunks_fts MATCH ? ORDER BY bm25(chunks_fts) LIMIT ?`,
25
+ WHERE chunks_fts MATCH ? AND c.kb_id = ? ORDER BY bm25(chunks_fts) LIMIT ?`,
35
26
  )
36
- .all(ftsQuery, limit) as BM25Result[];
27
+ .all(ftsQuery, kbId, limit) as BM25Result[];
28
+ }
29
+ return db
30
+ .prepare(
31
+ `SELECT c.id as chunkId, -bm25(chunks_fts) as score
32
+ FROM chunks_fts JOIN chunks c ON chunks_fts.rowid = c.rowid
33
+ WHERE chunks_fts MATCH ? ORDER BY bm25(chunks_fts) LIMIT ?`,
34
+ )
35
+ .all(ftsQuery, limit) as BM25Result[];
36
+ }
37
+
38
+ export function searchBM25(
39
+ db: Database.Database,
40
+ query: string,
41
+ limit = 50,
42
+ kbId?: string,
43
+ options: { allowOrFallback?: boolean } = {},
44
+ ): BM25Result[] {
45
+ const terms = prepareFtsTerms(query);
46
+ if (terms.length === 0) return [];
47
+ try {
48
+ const quotedTerms = terms.map(quoteFtsTerm);
49
+ const strict = runSearch(db, quotedTerms.join(" AND "), limit, kbId);
50
+ if (strict.length > 0 || terms.length === 1 || options.allowOrFallback === false) return strict;
51
+ return runSearch(db, quotedTerms.join(" OR "), limit, kbId);
37
52
  } catch {
38
53
  return [];
39
54
  }