pi-knowledge 0.3.3 → 0.3.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -59,8 +59,33 @@ const DEFAULT_IGNORE = [
59
59
  ".git",
60
60
  "dist",
61
61
  "build",
62
+ "bin",
63
+ "obj",
64
+ "out",
65
+ "target",
66
+ "coverage",
62
67
  ".next",
68
+ ".cache",
69
+ ".playwright",
63
70
  "__pycache__",
71
+ ".env",
72
+ ".env.*",
73
+ "*.pem",
74
+ "*.key",
75
+ "*.p12",
76
+ "*.pfx",
77
+ "*.crt",
78
+ "*.cert",
79
+ "*secret*",
80
+ "*secrets*",
81
+ "*credential*",
82
+ "*credentials*",
83
+ "docs/*eval-report*.md",
84
+ "docs/*evaluation-report*.md",
85
+ "docs/*knowledge-base*report*.md",
86
+ "*knowledge-base-evaluation-report*.md",
87
+ "setting*.json",
88
+ "appsettings*.json",
64
89
  "*.min.js",
65
90
  "*.min.css",
66
91
  "*.map",
@@ -179,6 +204,9 @@ function estimateTokens(text: string): number {
179
204
  return Math.ceil(text.length / 3);
180
205
  }
181
206
 
207
+ const MARKDOWN_TARGET_TOKENS = 450;
208
+ const TEXT_TARGET_TOKENS = 550;
209
+
182
210
  export function preTokenizeForFTS(content: string): string {
183
211
  return content
184
212
  .replace(/([a-z])([A-Z])/g, "$1 $2")
@@ -194,74 +222,108 @@ export function contentHash(content: string): string {
194
222
  return createHash("sha256").update(content).digest("hex");
195
223
  }
196
224
 
225
+ function normalizeHeading(heading: string): string {
226
+ return heading.replace(/^#{1,6}\s+/, "").trim();
227
+ }
228
+
229
+ function buildContextPrefix(
230
+ filePath: string,
231
+ fileType: string,
232
+ metadata: Record<string, string | number | null | undefined>,
233
+ ): string {
234
+ const parts = [`File: ${filePath}`, `Type: ${fileType}`];
235
+ const heading = typeof metadata.heading === "string" ? metadata.heading : "";
236
+ const breadcrumb = typeof metadata.breadcrumb === "string" ? metadata.breadcrumb : "";
237
+ const symbol = typeof metadata.function_name === "string" ? metadata.function_name : "";
238
+ if (breadcrumb) parts.push(`Section: ${breadcrumb}`);
239
+ else if (heading) parts.push(`Section: ${normalizeHeading(heading)}`);
240
+ if (symbol) parts.push(`Symbol: ${symbol}`);
241
+ return parts.join("\n");
242
+ }
243
+
244
+ export function buildChunkEmbeddingText(
245
+ chunk: Pick<ChunkInsert, "content" | "file_path" | "file_type" | "metadata_json">,
246
+ ): string {
247
+ let metadata: Record<string, string | number | null | undefined> = {};
248
+ try {
249
+ metadata = JSON.parse(chunk.metadata_json) as Record<string, string | number | null | undefined>;
250
+ } catch {
251
+ metadata = {};
252
+ }
253
+ return `${buildContextPrefix(chunk.file_path, chunk.file_type, metadata)}\n\n${chunk.content}`;
254
+ }
255
+
256
+ function makeChunk(
257
+ content: string,
258
+ filePath: string,
259
+ fileType: string,
260
+ startLine: number,
261
+ endLine: number,
262
+ metadata: Record<string, string | number | null | undefined> = {},
263
+ ): Omit<ChunkInsert, "kb_id"> {
264
+ const metadata_json = JSON.stringify(metadata);
265
+ const chunk = {
266
+ content_hash: contentHash(content),
267
+ content,
268
+ content_tokenized: "",
269
+ file_path: filePath,
270
+ file_type: fileType,
271
+ start_line: startLine,
272
+ end_line: endLine,
273
+ metadata_json,
274
+ };
275
+ return { ...chunk, content_tokenized: preTokenizeForFTS(buildChunkEmbeddingText(chunk)) };
276
+ }
277
+
197
278
  export function chunkMarkdown(content: string, filePath: string): Omit<ChunkInsert, "kb_id">[] {
198
279
  const lines = content.split("\n");
199
280
  const chunks: Omit<ChunkInsert, "kb_id">[] = [];
200
- let currentHeading = "";
281
+ const headingStack: string[] = [];
201
282
  let sectionLines: string[] = [];
202
283
  let startLine = 1;
284
+ let currentHeading = "";
285
+
286
+ function currentBreadcrumb(): string {
287
+ return headingStack.map(normalizeHeading).filter(Boolean).join(" > ");
288
+ }
289
+
290
+ function pushMarkdownChunk(text: string, start: number, end: number): void {
291
+ if (text.trim().length < 50) return;
292
+ chunks.push(
293
+ makeChunk(text.trim(), filePath, "markdown", start, end, {
294
+ heading: currentHeading,
295
+ breadcrumb: currentBreadcrumb(),
296
+ }),
297
+ );
298
+ }
203
299
 
204
300
  function flush(endLine: number): void {
205
301
  const text = sectionLines.join("\n").trim();
206
302
  if (text.length < 50) return;
207
303
 
208
304
  const fullText = currentHeading ? `${currentHeading}\n\n${text}` : text;
209
-
210
- if (estimateTokens(fullText) > 2000) {
211
- // Split large sections by paragraph
212
- const paragraphs = fullText.split(/\n\n+/);
213
- let buf: string[] = [];
214
- let bufStart = startLine;
215
- for (const para of paragraphs) {
216
- buf.push(para);
217
- if (estimateTokens(buf.join("\n\n")) > 1000) {
218
- const chunkText = buf.join("\n\n");
219
- chunks.push({
220
- content_hash: contentHash(chunkText),
221
- content: chunkText,
222
- content_tokenized: preTokenizeForFTS(chunkText),
223
- file_path: filePath,
224
- file_type: "markdown",
225
- start_line: bufStart,
226
- end_line: endLine,
227
- metadata_json: JSON.stringify({ heading: currentHeading }),
228
- });
229
- buf = [];
230
- bufStart = endLine;
231
- }
232
- }
233
- if (buf.length > 0) {
234
- const chunkText = buf.join("\n\n");
235
- if (chunkText.length >= 50) {
236
- chunks.push({
237
- content_hash: contentHash(chunkText),
238
- content: chunkText,
239
- content_tokenized: preTokenizeForFTS(chunkText),
240
- file_path: filePath,
241
- file_type: "markdown",
242
- start_line: bufStart,
243
- end_line: endLine,
244
- metadata_json: JSON.stringify({ heading: currentHeading }),
245
- });
246
- }
305
+ const paragraphs = fullText.split(/\n\n+/);
306
+ let buffer: string[] = [];
307
+ let bufferStart = startLine;
308
+
309
+ for (const para of paragraphs) {
310
+ const next = [...buffer, para].join("\n\n");
311
+ if (estimateTokens(next) > MARKDOWN_TARGET_TOKENS && buffer.length > 0) {
312
+ pushMarkdownChunk(buffer.join("\n\n"), bufferStart, endLine);
313
+ buffer = [];
314
+ bufferStart = endLine;
247
315
  }
248
- } else {
249
- chunks.push({
250
- content_hash: contentHash(fullText),
251
- content: fullText,
252
- content_tokenized: preTokenizeForFTS(fullText),
253
- file_path: filePath,
254
- file_type: "markdown",
255
- start_line: startLine,
256
- end_line: endLine,
257
- metadata_json: JSON.stringify({ heading: currentHeading }),
258
- });
316
+ buffer.push(para);
317
+ }
318
+
319
+ if (buffer.length > 0) {
320
+ pushMarkdownChunk(buffer.join("\n\n"), bufferStart, endLine);
259
321
  }
260
322
  }
261
323
 
262
324
  for (let i = 0; i < lines.length; i++) {
263
325
  const line = lines[i];
264
- const headingMatch = line.match(/^(#{1,4})\s+(.+)$/);
326
+ const headingMatch = line.match(/^(#{1,6})\s+(.+)$/);
265
327
 
266
328
  if (headingMatch && sectionLines.length > 0) {
267
329
  flush(i);
@@ -270,6 +332,9 @@ export function chunkMarkdown(content: string, filePath: string): Omit<ChunkInse
270
332
  }
271
333
 
272
334
  if (headingMatch) {
335
+ const level = headingMatch[1].length;
336
+ headingStack.splice(level - 1);
337
+ headingStack[level - 1] = line;
273
338
  currentHeading = line;
274
339
  } else {
275
340
  sectionLines.push(line);
@@ -294,26 +359,16 @@ export function chunkText(content: string, filePath: string): Omit<ChunkInsert,
294
359
  function flush(): void {
295
360
  const text = buffer.join("\n\n").trim();
296
361
  if (text.length < 50) return;
297
- chunks.push({
298
- content_hash: contentHash(text),
299
- content: text,
300
- content_tokenized: preTokenizeForFTS(text),
301
- file_path: filePath,
302
- file_type: fileType,
303
- start_line: lineOffset,
304
- end_line: lineOffset + text.split("\n").length,
305
- metadata_json: "{}",
306
- });
362
+ chunks.push(makeChunk(text, filePath, fileType, lineOffset, lineOffset + text.split("\n").length));
307
363
  lineOffset += text.split("\n").length + 1;
308
364
  }
309
365
 
310
366
  for (const para of paragraphs) {
311
367
  const paraTokens = estimateTokens(para);
312
- if (bufferTokens + paraTokens > 1000 && buffer.length > 0) {
368
+ if (bufferTokens + paraTokens > TEXT_TARGET_TOKENS && buffer.length > 0) {
313
369
  flush();
314
- // overlap: keep last paragraph
315
- buffer = buffer.length > 0 ? [buffer[buffer.length - 1]] : [];
316
- bufferTokens = buffer.length > 0 ? estimateTokens(buffer[0]) : 0;
370
+ buffer = [];
371
+ bufferTokens = 0;
317
372
  }
318
373
  buffer.push(para);
319
374
  bufferTokens += paraTokens;
@@ -342,18 +397,7 @@ export async function chunkFile(content: string, filePath: string): Promise<Omit
342
397
 
343
398
  // Fallback: if file has content but no chunks (too short for splitting), keep as single chunk
344
399
  if (chunks.length === 0 && content.trim().length > 10) {
345
- chunks = [
346
- {
347
- content_hash: contentHash(content),
348
- content: content.trim(),
349
- content_tokenized: preTokenizeForFTS(content),
350
- file_path: filePath,
351
- file_type: fileType,
352
- start_line: 1,
353
- end_line: content.split("\n").length,
354
- metadata_json: "{}",
355
- },
356
- ];
400
+ chunks = [makeChunk(content.trim(), filePath, fileType, 1, content.split("\n").length)];
357
401
  }
358
402
 
359
403
  return chunks;
@@ -1,5 +1,5 @@
1
1
  import type { ChunkInsert } from "../../storage/sqlite.ts";
2
- import { contentHash, preTokenizeForFTS } from "../chunker.ts";
2
+ import { buildChunkEmbeddingText, contentHash, preTokenizeForFTS } from "../chunker.ts";
3
3
 
4
4
  interface ASTNode {
5
5
  type: string;
@@ -191,16 +191,19 @@ export async function chunkWithAST(
191
191
  const tree = parser.parse(content);
192
192
  const fns = collectChunks(tree.rootNode as unknown as ASTNode, config);
193
193
  if (fns.length === 0) return [];
194
- return fns.map((fn) => ({
195
- content_hash: contentHash(fn.text),
196
- content: fn.text,
197
- content_tokenized: preTokenizeForFTS(fn.text),
198
- file_path: filePath,
199
- file_type: config.fileType,
200
- start_line: fn.start,
201
- end_line: fn.end,
202
- metadata_json: JSON.stringify({ function_name: fn.name }),
203
- }));
194
+ return fns.map((fn) => {
195
+ const chunk = {
196
+ content_hash: contentHash(fn.text),
197
+ content: fn.text,
198
+ content_tokenized: "",
199
+ file_path: filePath,
200
+ file_type: config.fileType,
201
+ start_line: fn.start,
202
+ end_line: fn.end,
203
+ metadata_json: JSON.stringify({ function_name: fn.name }),
204
+ };
205
+ return { ...chunk, content_tokenized: preTokenizeForFTS(buildChunkEmbeddingText(chunk)) };
206
+ });
204
207
  }
205
208
 
206
209
  export const SUPPORTED_LANGUAGES = new Set(Object.keys(LANGS));
@@ -1,39 +1,43 @@
1
1
  import type Database from "better-sqlite3";
2
- import { preTokenizeForFTS } from "../indexer/chunker.ts";
2
+ import { normalizedQueryText } from "./query.ts";
3
3
 
4
4
  export interface BM25Result {
5
5
  chunkId: string;
6
6
  score: number;
7
7
  }
8
8
 
9
- function prepareFtsQuery(query: string): string {
10
- let q = preTokenizeForFTS(query);
11
- q = q.replace(/[*"(){}[\]^~:+.#@!\\/<>|&$%]/g, " ");
12
- const terms = q.split(/\s+/).filter((t) => t.length > 0);
13
- if (terms.length === 0) return "";
14
- return terms.join(" AND ");
9
+ function prepareFtsTerms(query: string): string[] {
10
+ return normalizedQueryText(query)
11
+ .split(/\s+/)
12
+ .filter((t) => t.length > 0);
15
13
  }
16
14
 
17
- export function searchBM25(db: Database.Database, query: string, limit = 50, kbId?: string): BM25Result[] {
18
- const ftsQuery = prepareFtsQuery(query);
19
- if (!ftsQuery) return [];
20
- try {
21
- if (kbId) {
22
- return db
23
- .prepare(
24
- `SELECT c.id as chunkId, -bm25(chunks_fts) as score
25
- FROM chunks_fts JOIN chunks c ON chunks_fts.rowid = c.rowid
26
- WHERE chunks_fts MATCH ? AND c.kb_id = ? ORDER BY bm25(chunks_fts) LIMIT ?`,
27
- )
28
- .all(ftsQuery, kbId, limit) as BM25Result[];
29
- }
15
+ function runSearch(db: Database.Database, ftsQuery: string, limit: number, kbId?: string): BM25Result[] {
16
+ if (kbId) {
30
17
  return db
31
18
  .prepare(
32
19
  `SELECT c.id as chunkId, -bm25(chunks_fts) as score
33
20
  FROM chunks_fts JOIN chunks c ON chunks_fts.rowid = c.rowid
34
- WHERE chunks_fts MATCH ? ORDER BY bm25(chunks_fts) LIMIT ?`,
21
+ WHERE chunks_fts MATCH ? AND c.kb_id = ? ORDER BY bm25(chunks_fts) LIMIT ?`,
35
22
  )
36
- .all(ftsQuery, limit) as BM25Result[];
23
+ .all(ftsQuery, kbId, limit) as BM25Result[];
24
+ }
25
+ return db
26
+ .prepare(
27
+ `SELECT c.id as chunkId, -bm25(chunks_fts) as score
28
+ FROM chunks_fts JOIN chunks c ON chunks_fts.rowid = c.rowid
29
+ WHERE chunks_fts MATCH ? ORDER BY bm25(chunks_fts) LIMIT ?`,
30
+ )
31
+ .all(ftsQuery, limit) as BM25Result[];
32
+ }
33
+
34
+ export function searchBM25(db: Database.Database, query: string, limit = 50, kbId?: string): BM25Result[] {
35
+ const terms = prepareFtsTerms(query);
36
+ if (terms.length === 0) return [];
37
+ try {
38
+ const strict = runSearch(db, terms.join(" AND "), limit, kbId);
39
+ if (strict.length > 0 || terms.length === 1) return strict;
40
+ return runSearch(db, terms.join(" OR "), limit, kbId);
37
41
  } catch {
38
42
  return [];
39
43
  }
@@ -3,6 +3,19 @@ export interface FusionResult {
3
3
  score: number;
4
4
  }
5
5
 
6
+ function normalizeScores(list: Array<{ chunkId: string; score: number }>): Map<string, number> {
7
+ if (list.length === 0) return new Map();
8
+ const scores = list.map((item) => item.score);
9
+ const min = Math.min(...scores);
10
+ const max = Math.max(...scores);
11
+ return new Map(
12
+ list.map((item) => {
13
+ const normalized = max === min ? 1 : (item.score - min) / (max - min);
14
+ return [item.chunkId, normalized];
15
+ }),
16
+ );
17
+ }
18
+
6
19
  export function reciprocalRankFusion(lists: Array<{ chunkId: string; score: number }[]>, k = 60): FusionResult[] {
7
20
  const scores = new Map<string, number>();
8
21
  for (const list of lists) {
@@ -13,3 +26,23 @@ export function reciprocalRankFusion(lists: Array<{ chunkId: string; score: numb
13
26
  }
14
27
  return [...scores.entries()].map(([chunkId, score]) => ({ chunkId, score })).sort((a, b) => b.score - a.score);
15
28
  }
29
+
30
+ export function weightedScoreFusion(
31
+ bm25Results: Array<{ chunkId: string; score: number }>,
32
+ vectorResults: Array<{ chunkId: string; score: number }>,
33
+ weights = { bm25: 0.45, vector: 0.55, overlap: 0.15 },
34
+ ): FusionResult[] {
35
+ const bm25Scores = normalizeScores(bm25Results);
36
+ const vectorScores = normalizeScores(vectorResults);
37
+ const ids = new Set([...bm25Scores.keys(), ...vectorScores.keys()]);
38
+ const fused: FusionResult[] = [];
39
+
40
+ for (const chunkId of ids) {
41
+ const bm25 = bm25Scores.get(chunkId) ?? 0;
42
+ const vector = vectorScores.get(chunkId) ?? 0;
43
+ const overlap = bm25 > 0 && vector > 0 ? weights.overlap : 0;
44
+ fused.push({ chunkId, score: weights.bm25 * bm25 + weights.vector * vector + overlap });
45
+ }
46
+
47
+ return fused.sort((a, b) => b.score - a.score);
48
+ }
@@ -0,0 +1,59 @@
1
+ import { preTokenizeForFTS } from "../indexer/chunker.ts";
2
+
3
+ export const STOP_WORDS = new Set([
4
+ "the",
5
+ "and",
6
+ "or",
7
+ "to",
8
+ "of",
9
+ "in",
10
+ "on",
11
+ "for",
12
+ "with",
13
+ "how",
14
+ "does",
15
+ "what",
16
+ "when",
17
+ "where",
18
+ "why",
19
+ "this",
20
+ "that",
21
+ "from",
22
+ ]);
23
+
24
+ const TYPO_CORRECTIONS: Record<string, string> = {
25
+ ho: "how",
26
+ setpup: "setup",
27
+ setpu: "setup",
28
+ seting: "setting",
29
+ configuraiton: "configuration",
30
+ permisson: "permission",
31
+ permisions: "permissions",
32
+ };
33
+
34
+ export function stemToken(token: string): string {
35
+ if (token.length > 4 && token.endsWith("ies")) return `${token.slice(0, -3)}y`;
36
+ if (token.length > 5 && token.endsWith("ing")) return token.slice(0, -3);
37
+ if (token.length > 4 && token.endsWith("ed")) return token.slice(0, -2);
38
+ if (token.length > 3 && token.endsWith("s")) return token.slice(0, -1);
39
+ return token;
40
+ }
41
+
42
+ export function tokenizeForSearch(text: string): Set<string> {
43
+ const normalized = preTokenizeForFTS(text)
44
+ .toLowerCase()
45
+ .replace(/[-_*"(){}[\]^~:+.#@!\\/<>|&$%?]/g, " ");
46
+ const tokens = normalized
47
+ .split(/\s+/)
48
+ .map((token) => TYPO_CORRECTIONS[token] ?? token)
49
+ .filter((token) => token.length > 1 || /[\u3400-\u4dbf\u4e00-\u9fff]/.test(token));
50
+ return new Set(tokens);
51
+ }
52
+
53
+ export function signalTokens(tokens: Set<string>): Set<string> {
54
+ return new Set([...tokens].filter((token) => !STOP_WORDS.has(token)).map(stemToken));
55
+ }
56
+
57
+ export function normalizedQueryText(query: string): string {
58
+ return [...tokenizeForSearch(query)].join(" ");
59
+ }
@@ -0,0 +1,197 @@
1
+ import { buildChunkEmbeddingText } from "../indexer/chunker.ts";
2
+ import type { Chunk } from "../storage/sqlite.ts";
3
+ import { signalTokens, tokenizeForSearch } from "./query.ts";
4
+
5
+ export const MIN_HYBRID_SCORE = 0.18;
6
+
7
+ export const FILE_TYPE_ALIASES: Record<string, string> = {
8
+ csharp: "csharp",
9
+ cs: "csharp",
10
+ cpp: "cpp",
11
+ hpp: "cpp",
12
+ js: "javascript",
13
+ jsx: "javascript",
14
+ md: "markdown",
15
+ mdx: "markdown",
16
+ php: "php",
17
+ py: "python",
18
+ rb: "ruby",
19
+ sh: "shell",
20
+ ts: "typescript",
21
+ tsx: "typescript",
22
+ yml: "yaml",
23
+ };
24
+
25
+ const SOURCE_FILE_TYPES = new Set([
26
+ "c",
27
+ "cpp",
28
+ "csharp",
29
+ "go",
30
+ "java",
31
+ "javascript",
32
+ "php",
33
+ "python",
34
+ "ruby",
35
+ "rust",
36
+ "typescript",
37
+ ]);
38
+
39
+ const TEST_PATH_PATTERNS = ["_test.", ".test.", ".spec.", "/test/", "/tests/", "tests/", "__tests__/", "spec/"];
40
+
41
+ export interface RankingDiagnostics {
42
+ adjusted_score: number;
43
+ base_score: number;
44
+ coverage: number;
45
+ documentation_boost: number;
46
+ is_localization: boolean;
47
+ is_test: boolean;
48
+ localization_penalty: number;
49
+ path_boost: number;
50
+ source_boost: number;
51
+ test_intent: boolean;
52
+ user_guide_intent: boolean;
53
+ }
54
+
55
+ export function normalizeFileTypeFilter(fileType: string | undefined): string | undefined {
56
+ if (!fileType) return undefined;
57
+ const normalized = fileType.trim().toLowerCase();
58
+ return FILE_TYPE_ALIASES[normalized] ?? normalized;
59
+ }
60
+
61
+ export function isTestPath(filePath: string): boolean {
62
+ const lower = filePath.toLowerCase();
63
+ return TEST_PATH_PATTERNS.some((pattern) => lower.includes(pattern));
64
+ }
65
+
66
+ export function queryAsksForTests(queryTokens: Set<string>): boolean {
67
+ const signals = signalTokens(queryTokens);
68
+ return ["assert", "fixture", "spec", "test", "testing"].some((token) => signals.has(token));
69
+ }
70
+
71
+ function queryAsksForUserGuide(queryTokens: Set<string>): boolean {
72
+ const signals = signalTokens(queryTokens);
73
+ return ["config", "configuration", "install", "onboard", "quickstart", "setup", "start", "usage"].some((token) =>
74
+ signals.has(token),
75
+ );
76
+ }
77
+
78
+ function queryAsksForLocalization(queryTokens: Set<string>): boolean {
79
+ const signals = signalTokens(queryTokens);
80
+ return ["i18n", "lang", "language", "locale", "localization", "translation"].some((token) => signals.has(token));
81
+ }
82
+
83
+ export function isLocalizationPath(filePath: string, fileType: string): boolean {
84
+ if (!["json", "markdown", "text", "toml", "yaml"].includes(fileType)) return false;
85
+ const lower = filePath.toLowerCase();
86
+ return (
87
+ lower.includes("/locale/") ||
88
+ lower.startsWith("locale/") ||
89
+ lower.includes("/locales/") ||
90
+ lower.startsWith("locales/") ||
91
+ lower.includes("/lang/") ||
92
+ lower.startsWith("lang/") ||
93
+ lower.includes("/i18n/") ||
94
+ lower.startsWith("i18n/") ||
95
+ lower.includes("/translations/") ||
96
+ lower.startsWith("translations/")
97
+ );
98
+ }
99
+
100
+ export function queryCoverage(text: string, queryTokens: Set<string>): number {
101
+ const textTokens = signalTokens(tokenizeForSearch(text));
102
+ const signals = signalTokens(queryTokens);
103
+ if (signals.size === 0) return 0;
104
+ let matched = 0;
105
+ for (const token of signals) {
106
+ if (textTokens.has(token)) matched++;
107
+ }
108
+ return matched / signals.size;
109
+ }
110
+
111
+ export function pathTokenBoost(filePath: string, queryTokens: Set<string>): number {
112
+ const pathTokens = signalTokens(tokenizeForSearch(filePath.replace(/[./_-]/g, " ")));
113
+ const signals = signalTokens(queryTokens);
114
+ let matched = 0;
115
+ for (const token of signals) {
116
+ if (pathTokens.has(token)) matched++;
117
+ }
118
+ if (matched === 0) return 0;
119
+ return Math.min(0.45, 0.18 * matched);
120
+ }
121
+
122
+ export function basenameTokenBoost(filePath: string, queryTokens: Set<string>): number {
123
+ const basename = filePath.split("/").at(-1) ?? filePath;
124
+ const stem = basename.replace(/\.[^.]+$/, "").replace(/_test$|\.test$|\.spec$/i, "");
125
+ const baseTokens = signalTokens(tokenizeForSearch(stem.replace(/[._-]/g, " ")));
126
+ const signals = signalTokens(queryTokens);
127
+ if (baseTokens.size === 0 || signals.size === 0) return 0;
128
+ for (const token of signals) {
129
+ if (baseTokens.has(token)) return 0.32;
130
+ }
131
+ return 0;
132
+ }
133
+
134
+ export function sourceFileBoost(chunk: Pick<Chunk, "file_path" | "file_type">, queryTokens: Set<string>): number {
135
+ if (!SOURCE_FILE_TYPES.has(chunk.file_type)) return 0;
136
+ if (queryAsksForUserGuide(queryTokens)) return Math.min(0.05, pathTokenBoost(chunk.file_path, queryTokens));
137
+ const boost = basenameTokenBoost(chunk.file_path, queryTokens);
138
+ if (boost > 0) return boost;
139
+ const boostFromPath = pathTokenBoost(chunk.file_path, queryTokens);
140
+ return Math.min(0.22, boostFromPath);
141
+ }
142
+
143
+ function documentationBoost(chunk: Pick<Chunk, "file_path" | "file_type">, queryTokens: Set<string>): number {
144
+ if (!queryAsksForUserGuide(queryTokens)) return 0;
145
+ const lower = chunk.file_path.toLowerCase();
146
+ if (chunk.file_type !== "markdown") return 0;
147
+ if (
148
+ lower.includes("install") ||
149
+ lower.includes("readme") ||
150
+ lower.includes("getting-started") ||
151
+ lower.includes("quickstart")
152
+ ) {
153
+ return 0.85;
154
+ }
155
+ if (lower.startsWith("docs/")) return 0.35;
156
+ return 0;
157
+ }
158
+
159
+ export function scoreChunkForQuery(baseScore: number, chunk: Chunk, queryTokens: Set<string>): RankingDiagnostics {
160
+ const guideIntent = queryAsksForUserGuide(queryTokens);
161
+ const pathBoost =
162
+ guideIntent && SOURCE_FILE_TYPES.has(chunk.file_type)
163
+ ? Math.min(0.08, pathTokenBoost(chunk.file_path, queryTokens))
164
+ : pathTokenBoost(chunk.file_path, queryTokens);
165
+ const sourceBoost = sourceFileBoost(chunk, queryTokens);
166
+ const docBoost = documentationBoost(chunk, queryTokens);
167
+ const testIntent = queryAsksForTests(queryTokens);
168
+ const test = isTestPath(chunk.file_path);
169
+ const localization = isLocalizationPath(chunk.file_path, chunk.file_type);
170
+ const localizationPenalty = localization && !queryAsksForLocalization(queryTokens) ? 0.2 : 1;
171
+ let adjustedScore = baseScore + pathBoost + sourceBoost + docBoost;
172
+ if (test && !testIntent) adjustedScore *= 0.48;
173
+ if (!test && testIntent) adjustedScore *= 0.88;
174
+ adjustedScore *= localizationPenalty;
175
+ return {
176
+ adjusted_score: adjustedScore,
177
+ base_score: baseScore,
178
+ coverage: queryCoverage(buildChunkEmbeddingText(chunk), queryTokens),
179
+ documentation_boost: docBoost,
180
+ is_localization: localization,
181
+ is_test: test,
182
+ localization_penalty: localizationPenalty,
183
+ path_boost: pathBoost,
184
+ source_boost: sourceBoost,
185
+ test_intent: testIntent,
186
+ user_guide_intent: guideIntent,
187
+ };
188
+ }
189
+
190
+ export function hasEnoughLexicalEvidence(chunk: Chunk, queryTokens: Set<string>): boolean {
191
+ const signals = signalTokens(queryTokens);
192
+ if (signals.size <= 1) return true;
193
+ if (sourceFileBoost(chunk, queryTokens) >= 0.22) return true;
194
+ const coverage = queryCoverage(buildChunkEmbeddingText(chunk), queryTokens);
195
+ if (signals.size <= 3) return coverage >= 1 / signals.size;
196
+ return coverage >= 0.34;
197
+ }