pi-knowledge 0.3.3 → 0.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +36 -0
- package/README.md +16 -2
- package/index.ts +67 -7
- package/package.json +1 -1
- package/src/embedding/provider.ts +50 -12
- package/src/engine.ts +409 -67
- package/src/indexer/chunker.ts +121 -77
- package/src/indexer/chunkers/code-ast.ts +14 -11
- package/src/search/bm25.ts +26 -22
- package/src/search/fusion.ts +33 -0
- package/src/search/query.ts +59 -0
- package/src/search/ranking.ts +197 -0
- package/src/search/reranker.ts +49 -11
- package/src/storage/sqlite.ts +16 -0
package/src/indexer/chunker.ts
CHANGED
|
@@ -59,8 +59,33 @@ const DEFAULT_IGNORE = [
|
|
|
59
59
|
".git",
|
|
60
60
|
"dist",
|
|
61
61
|
"build",
|
|
62
|
+
"bin",
|
|
63
|
+
"obj",
|
|
64
|
+
"out",
|
|
65
|
+
"target",
|
|
66
|
+
"coverage",
|
|
62
67
|
".next",
|
|
68
|
+
".cache",
|
|
69
|
+
".playwright",
|
|
63
70
|
"__pycache__",
|
|
71
|
+
".env",
|
|
72
|
+
".env.*",
|
|
73
|
+
"*.pem",
|
|
74
|
+
"*.key",
|
|
75
|
+
"*.p12",
|
|
76
|
+
"*.pfx",
|
|
77
|
+
"*.crt",
|
|
78
|
+
"*.cert",
|
|
79
|
+
"*secret*",
|
|
80
|
+
"*secrets*",
|
|
81
|
+
"*credential*",
|
|
82
|
+
"*credentials*",
|
|
83
|
+
"docs/*eval-report*.md",
|
|
84
|
+
"docs/*evaluation-report*.md",
|
|
85
|
+
"docs/*knowledge-base*report*.md",
|
|
86
|
+
"*knowledge-base-evaluation-report*.md",
|
|
87
|
+
"setting*.json",
|
|
88
|
+
"appsettings*.json",
|
|
64
89
|
"*.min.js",
|
|
65
90
|
"*.min.css",
|
|
66
91
|
"*.map",
|
|
@@ -179,6 +204,9 @@ function estimateTokens(text: string): number {
|
|
|
179
204
|
return Math.ceil(text.length / 3);
|
|
180
205
|
}
|
|
181
206
|
|
|
207
|
+
const MARKDOWN_TARGET_TOKENS = 450;
|
|
208
|
+
const TEXT_TARGET_TOKENS = 550;
|
|
209
|
+
|
|
182
210
|
export function preTokenizeForFTS(content: string): string {
|
|
183
211
|
return content
|
|
184
212
|
.replace(/([a-z])([A-Z])/g, "$1 $2")
|
|
@@ -194,74 +222,108 @@ export function contentHash(content: string): string {
|
|
|
194
222
|
return createHash("sha256").update(content).digest("hex");
|
|
195
223
|
}
|
|
196
224
|
|
|
225
|
+
function normalizeHeading(heading: string): string {
|
|
226
|
+
return heading.replace(/^#{1,6}\s+/, "").trim();
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
function buildContextPrefix(
|
|
230
|
+
filePath: string,
|
|
231
|
+
fileType: string,
|
|
232
|
+
metadata: Record<string, string | number | null | undefined>,
|
|
233
|
+
): string {
|
|
234
|
+
const parts = [`File: ${filePath}`, `Type: ${fileType}`];
|
|
235
|
+
const heading = typeof metadata.heading === "string" ? metadata.heading : "";
|
|
236
|
+
const breadcrumb = typeof metadata.breadcrumb === "string" ? metadata.breadcrumb : "";
|
|
237
|
+
const symbol = typeof metadata.function_name === "string" ? metadata.function_name : "";
|
|
238
|
+
if (breadcrumb) parts.push(`Section: ${breadcrumb}`);
|
|
239
|
+
else if (heading) parts.push(`Section: ${normalizeHeading(heading)}`);
|
|
240
|
+
if (symbol) parts.push(`Symbol: ${symbol}`);
|
|
241
|
+
return parts.join("\n");
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
export function buildChunkEmbeddingText(
|
|
245
|
+
chunk: Pick<ChunkInsert, "content" | "file_path" | "file_type" | "metadata_json">,
|
|
246
|
+
): string {
|
|
247
|
+
let metadata: Record<string, string | number | null | undefined> = {};
|
|
248
|
+
try {
|
|
249
|
+
metadata = JSON.parse(chunk.metadata_json) as Record<string, string | number | null | undefined>;
|
|
250
|
+
} catch {
|
|
251
|
+
metadata = {};
|
|
252
|
+
}
|
|
253
|
+
return `${buildContextPrefix(chunk.file_path, chunk.file_type, metadata)}\n\n${chunk.content}`;
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
function makeChunk(
|
|
257
|
+
content: string,
|
|
258
|
+
filePath: string,
|
|
259
|
+
fileType: string,
|
|
260
|
+
startLine: number,
|
|
261
|
+
endLine: number,
|
|
262
|
+
metadata: Record<string, string | number | null | undefined> = {},
|
|
263
|
+
): Omit<ChunkInsert, "kb_id"> {
|
|
264
|
+
const metadata_json = JSON.stringify(metadata);
|
|
265
|
+
const chunk = {
|
|
266
|
+
content_hash: contentHash(content),
|
|
267
|
+
content,
|
|
268
|
+
content_tokenized: "",
|
|
269
|
+
file_path: filePath,
|
|
270
|
+
file_type: fileType,
|
|
271
|
+
start_line: startLine,
|
|
272
|
+
end_line: endLine,
|
|
273
|
+
metadata_json,
|
|
274
|
+
};
|
|
275
|
+
return { ...chunk, content_tokenized: preTokenizeForFTS(buildChunkEmbeddingText(chunk)) };
|
|
276
|
+
}
|
|
277
|
+
|
|
197
278
|
export function chunkMarkdown(content: string, filePath: string): Omit<ChunkInsert, "kb_id">[] {
|
|
198
279
|
const lines = content.split("\n");
|
|
199
280
|
const chunks: Omit<ChunkInsert, "kb_id">[] = [];
|
|
200
|
-
|
|
281
|
+
const headingStack: string[] = [];
|
|
201
282
|
let sectionLines: string[] = [];
|
|
202
283
|
let startLine = 1;
|
|
284
|
+
let currentHeading = "";
|
|
285
|
+
|
|
286
|
+
function currentBreadcrumb(): string {
|
|
287
|
+
return headingStack.map(normalizeHeading).filter(Boolean).join(" > ");
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
function pushMarkdownChunk(text: string, start: number, end: number): void {
|
|
291
|
+
if (text.trim().length < 50) return;
|
|
292
|
+
chunks.push(
|
|
293
|
+
makeChunk(text.trim(), filePath, "markdown", start, end, {
|
|
294
|
+
heading: currentHeading,
|
|
295
|
+
breadcrumb: currentBreadcrumb(),
|
|
296
|
+
}),
|
|
297
|
+
);
|
|
298
|
+
}
|
|
203
299
|
|
|
204
300
|
function flush(endLine: number): void {
|
|
205
301
|
const text = sectionLines.join("\n").trim();
|
|
206
302
|
if (text.length < 50) return;
|
|
207
303
|
|
|
208
304
|
const fullText = currentHeading ? `${currentHeading}\n\n${text}` : text;
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
chunks.push({
|
|
220
|
-
content_hash: contentHash(chunkText),
|
|
221
|
-
content: chunkText,
|
|
222
|
-
content_tokenized: preTokenizeForFTS(chunkText),
|
|
223
|
-
file_path: filePath,
|
|
224
|
-
file_type: "markdown",
|
|
225
|
-
start_line: bufStart,
|
|
226
|
-
end_line: endLine,
|
|
227
|
-
metadata_json: JSON.stringify({ heading: currentHeading }),
|
|
228
|
-
});
|
|
229
|
-
buf = [];
|
|
230
|
-
bufStart = endLine;
|
|
231
|
-
}
|
|
232
|
-
}
|
|
233
|
-
if (buf.length > 0) {
|
|
234
|
-
const chunkText = buf.join("\n\n");
|
|
235
|
-
if (chunkText.length >= 50) {
|
|
236
|
-
chunks.push({
|
|
237
|
-
content_hash: contentHash(chunkText),
|
|
238
|
-
content: chunkText,
|
|
239
|
-
content_tokenized: preTokenizeForFTS(chunkText),
|
|
240
|
-
file_path: filePath,
|
|
241
|
-
file_type: "markdown",
|
|
242
|
-
start_line: bufStart,
|
|
243
|
-
end_line: endLine,
|
|
244
|
-
metadata_json: JSON.stringify({ heading: currentHeading }),
|
|
245
|
-
});
|
|
246
|
-
}
|
|
305
|
+
const paragraphs = fullText.split(/\n\n+/);
|
|
306
|
+
let buffer: string[] = [];
|
|
307
|
+
let bufferStart = startLine;
|
|
308
|
+
|
|
309
|
+
for (const para of paragraphs) {
|
|
310
|
+
const next = [...buffer, para].join("\n\n");
|
|
311
|
+
if (estimateTokens(next) > MARKDOWN_TARGET_TOKENS && buffer.length > 0) {
|
|
312
|
+
pushMarkdownChunk(buffer.join("\n\n"), bufferStart, endLine);
|
|
313
|
+
buffer = [];
|
|
314
|
+
bufferStart = endLine;
|
|
247
315
|
}
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
file_path: filePath,
|
|
254
|
-
file_type: "markdown",
|
|
255
|
-
start_line: startLine,
|
|
256
|
-
end_line: endLine,
|
|
257
|
-
metadata_json: JSON.stringify({ heading: currentHeading }),
|
|
258
|
-
});
|
|
316
|
+
buffer.push(para);
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
if (buffer.length > 0) {
|
|
320
|
+
pushMarkdownChunk(buffer.join("\n\n"), bufferStart, endLine);
|
|
259
321
|
}
|
|
260
322
|
}
|
|
261
323
|
|
|
262
324
|
for (let i = 0; i < lines.length; i++) {
|
|
263
325
|
const line = lines[i];
|
|
264
|
-
const headingMatch = line.match(/^(#{1,
|
|
326
|
+
const headingMatch = line.match(/^(#{1,6})\s+(.+)$/);
|
|
265
327
|
|
|
266
328
|
if (headingMatch && sectionLines.length > 0) {
|
|
267
329
|
flush(i);
|
|
@@ -270,6 +332,9 @@ export function chunkMarkdown(content: string, filePath: string): Omit<ChunkInse
|
|
|
270
332
|
}
|
|
271
333
|
|
|
272
334
|
if (headingMatch) {
|
|
335
|
+
const level = headingMatch[1].length;
|
|
336
|
+
headingStack.splice(level - 1);
|
|
337
|
+
headingStack[level - 1] = line;
|
|
273
338
|
currentHeading = line;
|
|
274
339
|
} else {
|
|
275
340
|
sectionLines.push(line);
|
|
@@ -294,26 +359,16 @@ export function chunkText(content: string, filePath: string): Omit<ChunkInsert,
|
|
|
294
359
|
function flush(): void {
|
|
295
360
|
const text = buffer.join("\n\n").trim();
|
|
296
361
|
if (text.length < 50) return;
|
|
297
|
-
chunks.push(
|
|
298
|
-
content_hash: contentHash(text),
|
|
299
|
-
content: text,
|
|
300
|
-
content_tokenized: preTokenizeForFTS(text),
|
|
301
|
-
file_path: filePath,
|
|
302
|
-
file_type: fileType,
|
|
303
|
-
start_line: lineOffset,
|
|
304
|
-
end_line: lineOffset + text.split("\n").length,
|
|
305
|
-
metadata_json: "{}",
|
|
306
|
-
});
|
|
362
|
+
chunks.push(makeChunk(text, filePath, fileType, lineOffset, lineOffset + text.split("\n").length));
|
|
307
363
|
lineOffset += text.split("\n").length + 1;
|
|
308
364
|
}
|
|
309
365
|
|
|
310
366
|
for (const para of paragraphs) {
|
|
311
367
|
const paraTokens = estimateTokens(para);
|
|
312
|
-
if (bufferTokens + paraTokens >
|
|
368
|
+
if (bufferTokens + paraTokens > TEXT_TARGET_TOKENS && buffer.length > 0) {
|
|
313
369
|
flush();
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
bufferTokens = buffer.length > 0 ? estimateTokens(buffer[0]) : 0;
|
|
370
|
+
buffer = [];
|
|
371
|
+
bufferTokens = 0;
|
|
317
372
|
}
|
|
318
373
|
buffer.push(para);
|
|
319
374
|
bufferTokens += paraTokens;
|
|
@@ -342,18 +397,7 @@ export async function chunkFile(content: string, filePath: string): Promise<Omit
|
|
|
342
397
|
|
|
343
398
|
// Fallback: if file has content but no chunks (too short for splitting), keep as single chunk
|
|
344
399
|
if (chunks.length === 0 && content.trim().length > 10) {
|
|
345
|
-
chunks = [
|
|
346
|
-
{
|
|
347
|
-
content_hash: contentHash(content),
|
|
348
|
-
content: content.trim(),
|
|
349
|
-
content_tokenized: preTokenizeForFTS(content),
|
|
350
|
-
file_path: filePath,
|
|
351
|
-
file_type: fileType,
|
|
352
|
-
start_line: 1,
|
|
353
|
-
end_line: content.split("\n").length,
|
|
354
|
-
metadata_json: "{}",
|
|
355
|
-
},
|
|
356
|
-
];
|
|
400
|
+
chunks = [makeChunk(content.trim(), filePath, fileType, 1, content.split("\n").length)];
|
|
357
401
|
}
|
|
358
402
|
|
|
359
403
|
return chunks;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { ChunkInsert } from "../../storage/sqlite.ts";
|
|
2
|
-
import { contentHash, preTokenizeForFTS } from "../chunker.ts";
|
|
2
|
+
import { buildChunkEmbeddingText, contentHash, preTokenizeForFTS } from "../chunker.ts";
|
|
3
3
|
|
|
4
4
|
interface ASTNode {
|
|
5
5
|
type: string;
|
|
@@ -191,16 +191,19 @@ export async function chunkWithAST(
|
|
|
191
191
|
const tree = parser.parse(content);
|
|
192
192
|
const fns = collectChunks(tree.rootNode as unknown as ASTNode, config);
|
|
193
193
|
if (fns.length === 0) return [];
|
|
194
|
-
return fns.map((fn) =>
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
194
|
+
return fns.map((fn) => {
|
|
195
|
+
const chunk = {
|
|
196
|
+
content_hash: contentHash(fn.text),
|
|
197
|
+
content: fn.text,
|
|
198
|
+
content_tokenized: "",
|
|
199
|
+
file_path: filePath,
|
|
200
|
+
file_type: config.fileType,
|
|
201
|
+
start_line: fn.start,
|
|
202
|
+
end_line: fn.end,
|
|
203
|
+
metadata_json: JSON.stringify({ function_name: fn.name }),
|
|
204
|
+
};
|
|
205
|
+
return { ...chunk, content_tokenized: preTokenizeForFTS(buildChunkEmbeddingText(chunk)) };
|
|
206
|
+
});
|
|
204
207
|
}
|
|
205
208
|
|
|
206
209
|
export const SUPPORTED_LANGUAGES = new Set(Object.keys(LANGS));
|
package/src/search/bm25.ts
CHANGED
|
@@ -1,39 +1,43 @@
|
|
|
1
1
|
import type Database from "better-sqlite3";
|
|
2
|
-
import {
|
|
2
|
+
import { normalizedQueryText } from "./query.ts";
|
|
3
3
|
|
|
4
4
|
export interface BM25Result {
|
|
5
5
|
chunkId: string;
|
|
6
6
|
score: number;
|
|
7
7
|
}
|
|
8
8
|
|
|
9
|
-
function
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
if (terms.length === 0) return "";
|
|
14
|
-
return terms.join(" AND ");
|
|
9
|
+
function prepareFtsTerms(query: string): string[] {
|
|
10
|
+
return normalizedQueryText(query)
|
|
11
|
+
.split(/\s+/)
|
|
12
|
+
.filter((t) => t.length > 0);
|
|
15
13
|
}
|
|
16
14
|
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
if (!ftsQuery) return [];
|
|
20
|
-
try {
|
|
21
|
-
if (kbId) {
|
|
22
|
-
return db
|
|
23
|
-
.prepare(
|
|
24
|
-
`SELECT c.id as chunkId, -bm25(chunks_fts) as score
|
|
25
|
-
FROM chunks_fts JOIN chunks c ON chunks_fts.rowid = c.rowid
|
|
26
|
-
WHERE chunks_fts MATCH ? AND c.kb_id = ? ORDER BY bm25(chunks_fts) LIMIT ?`,
|
|
27
|
-
)
|
|
28
|
-
.all(ftsQuery, kbId, limit) as BM25Result[];
|
|
29
|
-
}
|
|
15
|
+
function runSearch(db: Database.Database, ftsQuery: string, limit: number, kbId?: string): BM25Result[] {
|
|
16
|
+
if (kbId) {
|
|
30
17
|
return db
|
|
31
18
|
.prepare(
|
|
32
19
|
`SELECT c.id as chunkId, -bm25(chunks_fts) as score
|
|
33
20
|
FROM chunks_fts JOIN chunks c ON chunks_fts.rowid = c.rowid
|
|
34
|
-
WHERE chunks_fts MATCH ? ORDER BY bm25(chunks_fts) LIMIT ?`,
|
|
21
|
+
WHERE chunks_fts MATCH ? AND c.kb_id = ? ORDER BY bm25(chunks_fts) LIMIT ?`,
|
|
35
22
|
)
|
|
36
|
-
.all(ftsQuery, limit) as BM25Result[];
|
|
23
|
+
.all(ftsQuery, kbId, limit) as BM25Result[];
|
|
24
|
+
}
|
|
25
|
+
return db
|
|
26
|
+
.prepare(
|
|
27
|
+
`SELECT c.id as chunkId, -bm25(chunks_fts) as score
|
|
28
|
+
FROM chunks_fts JOIN chunks c ON chunks_fts.rowid = c.rowid
|
|
29
|
+
WHERE chunks_fts MATCH ? ORDER BY bm25(chunks_fts) LIMIT ?`,
|
|
30
|
+
)
|
|
31
|
+
.all(ftsQuery, limit) as BM25Result[];
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function searchBM25(db: Database.Database, query: string, limit = 50, kbId?: string): BM25Result[] {
|
|
35
|
+
const terms = prepareFtsTerms(query);
|
|
36
|
+
if (terms.length === 0) return [];
|
|
37
|
+
try {
|
|
38
|
+
const strict = runSearch(db, terms.join(" AND "), limit, kbId);
|
|
39
|
+
if (strict.length > 0 || terms.length === 1) return strict;
|
|
40
|
+
return runSearch(db, terms.join(" OR "), limit, kbId);
|
|
37
41
|
} catch {
|
|
38
42
|
return [];
|
|
39
43
|
}
|
package/src/search/fusion.ts
CHANGED
|
@@ -3,6 +3,19 @@ export interface FusionResult {
|
|
|
3
3
|
score: number;
|
|
4
4
|
}
|
|
5
5
|
|
|
6
|
+
function normalizeScores(list: Array<{ chunkId: string; score: number }>): Map<string, number> {
|
|
7
|
+
if (list.length === 0) return new Map();
|
|
8
|
+
const scores = list.map((item) => item.score);
|
|
9
|
+
const min = Math.min(...scores);
|
|
10
|
+
const max = Math.max(...scores);
|
|
11
|
+
return new Map(
|
|
12
|
+
list.map((item) => {
|
|
13
|
+
const normalized = max === min ? 1 : (item.score - min) / (max - min);
|
|
14
|
+
return [item.chunkId, normalized];
|
|
15
|
+
}),
|
|
16
|
+
);
|
|
17
|
+
}
|
|
18
|
+
|
|
6
19
|
export function reciprocalRankFusion(lists: Array<{ chunkId: string; score: number }[]>, k = 60): FusionResult[] {
|
|
7
20
|
const scores = new Map<string, number>();
|
|
8
21
|
for (const list of lists) {
|
|
@@ -13,3 +26,23 @@ export function reciprocalRankFusion(lists: Array<{ chunkId: string; score: numb
|
|
|
13
26
|
}
|
|
14
27
|
return [...scores.entries()].map(([chunkId, score]) => ({ chunkId, score })).sort((a, b) => b.score - a.score);
|
|
15
28
|
}
|
|
29
|
+
|
|
30
|
+
export function weightedScoreFusion(
|
|
31
|
+
bm25Results: Array<{ chunkId: string; score: number }>,
|
|
32
|
+
vectorResults: Array<{ chunkId: string; score: number }>,
|
|
33
|
+
weights = { bm25: 0.45, vector: 0.55, overlap: 0.15 },
|
|
34
|
+
): FusionResult[] {
|
|
35
|
+
const bm25Scores = normalizeScores(bm25Results);
|
|
36
|
+
const vectorScores = normalizeScores(vectorResults);
|
|
37
|
+
const ids = new Set([...bm25Scores.keys(), ...vectorScores.keys()]);
|
|
38
|
+
const fused: FusionResult[] = [];
|
|
39
|
+
|
|
40
|
+
for (const chunkId of ids) {
|
|
41
|
+
const bm25 = bm25Scores.get(chunkId) ?? 0;
|
|
42
|
+
const vector = vectorScores.get(chunkId) ?? 0;
|
|
43
|
+
const overlap = bm25 > 0 && vector > 0 ? weights.overlap : 0;
|
|
44
|
+
fused.push({ chunkId, score: weights.bm25 * bm25 + weights.vector * vector + overlap });
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
return fused.sort((a, b) => b.score - a.score);
|
|
48
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { preTokenizeForFTS } from "../indexer/chunker.ts";
|
|
2
|
+
|
|
3
|
+
export const STOP_WORDS = new Set([
|
|
4
|
+
"the",
|
|
5
|
+
"and",
|
|
6
|
+
"or",
|
|
7
|
+
"to",
|
|
8
|
+
"of",
|
|
9
|
+
"in",
|
|
10
|
+
"on",
|
|
11
|
+
"for",
|
|
12
|
+
"with",
|
|
13
|
+
"how",
|
|
14
|
+
"does",
|
|
15
|
+
"what",
|
|
16
|
+
"when",
|
|
17
|
+
"where",
|
|
18
|
+
"why",
|
|
19
|
+
"this",
|
|
20
|
+
"that",
|
|
21
|
+
"from",
|
|
22
|
+
]);
|
|
23
|
+
|
|
24
|
+
const TYPO_CORRECTIONS: Record<string, string> = {
|
|
25
|
+
ho: "how",
|
|
26
|
+
setpup: "setup",
|
|
27
|
+
setpu: "setup",
|
|
28
|
+
seting: "setting",
|
|
29
|
+
configuraiton: "configuration",
|
|
30
|
+
permisson: "permission",
|
|
31
|
+
permisions: "permissions",
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
export function stemToken(token: string): string {
|
|
35
|
+
if (token.length > 4 && token.endsWith("ies")) return `${token.slice(0, -3)}y`;
|
|
36
|
+
if (token.length > 5 && token.endsWith("ing")) return token.slice(0, -3);
|
|
37
|
+
if (token.length > 4 && token.endsWith("ed")) return token.slice(0, -2);
|
|
38
|
+
if (token.length > 3 && token.endsWith("s")) return token.slice(0, -1);
|
|
39
|
+
return token;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export function tokenizeForSearch(text: string): Set<string> {
|
|
43
|
+
const normalized = preTokenizeForFTS(text)
|
|
44
|
+
.toLowerCase()
|
|
45
|
+
.replace(/[-_*"(){}[\]^~:+.#@!\\/<>|&$%?]/g, " ");
|
|
46
|
+
const tokens = normalized
|
|
47
|
+
.split(/\s+/)
|
|
48
|
+
.map((token) => TYPO_CORRECTIONS[token] ?? token)
|
|
49
|
+
.filter((token) => token.length > 1 || /[\u3400-\u4dbf\u4e00-\u9fff]/.test(token));
|
|
50
|
+
return new Set(tokens);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export function signalTokens(tokens: Set<string>): Set<string> {
|
|
54
|
+
return new Set([...tokens].filter((token) => !STOP_WORDS.has(token)).map(stemToken));
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export function normalizedQueryText(query: string): string {
|
|
58
|
+
return [...tokenizeForSearch(query)].join(" ");
|
|
59
|
+
}
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
import { buildChunkEmbeddingText } from "../indexer/chunker.ts";
|
|
2
|
+
import type { Chunk } from "../storage/sqlite.ts";
|
|
3
|
+
import { signalTokens, tokenizeForSearch } from "./query.ts";
|
|
4
|
+
|
|
5
|
+
export const MIN_HYBRID_SCORE = 0.18;
|
|
6
|
+
|
|
7
|
+
export const FILE_TYPE_ALIASES: Record<string, string> = {
|
|
8
|
+
csharp: "csharp",
|
|
9
|
+
cs: "csharp",
|
|
10
|
+
cpp: "cpp",
|
|
11
|
+
hpp: "cpp",
|
|
12
|
+
js: "javascript",
|
|
13
|
+
jsx: "javascript",
|
|
14
|
+
md: "markdown",
|
|
15
|
+
mdx: "markdown",
|
|
16
|
+
php: "php",
|
|
17
|
+
py: "python",
|
|
18
|
+
rb: "ruby",
|
|
19
|
+
sh: "shell",
|
|
20
|
+
ts: "typescript",
|
|
21
|
+
tsx: "typescript",
|
|
22
|
+
yml: "yaml",
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
const SOURCE_FILE_TYPES = new Set([
|
|
26
|
+
"c",
|
|
27
|
+
"cpp",
|
|
28
|
+
"csharp",
|
|
29
|
+
"go",
|
|
30
|
+
"java",
|
|
31
|
+
"javascript",
|
|
32
|
+
"php",
|
|
33
|
+
"python",
|
|
34
|
+
"ruby",
|
|
35
|
+
"rust",
|
|
36
|
+
"typescript",
|
|
37
|
+
]);
|
|
38
|
+
|
|
39
|
+
const TEST_PATH_PATTERNS = ["_test.", ".test.", ".spec.", "/test/", "/tests/", "tests/", "__tests__/", "spec/"];
|
|
40
|
+
|
|
41
|
+
export interface RankingDiagnostics {
|
|
42
|
+
adjusted_score: number;
|
|
43
|
+
base_score: number;
|
|
44
|
+
coverage: number;
|
|
45
|
+
documentation_boost: number;
|
|
46
|
+
is_localization: boolean;
|
|
47
|
+
is_test: boolean;
|
|
48
|
+
localization_penalty: number;
|
|
49
|
+
path_boost: number;
|
|
50
|
+
source_boost: number;
|
|
51
|
+
test_intent: boolean;
|
|
52
|
+
user_guide_intent: boolean;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export function normalizeFileTypeFilter(fileType: string | undefined): string | undefined {
|
|
56
|
+
if (!fileType) return undefined;
|
|
57
|
+
const normalized = fileType.trim().toLowerCase();
|
|
58
|
+
return FILE_TYPE_ALIASES[normalized] ?? normalized;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export function isTestPath(filePath: string): boolean {
|
|
62
|
+
const lower = filePath.toLowerCase();
|
|
63
|
+
return TEST_PATH_PATTERNS.some((pattern) => lower.includes(pattern));
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export function queryAsksForTests(queryTokens: Set<string>): boolean {
|
|
67
|
+
const signals = signalTokens(queryTokens);
|
|
68
|
+
return ["assert", "fixture", "spec", "test", "testing"].some((token) => signals.has(token));
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function queryAsksForUserGuide(queryTokens: Set<string>): boolean {
|
|
72
|
+
const signals = signalTokens(queryTokens);
|
|
73
|
+
return ["config", "configuration", "install", "onboard", "quickstart", "setup", "start", "usage"].some((token) =>
|
|
74
|
+
signals.has(token),
|
|
75
|
+
);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function queryAsksForLocalization(queryTokens: Set<string>): boolean {
|
|
79
|
+
const signals = signalTokens(queryTokens);
|
|
80
|
+
return ["i18n", "lang", "language", "locale", "localization", "translation"].some((token) => signals.has(token));
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export function isLocalizationPath(filePath: string, fileType: string): boolean {
|
|
84
|
+
if (!["json", "markdown", "text", "toml", "yaml"].includes(fileType)) return false;
|
|
85
|
+
const lower = filePath.toLowerCase();
|
|
86
|
+
return (
|
|
87
|
+
lower.includes("/locale/") ||
|
|
88
|
+
lower.startsWith("locale/") ||
|
|
89
|
+
lower.includes("/locales/") ||
|
|
90
|
+
lower.startsWith("locales/") ||
|
|
91
|
+
lower.includes("/lang/") ||
|
|
92
|
+
lower.startsWith("lang/") ||
|
|
93
|
+
lower.includes("/i18n/") ||
|
|
94
|
+
lower.startsWith("i18n/") ||
|
|
95
|
+
lower.includes("/translations/") ||
|
|
96
|
+
lower.startsWith("translations/")
|
|
97
|
+
);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export function queryCoverage(text: string, queryTokens: Set<string>): number {
|
|
101
|
+
const textTokens = signalTokens(tokenizeForSearch(text));
|
|
102
|
+
const signals = signalTokens(queryTokens);
|
|
103
|
+
if (signals.size === 0) return 0;
|
|
104
|
+
let matched = 0;
|
|
105
|
+
for (const token of signals) {
|
|
106
|
+
if (textTokens.has(token)) matched++;
|
|
107
|
+
}
|
|
108
|
+
return matched / signals.size;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
export function pathTokenBoost(filePath: string, queryTokens: Set<string>): number {
|
|
112
|
+
const pathTokens = signalTokens(tokenizeForSearch(filePath.replace(/[./_-]/g, " ")));
|
|
113
|
+
const signals = signalTokens(queryTokens);
|
|
114
|
+
let matched = 0;
|
|
115
|
+
for (const token of signals) {
|
|
116
|
+
if (pathTokens.has(token)) matched++;
|
|
117
|
+
}
|
|
118
|
+
if (matched === 0) return 0;
|
|
119
|
+
return Math.min(0.45, 0.18 * matched);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
export function basenameTokenBoost(filePath: string, queryTokens: Set<string>): number {
|
|
123
|
+
const basename = filePath.split("/").at(-1) ?? filePath;
|
|
124
|
+
const stem = basename.replace(/\.[^.]+$/, "").replace(/_test$|\.test$|\.spec$/i, "");
|
|
125
|
+
const baseTokens = signalTokens(tokenizeForSearch(stem.replace(/[._-]/g, " ")));
|
|
126
|
+
const signals = signalTokens(queryTokens);
|
|
127
|
+
if (baseTokens.size === 0 || signals.size === 0) return 0;
|
|
128
|
+
for (const token of signals) {
|
|
129
|
+
if (baseTokens.has(token)) return 0.32;
|
|
130
|
+
}
|
|
131
|
+
return 0;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
export function sourceFileBoost(chunk: Pick<Chunk, "file_path" | "file_type">, queryTokens: Set<string>): number {
|
|
135
|
+
if (!SOURCE_FILE_TYPES.has(chunk.file_type)) return 0;
|
|
136
|
+
if (queryAsksForUserGuide(queryTokens)) return Math.min(0.05, pathTokenBoost(chunk.file_path, queryTokens));
|
|
137
|
+
const boost = basenameTokenBoost(chunk.file_path, queryTokens);
|
|
138
|
+
if (boost > 0) return boost;
|
|
139
|
+
const boostFromPath = pathTokenBoost(chunk.file_path, queryTokens);
|
|
140
|
+
return Math.min(0.22, boostFromPath);
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
function documentationBoost(chunk: Pick<Chunk, "file_path" | "file_type">, queryTokens: Set<string>): number {
|
|
144
|
+
if (!queryAsksForUserGuide(queryTokens)) return 0;
|
|
145
|
+
const lower = chunk.file_path.toLowerCase();
|
|
146
|
+
if (chunk.file_type !== "markdown") return 0;
|
|
147
|
+
if (
|
|
148
|
+
lower.includes("install") ||
|
|
149
|
+
lower.includes("readme") ||
|
|
150
|
+
lower.includes("getting-started") ||
|
|
151
|
+
lower.includes("quickstart")
|
|
152
|
+
) {
|
|
153
|
+
return 0.85;
|
|
154
|
+
}
|
|
155
|
+
if (lower.startsWith("docs/")) return 0.35;
|
|
156
|
+
return 0;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
export function scoreChunkForQuery(baseScore: number, chunk: Chunk, queryTokens: Set<string>): RankingDiagnostics {
|
|
160
|
+
const guideIntent = queryAsksForUserGuide(queryTokens);
|
|
161
|
+
const pathBoost =
|
|
162
|
+
guideIntent && SOURCE_FILE_TYPES.has(chunk.file_type)
|
|
163
|
+
? Math.min(0.08, pathTokenBoost(chunk.file_path, queryTokens))
|
|
164
|
+
: pathTokenBoost(chunk.file_path, queryTokens);
|
|
165
|
+
const sourceBoost = sourceFileBoost(chunk, queryTokens);
|
|
166
|
+
const docBoost = documentationBoost(chunk, queryTokens);
|
|
167
|
+
const testIntent = queryAsksForTests(queryTokens);
|
|
168
|
+
const test = isTestPath(chunk.file_path);
|
|
169
|
+
const localization = isLocalizationPath(chunk.file_path, chunk.file_type);
|
|
170
|
+
const localizationPenalty = localization && !queryAsksForLocalization(queryTokens) ? 0.2 : 1;
|
|
171
|
+
let adjustedScore = baseScore + pathBoost + sourceBoost + docBoost;
|
|
172
|
+
if (test && !testIntent) adjustedScore *= 0.48;
|
|
173
|
+
if (!test && testIntent) adjustedScore *= 0.88;
|
|
174
|
+
adjustedScore *= localizationPenalty;
|
|
175
|
+
return {
|
|
176
|
+
adjusted_score: adjustedScore,
|
|
177
|
+
base_score: baseScore,
|
|
178
|
+
coverage: queryCoverage(buildChunkEmbeddingText(chunk), queryTokens),
|
|
179
|
+
documentation_boost: docBoost,
|
|
180
|
+
is_localization: localization,
|
|
181
|
+
is_test: test,
|
|
182
|
+
localization_penalty: localizationPenalty,
|
|
183
|
+
path_boost: pathBoost,
|
|
184
|
+
source_boost: sourceBoost,
|
|
185
|
+
test_intent: testIntent,
|
|
186
|
+
user_guide_intent: guideIntent,
|
|
187
|
+
};
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
export function hasEnoughLexicalEvidence(chunk: Chunk, queryTokens: Set<string>): boolean {
|
|
191
|
+
const signals = signalTokens(queryTokens);
|
|
192
|
+
if (signals.size <= 1) return true;
|
|
193
|
+
if (sourceFileBoost(chunk, queryTokens) >= 0.22) return true;
|
|
194
|
+
const coverage = queryCoverage(buildChunkEmbeddingText(chunk), queryTokens);
|
|
195
|
+
if (signals.size <= 3) return coverage >= 1 / signals.size;
|
|
196
|
+
return coverage >= 0.34;
|
|
197
|
+
}
|