pi-knowledge 0.3.4 → 0.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/README.md +16 -2
- package/index.ts +63 -6
- package/package.json +1 -1
- package/src/embedding/provider.ts +50 -12
- package/src/engine.ts +402 -67
- package/src/indexer/chunker.ts +100 -77
- package/src/indexer/chunkers/code-ast.ts +14 -11
- package/src/search/bm25.ts +26 -22
- package/src/search/fusion.ts +33 -0
- package/src/search/query.ts +59 -0
- package/src/search/ranking.ts +197 -0
- package/src/search/reranker.ts +49 -11
- package/src/storage/sqlite.ts +16 -0
package/src/indexer/chunker.ts
CHANGED
|
@@ -80,6 +80,10 @@ const DEFAULT_IGNORE = [
|
|
|
80
80
|
"*secrets*",
|
|
81
81
|
"*credential*",
|
|
82
82
|
"*credentials*",
|
|
83
|
+
"docs/*eval-report*.md",
|
|
84
|
+
"docs/*evaluation-report*.md",
|
|
85
|
+
"docs/*knowledge-base*report*.md",
|
|
86
|
+
"*knowledge-base-evaluation-report*.md",
|
|
83
87
|
"setting*.json",
|
|
84
88
|
"appsettings*.json",
|
|
85
89
|
"*.min.js",
|
|
@@ -200,6 +204,9 @@ function estimateTokens(text: string): number {
|
|
|
200
204
|
return Math.ceil(text.length / 3);
|
|
201
205
|
}
|
|
202
206
|
|
|
207
|
+
const MARKDOWN_TARGET_TOKENS = 450;
|
|
208
|
+
const TEXT_TARGET_TOKENS = 550;
|
|
209
|
+
|
|
203
210
|
export function preTokenizeForFTS(content: string): string {
|
|
204
211
|
return content
|
|
205
212
|
.replace(/([a-z])([A-Z])/g, "$1 $2")
|
|
@@ -215,74 +222,108 @@ export function contentHash(content: string): string {
|
|
|
215
222
|
return createHash("sha256").update(content).digest("hex");
|
|
216
223
|
}
|
|
217
224
|
|
|
225
|
+
function normalizeHeading(heading: string): string {
|
|
226
|
+
return heading.replace(/^#{1,6}\s+/, "").trim();
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
function buildContextPrefix(
|
|
230
|
+
filePath: string,
|
|
231
|
+
fileType: string,
|
|
232
|
+
metadata: Record<string, string | number | null | undefined>,
|
|
233
|
+
): string {
|
|
234
|
+
const parts = [`File: ${filePath}`, `Type: ${fileType}`];
|
|
235
|
+
const heading = typeof metadata.heading === "string" ? metadata.heading : "";
|
|
236
|
+
const breadcrumb = typeof metadata.breadcrumb === "string" ? metadata.breadcrumb : "";
|
|
237
|
+
const symbol = typeof metadata.function_name === "string" ? metadata.function_name : "";
|
|
238
|
+
if (breadcrumb) parts.push(`Section: ${breadcrumb}`);
|
|
239
|
+
else if (heading) parts.push(`Section: ${normalizeHeading(heading)}`);
|
|
240
|
+
if (symbol) parts.push(`Symbol: ${symbol}`);
|
|
241
|
+
return parts.join("\n");
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
export function buildChunkEmbeddingText(
|
|
245
|
+
chunk: Pick<ChunkInsert, "content" | "file_path" | "file_type" | "metadata_json">,
|
|
246
|
+
): string {
|
|
247
|
+
let metadata: Record<string, string | number | null | undefined> = {};
|
|
248
|
+
try {
|
|
249
|
+
metadata = JSON.parse(chunk.metadata_json) as Record<string, string | number | null | undefined>;
|
|
250
|
+
} catch {
|
|
251
|
+
metadata = {};
|
|
252
|
+
}
|
|
253
|
+
return `${buildContextPrefix(chunk.file_path, chunk.file_type, metadata)}\n\n${chunk.content}`;
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
function makeChunk(
|
|
257
|
+
content: string,
|
|
258
|
+
filePath: string,
|
|
259
|
+
fileType: string,
|
|
260
|
+
startLine: number,
|
|
261
|
+
endLine: number,
|
|
262
|
+
metadata: Record<string, string | number | null | undefined> = {},
|
|
263
|
+
): Omit<ChunkInsert, "kb_id"> {
|
|
264
|
+
const metadata_json = JSON.stringify(metadata);
|
|
265
|
+
const chunk = {
|
|
266
|
+
content_hash: contentHash(content),
|
|
267
|
+
content,
|
|
268
|
+
content_tokenized: "",
|
|
269
|
+
file_path: filePath,
|
|
270
|
+
file_type: fileType,
|
|
271
|
+
start_line: startLine,
|
|
272
|
+
end_line: endLine,
|
|
273
|
+
metadata_json,
|
|
274
|
+
};
|
|
275
|
+
return { ...chunk, content_tokenized: preTokenizeForFTS(buildChunkEmbeddingText(chunk)) };
|
|
276
|
+
}
|
|
277
|
+
|
|
218
278
|
export function chunkMarkdown(content: string, filePath: string): Omit<ChunkInsert, "kb_id">[] {
|
|
219
279
|
const lines = content.split("\n");
|
|
220
280
|
const chunks: Omit<ChunkInsert, "kb_id">[] = [];
|
|
221
|
-
|
|
281
|
+
const headingStack: string[] = [];
|
|
222
282
|
let sectionLines: string[] = [];
|
|
223
283
|
let startLine = 1;
|
|
284
|
+
let currentHeading = "";
|
|
285
|
+
|
|
286
|
+
function currentBreadcrumb(): string {
|
|
287
|
+
return headingStack.map(normalizeHeading).filter(Boolean).join(" > ");
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
function pushMarkdownChunk(text: string, start: number, end: number): void {
|
|
291
|
+
if (text.trim().length < 50) return;
|
|
292
|
+
chunks.push(
|
|
293
|
+
makeChunk(text.trim(), filePath, "markdown", start, end, {
|
|
294
|
+
heading: currentHeading,
|
|
295
|
+
breadcrumb: currentBreadcrumb(),
|
|
296
|
+
}),
|
|
297
|
+
);
|
|
298
|
+
}
|
|
224
299
|
|
|
225
300
|
function flush(endLine: number): void {
|
|
226
301
|
const text = sectionLines.join("\n").trim();
|
|
227
302
|
if (text.length < 50) return;
|
|
228
303
|
|
|
229
304
|
const fullText = currentHeading ? `${currentHeading}\n\n${text}` : text;
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
chunks.push({
|
|
241
|
-
content_hash: contentHash(chunkText),
|
|
242
|
-
content: chunkText,
|
|
243
|
-
content_tokenized: preTokenizeForFTS(chunkText),
|
|
244
|
-
file_path: filePath,
|
|
245
|
-
file_type: "markdown",
|
|
246
|
-
start_line: bufStart,
|
|
247
|
-
end_line: endLine,
|
|
248
|
-
metadata_json: JSON.stringify({ heading: currentHeading }),
|
|
249
|
-
});
|
|
250
|
-
buf = [];
|
|
251
|
-
bufStart = endLine;
|
|
252
|
-
}
|
|
253
|
-
}
|
|
254
|
-
if (buf.length > 0) {
|
|
255
|
-
const chunkText = buf.join("\n\n");
|
|
256
|
-
if (chunkText.length >= 50) {
|
|
257
|
-
chunks.push({
|
|
258
|
-
content_hash: contentHash(chunkText),
|
|
259
|
-
content: chunkText,
|
|
260
|
-
content_tokenized: preTokenizeForFTS(chunkText),
|
|
261
|
-
file_path: filePath,
|
|
262
|
-
file_type: "markdown",
|
|
263
|
-
start_line: bufStart,
|
|
264
|
-
end_line: endLine,
|
|
265
|
-
metadata_json: JSON.stringify({ heading: currentHeading }),
|
|
266
|
-
});
|
|
267
|
-
}
|
|
305
|
+
const paragraphs = fullText.split(/\n\n+/);
|
|
306
|
+
let buffer: string[] = [];
|
|
307
|
+
let bufferStart = startLine;
|
|
308
|
+
|
|
309
|
+
for (const para of paragraphs) {
|
|
310
|
+
const next = [...buffer, para].join("\n\n");
|
|
311
|
+
if (estimateTokens(next) > MARKDOWN_TARGET_TOKENS && buffer.length > 0) {
|
|
312
|
+
pushMarkdownChunk(buffer.join("\n\n"), bufferStart, endLine);
|
|
313
|
+
buffer = [];
|
|
314
|
+
bufferStart = endLine;
|
|
268
315
|
}
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
file_path: filePath,
|
|
275
|
-
file_type: "markdown",
|
|
276
|
-
start_line: startLine,
|
|
277
|
-
end_line: endLine,
|
|
278
|
-
metadata_json: JSON.stringify({ heading: currentHeading }),
|
|
279
|
-
});
|
|
316
|
+
buffer.push(para);
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
if (buffer.length > 0) {
|
|
320
|
+
pushMarkdownChunk(buffer.join("\n\n"), bufferStart, endLine);
|
|
280
321
|
}
|
|
281
322
|
}
|
|
282
323
|
|
|
283
324
|
for (let i = 0; i < lines.length; i++) {
|
|
284
325
|
const line = lines[i];
|
|
285
|
-
const headingMatch = line.match(/^(#{1,
|
|
326
|
+
const headingMatch = line.match(/^(#{1,6})\s+(.+)$/);
|
|
286
327
|
|
|
287
328
|
if (headingMatch && sectionLines.length > 0) {
|
|
288
329
|
flush(i);
|
|
@@ -291,6 +332,9 @@ export function chunkMarkdown(content: string, filePath: string): Omit<ChunkInse
|
|
|
291
332
|
}
|
|
292
333
|
|
|
293
334
|
if (headingMatch) {
|
|
335
|
+
const level = headingMatch[1].length;
|
|
336
|
+
headingStack.splice(level - 1);
|
|
337
|
+
headingStack[level - 1] = line;
|
|
294
338
|
currentHeading = line;
|
|
295
339
|
} else {
|
|
296
340
|
sectionLines.push(line);
|
|
@@ -315,26 +359,16 @@ export function chunkText(content: string, filePath: string): Omit<ChunkInsert,
|
|
|
315
359
|
function flush(): void {
|
|
316
360
|
const text = buffer.join("\n\n").trim();
|
|
317
361
|
if (text.length < 50) return;
|
|
318
|
-
chunks.push(
|
|
319
|
-
content_hash: contentHash(text),
|
|
320
|
-
content: text,
|
|
321
|
-
content_tokenized: preTokenizeForFTS(text),
|
|
322
|
-
file_path: filePath,
|
|
323
|
-
file_type: fileType,
|
|
324
|
-
start_line: lineOffset,
|
|
325
|
-
end_line: lineOffset + text.split("\n").length,
|
|
326
|
-
metadata_json: "{}",
|
|
327
|
-
});
|
|
362
|
+
chunks.push(makeChunk(text, filePath, fileType, lineOffset, lineOffset + text.split("\n").length));
|
|
328
363
|
lineOffset += text.split("\n").length + 1;
|
|
329
364
|
}
|
|
330
365
|
|
|
331
366
|
for (const para of paragraphs) {
|
|
332
367
|
const paraTokens = estimateTokens(para);
|
|
333
|
-
if (bufferTokens + paraTokens >
|
|
368
|
+
if (bufferTokens + paraTokens > TEXT_TARGET_TOKENS && buffer.length > 0) {
|
|
334
369
|
flush();
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
bufferTokens = buffer.length > 0 ? estimateTokens(buffer[0]) : 0;
|
|
370
|
+
buffer = [];
|
|
371
|
+
bufferTokens = 0;
|
|
338
372
|
}
|
|
339
373
|
buffer.push(para);
|
|
340
374
|
bufferTokens += paraTokens;
|
|
@@ -363,18 +397,7 @@ export async function chunkFile(content: string, filePath: string): Promise<Omit
|
|
|
363
397
|
|
|
364
398
|
// Fallback: if file has content but no chunks (too short for splitting), keep as single chunk
|
|
365
399
|
if (chunks.length === 0 && content.trim().length > 10) {
|
|
366
|
-
chunks = [
|
|
367
|
-
{
|
|
368
|
-
content_hash: contentHash(content),
|
|
369
|
-
content: content.trim(),
|
|
370
|
-
content_tokenized: preTokenizeForFTS(content),
|
|
371
|
-
file_path: filePath,
|
|
372
|
-
file_type: fileType,
|
|
373
|
-
start_line: 1,
|
|
374
|
-
end_line: content.split("\n").length,
|
|
375
|
-
metadata_json: "{}",
|
|
376
|
-
},
|
|
377
|
-
];
|
|
400
|
+
chunks = [makeChunk(content.trim(), filePath, fileType, 1, content.split("\n").length)];
|
|
378
401
|
}
|
|
379
402
|
|
|
380
403
|
return chunks;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { ChunkInsert } from "../../storage/sqlite.ts";
|
|
2
|
-
import { contentHash, preTokenizeForFTS } from "../chunker.ts";
|
|
2
|
+
import { buildChunkEmbeddingText, contentHash, preTokenizeForFTS } from "../chunker.ts";
|
|
3
3
|
|
|
4
4
|
interface ASTNode {
|
|
5
5
|
type: string;
|
|
@@ -191,16 +191,19 @@ export async function chunkWithAST(
|
|
|
191
191
|
const tree = parser.parse(content);
|
|
192
192
|
const fns = collectChunks(tree.rootNode as unknown as ASTNode, config);
|
|
193
193
|
if (fns.length === 0) return [];
|
|
194
|
-
return fns.map((fn) =>
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
194
|
+
return fns.map((fn) => {
|
|
195
|
+
const chunk = {
|
|
196
|
+
content_hash: contentHash(fn.text),
|
|
197
|
+
content: fn.text,
|
|
198
|
+
content_tokenized: "",
|
|
199
|
+
file_path: filePath,
|
|
200
|
+
file_type: config.fileType,
|
|
201
|
+
start_line: fn.start,
|
|
202
|
+
end_line: fn.end,
|
|
203
|
+
metadata_json: JSON.stringify({ function_name: fn.name }),
|
|
204
|
+
};
|
|
205
|
+
return { ...chunk, content_tokenized: preTokenizeForFTS(buildChunkEmbeddingText(chunk)) };
|
|
206
|
+
});
|
|
204
207
|
}
|
|
205
208
|
|
|
206
209
|
export const SUPPORTED_LANGUAGES = new Set(Object.keys(LANGS));
|
package/src/search/bm25.ts
CHANGED
|
@@ -1,39 +1,43 @@
|
|
|
1
1
|
import type Database from "better-sqlite3";
|
|
2
|
-
import {
|
|
2
|
+
import { normalizedQueryText } from "./query.ts";
|
|
3
3
|
|
|
4
4
|
export interface BM25Result {
|
|
5
5
|
chunkId: string;
|
|
6
6
|
score: number;
|
|
7
7
|
}
|
|
8
8
|
|
|
9
|
-
function
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
if (terms.length === 0) return "";
|
|
14
|
-
return terms.join(" AND ");
|
|
9
|
+
function prepareFtsTerms(query: string): string[] {
|
|
10
|
+
return normalizedQueryText(query)
|
|
11
|
+
.split(/\s+/)
|
|
12
|
+
.filter((t) => t.length > 0);
|
|
15
13
|
}
|
|
16
14
|
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
if (!ftsQuery) return [];
|
|
20
|
-
try {
|
|
21
|
-
if (kbId) {
|
|
22
|
-
return db
|
|
23
|
-
.prepare(
|
|
24
|
-
`SELECT c.id as chunkId, -bm25(chunks_fts) as score
|
|
25
|
-
FROM chunks_fts JOIN chunks c ON chunks_fts.rowid = c.rowid
|
|
26
|
-
WHERE chunks_fts MATCH ? AND c.kb_id = ? ORDER BY bm25(chunks_fts) LIMIT ?`,
|
|
27
|
-
)
|
|
28
|
-
.all(ftsQuery, kbId, limit) as BM25Result[];
|
|
29
|
-
}
|
|
15
|
+
function runSearch(db: Database.Database, ftsQuery: string, limit: number, kbId?: string): BM25Result[] {
|
|
16
|
+
if (kbId) {
|
|
30
17
|
return db
|
|
31
18
|
.prepare(
|
|
32
19
|
`SELECT c.id as chunkId, -bm25(chunks_fts) as score
|
|
33
20
|
FROM chunks_fts JOIN chunks c ON chunks_fts.rowid = c.rowid
|
|
34
|
-
WHERE chunks_fts MATCH ? ORDER BY bm25(chunks_fts) LIMIT ?`,
|
|
21
|
+
WHERE chunks_fts MATCH ? AND c.kb_id = ? ORDER BY bm25(chunks_fts) LIMIT ?`,
|
|
35
22
|
)
|
|
36
|
-
.all(ftsQuery, limit) as BM25Result[];
|
|
23
|
+
.all(ftsQuery, kbId, limit) as BM25Result[];
|
|
24
|
+
}
|
|
25
|
+
return db
|
|
26
|
+
.prepare(
|
|
27
|
+
`SELECT c.id as chunkId, -bm25(chunks_fts) as score
|
|
28
|
+
FROM chunks_fts JOIN chunks c ON chunks_fts.rowid = c.rowid
|
|
29
|
+
WHERE chunks_fts MATCH ? ORDER BY bm25(chunks_fts) LIMIT ?`,
|
|
30
|
+
)
|
|
31
|
+
.all(ftsQuery, limit) as BM25Result[];
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function searchBM25(db: Database.Database, query: string, limit = 50, kbId?: string): BM25Result[] {
|
|
35
|
+
const terms = prepareFtsTerms(query);
|
|
36
|
+
if (terms.length === 0) return [];
|
|
37
|
+
try {
|
|
38
|
+
const strict = runSearch(db, terms.join(" AND "), limit, kbId);
|
|
39
|
+
if (strict.length > 0 || terms.length === 1) return strict;
|
|
40
|
+
return runSearch(db, terms.join(" OR "), limit, kbId);
|
|
37
41
|
} catch {
|
|
38
42
|
return [];
|
|
39
43
|
}
|
package/src/search/fusion.ts
CHANGED
|
@@ -3,6 +3,19 @@ export interface FusionResult {
|
|
|
3
3
|
score: number;
|
|
4
4
|
}
|
|
5
5
|
|
|
6
|
+
function normalizeScores(list: Array<{ chunkId: string; score: number }>): Map<string, number> {
|
|
7
|
+
if (list.length === 0) return new Map();
|
|
8
|
+
const scores = list.map((item) => item.score);
|
|
9
|
+
const min = Math.min(...scores);
|
|
10
|
+
const max = Math.max(...scores);
|
|
11
|
+
return new Map(
|
|
12
|
+
list.map((item) => {
|
|
13
|
+
const normalized = max === min ? 1 : (item.score - min) / (max - min);
|
|
14
|
+
return [item.chunkId, normalized];
|
|
15
|
+
}),
|
|
16
|
+
);
|
|
17
|
+
}
|
|
18
|
+
|
|
6
19
|
export function reciprocalRankFusion(lists: Array<{ chunkId: string; score: number }[]>, k = 60): FusionResult[] {
|
|
7
20
|
const scores = new Map<string, number>();
|
|
8
21
|
for (const list of lists) {
|
|
@@ -13,3 +26,23 @@ export function reciprocalRankFusion(lists: Array<{ chunkId: string; score: numb
|
|
|
13
26
|
}
|
|
14
27
|
return [...scores.entries()].map(([chunkId, score]) => ({ chunkId, score })).sort((a, b) => b.score - a.score);
|
|
15
28
|
}
|
|
29
|
+
|
|
30
|
+
export function weightedScoreFusion(
|
|
31
|
+
bm25Results: Array<{ chunkId: string; score: number }>,
|
|
32
|
+
vectorResults: Array<{ chunkId: string; score: number }>,
|
|
33
|
+
weights = { bm25: 0.45, vector: 0.55, overlap: 0.15 },
|
|
34
|
+
): FusionResult[] {
|
|
35
|
+
const bm25Scores = normalizeScores(bm25Results);
|
|
36
|
+
const vectorScores = normalizeScores(vectorResults);
|
|
37
|
+
const ids = new Set([...bm25Scores.keys(), ...vectorScores.keys()]);
|
|
38
|
+
const fused: FusionResult[] = [];
|
|
39
|
+
|
|
40
|
+
for (const chunkId of ids) {
|
|
41
|
+
const bm25 = bm25Scores.get(chunkId) ?? 0;
|
|
42
|
+
const vector = vectorScores.get(chunkId) ?? 0;
|
|
43
|
+
const overlap = bm25 > 0 && vector > 0 ? weights.overlap : 0;
|
|
44
|
+
fused.push({ chunkId, score: weights.bm25 * bm25 + weights.vector * vector + overlap });
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
return fused.sort((a, b) => b.score - a.score);
|
|
48
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { preTokenizeForFTS } from "../indexer/chunker.ts";
|
|
2
|
+
|
|
3
|
+
export const STOP_WORDS = new Set([
|
|
4
|
+
"the",
|
|
5
|
+
"and",
|
|
6
|
+
"or",
|
|
7
|
+
"to",
|
|
8
|
+
"of",
|
|
9
|
+
"in",
|
|
10
|
+
"on",
|
|
11
|
+
"for",
|
|
12
|
+
"with",
|
|
13
|
+
"how",
|
|
14
|
+
"does",
|
|
15
|
+
"what",
|
|
16
|
+
"when",
|
|
17
|
+
"where",
|
|
18
|
+
"why",
|
|
19
|
+
"this",
|
|
20
|
+
"that",
|
|
21
|
+
"from",
|
|
22
|
+
]);
|
|
23
|
+
|
|
24
|
+
const TYPO_CORRECTIONS: Record<string, string> = {
|
|
25
|
+
ho: "how",
|
|
26
|
+
setpup: "setup",
|
|
27
|
+
setpu: "setup",
|
|
28
|
+
seting: "setting",
|
|
29
|
+
configuraiton: "configuration",
|
|
30
|
+
permisson: "permission",
|
|
31
|
+
permisions: "permissions",
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
export function stemToken(token: string): string {
|
|
35
|
+
if (token.length > 4 && token.endsWith("ies")) return `${token.slice(0, -3)}y`;
|
|
36
|
+
if (token.length > 5 && token.endsWith("ing")) return token.slice(0, -3);
|
|
37
|
+
if (token.length > 4 && token.endsWith("ed")) return token.slice(0, -2);
|
|
38
|
+
if (token.length > 3 && token.endsWith("s")) return token.slice(0, -1);
|
|
39
|
+
return token;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export function tokenizeForSearch(text: string): Set<string> {
|
|
43
|
+
const normalized = preTokenizeForFTS(text)
|
|
44
|
+
.toLowerCase()
|
|
45
|
+
.replace(/[-_*"(){}[\]^~:+.#@!\\/<>|&$%?]/g, " ");
|
|
46
|
+
const tokens = normalized
|
|
47
|
+
.split(/\s+/)
|
|
48
|
+
.map((token) => TYPO_CORRECTIONS[token] ?? token)
|
|
49
|
+
.filter((token) => token.length > 1 || /[\u3400-\u4dbf\u4e00-\u9fff]/.test(token));
|
|
50
|
+
return new Set(tokens);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export function signalTokens(tokens: Set<string>): Set<string> {
|
|
54
|
+
return new Set([...tokens].filter((token) => !STOP_WORDS.has(token)).map(stemToken));
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export function normalizedQueryText(query: string): string {
|
|
58
|
+
return [...tokenizeForSearch(query)].join(" ");
|
|
59
|
+
}
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
import { buildChunkEmbeddingText } from "../indexer/chunker.ts";
|
|
2
|
+
import type { Chunk } from "../storage/sqlite.ts";
|
|
3
|
+
import { signalTokens, tokenizeForSearch } from "./query.ts";
|
|
4
|
+
|
|
5
|
+
export const MIN_HYBRID_SCORE = 0.18;
|
|
6
|
+
|
|
7
|
+
export const FILE_TYPE_ALIASES: Record<string, string> = {
|
|
8
|
+
csharp: "csharp",
|
|
9
|
+
cs: "csharp",
|
|
10
|
+
cpp: "cpp",
|
|
11
|
+
hpp: "cpp",
|
|
12
|
+
js: "javascript",
|
|
13
|
+
jsx: "javascript",
|
|
14
|
+
md: "markdown",
|
|
15
|
+
mdx: "markdown",
|
|
16
|
+
php: "php",
|
|
17
|
+
py: "python",
|
|
18
|
+
rb: "ruby",
|
|
19
|
+
sh: "shell",
|
|
20
|
+
ts: "typescript",
|
|
21
|
+
tsx: "typescript",
|
|
22
|
+
yml: "yaml",
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
const SOURCE_FILE_TYPES = new Set([
|
|
26
|
+
"c",
|
|
27
|
+
"cpp",
|
|
28
|
+
"csharp",
|
|
29
|
+
"go",
|
|
30
|
+
"java",
|
|
31
|
+
"javascript",
|
|
32
|
+
"php",
|
|
33
|
+
"python",
|
|
34
|
+
"ruby",
|
|
35
|
+
"rust",
|
|
36
|
+
"typescript",
|
|
37
|
+
]);
|
|
38
|
+
|
|
39
|
+
const TEST_PATH_PATTERNS = ["_test.", ".test.", ".spec.", "/test/", "/tests/", "tests/", "__tests__/", "spec/"];
|
|
40
|
+
|
|
41
|
+
export interface RankingDiagnostics {
|
|
42
|
+
adjusted_score: number;
|
|
43
|
+
base_score: number;
|
|
44
|
+
coverage: number;
|
|
45
|
+
documentation_boost: number;
|
|
46
|
+
is_localization: boolean;
|
|
47
|
+
is_test: boolean;
|
|
48
|
+
localization_penalty: number;
|
|
49
|
+
path_boost: number;
|
|
50
|
+
source_boost: number;
|
|
51
|
+
test_intent: boolean;
|
|
52
|
+
user_guide_intent: boolean;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export function normalizeFileTypeFilter(fileType: string | undefined): string | undefined {
|
|
56
|
+
if (!fileType) return undefined;
|
|
57
|
+
const normalized = fileType.trim().toLowerCase();
|
|
58
|
+
return FILE_TYPE_ALIASES[normalized] ?? normalized;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export function isTestPath(filePath: string): boolean {
|
|
62
|
+
const lower = filePath.toLowerCase();
|
|
63
|
+
return TEST_PATH_PATTERNS.some((pattern) => lower.includes(pattern));
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export function queryAsksForTests(queryTokens: Set<string>): boolean {
|
|
67
|
+
const signals = signalTokens(queryTokens);
|
|
68
|
+
return ["assert", "fixture", "spec", "test", "testing"].some((token) => signals.has(token));
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function queryAsksForUserGuide(queryTokens: Set<string>): boolean {
|
|
72
|
+
const signals = signalTokens(queryTokens);
|
|
73
|
+
return ["config", "configuration", "install", "onboard", "quickstart", "setup", "start", "usage"].some((token) =>
|
|
74
|
+
signals.has(token),
|
|
75
|
+
);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function queryAsksForLocalization(queryTokens: Set<string>): boolean {
|
|
79
|
+
const signals = signalTokens(queryTokens);
|
|
80
|
+
return ["i18n", "lang", "language", "locale", "localization", "translation"].some((token) => signals.has(token));
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export function isLocalizationPath(filePath: string, fileType: string): boolean {
|
|
84
|
+
if (!["json", "markdown", "text", "toml", "yaml"].includes(fileType)) return false;
|
|
85
|
+
const lower = filePath.toLowerCase();
|
|
86
|
+
return (
|
|
87
|
+
lower.includes("/locale/") ||
|
|
88
|
+
lower.startsWith("locale/") ||
|
|
89
|
+
lower.includes("/locales/") ||
|
|
90
|
+
lower.startsWith("locales/") ||
|
|
91
|
+
lower.includes("/lang/") ||
|
|
92
|
+
lower.startsWith("lang/") ||
|
|
93
|
+
lower.includes("/i18n/") ||
|
|
94
|
+
lower.startsWith("i18n/") ||
|
|
95
|
+
lower.includes("/translations/") ||
|
|
96
|
+
lower.startsWith("translations/")
|
|
97
|
+
);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export function queryCoverage(text: string, queryTokens: Set<string>): number {
|
|
101
|
+
const textTokens = signalTokens(tokenizeForSearch(text));
|
|
102
|
+
const signals = signalTokens(queryTokens);
|
|
103
|
+
if (signals.size === 0) return 0;
|
|
104
|
+
let matched = 0;
|
|
105
|
+
for (const token of signals) {
|
|
106
|
+
if (textTokens.has(token)) matched++;
|
|
107
|
+
}
|
|
108
|
+
return matched / signals.size;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
export function pathTokenBoost(filePath: string, queryTokens: Set<string>): number {
|
|
112
|
+
const pathTokens = signalTokens(tokenizeForSearch(filePath.replace(/[./_-]/g, " ")));
|
|
113
|
+
const signals = signalTokens(queryTokens);
|
|
114
|
+
let matched = 0;
|
|
115
|
+
for (const token of signals) {
|
|
116
|
+
if (pathTokens.has(token)) matched++;
|
|
117
|
+
}
|
|
118
|
+
if (matched === 0) return 0;
|
|
119
|
+
return Math.min(0.45, 0.18 * matched);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
export function basenameTokenBoost(filePath: string, queryTokens: Set<string>): number {
|
|
123
|
+
const basename = filePath.split("/").at(-1) ?? filePath;
|
|
124
|
+
const stem = basename.replace(/\.[^.]+$/, "").replace(/_test$|\.test$|\.spec$/i, "");
|
|
125
|
+
const baseTokens = signalTokens(tokenizeForSearch(stem.replace(/[._-]/g, " ")));
|
|
126
|
+
const signals = signalTokens(queryTokens);
|
|
127
|
+
if (baseTokens.size === 0 || signals.size === 0) return 0;
|
|
128
|
+
for (const token of signals) {
|
|
129
|
+
if (baseTokens.has(token)) return 0.32;
|
|
130
|
+
}
|
|
131
|
+
return 0;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
export function sourceFileBoost(chunk: Pick<Chunk, "file_path" | "file_type">, queryTokens: Set<string>): number {
|
|
135
|
+
if (!SOURCE_FILE_TYPES.has(chunk.file_type)) return 0;
|
|
136
|
+
if (queryAsksForUserGuide(queryTokens)) return Math.min(0.05, pathTokenBoost(chunk.file_path, queryTokens));
|
|
137
|
+
const boost = basenameTokenBoost(chunk.file_path, queryTokens);
|
|
138
|
+
if (boost > 0) return boost;
|
|
139
|
+
const boostFromPath = pathTokenBoost(chunk.file_path, queryTokens);
|
|
140
|
+
return Math.min(0.22, boostFromPath);
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
function documentationBoost(chunk: Pick<Chunk, "file_path" | "file_type">, queryTokens: Set<string>): number {
|
|
144
|
+
if (!queryAsksForUserGuide(queryTokens)) return 0;
|
|
145
|
+
const lower = chunk.file_path.toLowerCase();
|
|
146
|
+
if (chunk.file_type !== "markdown") return 0;
|
|
147
|
+
if (
|
|
148
|
+
lower.includes("install") ||
|
|
149
|
+
lower.includes("readme") ||
|
|
150
|
+
lower.includes("getting-started") ||
|
|
151
|
+
lower.includes("quickstart")
|
|
152
|
+
) {
|
|
153
|
+
return 0.85;
|
|
154
|
+
}
|
|
155
|
+
if (lower.startsWith("docs/")) return 0.35;
|
|
156
|
+
return 0;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
export function scoreChunkForQuery(baseScore: number, chunk: Chunk, queryTokens: Set<string>): RankingDiagnostics {
|
|
160
|
+
const guideIntent = queryAsksForUserGuide(queryTokens);
|
|
161
|
+
const pathBoost =
|
|
162
|
+
guideIntent && SOURCE_FILE_TYPES.has(chunk.file_type)
|
|
163
|
+
? Math.min(0.08, pathTokenBoost(chunk.file_path, queryTokens))
|
|
164
|
+
: pathTokenBoost(chunk.file_path, queryTokens);
|
|
165
|
+
const sourceBoost = sourceFileBoost(chunk, queryTokens);
|
|
166
|
+
const docBoost = documentationBoost(chunk, queryTokens);
|
|
167
|
+
const testIntent = queryAsksForTests(queryTokens);
|
|
168
|
+
const test = isTestPath(chunk.file_path);
|
|
169
|
+
const localization = isLocalizationPath(chunk.file_path, chunk.file_type);
|
|
170
|
+
const localizationPenalty = localization && !queryAsksForLocalization(queryTokens) ? 0.2 : 1;
|
|
171
|
+
let adjustedScore = baseScore + pathBoost + sourceBoost + docBoost;
|
|
172
|
+
if (test && !testIntent) adjustedScore *= 0.48;
|
|
173
|
+
if (!test && testIntent) adjustedScore *= 0.88;
|
|
174
|
+
adjustedScore *= localizationPenalty;
|
|
175
|
+
return {
|
|
176
|
+
adjusted_score: adjustedScore,
|
|
177
|
+
base_score: baseScore,
|
|
178
|
+
coverage: queryCoverage(buildChunkEmbeddingText(chunk), queryTokens),
|
|
179
|
+
documentation_boost: docBoost,
|
|
180
|
+
is_localization: localization,
|
|
181
|
+
is_test: test,
|
|
182
|
+
localization_penalty: localizationPenalty,
|
|
183
|
+
path_boost: pathBoost,
|
|
184
|
+
source_boost: sourceBoost,
|
|
185
|
+
test_intent: testIntent,
|
|
186
|
+
user_guide_intent: guideIntent,
|
|
187
|
+
};
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
export function hasEnoughLexicalEvidence(chunk: Chunk, queryTokens: Set<string>): boolean {
|
|
191
|
+
const signals = signalTokens(queryTokens);
|
|
192
|
+
if (signals.size <= 1) return true;
|
|
193
|
+
if (sourceFileBoost(chunk, queryTokens) >= 0.22) return true;
|
|
194
|
+
const coverage = queryCoverage(buildChunkEmbeddingText(chunk), queryTokens);
|
|
195
|
+
if (signals.size <= 3) return coverage >= 1 / signals.size;
|
|
196
|
+
return coverage >= 0.34;
|
|
197
|
+
}
|