pi-knowledge 0.4.2 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/README.md +4 -2
- package/dist/index.d.ts +38 -0
- package/dist/index.js +435 -0
- package/dist/src/diagnostics/health.d.ts +19 -0
- package/dist/src/diagnostics/health.js +110 -0
- package/dist/src/embedding/provider.d.ts +5 -0
- package/dist/src/embedding/provider.js +99 -0
- package/dist/src/embedding/vectors.d.ts +15 -0
- package/dist/src/embedding/vectors.js +111 -0
- package/dist/src/engine.d.ts +90 -0
- package/dist/src/engine.js +1370 -0
- package/dist/src/indexer/chunker.d.ts +53 -0
- package/dist/src/indexer/chunker.js +498 -0
- package/dist/src/indexer/chunkers/code-ast.d.ts +3 -0
- package/dist/src/indexer/chunkers/code-ast.js +179 -0
- package/dist/src/model-worker-client.d.ts +10 -0
- package/dist/src/model-worker-client.js +138 -0
- package/dist/src/model-worker.d.ts +1 -0
- package/dist/src/model-worker.js +61 -0
- package/dist/src/search/bm25.d.ts +8 -0
- package/dist/src/search/bm25.js +38 -0
- package/dist/src/search/fusion.d.ts +19 -0
- package/dist/src/search/fusion.js +34 -0
- package/dist/src/search/query.d.ts +5 -0
- package/dist/src/search/query.js +57 -0
- package/dist/src/search/ranking.d.ts +27 -0
- package/dist/src/search/ranking.js +180 -0
- package/dist/src/search/reranker.d.ts +10 -0
- package/dist/src/search/reranker.js +66 -0
- package/dist/src/search/vector.d.ts +14 -0
- package/dist/src/search/vector.js +58 -0
- package/dist/src/storage/sqlite.d.ts +99 -0
- package/dist/src/storage/sqlite.js +330 -0
- package/dist/src/watcher/file-watcher.d.ts +4 -0
- package/dist/src/watcher/file-watcher.js +108 -0
- package/extension.js +7 -0
- package/package.json +10 -5
- package/index.ts +0 -500
- package/src/diagnostics/health.ts +0 -145
- package/src/embedding/provider.ts +0 -106
- package/src/embedding/vectors.ts +0 -119
- package/src/engine.ts +0 -1592
- package/src/indexer/chunker.ts +0 -612
- package/src/indexer/chunkers/code-ast.ts +0 -209
- package/src/model-worker-client.ts +0 -146
- package/src/model-worker.ts +0 -88
- package/src/search/bm25.ts +0 -55
- package/src/search/fusion.ts +0 -48
- package/src/search/query.ts +0 -59
- package/src/search/ranking.ts +0 -201
- package/src/search/reranker.ts +0 -74
- package/src/search/vector.ts +0 -81
- package/src/storage/sqlite.ts +0 -489
- package/src/watcher/file-watcher.ts +0 -116
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
import { buildChunkEmbeddingText, contentHash, preTokenizeForFTS } from "../chunker.js";
|
|
2
|
+
const LANGS = {
|
|
3
|
+
typescript: {
|
|
4
|
+
grammar: async () => {
|
|
5
|
+
const m = await import("tree-sitter-typescript");
|
|
6
|
+
return m.typescript ?? m.default?.typescript;
|
|
7
|
+
},
|
|
8
|
+
fnTypes: new Set(["function_declaration", "arrow_function", "generator_function_declaration"]),
|
|
9
|
+
classTypes: new Set(["class_declaration", "interface_declaration"]),
|
|
10
|
+
classBody: "class_body",
|
|
11
|
+
fileType: "typescript",
|
|
12
|
+
},
|
|
13
|
+
javascript: {
|
|
14
|
+
grammar: async () => {
|
|
15
|
+
const m = await import("tree-sitter-typescript");
|
|
16
|
+
return m.typescript ?? m.default?.typescript;
|
|
17
|
+
},
|
|
18
|
+
fnTypes: new Set(["function_declaration", "arrow_function", "generator_function_declaration"]),
|
|
19
|
+
classTypes: new Set(["class_declaration"]),
|
|
20
|
+
classBody: "class_body",
|
|
21
|
+
fileType: "javascript",
|
|
22
|
+
},
|
|
23
|
+
python: {
|
|
24
|
+
grammar: async () => {
|
|
25
|
+
const m = await import("tree-sitter-python");
|
|
26
|
+
return m.default ?? m;
|
|
27
|
+
},
|
|
28
|
+
fnTypes: new Set(["function_definition"]),
|
|
29
|
+
classTypes: new Set(["class_definition"]),
|
|
30
|
+
classBody: "block",
|
|
31
|
+
fileType: "python",
|
|
32
|
+
},
|
|
33
|
+
go: {
|
|
34
|
+
grammar: async () => {
|
|
35
|
+
const m = await import("tree-sitter-go");
|
|
36
|
+
return m.default ?? m;
|
|
37
|
+
},
|
|
38
|
+
fnTypes: new Set(["function_declaration", "method_declaration"]),
|
|
39
|
+
classTypes: new Set(["type_declaration"]),
|
|
40
|
+
fileType: "go",
|
|
41
|
+
},
|
|
42
|
+
rust: {
|
|
43
|
+
grammar: async () => {
|
|
44
|
+
const m = await import("tree-sitter-rust");
|
|
45
|
+
return m.default ?? m;
|
|
46
|
+
},
|
|
47
|
+
fnTypes: new Set(["function_item"]),
|
|
48
|
+
classTypes: new Set(["struct_item", "enum_item"]),
|
|
49
|
+
methodContainer: "impl_item",
|
|
50
|
+
fileType: "rust",
|
|
51
|
+
},
|
|
52
|
+
java: {
|
|
53
|
+
grammar: async () => {
|
|
54
|
+
const m = await import("tree-sitter-java");
|
|
55
|
+
return m.default ?? m;
|
|
56
|
+
},
|
|
57
|
+
fnTypes: new Set(["method_declaration", "constructor_declaration"]),
|
|
58
|
+
classTypes: new Set(["class_declaration", "interface_declaration", "enum_declaration"]),
|
|
59
|
+
classBody: "class_body",
|
|
60
|
+
fileType: "java",
|
|
61
|
+
},
|
|
62
|
+
};
|
|
63
|
+
function getName(node) {
|
|
64
|
+
return node.childForFieldName("name")?.text ?? node.childForFieldName("type")?.text ?? "anonymous";
|
|
65
|
+
}
|
|
66
|
+
function collectChunks(root, config) {
|
|
67
|
+
const chunks = [];
|
|
68
|
+
function walk(node) {
|
|
69
|
+
if (node.type === "export_statement") {
|
|
70
|
+
for (const child of node.children) {
|
|
71
|
+
if (config.fnTypes.has(child.type)) {
|
|
72
|
+
chunks.push({
|
|
73
|
+
name: getName(child),
|
|
74
|
+
text: node.text,
|
|
75
|
+
start: node.startPosition.row + 1,
|
|
76
|
+
end: node.endPosition.row + 1,
|
|
77
|
+
});
|
|
78
|
+
return;
|
|
79
|
+
}
|
|
80
|
+
if (config.classTypes.has(child.type)) {
|
|
81
|
+
walk(child);
|
|
82
|
+
return;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
if (node.type === "decorated_definition") {
|
|
87
|
+
for (const child of node.children) {
|
|
88
|
+
if (config.fnTypes.has(child.type) || config.classTypes.has(child.type)) {
|
|
89
|
+
chunks.push({
|
|
90
|
+
name: getName(child),
|
|
91
|
+
text: node.text,
|
|
92
|
+
start: node.startPosition.row + 1,
|
|
93
|
+
end: node.endPosition.row + 1,
|
|
94
|
+
});
|
|
95
|
+
return;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
if (config.fnTypes.has(node.type)) {
|
|
100
|
+
chunks.push({
|
|
101
|
+
name: getName(node),
|
|
102
|
+
text: node.text,
|
|
103
|
+
start: node.startPosition.row + 1,
|
|
104
|
+
end: node.endPosition.row + 1,
|
|
105
|
+
});
|
|
106
|
+
return;
|
|
107
|
+
}
|
|
108
|
+
if (config.classTypes.has(node.type)) {
|
|
109
|
+
const name = getName(node);
|
|
110
|
+
let hasChildren = false;
|
|
111
|
+
for (const child of node.children) {
|
|
112
|
+
if (config.classBody && child.type === config.classBody) {
|
|
113
|
+
for (const m of child.children) {
|
|
114
|
+
if (config.fnTypes.has(m.type) || m.type === "method_definition") {
|
|
115
|
+
hasChildren = true;
|
|
116
|
+
chunks.push({
|
|
117
|
+
name: `${name}.${getName(m)}`,
|
|
118
|
+
text: m.text,
|
|
119
|
+
start: m.startPosition.row + 1,
|
|
120
|
+
end: m.endPosition.row + 1,
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
if (!hasChildren)
|
|
127
|
+
chunks.push({ name, text: node.text, start: node.startPosition.row + 1, end: node.endPosition.row + 1 });
|
|
128
|
+
return;
|
|
129
|
+
}
|
|
130
|
+
if (config.methodContainer && node.type === config.methodContainer) {
|
|
131
|
+
const typeName = getName(node);
|
|
132
|
+
for (const child of node.children) {
|
|
133
|
+
if (child.type === "declaration_list") {
|
|
134
|
+
for (const m of child.children) {
|
|
135
|
+
if (config.fnTypes.has(m.type))
|
|
136
|
+
chunks.push({
|
|
137
|
+
name: `${typeName}.${getName(m)}`,
|
|
138
|
+
text: m.text,
|
|
139
|
+
start: m.startPosition.row + 1,
|
|
140
|
+
end: m.endPosition.row + 1,
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
for (const child of node.children)
|
|
148
|
+
walk(child);
|
|
149
|
+
}
|
|
150
|
+
walk(root);
|
|
151
|
+
return chunks;
|
|
152
|
+
}
|
|
153
|
+
export async function chunkWithAST(content, filePath, language) {
|
|
154
|
+
const config = LANGS[language];
|
|
155
|
+
if (!config)
|
|
156
|
+
return [];
|
|
157
|
+
const Parser = (await import("tree-sitter")).default;
|
|
158
|
+
const grammar = await config.grammar();
|
|
159
|
+
const parser = new Parser();
|
|
160
|
+
parser.setLanguage(grammar);
|
|
161
|
+
const tree = parser.parse(content);
|
|
162
|
+
const fns = collectChunks(tree.rootNode, config);
|
|
163
|
+
if (fns.length === 0)
|
|
164
|
+
return [];
|
|
165
|
+
return fns.map((fn) => {
|
|
166
|
+
const chunk = {
|
|
167
|
+
content_hash: contentHash(fn.text),
|
|
168
|
+
content: fn.text,
|
|
169
|
+
content_tokenized: "",
|
|
170
|
+
file_path: filePath,
|
|
171
|
+
file_type: config.fileType,
|
|
172
|
+
start_line: fn.start,
|
|
173
|
+
end_line: fn.end,
|
|
174
|
+
metadata_json: JSON.stringify({ function_name: fn.name }),
|
|
175
|
+
};
|
|
176
|
+
return { ...chunk, content_tokenized: preTokenizeForFTS(buildChunkEmbeddingText(chunk)) };
|
|
177
|
+
});
|
|
178
|
+
}
|
|
179
|
+
export const SUPPORTED_LANGUAGES = new Set(Object.keys(LANGS));
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export declare function embedInModelWorker(texts: string[], prefix: "query" | "passage", signal?: AbortSignal): Promise<Float32Array[]>;
|
|
2
|
+
export interface RerankWorkerCandidate {
|
|
3
|
+
chunkId: string;
|
|
4
|
+
content: string;
|
|
5
|
+
}
|
|
6
|
+
export declare function rerankInModelWorker(query: string, candidates: RerankWorkerCandidate[], topK: number): Promise<Array<{
|
|
7
|
+
chunkId: string;
|
|
8
|
+
score: number;
|
|
9
|
+
}>>;
|
|
10
|
+
export declare function shutdownModelWorker(): void;
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
import { fork } from "node:child_process";
|
|
2
|
+
import { basename } from "node:path";
|
|
3
|
+
import { fileURLToPath } from "node:url";
|
|
4
|
+
let worker = null;
|
|
5
|
+
let nextRequestId = 1;
|
|
6
|
+
const pending = new Map();
|
|
7
|
+
function rejectPending(error) {
|
|
8
|
+
for (const request of pending.values()) {
|
|
9
|
+
request.reject(error);
|
|
10
|
+
}
|
|
11
|
+
pending.clear();
|
|
12
|
+
}
|
|
13
|
+
function getWorkerPath() {
|
|
14
|
+
const workerFile = fileURLToPath(import.meta.url).endsWith(".js") ? "model-worker.js" : "model-worker.ts";
|
|
15
|
+
return fileURLToPath(new URL(`./${workerFile}`, import.meta.url));
|
|
16
|
+
}
|
|
17
|
+
function getWorkerExecArgv() {
|
|
18
|
+
return fileURLToPath(import.meta.url).endsWith(".js") ? [] : ["--experimental-strip-types"];
|
|
19
|
+
}
|
|
20
|
+
function getNodeExecPath() {
|
|
21
|
+
const configured = process.env.PI_KNOWLEDGE_NODE_PATH?.trim();
|
|
22
|
+
if (configured)
|
|
23
|
+
return configured;
|
|
24
|
+
const execName = basename(process.execPath).toLowerCase();
|
|
25
|
+
return execName === "node" || execName === "node.exe" ? process.execPath : "node";
|
|
26
|
+
}
|
|
27
|
+
function getWorker() {
|
|
28
|
+
if (worker?.connected)
|
|
29
|
+
return worker;
|
|
30
|
+
const workerPath = getWorkerPath();
|
|
31
|
+
worker = fork(workerPath, {
|
|
32
|
+
execPath: getNodeExecPath(),
|
|
33
|
+
execArgv: getWorkerExecArgv(),
|
|
34
|
+
stdio: ["ignore", "ignore", "ignore", "ipc"],
|
|
35
|
+
env: process.env,
|
|
36
|
+
});
|
|
37
|
+
const child = worker;
|
|
38
|
+
worker.on("message", (message) => {
|
|
39
|
+
const request = pending.get(message.id);
|
|
40
|
+
if (!request)
|
|
41
|
+
return;
|
|
42
|
+
pending.delete(message.id);
|
|
43
|
+
if (message.error) {
|
|
44
|
+
request.reject(new Error(message.error));
|
|
45
|
+
}
|
|
46
|
+
else {
|
|
47
|
+
request.resolve(message.result);
|
|
48
|
+
}
|
|
49
|
+
});
|
|
50
|
+
worker.on("exit", (code, signal) => {
|
|
51
|
+
if (worker !== child)
|
|
52
|
+
return;
|
|
53
|
+
worker = null;
|
|
54
|
+
if (pending.size > 0) {
|
|
55
|
+
rejectPending(new Error(`Model worker exited before responding (code ${code ?? "null"}, signal ${signal ?? "null"})`));
|
|
56
|
+
}
|
|
57
|
+
});
|
|
58
|
+
worker.on("error", (error) => {
|
|
59
|
+
if (worker !== child)
|
|
60
|
+
return;
|
|
61
|
+
worker = null;
|
|
62
|
+
rejectPending(error);
|
|
63
|
+
});
|
|
64
|
+
return worker;
|
|
65
|
+
}
|
|
66
|
+
async function requestModelWorker(message, signal) {
|
|
67
|
+
if (signal?.aborted)
|
|
68
|
+
throw new Error("Cancelled");
|
|
69
|
+
const child = getWorker();
|
|
70
|
+
const id = nextRequestId++;
|
|
71
|
+
return new Promise((resolve, reject) => {
|
|
72
|
+
let abortHandler;
|
|
73
|
+
pending.set(id, { resolve, reject });
|
|
74
|
+
if (signal) {
|
|
75
|
+
abortHandler = () => {
|
|
76
|
+
pending.delete(id);
|
|
77
|
+
reject(new Error("Cancelled"));
|
|
78
|
+
shutdownModelWorker();
|
|
79
|
+
};
|
|
80
|
+
signal.addEventListener("abort", abortHandler, { once: true });
|
|
81
|
+
}
|
|
82
|
+
const cleanup = () => {
|
|
83
|
+
if (abortHandler)
|
|
84
|
+
signal?.removeEventListener("abort", abortHandler);
|
|
85
|
+
};
|
|
86
|
+
const originalResolve = resolve;
|
|
87
|
+
const originalReject = reject;
|
|
88
|
+
pending.set(id, {
|
|
89
|
+
resolve(value) {
|
|
90
|
+
cleanup();
|
|
91
|
+
originalResolve(value);
|
|
92
|
+
},
|
|
93
|
+
reject(error) {
|
|
94
|
+
cleanup();
|
|
95
|
+
originalReject(error);
|
|
96
|
+
},
|
|
97
|
+
});
|
|
98
|
+
child.send({ id, ...message }, (error) => {
|
|
99
|
+
if (!error)
|
|
100
|
+
return;
|
|
101
|
+
pending.delete(id);
|
|
102
|
+
cleanup();
|
|
103
|
+
reject(error);
|
|
104
|
+
});
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
export async function embedInModelWorker(texts, prefix, signal) {
|
|
108
|
+
const result = await requestModelWorker({ type: "embed", texts, prefix }, signal);
|
|
109
|
+
if (!Array.isArray(result))
|
|
110
|
+
throw new Error("Invalid embedding worker response");
|
|
111
|
+
return result.map((vector) => {
|
|
112
|
+
if (!Array.isArray(vector))
|
|
113
|
+
throw new Error("Invalid embedding vector from worker");
|
|
114
|
+
return new Float32Array(vector);
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
export async function rerankInModelWorker(query, candidates, topK) {
|
|
118
|
+
const result = await requestModelWorker({ type: "rerank", query, candidates, topK });
|
|
119
|
+
if (!Array.isArray(result))
|
|
120
|
+
throw new Error("Invalid reranker worker response");
|
|
121
|
+
return result.map((item) => {
|
|
122
|
+
if (typeof item !== "object" ||
|
|
123
|
+
item === null ||
|
|
124
|
+
typeof item.chunkId !== "string" ||
|
|
125
|
+
typeof item.score !== "number") {
|
|
126
|
+
throw new Error("Invalid reranker result from worker");
|
|
127
|
+
}
|
|
128
|
+
return item;
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
export function shutdownModelWorker() {
|
|
132
|
+
const child = worker;
|
|
133
|
+
worker = null;
|
|
134
|
+
rejectPending(new Error("Model worker shut down"));
|
|
135
|
+
if (child && !child.killed) {
|
|
136
|
+
child.kill("SIGKILL");
|
|
137
|
+
}
|
|
138
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { join } from "node:path";
|
|
2
|
+
import { getDefaultKnowledgeDir } from "./storage/sqlite.js";
|
|
3
|
+
let embeddingPipeline = null;
|
|
4
|
+
let rerankerPipeline = null;
|
|
5
|
+
function getModelCacheDir() {
|
|
6
|
+
return process.env.PI_KNOWLEDGE_MODEL_CACHE_DIR ?? join(getDefaultKnowledgeDir(), "models");
|
|
7
|
+
}
|
|
8
|
+
async function loadEmbeddingPipeline() {
|
|
9
|
+
if (embeddingPipeline)
|
|
10
|
+
return embeddingPipeline;
|
|
11
|
+
const { pipeline, env } = await import("@huggingface/transformers");
|
|
12
|
+
env.cacheDir = getModelCacheDir();
|
|
13
|
+
const createPipeline = pipeline;
|
|
14
|
+
const loaded = (await createPipeline("feature-extraction", "Xenova/multilingual-e5-small", {
|
|
15
|
+
quantized: true,
|
|
16
|
+
dtype: "fp32",
|
|
17
|
+
}));
|
|
18
|
+
embeddingPipeline = loaded;
|
|
19
|
+
return loaded;
|
|
20
|
+
}
|
|
21
|
+
async function loadRerankerPipeline() {
|
|
22
|
+
if (rerankerPipeline)
|
|
23
|
+
return rerankerPipeline;
|
|
24
|
+
const { pipeline, env } = await import("@huggingface/transformers");
|
|
25
|
+
env.cacheDir = getModelCacheDir();
|
|
26
|
+
const createPipeline = pipeline;
|
|
27
|
+
const loaded = (await createPipeline("text-classification", "Xenova/ms-marco-MiniLM-L-4-v2"));
|
|
28
|
+
rerankerPipeline = loaded;
|
|
29
|
+
return loaded;
|
|
30
|
+
}
|
|
31
|
+
async function handleEmbed(request) {
|
|
32
|
+
const pipe = await loadEmbeddingPipeline();
|
|
33
|
+
const vectors = [];
|
|
34
|
+
for (const text of request.texts) {
|
|
35
|
+
const output = await pipe(`${request.prefix}: ${text}`, { pooling: "mean", normalize: true });
|
|
36
|
+
vectors.push(Array.from(output.data));
|
|
37
|
+
}
|
|
38
|
+
return vectors;
|
|
39
|
+
}
|
|
40
|
+
async function handleRerank(request) {
|
|
41
|
+
const pipe = await loadRerankerPipeline();
|
|
42
|
+
const results = [];
|
|
43
|
+
for (const candidate of request.candidates) {
|
|
44
|
+
const output = await pipe({ text: request.query, text_pair: candidate.content });
|
|
45
|
+
const score = Array.isArray(output) ? (output[0]?.score ?? 0) : (output?.score ?? 0);
|
|
46
|
+
results.push({ chunkId: candidate.chunkId, score });
|
|
47
|
+
}
|
|
48
|
+
results.sort((a, b) => b.score - a.score);
|
|
49
|
+
return results.slice(0, request.topK);
|
|
50
|
+
}
|
|
51
|
+
process.on("message", (request) => {
|
|
52
|
+
void (async () => {
|
|
53
|
+
try {
|
|
54
|
+
const result = request.type === "embed" ? await handleEmbed(request) : await handleRerank(request);
|
|
55
|
+
process.send?.({ id: request.id, result });
|
|
56
|
+
}
|
|
57
|
+
catch (error) {
|
|
58
|
+
process.send?.({ id: request.id, error: error instanceof Error ? error.message : String(error) });
|
|
59
|
+
}
|
|
60
|
+
})();
|
|
61
|
+
});
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type Database from "better-sqlite3";
|
|
2
|
+
export interface BM25Result {
|
|
3
|
+
chunkId: string;
|
|
4
|
+
score: number;
|
|
5
|
+
}
|
|
6
|
+
export declare function searchBM25(db: Database.Database, query: string, limit?: number, kbId?: string, options?: {
|
|
7
|
+
allowOrFallback?: boolean;
|
|
8
|
+
}): BM25Result[];
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { normalizedQueryText } from "./query.js";
|
|
2
|
+
function prepareFtsTerms(query) {
|
|
3
|
+
return normalizedQueryText(query)
|
|
4
|
+
.split(/\s+/)
|
|
5
|
+
.filter((t) => t.length > 0);
|
|
6
|
+
}
|
|
7
|
+
function quoteFtsTerm(term) {
|
|
8
|
+
return `"${term.replace(/"/g, '""')}"`;
|
|
9
|
+
}
|
|
10
|
+
function runSearch(db, ftsQuery, limit, kbId) {
|
|
11
|
+
if (kbId) {
|
|
12
|
+
return db
|
|
13
|
+
.prepare(`SELECT c.id as chunkId, -bm25(chunks_fts) as score
|
|
14
|
+
FROM chunks_fts JOIN chunks c ON chunks_fts.rowid = c.rowid
|
|
15
|
+
WHERE chunks_fts MATCH ? AND c.kb_id = ? ORDER BY bm25(chunks_fts) LIMIT ?`)
|
|
16
|
+
.all(ftsQuery, kbId, limit);
|
|
17
|
+
}
|
|
18
|
+
return db
|
|
19
|
+
.prepare(`SELECT c.id as chunkId, -bm25(chunks_fts) as score
|
|
20
|
+
FROM chunks_fts JOIN chunks c ON chunks_fts.rowid = c.rowid
|
|
21
|
+
WHERE chunks_fts MATCH ? ORDER BY bm25(chunks_fts) LIMIT ?`)
|
|
22
|
+
.all(ftsQuery, limit);
|
|
23
|
+
}
|
|
24
|
+
export function searchBM25(db, query, limit = 50, kbId, options = {}) {
|
|
25
|
+
const terms = prepareFtsTerms(query);
|
|
26
|
+
if (terms.length === 0)
|
|
27
|
+
return [];
|
|
28
|
+
try {
|
|
29
|
+
const quotedTerms = terms.map(quoteFtsTerm);
|
|
30
|
+
const strict = runSearch(db, quotedTerms.join(" AND "), limit, kbId);
|
|
31
|
+
if (strict.length > 0 || terms.length === 1 || options.allowOrFallback === false)
|
|
32
|
+
return strict;
|
|
33
|
+
return runSearch(db, quotedTerms.join(" OR "), limit, kbId);
|
|
34
|
+
}
|
|
35
|
+
catch {
|
|
36
|
+
return [];
|
|
37
|
+
}
|
|
38
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
export interface FusionResult {
|
|
2
|
+
chunkId: string;
|
|
3
|
+
score: number;
|
|
4
|
+
}
|
|
5
|
+
export declare function reciprocalRankFusion(lists: Array<{
|
|
6
|
+
chunkId: string;
|
|
7
|
+
score: number;
|
|
8
|
+
}[]>, k?: number): FusionResult[];
|
|
9
|
+
export declare function weightedScoreFusion(bm25Results: Array<{
|
|
10
|
+
chunkId: string;
|
|
11
|
+
score: number;
|
|
12
|
+
}>, vectorResults: Array<{
|
|
13
|
+
chunkId: string;
|
|
14
|
+
score: number;
|
|
15
|
+
}>, weights?: {
|
|
16
|
+
bm25: number;
|
|
17
|
+
vector: number;
|
|
18
|
+
overlap: number;
|
|
19
|
+
}): FusionResult[];
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
function normalizeScores(list) {
|
|
2
|
+
if (list.length === 0)
|
|
3
|
+
return new Map();
|
|
4
|
+
const scores = list.map((item) => item.score);
|
|
5
|
+
const min = Math.min(...scores);
|
|
6
|
+
const max = Math.max(...scores);
|
|
7
|
+
return new Map(list.map((item) => {
|
|
8
|
+
const normalized = max === min ? 1 : (item.score - min) / (max - min);
|
|
9
|
+
return [item.chunkId, normalized];
|
|
10
|
+
}));
|
|
11
|
+
}
|
|
12
|
+
export function reciprocalRankFusion(lists, k = 60) {
|
|
13
|
+
const scores = new Map();
|
|
14
|
+
for (const list of lists) {
|
|
15
|
+
for (let rank = 0; rank < list.length; rank++) {
|
|
16
|
+
const { chunkId } = list[rank];
|
|
17
|
+
scores.set(chunkId, (scores.get(chunkId) ?? 0) + 1 / (k + rank + 1));
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
return [...scores.entries()].map(([chunkId, score]) => ({ chunkId, score })).sort((a, b) => b.score - a.score);
|
|
21
|
+
}
|
|
22
|
+
export function weightedScoreFusion(bm25Results, vectorResults, weights = { bm25: 0.45, vector: 0.55, overlap: 0.15 }) {
|
|
23
|
+
const bm25Scores = normalizeScores(bm25Results);
|
|
24
|
+
const vectorScores = normalizeScores(vectorResults);
|
|
25
|
+
const ids = new Set([...bm25Scores.keys(), ...vectorScores.keys()]);
|
|
26
|
+
const fused = [];
|
|
27
|
+
for (const chunkId of ids) {
|
|
28
|
+
const bm25 = bm25Scores.get(chunkId) ?? 0;
|
|
29
|
+
const vector = vectorScores.get(chunkId) ?? 0;
|
|
30
|
+
const overlap = bm25 > 0 && vector > 0 ? weights.overlap : 0;
|
|
31
|
+
fused.push({ chunkId, score: weights.bm25 * bm25 + weights.vector * vector + overlap });
|
|
32
|
+
}
|
|
33
|
+
return fused.sort((a, b) => b.score - a.score);
|
|
34
|
+
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
export declare const STOP_WORDS: Set<string>;
|
|
2
|
+
export declare function stemToken(token: string): string;
|
|
3
|
+
export declare function tokenizeForSearch(text: string): Set<string>;
|
|
4
|
+
export declare function signalTokens(tokens: Set<string>): Set<string>;
|
|
5
|
+
export declare function normalizedQueryText(query: string): string;
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { preTokenizeForFTS } from "../indexer/chunker.js";
|
|
2
|
+
export const STOP_WORDS = new Set([
|
|
3
|
+
"the",
|
|
4
|
+
"and",
|
|
5
|
+
"or",
|
|
6
|
+
"to",
|
|
7
|
+
"of",
|
|
8
|
+
"in",
|
|
9
|
+
"on",
|
|
10
|
+
"for",
|
|
11
|
+
"with",
|
|
12
|
+
"how",
|
|
13
|
+
"does",
|
|
14
|
+
"what",
|
|
15
|
+
"when",
|
|
16
|
+
"where",
|
|
17
|
+
"why",
|
|
18
|
+
"this",
|
|
19
|
+
"that",
|
|
20
|
+
"from",
|
|
21
|
+
]);
|
|
22
|
+
const TYPO_CORRECTIONS = {
|
|
23
|
+
ho: "how",
|
|
24
|
+
setpup: "setup",
|
|
25
|
+
setpu: "setup",
|
|
26
|
+
seting: "setting",
|
|
27
|
+
configuraiton: "configuration",
|
|
28
|
+
permisson: "permission",
|
|
29
|
+
permisions: "permissions",
|
|
30
|
+
};
|
|
31
|
+
export function stemToken(token) {
|
|
32
|
+
if (token.length > 4 && token.endsWith("ies"))
|
|
33
|
+
return `${token.slice(0, -3)}y`;
|
|
34
|
+
if (token.length > 5 && token.endsWith("ing"))
|
|
35
|
+
return token.slice(0, -3);
|
|
36
|
+
if (token.length > 4 && token.endsWith("ed"))
|
|
37
|
+
return token.slice(0, -2);
|
|
38
|
+
if (token.length > 3 && token.endsWith("s"))
|
|
39
|
+
return token.slice(0, -1);
|
|
40
|
+
return token;
|
|
41
|
+
}
|
|
42
|
+
export function tokenizeForSearch(text) {
|
|
43
|
+
const normalized = preTokenizeForFTS(text)
|
|
44
|
+
.toLowerCase()
|
|
45
|
+
.replace(/[-_*"(){}[\]^~:+.#@!\\/<>|&$%?]/g, " ");
|
|
46
|
+
const tokens = normalized
|
|
47
|
+
.split(/\s+/)
|
|
48
|
+
.map((token) => TYPO_CORRECTIONS[token] ?? token)
|
|
49
|
+
.filter((token) => token.length > 1 || /[\u3400-\u4dbf\u4e00-\u9fff]/.test(token));
|
|
50
|
+
return new Set(tokens);
|
|
51
|
+
}
|
|
52
|
+
export function signalTokens(tokens) {
|
|
53
|
+
return new Set([...tokens].filter((token) => !STOP_WORDS.has(token)).map(stemToken));
|
|
54
|
+
}
|
|
55
|
+
export function normalizedQueryText(query) {
|
|
56
|
+
return [...tokenizeForSearch(query)].join(" ");
|
|
57
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { Chunk } from "../storage/sqlite.ts";
|
|
2
|
+
export declare const MIN_HYBRID_SCORE = 0.18;
|
|
3
|
+
export declare const FILE_TYPE_ALIASES: Record<string, string>;
|
|
4
|
+
export interface RankingDiagnostics {
|
|
5
|
+
adjusted_score: number;
|
|
6
|
+
base_score: number;
|
|
7
|
+
coverage: number;
|
|
8
|
+
documentation_boost: number;
|
|
9
|
+
is_localization: boolean;
|
|
10
|
+
is_test: boolean;
|
|
11
|
+
localization_penalty: number;
|
|
12
|
+
path_boost: number;
|
|
13
|
+
source_boost: number;
|
|
14
|
+
test_intent: boolean;
|
|
15
|
+
user_guide_intent: boolean;
|
|
16
|
+
}
|
|
17
|
+
export declare function normalizeFileTypeFilter(fileType: string | undefined): string | undefined;
|
|
18
|
+
export declare function isTestPath(filePath: string): boolean;
|
|
19
|
+
export declare function queryAsksForTests(queryTokens: Set<string>): boolean;
|
|
20
|
+
export declare function isLocalizationPath(filePath: string, fileType: string): boolean;
|
|
21
|
+
export declare function queryCoverage(text: string, queryTokens: Set<string>): number;
|
|
22
|
+
export declare function hasAnyLexicalEvidence(text: string, queryTokens: Set<string>): boolean;
|
|
23
|
+
export declare function pathTokenBoost(filePath: string, queryTokens: Set<string>): number;
|
|
24
|
+
export declare function basenameTokenBoost(filePath: string, queryTokens: Set<string>): number;
|
|
25
|
+
export declare function sourceFileBoost(chunk: Pick<Chunk, "file_path" | "file_type">, queryTokens: Set<string>): number;
|
|
26
|
+
export declare function scoreChunkForQuery(baseScore: number, chunk: Chunk, queryTokens: Set<string>): RankingDiagnostics;
|
|
27
|
+
export declare function hasEnoughLexicalEvidence(chunk: Chunk, queryTokens: Set<string>): boolean;
|