pi-knowledge 0.4.2 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/README.md +4 -2
- package/dist/index.d.ts +38 -0
- package/dist/index.js +435 -0
- package/dist/src/diagnostics/health.d.ts +19 -0
- package/dist/src/diagnostics/health.js +110 -0
- package/dist/src/embedding/provider.d.ts +5 -0
- package/dist/src/embedding/provider.js +99 -0
- package/dist/src/embedding/vectors.d.ts +15 -0
- package/dist/src/embedding/vectors.js +111 -0
- package/dist/src/engine.d.ts +90 -0
- package/dist/src/engine.js +1370 -0
- package/dist/src/indexer/chunker.d.ts +53 -0
- package/dist/src/indexer/chunker.js +498 -0
- package/dist/src/indexer/chunkers/code-ast.d.ts +3 -0
- package/dist/src/indexer/chunkers/code-ast.js +179 -0
- package/dist/src/model-worker-client.d.ts +10 -0
- package/dist/src/model-worker-client.js +138 -0
- package/dist/src/model-worker.d.ts +1 -0
- package/dist/src/model-worker.js +61 -0
- package/dist/src/search/bm25.d.ts +8 -0
- package/dist/src/search/bm25.js +38 -0
- package/dist/src/search/fusion.d.ts +19 -0
- package/dist/src/search/fusion.js +34 -0
- package/dist/src/search/query.d.ts +5 -0
- package/dist/src/search/query.js +57 -0
- package/dist/src/search/ranking.d.ts +27 -0
- package/dist/src/search/ranking.js +180 -0
- package/dist/src/search/reranker.d.ts +10 -0
- package/dist/src/search/reranker.js +66 -0
- package/dist/src/search/vector.d.ts +14 -0
- package/dist/src/search/vector.js +58 -0
- package/dist/src/storage/sqlite.d.ts +99 -0
- package/dist/src/storage/sqlite.js +330 -0
- package/dist/src/watcher/file-watcher.d.ts +4 -0
- package/dist/src/watcher/file-watcher.js +108 -0
- package/extension.js +7 -0
- package/package.json +10 -5
- package/index.ts +0 -500
- package/src/diagnostics/health.ts +0 -145
- package/src/embedding/provider.ts +0 -106
- package/src/embedding/vectors.ts +0 -119
- package/src/engine.ts +0 -1592
- package/src/indexer/chunker.ts +0 -612
- package/src/indexer/chunkers/code-ast.ts +0 -209
- package/src/model-worker-client.ts +0 -146
- package/src/model-worker.ts +0 -88
- package/src/search/bm25.ts +0 -55
- package/src/search/fusion.ts +0 -48
- package/src/search/query.ts +0 -59
- package/src/search/ranking.ts +0 -201
- package/src/search/reranker.ts +0 -74
- package/src/search/vector.ts +0 -81
- package/src/storage/sqlite.ts +0 -489
- package/src/watcher/file-watcher.ts +0 -116
|
@@ -1,209 +0,0 @@
|
|
|
1
|
-
import type { ChunkInsert } from "../../storage/sqlite.ts";
|
|
2
|
-
import { buildChunkEmbeddingText, contentHash, preTokenizeForFTS } from "../chunker.ts";
|
|
3
|
-
|
|
4
|
-
interface ASTNode {
|
|
5
|
-
type: string;
|
|
6
|
-
text: string;
|
|
7
|
-
startPosition: { row: number };
|
|
8
|
-
endPosition: { row: number };
|
|
9
|
-
children: ASTNode[];
|
|
10
|
-
childForFieldName(name: string): ASTNode | null;
|
|
11
|
-
}
|
|
12
|
-
|
|
13
|
-
type LangConfig = {
|
|
14
|
-
grammar: () => Promise<unknown>;
|
|
15
|
-
fnTypes: Set<string>;
|
|
16
|
-
classTypes: Set<string>;
|
|
17
|
-
methodContainer?: string;
|
|
18
|
-
classBody?: string;
|
|
19
|
-
fileType: string;
|
|
20
|
-
};
|
|
21
|
-
|
|
22
|
-
const LANGS: Record<string, LangConfig> = {
|
|
23
|
-
typescript: {
|
|
24
|
-
grammar: async () => {
|
|
25
|
-
const m = await import("tree-sitter-typescript");
|
|
26
|
-
return m.typescript ?? m.default?.typescript;
|
|
27
|
-
},
|
|
28
|
-
fnTypes: new Set(["function_declaration", "arrow_function", "generator_function_declaration"]),
|
|
29
|
-
classTypes: new Set(["class_declaration", "interface_declaration"]),
|
|
30
|
-
classBody: "class_body",
|
|
31
|
-
fileType: "typescript",
|
|
32
|
-
},
|
|
33
|
-
javascript: {
|
|
34
|
-
grammar: async () => {
|
|
35
|
-
const m = await import("tree-sitter-typescript");
|
|
36
|
-
return m.typescript ?? m.default?.typescript;
|
|
37
|
-
},
|
|
38
|
-
fnTypes: new Set(["function_declaration", "arrow_function", "generator_function_declaration"]),
|
|
39
|
-
classTypes: new Set(["class_declaration"]),
|
|
40
|
-
classBody: "class_body",
|
|
41
|
-
fileType: "javascript",
|
|
42
|
-
},
|
|
43
|
-
python: {
|
|
44
|
-
grammar: async () => {
|
|
45
|
-
const m = await import("tree-sitter-python");
|
|
46
|
-
return m.default ?? m;
|
|
47
|
-
},
|
|
48
|
-
fnTypes: new Set(["function_definition"]),
|
|
49
|
-
classTypes: new Set(["class_definition"]),
|
|
50
|
-
classBody: "block",
|
|
51
|
-
fileType: "python",
|
|
52
|
-
},
|
|
53
|
-
go: {
|
|
54
|
-
grammar: async () => {
|
|
55
|
-
const m = await import("tree-sitter-go");
|
|
56
|
-
return m.default ?? m;
|
|
57
|
-
},
|
|
58
|
-
fnTypes: new Set(["function_declaration", "method_declaration"]),
|
|
59
|
-
classTypes: new Set(["type_declaration"]),
|
|
60
|
-
fileType: "go",
|
|
61
|
-
},
|
|
62
|
-
rust: {
|
|
63
|
-
grammar: async () => {
|
|
64
|
-
const m = await import("tree-sitter-rust");
|
|
65
|
-
return m.default ?? m;
|
|
66
|
-
},
|
|
67
|
-
fnTypes: new Set(["function_item"]),
|
|
68
|
-
classTypes: new Set(["struct_item", "enum_item"]),
|
|
69
|
-
methodContainer: "impl_item",
|
|
70
|
-
fileType: "rust",
|
|
71
|
-
},
|
|
72
|
-
java: {
|
|
73
|
-
grammar: async () => {
|
|
74
|
-
const m = await import("tree-sitter-java");
|
|
75
|
-
return m.default ?? m;
|
|
76
|
-
},
|
|
77
|
-
fnTypes: new Set(["method_declaration", "constructor_declaration"]),
|
|
78
|
-
classTypes: new Set(["class_declaration", "interface_declaration", "enum_declaration"]),
|
|
79
|
-
classBody: "class_body",
|
|
80
|
-
fileType: "java",
|
|
81
|
-
},
|
|
82
|
-
};
|
|
83
|
-
|
|
84
|
-
function getName(node: ASTNode): string {
|
|
85
|
-
return node.childForFieldName("name")?.text ?? node.childForFieldName("type")?.text ?? "anonymous";
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
function collectChunks(
|
|
89
|
-
root: ASTNode,
|
|
90
|
-
config: LangConfig,
|
|
91
|
-
): Array<{ name: string; text: string; start: number; end: number }> {
|
|
92
|
-
const chunks: Array<{ name: string; text: string; start: number; end: number }> = [];
|
|
93
|
-
|
|
94
|
-
function walk(node: ASTNode) {
|
|
95
|
-
if (node.type === "export_statement") {
|
|
96
|
-
for (const child of node.children) {
|
|
97
|
-
if (config.fnTypes.has(child.type)) {
|
|
98
|
-
chunks.push({
|
|
99
|
-
name: getName(child),
|
|
100
|
-
text: node.text,
|
|
101
|
-
start: node.startPosition.row + 1,
|
|
102
|
-
end: node.endPosition.row + 1,
|
|
103
|
-
});
|
|
104
|
-
return;
|
|
105
|
-
}
|
|
106
|
-
if (config.classTypes.has(child.type)) {
|
|
107
|
-
walk(child);
|
|
108
|
-
return;
|
|
109
|
-
}
|
|
110
|
-
}
|
|
111
|
-
}
|
|
112
|
-
if (node.type === "decorated_definition") {
|
|
113
|
-
for (const child of node.children) {
|
|
114
|
-
if (config.fnTypes.has(child.type) || config.classTypes.has(child.type)) {
|
|
115
|
-
chunks.push({
|
|
116
|
-
name: getName(child),
|
|
117
|
-
text: node.text,
|
|
118
|
-
start: node.startPosition.row + 1,
|
|
119
|
-
end: node.endPosition.row + 1,
|
|
120
|
-
});
|
|
121
|
-
return;
|
|
122
|
-
}
|
|
123
|
-
}
|
|
124
|
-
}
|
|
125
|
-
if (config.fnTypes.has(node.type)) {
|
|
126
|
-
chunks.push({
|
|
127
|
-
name: getName(node),
|
|
128
|
-
text: node.text,
|
|
129
|
-
start: node.startPosition.row + 1,
|
|
130
|
-
end: node.endPosition.row + 1,
|
|
131
|
-
});
|
|
132
|
-
return;
|
|
133
|
-
}
|
|
134
|
-
if (config.classTypes.has(node.type)) {
|
|
135
|
-
const name = getName(node);
|
|
136
|
-
let hasChildren = false;
|
|
137
|
-
for (const child of node.children) {
|
|
138
|
-
if (config.classBody && child.type === config.classBody) {
|
|
139
|
-
for (const m of child.children) {
|
|
140
|
-
if (config.fnTypes.has(m.type) || m.type === "method_definition") {
|
|
141
|
-
hasChildren = true;
|
|
142
|
-
chunks.push({
|
|
143
|
-
name: `${name}.${getName(m)}`,
|
|
144
|
-
text: m.text,
|
|
145
|
-
start: m.startPosition.row + 1,
|
|
146
|
-
end: m.endPosition.row + 1,
|
|
147
|
-
});
|
|
148
|
-
}
|
|
149
|
-
}
|
|
150
|
-
}
|
|
151
|
-
}
|
|
152
|
-
if (!hasChildren)
|
|
153
|
-
chunks.push({ name, text: node.text, start: node.startPosition.row + 1, end: node.endPosition.row + 1 });
|
|
154
|
-
return;
|
|
155
|
-
}
|
|
156
|
-
if (config.methodContainer && node.type === config.methodContainer) {
|
|
157
|
-
const typeName = getName(node);
|
|
158
|
-
for (const child of node.children) {
|
|
159
|
-
if (child.type === "declaration_list") {
|
|
160
|
-
for (const m of child.children) {
|
|
161
|
-
if (config.fnTypes.has(m.type))
|
|
162
|
-
chunks.push({
|
|
163
|
-
name: `${typeName}.${getName(m)}`,
|
|
164
|
-
text: m.text,
|
|
165
|
-
start: m.startPosition.row + 1,
|
|
166
|
-
end: m.endPosition.row + 1,
|
|
167
|
-
});
|
|
168
|
-
}
|
|
169
|
-
}
|
|
170
|
-
}
|
|
171
|
-
return;
|
|
172
|
-
}
|
|
173
|
-
for (const child of node.children) walk(child);
|
|
174
|
-
}
|
|
175
|
-
|
|
176
|
-
walk(root);
|
|
177
|
-
return chunks;
|
|
178
|
-
}
|
|
179
|
-
|
|
180
|
-
export async function chunkWithAST(
|
|
181
|
-
content: string,
|
|
182
|
-
filePath: string,
|
|
183
|
-
language: string,
|
|
184
|
-
): Promise<Omit<ChunkInsert, "kb_id">[]> {
|
|
185
|
-
const config = LANGS[language];
|
|
186
|
-
if (!config) return [];
|
|
187
|
-
const Parser = (await import("tree-sitter")).default;
|
|
188
|
-
const grammar = await config.grammar();
|
|
189
|
-
const parser = new Parser();
|
|
190
|
-
parser.setLanguage(grammar);
|
|
191
|
-
const tree = parser.parse(content);
|
|
192
|
-
const fns = collectChunks(tree.rootNode as unknown as ASTNode, config);
|
|
193
|
-
if (fns.length === 0) return [];
|
|
194
|
-
return fns.map((fn) => {
|
|
195
|
-
const chunk = {
|
|
196
|
-
content_hash: contentHash(fn.text),
|
|
197
|
-
content: fn.text,
|
|
198
|
-
content_tokenized: "",
|
|
199
|
-
file_path: filePath,
|
|
200
|
-
file_type: config.fileType,
|
|
201
|
-
start_line: fn.start,
|
|
202
|
-
end_line: fn.end,
|
|
203
|
-
metadata_json: JSON.stringify({ function_name: fn.name }),
|
|
204
|
-
};
|
|
205
|
-
return { ...chunk, content_tokenized: preTokenizeForFTS(buildChunkEmbeddingText(chunk)) };
|
|
206
|
-
});
|
|
207
|
-
}
|
|
208
|
-
|
|
209
|
-
export const SUPPORTED_LANGUAGES = new Set(Object.keys(LANGS));
|
|
@@ -1,146 +0,0 @@
|
|
|
1
|
-
import { type ChildProcess, fork } from "node:child_process";
|
|
2
|
-
import { fileURLToPath } from "node:url";
|
|
3
|
-
|
|
4
|
-
type WorkerResponse = {
|
|
5
|
-
id: number;
|
|
6
|
-
result?: unknown;
|
|
7
|
-
error?: string;
|
|
8
|
-
};
|
|
9
|
-
|
|
10
|
-
type PendingRequest = {
|
|
11
|
-
resolve: (value: unknown) => void;
|
|
12
|
-
reject: (reason: Error) => void;
|
|
13
|
-
};
|
|
14
|
-
|
|
15
|
-
let worker: ChildProcess | null = null;
|
|
16
|
-
let nextRequestId = 1;
|
|
17
|
-
const pending = new Map<number, PendingRequest>();
|
|
18
|
-
|
|
19
|
-
function rejectPending(error: Error): void {
|
|
20
|
-
for (const request of pending.values()) {
|
|
21
|
-
request.reject(error);
|
|
22
|
-
}
|
|
23
|
-
pending.clear();
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
function getWorker(): ChildProcess {
|
|
27
|
-
if (worker?.connected) return worker;
|
|
28
|
-
const workerPath = fileURLToPath(new URL("./model-worker.ts", import.meta.url));
|
|
29
|
-
worker = fork(workerPath, {
|
|
30
|
-
execArgv: ["--experimental-strip-types"],
|
|
31
|
-
stdio: ["ignore", "ignore", "ignore", "ipc"],
|
|
32
|
-
env: process.env,
|
|
33
|
-
});
|
|
34
|
-
const child = worker;
|
|
35
|
-
worker.on("message", (message: WorkerResponse) => {
|
|
36
|
-
const request = pending.get(message.id);
|
|
37
|
-
if (!request) return;
|
|
38
|
-
pending.delete(message.id);
|
|
39
|
-
if (message.error) {
|
|
40
|
-
request.reject(new Error(message.error));
|
|
41
|
-
} else {
|
|
42
|
-
request.resolve(message.result);
|
|
43
|
-
}
|
|
44
|
-
});
|
|
45
|
-
worker.on("exit", (code, signal) => {
|
|
46
|
-
if (worker !== child) return;
|
|
47
|
-
worker = null;
|
|
48
|
-
if (pending.size > 0) {
|
|
49
|
-
rejectPending(
|
|
50
|
-
new Error(`Model worker exited before responding (code ${code ?? "null"}, signal ${signal ?? "null"})`),
|
|
51
|
-
);
|
|
52
|
-
}
|
|
53
|
-
});
|
|
54
|
-
worker.on("error", (error) => {
|
|
55
|
-
if (worker !== child) return;
|
|
56
|
-
worker = null;
|
|
57
|
-
rejectPending(error);
|
|
58
|
-
});
|
|
59
|
-
return worker;
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
async function requestModelWorker(message: Record<string, unknown>, signal?: AbortSignal): Promise<unknown> {
|
|
63
|
-
if (signal?.aborted) throw new Error("Cancelled");
|
|
64
|
-
const child = getWorker();
|
|
65
|
-
const id = nextRequestId++;
|
|
66
|
-
return new Promise((resolve, reject) => {
|
|
67
|
-
let abortHandler: (() => void) | undefined;
|
|
68
|
-
pending.set(id, { resolve, reject });
|
|
69
|
-
if (signal) {
|
|
70
|
-
abortHandler = () => {
|
|
71
|
-
pending.delete(id);
|
|
72
|
-
reject(new Error("Cancelled"));
|
|
73
|
-
shutdownModelWorker();
|
|
74
|
-
};
|
|
75
|
-
signal.addEventListener("abort", abortHandler, { once: true });
|
|
76
|
-
}
|
|
77
|
-
const cleanup = (): void => {
|
|
78
|
-
if (abortHandler) signal?.removeEventListener("abort", abortHandler);
|
|
79
|
-
};
|
|
80
|
-
const originalResolve = resolve;
|
|
81
|
-
const originalReject = reject;
|
|
82
|
-
pending.set(id, {
|
|
83
|
-
resolve(value) {
|
|
84
|
-
cleanup();
|
|
85
|
-
originalResolve(value);
|
|
86
|
-
},
|
|
87
|
-
reject(error) {
|
|
88
|
-
cleanup();
|
|
89
|
-
originalReject(error);
|
|
90
|
-
},
|
|
91
|
-
});
|
|
92
|
-
child.send({ id, ...message }, (error) => {
|
|
93
|
-
if (!error) return;
|
|
94
|
-
pending.delete(id);
|
|
95
|
-
cleanup();
|
|
96
|
-
reject(error);
|
|
97
|
-
});
|
|
98
|
-
});
|
|
99
|
-
}
|
|
100
|
-
|
|
101
|
-
export async function embedInModelWorker(
|
|
102
|
-
texts: string[],
|
|
103
|
-
prefix: "query" | "passage",
|
|
104
|
-
signal?: AbortSignal,
|
|
105
|
-
): Promise<Float32Array[]> {
|
|
106
|
-
const result = await requestModelWorker({ type: "embed", texts, prefix }, signal);
|
|
107
|
-
if (!Array.isArray(result)) throw new Error("Invalid embedding worker response");
|
|
108
|
-
return result.map((vector) => {
|
|
109
|
-
if (!Array.isArray(vector)) throw new Error("Invalid embedding vector from worker");
|
|
110
|
-
return new Float32Array(vector);
|
|
111
|
-
});
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
export interface RerankWorkerCandidate {
|
|
115
|
-
chunkId: string;
|
|
116
|
-
content: string;
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
export async function rerankInModelWorker(
|
|
120
|
-
query: string,
|
|
121
|
-
candidates: RerankWorkerCandidate[],
|
|
122
|
-
topK: number,
|
|
123
|
-
): Promise<Array<{ chunkId: string; score: number }>> {
|
|
124
|
-
const result = await requestModelWorker({ type: "rerank", query, candidates, topK });
|
|
125
|
-
if (!Array.isArray(result)) throw new Error("Invalid reranker worker response");
|
|
126
|
-
return result.map((item) => {
|
|
127
|
-
if (
|
|
128
|
-
typeof item !== "object" ||
|
|
129
|
-
item === null ||
|
|
130
|
-
typeof (item as { chunkId?: unknown }).chunkId !== "string" ||
|
|
131
|
-
typeof (item as { score?: unknown }).score !== "number"
|
|
132
|
-
) {
|
|
133
|
-
throw new Error("Invalid reranker result from worker");
|
|
134
|
-
}
|
|
135
|
-
return item as { chunkId: string; score: number };
|
|
136
|
-
});
|
|
137
|
-
}
|
|
138
|
-
|
|
139
|
-
export function shutdownModelWorker(): void {
|
|
140
|
-
const child = worker;
|
|
141
|
-
worker = null;
|
|
142
|
-
rejectPending(new Error("Model worker shut down"));
|
|
143
|
-
if (child && !child.killed) {
|
|
144
|
-
child.kill("SIGKILL");
|
|
145
|
-
}
|
|
146
|
-
}
|
package/src/model-worker.ts
DELETED
|
@@ -1,88 +0,0 @@
|
|
|
1
|
-
import { join } from "node:path";
|
|
2
|
-
import { getDefaultKnowledgeDir } from "./storage/sqlite.ts";
|
|
3
|
-
|
|
4
|
-
type FeatureExtractionPipeline = (
|
|
5
|
-
input: string,
|
|
6
|
-
options: { pooling: "mean"; normalize: true },
|
|
7
|
-
) => Promise<{ data: ArrayLike<number> }>;
|
|
8
|
-
|
|
9
|
-
type RerankerPipeline = (input: {
|
|
10
|
-
text: string;
|
|
11
|
-
text_pair: string;
|
|
12
|
-
}) => Promise<{ score?: number } | Array<{ score?: number }>>;
|
|
13
|
-
|
|
14
|
-
type EmbedRequest = {
|
|
15
|
-
id: number;
|
|
16
|
-
type: "embed";
|
|
17
|
-
texts: string[];
|
|
18
|
-
prefix: "query" | "passage";
|
|
19
|
-
};
|
|
20
|
-
|
|
21
|
-
type RerankRequest = {
|
|
22
|
-
id: number;
|
|
23
|
-
type: "rerank";
|
|
24
|
-
query: string;
|
|
25
|
-
candidates: Array<{ chunkId: string; content: string }>;
|
|
26
|
-
topK: number;
|
|
27
|
-
};
|
|
28
|
-
|
|
29
|
-
type WorkerRequest = EmbedRequest | RerankRequest;
|
|
30
|
-
|
|
31
|
-
let embeddingPipeline: FeatureExtractionPipeline | null = null;
|
|
32
|
-
let rerankerPipeline: RerankerPipeline | null = null;
|
|
33
|
-
|
|
34
|
-
function getModelCacheDir(): string {
|
|
35
|
-
return process.env.PI_KNOWLEDGE_MODEL_CACHE_DIR ?? join(getDefaultKnowledgeDir(), "models");
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
async function loadEmbeddingPipeline(): Promise<FeatureExtractionPipeline> {
|
|
39
|
-
if (embeddingPipeline) return embeddingPipeline;
|
|
40
|
-
const { pipeline, env } = await import("@huggingface/transformers");
|
|
41
|
-
env.cacheDir = getModelCacheDir();
|
|
42
|
-
embeddingPipeline = await pipeline("feature-extraction", "Xenova/multilingual-e5-small", {
|
|
43
|
-
quantized: true,
|
|
44
|
-
dtype: "fp32",
|
|
45
|
-
});
|
|
46
|
-
return embeddingPipeline;
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
async function loadRerankerPipeline(): Promise<RerankerPipeline> {
|
|
50
|
-
if (rerankerPipeline) return rerankerPipeline;
|
|
51
|
-
const { pipeline, env } = await import("@huggingface/transformers");
|
|
52
|
-
env.cacheDir = getModelCacheDir();
|
|
53
|
-
rerankerPipeline = await pipeline("text-classification", "Xenova/ms-marco-MiniLM-L-4-v2");
|
|
54
|
-
return rerankerPipeline;
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
async function handleEmbed(request: EmbedRequest): Promise<number[][]> {
|
|
58
|
-
const pipe = await loadEmbeddingPipeline();
|
|
59
|
-
const vectors: number[][] = [];
|
|
60
|
-
for (const text of request.texts) {
|
|
61
|
-
const output = await pipe(`${request.prefix}: ${text}`, { pooling: "mean", normalize: true });
|
|
62
|
-
vectors.push(Array.from(output.data));
|
|
63
|
-
}
|
|
64
|
-
return vectors;
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
async function handleRerank(request: RerankRequest): Promise<Array<{ chunkId: string; score: number }>> {
|
|
68
|
-
const pipe = await loadRerankerPipeline();
|
|
69
|
-
const results: Array<{ chunkId: string; score: number }> = [];
|
|
70
|
-
for (const candidate of request.candidates) {
|
|
71
|
-
const output = await pipe({ text: request.query, text_pair: candidate.content });
|
|
72
|
-
const score = Array.isArray(output) ? (output[0]?.score ?? 0) : (output?.score ?? 0);
|
|
73
|
-
results.push({ chunkId: candidate.chunkId, score });
|
|
74
|
-
}
|
|
75
|
-
results.sort((a, b) => b.score - a.score);
|
|
76
|
-
return results.slice(0, request.topK);
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
process.on("message", (request: WorkerRequest) => {
|
|
80
|
-
void (async () => {
|
|
81
|
-
try {
|
|
82
|
-
const result = request.type === "embed" ? await handleEmbed(request) : await handleRerank(request);
|
|
83
|
-
process.send?.({ id: request.id, result });
|
|
84
|
-
} catch (error) {
|
|
85
|
-
process.send?.({ id: request.id, error: error instanceof Error ? error.message : String(error) });
|
|
86
|
-
}
|
|
87
|
-
})();
|
|
88
|
-
});
|
package/src/search/bm25.ts
DELETED
|
@@ -1,55 +0,0 @@
|
|
|
1
|
-
import type Database from "better-sqlite3";
|
|
2
|
-
import { normalizedQueryText } from "./query.ts";
|
|
3
|
-
|
|
4
|
-
export interface BM25Result {
|
|
5
|
-
chunkId: string;
|
|
6
|
-
score: number;
|
|
7
|
-
}
|
|
8
|
-
|
|
9
|
-
function prepareFtsTerms(query: string): string[] {
|
|
10
|
-
return normalizedQueryText(query)
|
|
11
|
-
.split(/\s+/)
|
|
12
|
-
.filter((t) => t.length > 0);
|
|
13
|
-
}
|
|
14
|
-
|
|
15
|
-
function quoteFtsTerm(term: string): string {
|
|
16
|
-
return `"${term.replace(/"/g, '""')}"`;
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
function runSearch(db: Database.Database, ftsQuery: string, limit: number, kbId?: string): BM25Result[] {
|
|
20
|
-
if (kbId) {
|
|
21
|
-
return db
|
|
22
|
-
.prepare(
|
|
23
|
-
`SELECT c.id as chunkId, -bm25(chunks_fts) as score
|
|
24
|
-
FROM chunks_fts JOIN chunks c ON chunks_fts.rowid = c.rowid
|
|
25
|
-
WHERE chunks_fts MATCH ? AND c.kb_id = ? ORDER BY bm25(chunks_fts) LIMIT ?`,
|
|
26
|
-
)
|
|
27
|
-
.all(ftsQuery, kbId, limit) as BM25Result[];
|
|
28
|
-
}
|
|
29
|
-
return db
|
|
30
|
-
.prepare(
|
|
31
|
-
`SELECT c.id as chunkId, -bm25(chunks_fts) as score
|
|
32
|
-
FROM chunks_fts JOIN chunks c ON chunks_fts.rowid = c.rowid
|
|
33
|
-
WHERE chunks_fts MATCH ? ORDER BY bm25(chunks_fts) LIMIT ?`,
|
|
34
|
-
)
|
|
35
|
-
.all(ftsQuery, limit) as BM25Result[];
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
export function searchBM25(
|
|
39
|
-
db: Database.Database,
|
|
40
|
-
query: string,
|
|
41
|
-
limit = 50,
|
|
42
|
-
kbId?: string,
|
|
43
|
-
options: { allowOrFallback?: boolean } = {},
|
|
44
|
-
): BM25Result[] {
|
|
45
|
-
const terms = prepareFtsTerms(query);
|
|
46
|
-
if (terms.length === 0) return [];
|
|
47
|
-
try {
|
|
48
|
-
const quotedTerms = terms.map(quoteFtsTerm);
|
|
49
|
-
const strict = runSearch(db, quotedTerms.join(" AND "), limit, kbId);
|
|
50
|
-
if (strict.length > 0 || terms.length === 1 || options.allowOrFallback === false) return strict;
|
|
51
|
-
return runSearch(db, quotedTerms.join(" OR "), limit, kbId);
|
|
52
|
-
} catch {
|
|
53
|
-
return [];
|
|
54
|
-
}
|
|
55
|
-
}
|
package/src/search/fusion.ts
DELETED
|
@@ -1,48 +0,0 @@
|
|
|
1
|
-
export interface FusionResult {
|
|
2
|
-
chunkId: string;
|
|
3
|
-
score: number;
|
|
4
|
-
}
|
|
5
|
-
|
|
6
|
-
function normalizeScores(list: Array<{ chunkId: string; score: number }>): Map<string, number> {
|
|
7
|
-
if (list.length === 0) return new Map();
|
|
8
|
-
const scores = list.map((item) => item.score);
|
|
9
|
-
const min = Math.min(...scores);
|
|
10
|
-
const max = Math.max(...scores);
|
|
11
|
-
return new Map(
|
|
12
|
-
list.map((item) => {
|
|
13
|
-
const normalized = max === min ? 1 : (item.score - min) / (max - min);
|
|
14
|
-
return [item.chunkId, normalized];
|
|
15
|
-
}),
|
|
16
|
-
);
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
export function reciprocalRankFusion(lists: Array<{ chunkId: string; score: number }[]>, k = 60): FusionResult[] {
|
|
20
|
-
const scores = new Map<string, number>();
|
|
21
|
-
for (const list of lists) {
|
|
22
|
-
for (let rank = 0; rank < list.length; rank++) {
|
|
23
|
-
const { chunkId } = list[rank];
|
|
24
|
-
scores.set(chunkId, (scores.get(chunkId) ?? 0) + 1 / (k + rank + 1));
|
|
25
|
-
}
|
|
26
|
-
}
|
|
27
|
-
return [...scores.entries()].map(([chunkId, score]) => ({ chunkId, score })).sort((a, b) => b.score - a.score);
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
export function weightedScoreFusion(
|
|
31
|
-
bm25Results: Array<{ chunkId: string; score: number }>,
|
|
32
|
-
vectorResults: Array<{ chunkId: string; score: number }>,
|
|
33
|
-
weights = { bm25: 0.45, vector: 0.55, overlap: 0.15 },
|
|
34
|
-
): FusionResult[] {
|
|
35
|
-
const bm25Scores = normalizeScores(bm25Results);
|
|
36
|
-
const vectorScores = normalizeScores(vectorResults);
|
|
37
|
-
const ids = new Set([...bm25Scores.keys(), ...vectorScores.keys()]);
|
|
38
|
-
const fused: FusionResult[] = [];
|
|
39
|
-
|
|
40
|
-
for (const chunkId of ids) {
|
|
41
|
-
const bm25 = bm25Scores.get(chunkId) ?? 0;
|
|
42
|
-
const vector = vectorScores.get(chunkId) ?? 0;
|
|
43
|
-
const overlap = bm25 > 0 && vector > 0 ? weights.overlap : 0;
|
|
44
|
-
fused.push({ chunkId, score: weights.bm25 * bm25 + weights.vector * vector + overlap });
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
return fused.sort((a, b) => b.score - a.score);
|
|
48
|
-
}
|
package/src/search/query.ts
DELETED
|
@@ -1,59 +0,0 @@
|
|
|
1
|
-
import { preTokenizeForFTS } from "../indexer/chunker.ts";
|
|
2
|
-
|
|
3
|
-
export const STOP_WORDS = new Set([
|
|
4
|
-
"the",
|
|
5
|
-
"and",
|
|
6
|
-
"or",
|
|
7
|
-
"to",
|
|
8
|
-
"of",
|
|
9
|
-
"in",
|
|
10
|
-
"on",
|
|
11
|
-
"for",
|
|
12
|
-
"with",
|
|
13
|
-
"how",
|
|
14
|
-
"does",
|
|
15
|
-
"what",
|
|
16
|
-
"when",
|
|
17
|
-
"where",
|
|
18
|
-
"why",
|
|
19
|
-
"this",
|
|
20
|
-
"that",
|
|
21
|
-
"from",
|
|
22
|
-
]);
|
|
23
|
-
|
|
24
|
-
const TYPO_CORRECTIONS: Record<string, string> = {
|
|
25
|
-
ho: "how",
|
|
26
|
-
setpup: "setup",
|
|
27
|
-
setpu: "setup",
|
|
28
|
-
seting: "setting",
|
|
29
|
-
configuraiton: "configuration",
|
|
30
|
-
permisson: "permission",
|
|
31
|
-
permisions: "permissions",
|
|
32
|
-
};
|
|
33
|
-
|
|
34
|
-
export function stemToken(token: string): string {
|
|
35
|
-
if (token.length > 4 && token.endsWith("ies")) return `${token.slice(0, -3)}y`;
|
|
36
|
-
if (token.length > 5 && token.endsWith("ing")) return token.slice(0, -3);
|
|
37
|
-
if (token.length > 4 && token.endsWith("ed")) return token.slice(0, -2);
|
|
38
|
-
if (token.length > 3 && token.endsWith("s")) return token.slice(0, -1);
|
|
39
|
-
return token;
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
export function tokenizeForSearch(text: string): Set<string> {
|
|
43
|
-
const normalized = preTokenizeForFTS(text)
|
|
44
|
-
.toLowerCase()
|
|
45
|
-
.replace(/[-_*"(){}[\]^~:+.#@!\\/<>|&$%?]/g, " ");
|
|
46
|
-
const tokens = normalized
|
|
47
|
-
.split(/\s+/)
|
|
48
|
-
.map((token) => TYPO_CORRECTIONS[token] ?? token)
|
|
49
|
-
.filter((token) => token.length > 1 || /[\u3400-\u4dbf\u4e00-\u9fff]/.test(token));
|
|
50
|
-
return new Set(tokens);
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
export function signalTokens(tokens: Set<string>): Set<string> {
|
|
54
|
-
return new Set([...tokens].filter((token) => !STOP_WORDS.has(token)).map(stemToken));
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
export function normalizedQueryText(query: string): string {
|
|
58
|
-
return [...tokenizeForSearch(query)].join(" ");
|
|
59
|
-
}
|