pi-knowledge 0.2.3 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +43 -0
- package/README.md +23 -1
- package/index.ts +153 -34
- package/package.json +60 -60
- package/src/diagnostics/health.ts +11 -10
- package/src/embedding/provider.ts +36 -9
- package/src/embedding/vectors.ts +1 -1
- package/src/engine.ts +212 -85
- package/src/indexer/chunker.ts +115 -39
- package/src/indexer/chunkers/code-ast.ts +77 -17
- package/src/search/bm25.ts +19 -10
- package/src/search/fusion.ts +5 -4
- package/src/search/reranker.ts +31 -7
- package/src/search/vector.ts +10 -2
- package/src/storage/sqlite.ts +27 -8
- package/src/watcher/file-watcher.ts +92 -12
package/src/indexer/chunker.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
-
import {
|
|
2
|
+
import { type Dirent, existsSync, readdirSync, readFileSync, statSync } from "node:fs";
|
|
3
3
|
import { extname, join, relative, sep } from "node:path";
|
|
4
4
|
import ignore from "ignore";
|
|
5
5
|
import type { ChunkInsert } from "../storage/sqlite.ts";
|
|
@@ -7,25 +7,74 @@ import type { ChunkInsert } from "../storage/sqlite.ts";
|
|
|
7
7
|
const MAX_FILE_SIZE = 10 * 1024 * 1024; // 10 MB
|
|
8
8
|
|
|
9
9
|
const BINARY_EXTENSIONS = new Set([
|
|
10
|
-
".png",
|
|
11
|
-
".
|
|
12
|
-
".
|
|
13
|
-
".
|
|
14
|
-
".
|
|
15
|
-
".
|
|
16
|
-
".
|
|
17
|
-
".
|
|
10
|
+
".png",
|
|
11
|
+
".jpg",
|
|
12
|
+
".jpeg",
|
|
13
|
+
".gif",
|
|
14
|
+
".ico",
|
|
15
|
+
".bmp",
|
|
16
|
+
".webp",
|
|
17
|
+
".svg",
|
|
18
|
+
".woff",
|
|
19
|
+
".woff2",
|
|
20
|
+
".ttf",
|
|
21
|
+
".eot",
|
|
22
|
+
".otf",
|
|
23
|
+
".zip",
|
|
24
|
+
".gz",
|
|
25
|
+
".tar",
|
|
26
|
+
".bz2",
|
|
27
|
+
".7z",
|
|
28
|
+
".rar",
|
|
29
|
+
".pdf",
|
|
30
|
+
".doc",
|
|
31
|
+
".docx",
|
|
32
|
+
".xls",
|
|
33
|
+
".xlsx",
|
|
34
|
+
".ppt",
|
|
35
|
+
".exe",
|
|
36
|
+
".dll",
|
|
37
|
+
".so",
|
|
38
|
+
".dylib",
|
|
39
|
+
".node",
|
|
40
|
+
".db",
|
|
41
|
+
".sqlite",
|
|
42
|
+
".bin",
|
|
43
|
+
".dat",
|
|
44
|
+
".lock",
|
|
45
|
+
".mp3",
|
|
46
|
+
".mp4",
|
|
47
|
+
".wav",
|
|
48
|
+
".avi",
|
|
49
|
+
".mov",
|
|
50
|
+
".webm",
|
|
51
|
+
".wasm",
|
|
52
|
+
".o",
|
|
53
|
+
".a",
|
|
54
|
+
".lib",
|
|
18
55
|
]);
|
|
19
56
|
|
|
20
57
|
const DEFAULT_IGNORE = [
|
|
21
|
-
"node_modules",
|
|
22
|
-
"
|
|
23
|
-
"
|
|
58
|
+
"node_modules",
|
|
59
|
+
".git",
|
|
60
|
+
"dist",
|
|
61
|
+
"build",
|
|
62
|
+
".next",
|
|
63
|
+
"__pycache__",
|
|
64
|
+
"*.min.js",
|
|
65
|
+
"*.min.css",
|
|
66
|
+
"*.map",
|
|
67
|
+
"*.lock",
|
|
68
|
+
"package-lock.json",
|
|
69
|
+
".DS_Store",
|
|
70
|
+
"Thumbs.db",
|
|
71
|
+
"*.pyc",
|
|
72
|
+
"*.class",
|
|
24
73
|
];
|
|
25
74
|
|
|
26
75
|
export interface ScannedFile {
|
|
27
|
-
path: string;
|
|
28
|
-
relPath: string;
|
|
76
|
+
path: string; // absolute path
|
|
77
|
+
relPath: string; // relative to root
|
|
29
78
|
content: string;
|
|
30
79
|
fileType: string;
|
|
31
80
|
}
|
|
@@ -33,16 +82,36 @@ export interface ScannedFile {
|
|
|
33
82
|
function detectFileType(filePath: string): string {
|
|
34
83
|
const ext = extname(filePath).toLowerCase();
|
|
35
84
|
const map: Record<string, string> = {
|
|
36
|
-
".ts": "typescript",
|
|
37
|
-
".
|
|
38
|
-
".
|
|
39
|
-
".
|
|
40
|
-
".
|
|
41
|
-
".
|
|
42
|
-
".
|
|
43
|
-
".
|
|
44
|
-
".
|
|
45
|
-
".
|
|
85
|
+
".ts": "typescript",
|
|
86
|
+
".tsx": "typescript",
|
|
87
|
+
".js": "javascript",
|
|
88
|
+
".jsx": "javascript",
|
|
89
|
+
".mjs": "javascript",
|
|
90
|
+
".py": "python",
|
|
91
|
+
".go": "go",
|
|
92
|
+
".rs": "rust",
|
|
93
|
+
".java": "java",
|
|
94
|
+
".c": "c",
|
|
95
|
+
".cpp": "cpp",
|
|
96
|
+
".h": "c",
|
|
97
|
+
".hpp": "cpp",
|
|
98
|
+
".md": "markdown",
|
|
99
|
+
".mdx": "markdown",
|
|
100
|
+
".json": "json",
|
|
101
|
+
".yaml": "yaml",
|
|
102
|
+
".yml": "yaml",
|
|
103
|
+
".toml": "toml",
|
|
104
|
+
".html": "html",
|
|
105
|
+
".css": "css",
|
|
106
|
+
".scss": "css",
|
|
107
|
+
".sh": "shell",
|
|
108
|
+
".bash": "shell",
|
|
109
|
+
".zsh": "shell",
|
|
110
|
+
".sql": "sql",
|
|
111
|
+
".graphql": "graphql",
|
|
112
|
+
".txt": "text",
|
|
113
|
+
".csv": "text",
|
|
114
|
+
".log": "text",
|
|
46
115
|
};
|
|
47
116
|
return map[ext] ?? "text";
|
|
48
117
|
}
|
|
@@ -70,9 +139,12 @@ export function walkDir(dirPath: string): ScannedFile[] {
|
|
|
70
139
|
const results: ScannedFile[] = [];
|
|
71
140
|
|
|
72
141
|
function walk(dir: string): void {
|
|
73
|
-
let entries;
|
|
74
|
-
try {
|
|
75
|
-
|
|
142
|
+
let entries: Dirent[];
|
|
143
|
+
try {
|
|
144
|
+
entries = readdirSync(dir, { withFileTypes: true });
|
|
145
|
+
} catch {
|
|
146
|
+
return;
|
|
147
|
+
}
|
|
76
148
|
for (const entry of entries) {
|
|
77
149
|
const fullPath = join(dir, entry.name);
|
|
78
150
|
const relPath = relative(dirPath, fullPath).split(sep).join("/");
|
|
@@ -80,7 +152,7 @@ export function walkDir(dirPath: string): ScannedFile[] {
|
|
|
80
152
|
if (ig.ignores(relPath)) continue;
|
|
81
153
|
|
|
82
154
|
if (entry.isDirectory()) {
|
|
83
|
-
if (ig.ignores(relPath
|
|
155
|
+
if (ig.ignores(`${relPath}/`)) continue;
|
|
84
156
|
walk(fullPath);
|
|
85
157
|
} else if (entry.isFile()) {
|
|
86
158
|
const stat = statSync(fullPath);
|
|
@@ -261,23 +333,27 @@ export async function chunkFile(content: string, filePath: string): Promise<Omit
|
|
|
261
333
|
try {
|
|
262
334
|
const { chunkWithAST } = await import("./chunkers/code-ast.ts");
|
|
263
335
|
chunks = await chunkWithAST(content, filePath, fileType);
|
|
264
|
-
} catch {
|
|
336
|
+
} catch {
|
|
337
|
+
/* fallback below */
|
|
338
|
+
}
|
|
265
339
|
}
|
|
266
340
|
|
|
267
341
|
if (chunks.length === 0) chunks = chunkText(content, filePath);
|
|
268
342
|
|
|
269
343
|
// Fallback: if file has content but no chunks (too short for splitting), keep as single chunk
|
|
270
344
|
if (chunks.length === 0 && content.trim().length > 10) {
|
|
271
|
-
chunks = [
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
345
|
+
chunks = [
|
|
346
|
+
{
|
|
347
|
+
content_hash: contentHash(content),
|
|
348
|
+
content: content.trim(),
|
|
349
|
+
content_tokenized: preTokenizeForFTS(content),
|
|
350
|
+
file_path: filePath,
|
|
351
|
+
file_type: fileType,
|
|
352
|
+
start_line: 1,
|
|
353
|
+
end_line: content.split("\n").length,
|
|
354
|
+
metadata_json: "{}",
|
|
355
|
+
},
|
|
356
|
+
];
|
|
281
357
|
}
|
|
282
358
|
|
|
283
359
|
return chunks;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { contentHash, preTokenizeForFTS } from "../chunker.ts";
|
|
2
1
|
import type { ChunkInsert } from "../../storage/sqlite.ts";
|
|
2
|
+
import { contentHash, preTokenizeForFTS } from "../chunker.ts";
|
|
3
3
|
|
|
4
4
|
interface ASTNode {
|
|
5
5
|
type: string;
|
|
@@ -11,7 +11,7 @@ interface ASTNode {
|
|
|
11
11
|
}
|
|
12
12
|
|
|
13
13
|
type LangConfig = {
|
|
14
|
-
grammar: () => Promise<
|
|
14
|
+
grammar: () => Promise<unknown>;
|
|
15
15
|
fnTypes: Set<string>;
|
|
16
16
|
classTypes: Set<string>;
|
|
17
17
|
methodContainer?: string;
|
|
@@ -21,41 +21,59 @@ type LangConfig = {
|
|
|
21
21
|
|
|
22
22
|
const LANGS: Record<string, LangConfig> = {
|
|
23
23
|
typescript: {
|
|
24
|
-
grammar: async () => {
|
|
24
|
+
grammar: async () => {
|
|
25
|
+
const m = await import("tree-sitter-typescript");
|
|
26
|
+
return m.typescript ?? m.default?.typescript;
|
|
27
|
+
},
|
|
25
28
|
fnTypes: new Set(["function_declaration", "arrow_function", "generator_function_declaration"]),
|
|
26
29
|
classTypes: new Set(["class_declaration", "interface_declaration"]),
|
|
27
30
|
classBody: "class_body",
|
|
28
31
|
fileType: "typescript",
|
|
29
32
|
},
|
|
30
33
|
javascript: {
|
|
31
|
-
grammar: async () => {
|
|
34
|
+
grammar: async () => {
|
|
35
|
+
const m = await import("tree-sitter-typescript");
|
|
36
|
+
return m.typescript ?? m.default?.typescript;
|
|
37
|
+
},
|
|
32
38
|
fnTypes: new Set(["function_declaration", "arrow_function", "generator_function_declaration"]),
|
|
33
39
|
classTypes: new Set(["class_declaration"]),
|
|
34
40
|
classBody: "class_body",
|
|
35
41
|
fileType: "javascript",
|
|
36
42
|
},
|
|
37
43
|
python: {
|
|
38
|
-
grammar: async () => {
|
|
44
|
+
grammar: async () => {
|
|
45
|
+
const m = await import("tree-sitter-python");
|
|
46
|
+
return m.default ?? m;
|
|
47
|
+
},
|
|
39
48
|
fnTypes: new Set(["function_definition"]),
|
|
40
49
|
classTypes: new Set(["class_definition"]),
|
|
41
50
|
classBody: "block",
|
|
42
51
|
fileType: "python",
|
|
43
52
|
},
|
|
44
53
|
go: {
|
|
45
|
-
grammar: async () => {
|
|
54
|
+
grammar: async () => {
|
|
55
|
+
const m = await import("tree-sitter-go");
|
|
56
|
+
return m.default ?? m;
|
|
57
|
+
},
|
|
46
58
|
fnTypes: new Set(["function_declaration", "method_declaration"]),
|
|
47
59
|
classTypes: new Set(["type_declaration"]),
|
|
48
60
|
fileType: "go",
|
|
49
61
|
},
|
|
50
62
|
rust: {
|
|
51
|
-
grammar: async () => {
|
|
63
|
+
grammar: async () => {
|
|
64
|
+
const m = await import("tree-sitter-rust");
|
|
65
|
+
return m.default ?? m;
|
|
66
|
+
},
|
|
52
67
|
fnTypes: new Set(["function_item"]),
|
|
53
68
|
classTypes: new Set(["struct_item", "enum_item"]),
|
|
54
69
|
methodContainer: "impl_item",
|
|
55
70
|
fileType: "rust",
|
|
56
71
|
},
|
|
57
72
|
java: {
|
|
58
|
-
grammar: async () => {
|
|
73
|
+
grammar: async () => {
|
|
74
|
+
const m = await import("tree-sitter-java");
|
|
75
|
+
return m.default ?? m;
|
|
76
|
+
},
|
|
59
77
|
fnTypes: new Set(["method_declaration", "constructor_declaration"]),
|
|
60
78
|
classTypes: new Set(["class_declaration", "interface_declaration", "enum_declaration"]),
|
|
61
79
|
classBody: "class_body",
|
|
@@ -67,25 +85,51 @@ function getName(node: ASTNode): string {
|
|
|
67
85
|
return node.childForFieldName("name")?.text ?? node.childForFieldName("type")?.text ?? "anonymous";
|
|
68
86
|
}
|
|
69
87
|
|
|
70
|
-
function collectChunks(
|
|
88
|
+
function collectChunks(
|
|
89
|
+
root: ASTNode,
|
|
90
|
+
config: LangConfig,
|
|
91
|
+
): Array<{ name: string; text: string; start: number; end: number }> {
|
|
71
92
|
const chunks: Array<{ name: string; text: string; start: number; end: number }> = [];
|
|
72
93
|
|
|
73
94
|
function walk(node: ASTNode) {
|
|
74
95
|
if (node.type === "export_statement") {
|
|
75
96
|
for (const child of node.children) {
|
|
76
|
-
if (config.fnTypes.has(child.type)) {
|
|
77
|
-
|
|
97
|
+
if (config.fnTypes.has(child.type)) {
|
|
98
|
+
chunks.push({
|
|
99
|
+
name: getName(child),
|
|
100
|
+
text: node.text,
|
|
101
|
+
start: node.startPosition.row + 1,
|
|
102
|
+
end: node.endPosition.row + 1,
|
|
103
|
+
});
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
if (config.classTypes.has(child.type)) {
|
|
107
|
+
walk(child);
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
78
110
|
}
|
|
79
111
|
}
|
|
80
112
|
if (node.type === "decorated_definition") {
|
|
81
113
|
for (const child of node.children) {
|
|
82
114
|
if (config.fnTypes.has(child.type) || config.classTypes.has(child.type)) {
|
|
83
|
-
chunks.push({
|
|
115
|
+
chunks.push({
|
|
116
|
+
name: getName(child),
|
|
117
|
+
text: node.text,
|
|
118
|
+
start: node.startPosition.row + 1,
|
|
119
|
+
end: node.endPosition.row + 1,
|
|
120
|
+
});
|
|
121
|
+
return;
|
|
84
122
|
}
|
|
85
123
|
}
|
|
86
124
|
}
|
|
87
125
|
if (config.fnTypes.has(node.type)) {
|
|
88
|
-
chunks.push({
|
|
126
|
+
chunks.push({
|
|
127
|
+
name: getName(node),
|
|
128
|
+
text: node.text,
|
|
129
|
+
start: node.startPosition.row + 1,
|
|
130
|
+
end: node.endPosition.row + 1,
|
|
131
|
+
});
|
|
132
|
+
return;
|
|
89
133
|
}
|
|
90
134
|
if (config.classTypes.has(node.type)) {
|
|
91
135
|
const name = getName(node);
|
|
@@ -95,12 +139,18 @@ function collectChunks(root: ASTNode, config: LangConfig): Array<{ name: string;
|
|
|
95
139
|
for (const m of child.children) {
|
|
96
140
|
if (config.fnTypes.has(m.type) || m.type === "method_definition") {
|
|
97
141
|
hasChildren = true;
|
|
98
|
-
chunks.push({
|
|
142
|
+
chunks.push({
|
|
143
|
+
name: `${name}.${getName(m)}`,
|
|
144
|
+
text: m.text,
|
|
145
|
+
start: m.startPosition.row + 1,
|
|
146
|
+
end: m.endPosition.row + 1,
|
|
147
|
+
});
|
|
99
148
|
}
|
|
100
149
|
}
|
|
101
150
|
}
|
|
102
151
|
}
|
|
103
|
-
if (!hasChildren)
|
|
152
|
+
if (!hasChildren)
|
|
153
|
+
chunks.push({ name, text: node.text, start: node.startPosition.row + 1, end: node.endPosition.row + 1 });
|
|
104
154
|
return;
|
|
105
155
|
}
|
|
106
156
|
if (config.methodContainer && node.type === config.methodContainer) {
|
|
@@ -108,7 +158,13 @@ function collectChunks(root: ASTNode, config: LangConfig): Array<{ name: string;
|
|
|
108
158
|
for (const child of node.children) {
|
|
109
159
|
if (child.type === "declaration_list") {
|
|
110
160
|
for (const m of child.children) {
|
|
111
|
-
if (config.fnTypes.has(m.type))
|
|
161
|
+
if (config.fnTypes.has(m.type))
|
|
162
|
+
chunks.push({
|
|
163
|
+
name: `${typeName}.${getName(m)}`,
|
|
164
|
+
text: m.text,
|
|
165
|
+
start: m.startPosition.row + 1,
|
|
166
|
+
end: m.endPosition.row + 1,
|
|
167
|
+
});
|
|
112
168
|
}
|
|
113
169
|
}
|
|
114
170
|
}
|
|
@@ -121,7 +177,11 @@ function collectChunks(root: ASTNode, config: LangConfig): Array<{ name: string;
|
|
|
121
177
|
return chunks;
|
|
122
178
|
}
|
|
123
179
|
|
|
124
|
-
export async function chunkWithAST(
|
|
180
|
+
export async function chunkWithAST(
|
|
181
|
+
content: string,
|
|
182
|
+
filePath: string,
|
|
183
|
+
language: string,
|
|
184
|
+
): Promise<Omit<ChunkInsert, "kb_id">[]> {
|
|
125
185
|
const config = LANGS[language];
|
|
126
186
|
if (!config) return [];
|
|
127
187
|
const Parser = (await import("tree-sitter")).default;
|
package/src/search/bm25.ts
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
import type Database from "better-sqlite3";
|
|
2
2
|
import { preTokenizeForFTS } from "../indexer/chunker.ts";
|
|
3
3
|
|
|
4
|
-
export interface BM25Result {
|
|
4
|
+
export interface BM25Result {
|
|
5
|
+
chunkId: string;
|
|
6
|
+
score: number;
|
|
7
|
+
}
|
|
5
8
|
|
|
6
9
|
function prepareFtsQuery(query: string): string {
|
|
7
10
|
let q = preTokenizeForFTS(query);
|
|
@@ -16,16 +19,22 @@ export function searchBM25(db: Database.Database, query: string, limit = 50, kbI
|
|
|
16
19
|
if (!ftsQuery) return [];
|
|
17
20
|
try {
|
|
18
21
|
if (kbId) {
|
|
19
|
-
return db
|
|
20
|
-
|
|
22
|
+
return db
|
|
23
|
+
.prepare(
|
|
24
|
+
`SELECT c.id as chunkId, -bm25(chunks_fts) as score
|
|
21
25
|
FROM chunks_fts JOIN chunks c ON chunks_fts.rowid = c.rowid
|
|
22
|
-
WHERE chunks_fts MATCH ? AND c.kb_id = ? ORDER BY
|
|
23
|
-
|
|
26
|
+
WHERE chunks_fts MATCH ? AND c.kb_id = ? ORDER BY bm25(chunks_fts) LIMIT ?`,
|
|
27
|
+
)
|
|
28
|
+
.all(ftsQuery, kbId, limit) as BM25Result[];
|
|
24
29
|
}
|
|
25
|
-
return db
|
|
26
|
-
|
|
30
|
+
return db
|
|
31
|
+
.prepare(
|
|
32
|
+
`SELECT c.id as chunkId, -bm25(chunks_fts) as score
|
|
27
33
|
FROM chunks_fts JOIN chunks c ON chunks_fts.rowid = c.rowid
|
|
28
|
-
WHERE chunks_fts MATCH ? ORDER BY
|
|
29
|
-
|
|
30
|
-
|
|
34
|
+
WHERE chunks_fts MATCH ? ORDER BY bm25(chunks_fts) LIMIT ?`,
|
|
35
|
+
)
|
|
36
|
+
.all(ftsQuery, limit) as BM25Result[];
|
|
37
|
+
} catch {
|
|
38
|
+
return [];
|
|
39
|
+
}
|
|
31
40
|
}
|
package/src/search/fusion.ts
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
1
|
-
export interface FusionResult {
|
|
1
|
+
export interface FusionResult {
|
|
2
|
+
chunkId: string;
|
|
3
|
+
score: number;
|
|
4
|
+
}
|
|
2
5
|
|
|
3
6
|
export function reciprocalRankFusion(lists: Array<{ chunkId: string; score: number }[]>, k = 60): FusionResult[] {
|
|
4
7
|
const scores = new Map<string, number>();
|
|
@@ -8,7 +11,5 @@ export function reciprocalRankFusion(lists: Array<{ chunkId: string; score: numb
|
|
|
8
11
|
scores.set(chunkId, (scores.get(chunkId) ?? 0) + 1 / (k + rank + 1));
|
|
9
12
|
}
|
|
10
13
|
}
|
|
11
|
-
return [...scores.entries()]
|
|
12
|
-
.map(([chunkId, score]) => ({ chunkId, score }))
|
|
13
|
-
.sort((a, b) => b.score - a.score);
|
|
14
|
+
return [...scores.entries()].map(([chunkId, score]) => ({ chunkId, score })).sort((a, b) => b.score - a.score);
|
|
14
15
|
}
|
package/src/search/reranker.ts
CHANGED
|
@@ -1,12 +1,19 @@
|
|
|
1
1
|
import { join } from "node:path";
|
|
2
2
|
import { getDefaultKnowledgeDir } from "../storage/sqlite.ts";
|
|
3
3
|
|
|
4
|
-
|
|
4
|
+
type RerankerPipeline = {
|
|
5
|
+
(input: { text: string; text_pair: string }): Promise<{ score?: number } | Array<{ score?: number }>>;
|
|
6
|
+
dispose(): Promise<void> | void;
|
|
7
|
+
};
|
|
8
|
+
|
|
9
|
+
let rerankerPipeline: RerankerPipeline | null = null;
|
|
5
10
|
let disposeTimer: ReturnType<typeof setTimeout> | null = null;
|
|
6
|
-
|
|
11
|
+
let disposePromise: Promise<void> | null = null;
|
|
12
|
+
const IDLE_MS = 30_000;
|
|
7
13
|
|
|
8
|
-
async function load(): Promise<
|
|
14
|
+
async function load(): Promise<RerankerPipeline> {
|
|
9
15
|
if (rerankerPipeline) return rerankerPipeline;
|
|
16
|
+
if (disposePromise) await disposePromise;
|
|
10
17
|
const { pipeline, env } = await import("@huggingface/transformers");
|
|
11
18
|
env.cacheDir = join(getDefaultKnowledgeDir(), "models");
|
|
12
19
|
rerankerPipeline = await pipeline("text-classification", "Xenova/ms-marco-MiniLM-L-4-v2");
|
|
@@ -19,13 +26,30 @@ function resetTimer(): void {
|
|
|
19
26
|
}
|
|
20
27
|
|
|
21
28
|
export async function disposeReranker(): Promise<void> {
|
|
22
|
-
if (disposeTimer) {
|
|
23
|
-
|
|
29
|
+
if (disposeTimer) {
|
|
30
|
+
clearTimeout(disposeTimer);
|
|
31
|
+
disposeTimer = null;
|
|
32
|
+
}
|
|
33
|
+
if (disposePromise) return disposePromise;
|
|
34
|
+
const instance = rerankerPipeline;
|
|
35
|
+
rerankerPipeline = null;
|
|
36
|
+
if (!instance) return;
|
|
37
|
+
disposePromise = Promise.resolve(instance.dispose()).finally(() => {
|
|
38
|
+
disposePromise = null;
|
|
39
|
+
});
|
|
40
|
+
return disposePromise;
|
|
24
41
|
}
|
|
25
42
|
|
|
26
|
-
export interface RerankCandidate {
|
|
43
|
+
export interface RerankCandidate {
|
|
44
|
+
chunkId: string;
|
|
45
|
+
content: string;
|
|
46
|
+
}
|
|
27
47
|
|
|
28
|
-
export async function rerank(
|
|
48
|
+
export async function rerank(
|
|
49
|
+
query: string,
|
|
50
|
+
candidates: RerankCandidate[],
|
|
51
|
+
topK: number,
|
|
52
|
+
): Promise<Array<{ chunkId: string; score: number }>> {
|
|
29
53
|
if (candidates.length === 0) return [];
|
|
30
54
|
const pipe = await load();
|
|
31
55
|
resetTimer();
|
package/src/search/vector.ts
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
1
|
-
export interface VectorResult {
|
|
1
|
+
export interface VectorResult {
|
|
2
|
+
chunkId: string;
|
|
3
|
+
score: number;
|
|
4
|
+
}
|
|
2
5
|
|
|
3
6
|
function cosine(a: Float32Array, b: Float32Array): number {
|
|
4
7
|
let dot = 0;
|
|
@@ -6,7 +9,12 @@ function cosine(a: Float32Array, b: Float32Array): number {
|
|
|
6
9
|
return dot;
|
|
7
10
|
}
|
|
8
11
|
|
|
9
|
-
export function searchVector(
|
|
12
|
+
export function searchVector(
|
|
13
|
+
queryVec: Float32Array,
|
|
14
|
+
vectors: Float32Array[],
|
|
15
|
+
chunkIds: string[],
|
|
16
|
+
limit = 50,
|
|
17
|
+
): VectorResult[] {
|
|
10
18
|
const scores = vectors.map((v, i) => ({ chunkId: chunkIds[i], score: cosine(queryVec, v) }));
|
|
11
19
|
scores.sort((a, b) => b.score - a.score);
|
|
12
20
|
return scores.slice(0, limit);
|
package/src/storage/sqlite.ts
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
2
|
import { existsSync, mkdirSync } from "node:fs";
|
|
3
|
-
import { join } from "node:path";
|
|
4
3
|
import { homedir } from "node:os";
|
|
5
|
-
import {
|
|
4
|
+
import { join } from "node:path";
|
|
5
|
+
import Database from "better-sqlite3";
|
|
6
6
|
|
|
7
7
|
export interface KnowledgeBase {
|
|
8
8
|
id: string;
|
|
9
9
|
name: string;
|
|
10
10
|
description: string | null;
|
|
11
11
|
source_path: string | null;
|
|
12
|
-
source_type: "file" | "directory" | "text";
|
|
12
|
+
source_type: "file" | "directory" | "text" | "url";
|
|
13
13
|
created_at: number;
|
|
14
14
|
updated_at: number;
|
|
15
15
|
chunk_count: number;
|
|
@@ -147,7 +147,9 @@ export function createKB(
|
|
|
147
147
|
`INSERT INTO knowledge_bases (id, name, description, source_path, source_type, created_at, updated_at)
|
|
148
148
|
VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
|
149
149
|
).run(id, opts.name, opts.description ?? null, opts.source_path ?? null, opts.source_type, now, now);
|
|
150
|
-
|
|
150
|
+
const kb = getKB(db, id);
|
|
151
|
+
if (!kb) throw new Error(`Failed to create knowledge base: ${id}`);
|
|
152
|
+
return kb;
|
|
151
153
|
}
|
|
152
154
|
|
|
153
155
|
export function getKB(db: Database.Database, id: string): KnowledgeBase | undefined {
|
|
@@ -190,7 +192,19 @@ export function insertChunks(db: Database.Database, kbId: string, chunks: ChunkI
|
|
|
190
192
|
const now = Date.now();
|
|
191
193
|
const insertMany = db.transaction((items: ChunkInsert[]) => {
|
|
192
194
|
for (const c of items) {
|
|
193
|
-
stmt.run(
|
|
195
|
+
stmt.run(
|
|
196
|
+
randomUUID(),
|
|
197
|
+
kbId,
|
|
198
|
+
c.content_hash,
|
|
199
|
+
c.content,
|
|
200
|
+
c.content_tokenized,
|
|
201
|
+
c.file_path,
|
|
202
|
+
c.file_type,
|
|
203
|
+
c.start_line,
|
|
204
|
+
c.end_line,
|
|
205
|
+
c.metadata_json,
|
|
206
|
+
now,
|
|
207
|
+
);
|
|
194
208
|
}
|
|
195
209
|
});
|
|
196
210
|
insertMany(chunks);
|
|
@@ -224,7 +238,10 @@ export function deleteChunksByIds(db: Database.Database, ids: string[]): void {
|
|
|
224
238
|
}
|
|
225
239
|
|
|
226
240
|
export function getChunkHashesByKB(db: Database.Database, kbId: string): Map<string, string> {
|
|
227
|
-
const rows = db.prepare("SELECT id, content_hash FROM chunks WHERE kb_id = ? ORDER BY rowid").all(kbId) as {
|
|
241
|
+
const rows = db.prepare("SELECT id, content_hash FROM chunks WHERE kb_id = ? ORDER BY rowid").all(kbId) as {
|
|
242
|
+
id: string;
|
|
243
|
+
content_hash: string;
|
|
244
|
+
}[];
|
|
228
245
|
return new Map(rows.map((r) => [r.content_hash, r.id]));
|
|
229
246
|
}
|
|
230
247
|
|
|
@@ -234,6 +251,8 @@ export function getChunkCount(db: Database.Database, kbId: string): number {
|
|
|
234
251
|
}
|
|
235
252
|
|
|
236
253
|
export function getFileCount(db: Database.Database, kbId: string): number {
|
|
237
|
-
const row = db.prepare("SELECT COUNT(DISTINCT file_path) as count FROM chunks WHERE kb_id = ?").get(kbId) as {
|
|
254
|
+
const row = db.prepare("SELECT COUNT(DISTINCT file_path) as count FROM chunks WHERE kb_id = ?").get(kbId) as {
|
|
255
|
+
count: number;
|
|
256
|
+
};
|
|
238
257
|
return row.count;
|
|
239
258
|
}
|