pi-knowledge 0.4.6 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/README.md +19 -8
- package/dist/index.js +129 -20
- package/dist/src/diagnostics/health.d.ts +1 -1
- package/dist/src/diagnostics/health.js +12 -1
- package/dist/src/embedding/provider.d.ts +1 -1
- package/dist/src/embedding/provider.js +12 -4
- package/dist/src/engine.d.ts +63 -10
- package/dist/src/engine.js +538 -164
- package/dist/src/indexer/symbols.d.ts +2 -0
- package/dist/src/indexer/symbols.js +175 -0
- package/dist/src/model-worker-client.d.ts +1 -1
- package/dist/src/model-worker-client.js +7 -7
- package/dist/src/search/reranker.d.ts +1 -1
- package/dist/src/search/reranker.js +2 -2
- package/dist/src/search/vector.d.ts +1 -1
- package/dist/src/search/vector.js +5 -1
- package/dist/src/storage/sqlite.d.ts +36 -1
- package/dist/src/storage/sqlite.js +213 -8
- package/dist/src/watcher/file-watcher.d.ts +2 -1
- package/dist/src/watcher/file-watcher.js +20 -30
- package/docs/configuration.md +8 -2
- package/package.json +4 -2
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
import { extname } from "node:path";
|
|
2
|
+
const CODE_EXTENSIONS = new Set([".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".py", ".go", ".rs", ".java"]);
|
|
3
|
+
const CONFIG_EXTENSIONS = new Set([".json", ".yaml", ".yml", ".toml", ".env"]);
|
|
4
|
+
function lineNumberAt(content, offset) {
|
|
5
|
+
let line = 1;
|
|
6
|
+
for (let i = 0; i < offset; i++) {
|
|
7
|
+
if (content.charCodeAt(i) === 10)
|
|
8
|
+
line++;
|
|
9
|
+
}
|
|
10
|
+
return line;
|
|
11
|
+
}
|
|
12
|
+
function lineEnd(content, startLine, maxLines = 12) {
|
|
13
|
+
return Math.min(startLine + maxLines, content.split("\n").length);
|
|
14
|
+
}
|
|
15
|
+
function makeSymbol(name, kind, filePath, fileType, startLine, options = {}) {
|
|
16
|
+
const trimmed = name.trim();
|
|
17
|
+
if (!trimmed)
|
|
18
|
+
return undefined;
|
|
19
|
+
return {
|
|
20
|
+
name: trimmed,
|
|
21
|
+
kind,
|
|
22
|
+
file_path: filePath,
|
|
23
|
+
file_type: fileType,
|
|
24
|
+
start_line: startLine,
|
|
25
|
+
end_line: options.endLine ?? startLine,
|
|
26
|
+
container_name: options.containerName ?? null,
|
|
27
|
+
signature: options.signature ?? null,
|
|
28
|
+
text: options.text ?? trimmed,
|
|
29
|
+
metadata_json: JSON.stringify(options.metadata ?? {}),
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
function pushSymbol(symbols, symbol) {
|
|
33
|
+
if (symbol)
|
|
34
|
+
symbols.push(symbol);
|
|
35
|
+
}
|
|
36
|
+
function extractMarkdownSymbols(content, filePath, fileType) {
|
|
37
|
+
const symbols = [];
|
|
38
|
+
const lines = content.split("\n");
|
|
39
|
+
for (let index = 0; index < lines.length; index++) {
|
|
40
|
+
const match = /^(#{1,6})\s+(.+?)\s*$/.exec(lines[index]);
|
|
41
|
+
if (!match)
|
|
42
|
+
continue;
|
|
43
|
+
pushSymbol(symbols, makeSymbol(match[2].replace(/\s+#*$/, ""), "heading", filePath, fileType, index + 1, {
|
|
44
|
+
text: lines[index],
|
|
45
|
+
metadata: { depth: match[1].length },
|
|
46
|
+
}));
|
|
47
|
+
}
|
|
48
|
+
return symbols;
|
|
49
|
+
}
|
|
50
|
+
function extractConfigSymbols(content, filePath, fileType) {
|
|
51
|
+
const symbols = [];
|
|
52
|
+
const lines = content.split("\n");
|
|
53
|
+
const ext = extname(filePath).toLowerCase();
|
|
54
|
+
for (let index = 0; index < lines.length; index++) {
|
|
55
|
+
const line = lines[index];
|
|
56
|
+
if (ext === ".json") {
|
|
57
|
+
const match = /^\s*"([^"]+)"\s*:/.exec(line);
|
|
58
|
+
pushSymbol(symbols, match ? makeSymbol(match[1], "config_key", filePath, fileType, index + 1, { text: line.trim() }) : undefined);
|
|
59
|
+
continue;
|
|
60
|
+
}
|
|
61
|
+
if (ext === ".yaml" || ext === ".yml") {
|
|
62
|
+
const match = /^\s*([A-Za-z0-9_.-]+)\s*:/.exec(line);
|
|
63
|
+
pushSymbol(symbols, match ? makeSymbol(match[1], "config_key", filePath, fileType, index + 1, { text: line.trim() }) : undefined);
|
|
64
|
+
continue;
|
|
65
|
+
}
|
|
66
|
+
if (ext === ".toml") {
|
|
67
|
+
const section = /^\s*\[+([A-Za-z0-9_.-]+)\]+/.exec(line);
|
|
68
|
+
const key = /^\s*([A-Za-z0-9_.-]+)\s*=/.exec(line);
|
|
69
|
+
const name = section?.[1] ?? key?.[1];
|
|
70
|
+
pushSymbol(symbols, name ? makeSymbol(name, "config_key", filePath, fileType, index + 1, { text: line.trim() }) : undefined);
|
|
71
|
+
continue;
|
|
72
|
+
}
|
|
73
|
+
const env = /^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=/.exec(line);
|
|
74
|
+
pushSymbol(symbols, env ? makeSymbol(env[1], "env_var", filePath, fileType, index + 1, { text: line.trim() }) : undefined);
|
|
75
|
+
}
|
|
76
|
+
return symbols;
|
|
77
|
+
}
|
|
78
|
+
function extractCodeSymbols(content, filePath, fileType) {
|
|
79
|
+
const symbols = [];
|
|
80
|
+
const patterns = [
|
|
81
|
+
{
|
|
82
|
+
regex: /\b(?:export\s+)?(?:async\s+)?function\s+([A-Za-z_$][\w$]*)\s*\([^)]*\)/g,
|
|
83
|
+
kind: "function",
|
|
84
|
+
nameIndex: 1,
|
|
85
|
+
},
|
|
86
|
+
{ regex: /\b(?:export\s+)?class\s+([A-Za-z_$][\w$]*)\b/g, kind: "class", nameIndex: 1 },
|
|
87
|
+
{ regex: /\b(?:export\s+)?interface\s+([A-Za-z_$][\w$]*)\b/g, kind: "interface", nameIndex: 1 },
|
|
88
|
+
{ regex: /\b(?:export\s+)?type\s+([A-Za-z_$][\w$]*)\b/g, kind: "type", nameIndex: 1 },
|
|
89
|
+
{
|
|
90
|
+
regex: /\b(?:export\s+)?(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*(?:async\s*)?(?:\([^)]*\)|[A-Za-z_$][\w$]*)\s*=>/g,
|
|
91
|
+
kind: "function",
|
|
92
|
+
nameIndex: 1,
|
|
93
|
+
},
|
|
94
|
+
{ regex: /\b(?:export\s+)?(?:const|let|var)\s+([A-Za-z_$][\w$]*)\b/g, kind: "variable", nameIndex: 1 },
|
|
95
|
+
{ regex: /\bfunc\s+(?:\([^)]+\)\s*)?([A-Za-z_][\w]*)\s*\([^)]*\)/g, kind: "function", nameIndex: 1 },
|
|
96
|
+
{ regex: /\btype\s+([A-Za-z_][\w]*)\s+(?:struct|interface)\b/g, kind: "type", nameIndex: 1 },
|
|
97
|
+
{ regex: /\bdef\s+([A-Za-z_][\w]*)\s*\([^)]*\)\s*:/g, kind: "function", nameIndex: 1 },
|
|
98
|
+
{ regex: /\bclass\s+([A-Za-z_][\w]*)\b/g, kind: "class", nameIndex: 1 },
|
|
99
|
+
{ regex: /\bfn\s+([A-Za-z_][\w]*)\s*\([^)]*\)/g, kind: "function", nameIndex: 1 },
|
|
100
|
+
{ regex: /\b(?:struct|enum|trait)\s+([A-Za-z_][\w]*)\b/g, kind: "type", nameIndex: 1 },
|
|
101
|
+
];
|
|
102
|
+
const seen = new Set();
|
|
103
|
+
for (const pattern of patterns) {
|
|
104
|
+
for (const match of content.matchAll(pattern.regex)) {
|
|
105
|
+
const name = match[pattern.nameIndex];
|
|
106
|
+
const index = match.index ?? 0;
|
|
107
|
+
const startLine = lineNumberAt(content, index);
|
|
108
|
+
const key = `${pattern.kind}:${name}:${startLine}`;
|
|
109
|
+
if (seen.has(key))
|
|
110
|
+
continue;
|
|
111
|
+
seen.add(key);
|
|
112
|
+
pushSymbol(symbols, makeSymbol(name, pattern.kind, filePath, fileType, startLine, {
|
|
113
|
+
endLine: lineEnd(content, startLine),
|
|
114
|
+
signature: match[0].split("\n")[0].trim(),
|
|
115
|
+
text: match[0].split("\n")[0].trim(),
|
|
116
|
+
}));
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
return symbols;
|
|
120
|
+
}
|
|
121
|
+
function extractRouteSymbols(content, filePath, fileType) {
|
|
122
|
+
const symbols = [];
|
|
123
|
+
const patterns = [
|
|
124
|
+
{
|
|
125
|
+
regex: /\b(?:app|router|route|server|api|hono)\s*\.\s*(get|post|put|patch|delete|del|options|head|all|use)\s*\(\s*["'`]([^"'`]+)["'`]/g,
|
|
126
|
+
methodIndex: 1,
|
|
127
|
+
pathIndex: 2,
|
|
128
|
+
},
|
|
129
|
+
{
|
|
130
|
+
regex: /\b(?:http\.)?HandleFunc\s*\(\s*["'`]([^"'`]+)["'`]/g,
|
|
131
|
+
pathIndex: 1,
|
|
132
|
+
defaultMethod: "ANY",
|
|
133
|
+
},
|
|
134
|
+
{
|
|
135
|
+
regex: /\b(?:http\.)?Handle\s*\(\s*["'`]([^"'`]+)["'`]/g,
|
|
136
|
+
pathIndex: 1,
|
|
137
|
+
defaultMethod: "ANY",
|
|
138
|
+
},
|
|
139
|
+
];
|
|
140
|
+
const seen = new Set();
|
|
141
|
+
for (const pattern of patterns) {
|
|
142
|
+
for (const match of content.matchAll(pattern.regex)) {
|
|
143
|
+
const rawMethod = pattern.methodIndex ? match[pattern.methodIndex] : pattern.defaultMethod;
|
|
144
|
+
const method = rawMethod === "del" ? "DELETE" : (rawMethod ?? "ANY").toUpperCase();
|
|
145
|
+
const path = match[pattern.pathIndex];
|
|
146
|
+
const index = match.index ?? 0;
|
|
147
|
+
const startLine = lineNumberAt(content, index);
|
|
148
|
+
const name = `${method} ${path}`;
|
|
149
|
+
const key = `${name}:${startLine}`;
|
|
150
|
+
if (seen.has(key))
|
|
151
|
+
continue;
|
|
152
|
+
seen.add(key);
|
|
153
|
+
pushSymbol(symbols, makeSymbol(name, "route", filePath, fileType, startLine, {
|
|
154
|
+
signature: match[0].split("\n")[0].trim(),
|
|
155
|
+
text: match[0].split("\n")[0].trim(),
|
|
156
|
+
metadata: { method, path },
|
|
157
|
+
}));
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
return symbols;
|
|
161
|
+
}
|
|
162
|
+
export function extractSymbols(content, filePath, fileType) {
|
|
163
|
+
const ext = extname(filePath).toLowerCase();
|
|
164
|
+
const symbols = [];
|
|
165
|
+
if (fileType === "markdown" || ext === ".md" || ext === ".mdx")
|
|
166
|
+
symbols.push(...extractMarkdownSymbols(content, filePath, fileType));
|
|
167
|
+
if (CONFIG_EXTENSIONS.has(ext) || filePath.split("/").pop()?.startsWith(".env")) {
|
|
168
|
+
symbols.push(...extractConfigSymbols(content, filePath, fileType));
|
|
169
|
+
}
|
|
170
|
+
if (CODE_EXTENSIONS.has(ext) || ["typescript", "javascript", "python", "go", "rust", "java"].includes(fileType)) {
|
|
171
|
+
symbols.push(...extractCodeSymbols(content, filePath, fileType));
|
|
172
|
+
symbols.push(...extractRouteSymbols(content, filePath, fileType));
|
|
173
|
+
}
|
|
174
|
+
return symbols;
|
|
175
|
+
}
|
|
@@ -3,7 +3,7 @@ export interface RerankWorkerCandidate {
|
|
|
3
3
|
chunkId: string;
|
|
4
4
|
content: string;
|
|
5
5
|
}
|
|
6
|
-
export declare function rerankInModelWorker(query: string, candidates: RerankWorkerCandidate[], topK: number): Promise<Array<{
|
|
6
|
+
export declare function rerankInModelWorker(query: string, candidates: RerankWorkerCandidate[], topK: number, signal?: AbortSignal): Promise<Array<{
|
|
7
7
|
chunkId: string;
|
|
8
8
|
score: number;
|
|
9
9
|
}>>;
|
|
@@ -86,19 +86,19 @@ async function requestModelWorker(message, signal) {
|
|
|
86
86
|
const id = nextRequestId++;
|
|
87
87
|
return new Promise((resolve, reject) => {
|
|
88
88
|
let abortHandler;
|
|
89
|
+
const cleanup = () => {
|
|
90
|
+
if (abortHandler)
|
|
91
|
+
signal?.removeEventListener("abort", abortHandler);
|
|
92
|
+
};
|
|
89
93
|
pending.set(id, { resolve, reject });
|
|
90
94
|
if (signal) {
|
|
91
95
|
abortHandler = () => {
|
|
92
96
|
pending.delete(id);
|
|
97
|
+
cleanup();
|
|
93
98
|
reject(new Error("Cancelled"));
|
|
94
|
-
shutdownModelWorker();
|
|
95
99
|
};
|
|
96
100
|
signal.addEventListener("abort", abortHandler, { once: true });
|
|
97
101
|
}
|
|
98
|
-
const cleanup = () => {
|
|
99
|
-
if (abortHandler)
|
|
100
|
-
signal?.removeEventListener("abort", abortHandler);
|
|
101
|
-
};
|
|
102
102
|
const originalResolve = resolve;
|
|
103
103
|
const originalReject = reject;
|
|
104
104
|
pending.set(id, {
|
|
@@ -130,8 +130,8 @@ export async function embedInModelWorker(texts, prefix, signal) {
|
|
|
130
130
|
return new Float32Array(vector);
|
|
131
131
|
});
|
|
132
132
|
}
|
|
133
|
-
export async function rerankInModelWorker(query, candidates, topK) {
|
|
134
|
-
const result = await requestModelWorker({ type: "rerank", query, candidates, topK });
|
|
133
|
+
export async function rerankInModelWorker(query, candidates, topK, signal) {
|
|
134
|
+
const result = await requestModelWorker({ type: "rerank", query, candidates, topK }, signal);
|
|
135
135
|
if (!Array.isArray(result))
|
|
136
136
|
throw new Error("Invalid reranker worker response");
|
|
137
137
|
return result.map((item) => {
|
|
@@ -4,7 +4,7 @@ export interface RerankCandidate {
|
|
|
4
4
|
chunkId: string;
|
|
5
5
|
content: string;
|
|
6
6
|
}
|
|
7
|
-
export declare function rerank(query: string, candidates: RerankCandidate[], topK: number): Promise<Array<{
|
|
7
|
+
export declare function rerank(query: string, candidates: RerankCandidate[], topK: number, signal?: AbortSignal): Promise<Array<{
|
|
8
8
|
chunkId: string;
|
|
9
9
|
score: number;
|
|
10
10
|
}>>;
|
|
@@ -53,12 +53,12 @@ export async function prepareRerankerForShutdown() {
|
|
|
53
53
|
clearTimer();
|
|
54
54
|
await waitForNoActiveRuns();
|
|
55
55
|
}
|
|
56
|
-
export async function rerank(query, candidates, topK) {
|
|
56
|
+
export async function rerank(query, candidates, topK, signal) {
|
|
57
57
|
if (candidates.length === 0)
|
|
58
58
|
return [];
|
|
59
59
|
beginRun();
|
|
60
60
|
try {
|
|
61
|
-
return await rerankInModelWorker(query, candidates, topK);
|
|
61
|
+
return await rerankInModelWorker(query, candidates, topK, signal);
|
|
62
62
|
}
|
|
63
63
|
finally {
|
|
64
64
|
endRun();
|
|
@@ -10,5 +10,5 @@ type ChunkIdSource = Iterable<string | {
|
|
|
10
10
|
id: string;
|
|
11
11
|
}>;
|
|
12
12
|
export declare function searchVector(queryVec: Float32Array, vectors: Float32Array[], chunkIds: string[], limit?: number): VectorResult[];
|
|
13
|
-
export declare function searchVectorFile(queryVec: Float32Array, vectorPath: string, chunkIds: ChunkIdSource, limit?: number): VectorFileSearchResult;
|
|
13
|
+
export declare function searchVectorFile(queryVec: Float32Array, vectorPath: string, chunkIds: ChunkIdSource, limit?: number, signal?: AbortSignal): VectorFileSearchResult;
|
|
14
14
|
export {};
|
|
@@ -25,7 +25,9 @@ export function searchVector(queryVec, vectors, chunkIds, limit = 50) {
|
|
|
25
25
|
scores.sort((a, b) => b.score - a.score);
|
|
26
26
|
return scores.slice(0, limit);
|
|
27
27
|
}
|
|
28
|
-
export function searchVectorFile(queryVec, vectorPath, chunkIds, limit = 50) {
|
|
28
|
+
export function searchVectorFile(queryVec, vectorPath, chunkIds, limit = 50, signal) {
|
|
29
|
+
if (signal?.aborted)
|
|
30
|
+
throw new Error("Cancelled");
|
|
29
31
|
const reader = openVectorReader(vectorPath);
|
|
30
32
|
if (!reader)
|
|
31
33
|
return { results: [], vectorsByChunkId: new Map() };
|
|
@@ -36,6 +38,8 @@ export function searchVectorFile(queryVec, vectorPath, chunkIds, limit = 50) {
|
|
|
36
38
|
const top = [];
|
|
37
39
|
let i = 0;
|
|
38
40
|
for (const chunkIdRef of chunkIds) {
|
|
41
|
+
if (signal?.aborted)
|
|
42
|
+
throw new Error("Cancelled");
|
|
39
43
|
if (i >= reader.count)
|
|
40
44
|
break;
|
|
41
45
|
const vectorIndex = i;
|
|
@@ -45,6 +45,23 @@ export interface Chunk {
|
|
|
45
45
|
indexed_at: number;
|
|
46
46
|
}
|
|
47
47
|
export type ChunkInsert = Omit<Chunk, "id" | "kb_id" | "indexed_at">;
|
|
48
|
+
export interface KnowledgeSymbol {
|
|
49
|
+
id: string;
|
|
50
|
+
kb_id: string;
|
|
51
|
+
name: string;
|
|
52
|
+
normalized_name: string;
|
|
53
|
+
kind: "function" | "class" | "interface" | "type" | "variable" | "heading" | "config_key" | "env_var" | "route";
|
|
54
|
+
file_path: string;
|
|
55
|
+
file_type: string;
|
|
56
|
+
start_line: number;
|
|
57
|
+
end_line: number;
|
|
58
|
+
container_name: string | null;
|
|
59
|
+
signature: string | null;
|
|
60
|
+
text: string;
|
|
61
|
+
metadata_json: string;
|
|
62
|
+
indexed_at: number;
|
|
63
|
+
}
|
|
64
|
+
export type KnowledgeSymbolInsert = Omit<KnowledgeSymbol, "id" | "kb_id" | "indexed_at" | "normalized_name">;
|
|
48
65
|
export declare function getDefaultKnowledgeDir(): string;
|
|
49
66
|
export declare function resolveHostKnowledgeDir(hostRoot: string, options?: {
|
|
50
67
|
legacyPiDir?: string;
|
|
@@ -78,7 +95,7 @@ export declare function updateIndexingJob(db: Database.Database, kbId: string, p
|
|
|
78
95
|
}): void;
|
|
79
96
|
export declare function finishIndexingJob(db: Database.Database, kbId: string, status: Extract<IndexingJob["status"], "succeeded" | "failed" | "cancelled">, message: string, errorMessage?: string): void;
|
|
80
97
|
export declare function getIndexingJob(db: Database.Database, kbId: string): IndexingJob | undefined;
|
|
81
|
-
export declare function insertChunks(db: Database.Database, kbId: string, chunks: ChunkInsert[]):
|
|
98
|
+
export declare function insertChunks(db: Database.Database, kbId: string, chunks: ChunkInsert[]): string[];
|
|
82
99
|
export declare function getChunksByKB(db: Database.Database, kbId: string): Chunk[];
|
|
83
100
|
export declare function iterateChunksByKB(db: Database.Database, kbId: string): IterableIterator<Chunk>;
|
|
84
101
|
export declare function getChunkIdsByKB(db: Database.Database, kbId: string): string[];
|
|
@@ -97,3 +114,21 @@ export declare function iterateChunkHashesByKB(db: Database.Database, kbId: stri
|
|
|
97
114
|
}>;
|
|
98
115
|
export declare function getChunkCount(db: Database.Database, kbId: string): number;
|
|
99
116
|
export declare function getFileCount(db: Database.Database, kbId: string): number;
|
|
117
|
+
export declare function insertSymbols(db: Database.Database, kbId: string, symbols: KnowledgeSymbolInsert[]): void;
|
|
118
|
+
export declare function deleteSymbolsByKB(db: Database.Database, kbId: string): void;
|
|
119
|
+
export declare function deleteSymbolsByFile(db: Database.Database, kbId: string, filePath: string): void;
|
|
120
|
+
export declare function searchSymbols(db: Database.Database, query: string, options?: {
|
|
121
|
+
kbId?: string;
|
|
122
|
+
kind?: KnowledgeSymbol["kind"];
|
|
123
|
+
filePattern?: string;
|
|
124
|
+
limit?: number;
|
|
125
|
+
offset?: number;
|
|
126
|
+
exact?: boolean;
|
|
127
|
+
}): KnowledgeSymbol[];
|
|
128
|
+
export declare function countSymbols(db: Database.Database, query: string, options?: {
|
|
129
|
+
kbId?: string;
|
|
130
|
+
kind?: KnowledgeSymbol["kind"];
|
|
131
|
+
filePattern?: string;
|
|
132
|
+
exact?: boolean;
|
|
133
|
+
}): number;
|
|
134
|
+
export declare function getSymbolCount(db: Database.Database, kbId: string): number;
|
|
@@ -4,7 +4,8 @@ import { createRequire } from "node:module";
|
|
|
4
4
|
import { homedir } from "node:os";
|
|
5
5
|
import { basename, dirname, join } from "node:path";
|
|
6
6
|
import { fileURLToPath } from "node:url";
|
|
7
|
-
const SCHEMA_VERSION =
|
|
7
|
+
const SCHEMA_VERSION = 4;
|
|
8
|
+
const ITERATION_BATCH_SIZE = 500;
|
|
8
9
|
const SCHEMA_SQL = `
|
|
9
10
|
CREATE TABLE IF NOT EXISTS schema_version (
|
|
10
11
|
version INTEGER NOT NULL
|
|
@@ -63,6 +64,28 @@ CREATE TABLE IF NOT EXISTS indexing_jobs (
|
|
|
63
64
|
CREATE INDEX IF NOT EXISTS idx_chunks_kb_id ON chunks(kb_id);
|
|
64
65
|
CREATE INDEX IF NOT EXISTS idx_chunks_content_hash ON chunks(content_hash);
|
|
65
66
|
|
|
67
|
+
CREATE TABLE IF NOT EXISTS symbols (
|
|
68
|
+
id TEXT PRIMARY KEY,
|
|
69
|
+
kb_id TEXT NOT NULL,
|
|
70
|
+
name TEXT NOT NULL,
|
|
71
|
+
normalized_name TEXT NOT NULL,
|
|
72
|
+
kind TEXT NOT NULL,
|
|
73
|
+
file_path TEXT NOT NULL,
|
|
74
|
+
file_type TEXT NOT NULL DEFAULT 'text',
|
|
75
|
+
start_line INTEGER NOT NULL DEFAULT 0,
|
|
76
|
+
end_line INTEGER NOT NULL DEFAULT 0,
|
|
77
|
+
container_name TEXT,
|
|
78
|
+
signature TEXT,
|
|
79
|
+
text TEXT NOT NULL DEFAULT '',
|
|
80
|
+
metadata_json TEXT NOT NULL DEFAULT '{}',
|
|
81
|
+
indexed_at INTEGER NOT NULL,
|
|
82
|
+
FOREIGN KEY (kb_id) REFERENCES knowledge_bases(id) ON DELETE CASCADE
|
|
83
|
+
);
|
|
84
|
+
|
|
85
|
+
CREATE INDEX IF NOT EXISTS idx_symbols_kb_name ON symbols(kb_id, normalized_name);
|
|
86
|
+
CREATE INDEX IF NOT EXISTS idx_symbols_kb_kind ON symbols(kb_id, kind);
|
|
87
|
+
CREATE INDEX IF NOT EXISTS idx_symbols_file ON symbols(kb_id, file_path);
|
|
88
|
+
|
|
66
89
|
CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
|
|
67
90
|
content_tokenized,
|
|
68
91
|
content=chunks,
|
|
@@ -189,7 +212,7 @@ export function openDatabase(knowledgeDir) {
|
|
|
189
212
|
const nativeBinding = isBunRuntime() ? undefined : findBetterSqlite3NativeBinding();
|
|
190
213
|
const db = nativeBinding ? new Database(dbPath, { nativeBinding }) : new Database(dbPath);
|
|
191
214
|
applyPragma(db, "journal_mode = WAL");
|
|
192
|
-
applyPragma(db, "busy_timeout =
|
|
215
|
+
applyPragma(db, "busy_timeout = 60000");
|
|
193
216
|
applyPragma(db, "foreign_keys = ON");
|
|
194
217
|
const hasVersion = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='schema_version'").get();
|
|
195
218
|
if (!hasVersion) {
|
|
@@ -238,6 +261,31 @@ CREATE TABLE IF NOT EXISTS indexing_jobs (
|
|
|
238
261
|
}
|
|
239
262
|
continue;
|
|
240
263
|
}
|
|
264
|
+
if (v === 4) {
|
|
265
|
+
db.exec(`
|
|
266
|
+
CREATE TABLE IF NOT EXISTS symbols (
|
|
267
|
+
id TEXT PRIMARY KEY,
|
|
268
|
+
kb_id TEXT NOT NULL,
|
|
269
|
+
name TEXT NOT NULL,
|
|
270
|
+
normalized_name TEXT NOT NULL,
|
|
271
|
+
kind TEXT NOT NULL,
|
|
272
|
+
file_path TEXT NOT NULL,
|
|
273
|
+
file_type TEXT NOT NULL DEFAULT 'text',
|
|
274
|
+
start_line INTEGER NOT NULL DEFAULT 0,
|
|
275
|
+
end_line INTEGER NOT NULL DEFAULT 0,
|
|
276
|
+
container_name TEXT,
|
|
277
|
+
signature TEXT,
|
|
278
|
+
text TEXT NOT NULL DEFAULT '',
|
|
279
|
+
metadata_json TEXT NOT NULL DEFAULT '{}',
|
|
280
|
+
indexed_at INTEGER NOT NULL,
|
|
281
|
+
FOREIGN KEY (kb_id) REFERENCES knowledge_bases(id) ON DELETE CASCADE
|
|
282
|
+
);
|
|
283
|
+
CREATE INDEX IF NOT EXISTS idx_symbols_kb_name ON symbols(kb_id, normalized_name);
|
|
284
|
+
CREATE INDEX IF NOT EXISTS idx_symbols_kb_kind ON symbols(kb_id, kind);
|
|
285
|
+
CREATE INDEX IF NOT EXISTS idx_symbols_file ON symbols(kb_id, file_path);
|
|
286
|
+
`);
|
|
287
|
+
continue;
|
|
288
|
+
}
|
|
241
289
|
if (migrations[v])
|
|
242
290
|
db.exec(migrations[v]);
|
|
243
291
|
}
|
|
@@ -264,6 +312,7 @@ export function listKBs(db) {
|
|
|
264
312
|
}
|
|
265
313
|
export function deleteKB(db, id) {
|
|
266
314
|
db.prepare("DELETE FROM indexing_jobs WHERE kb_id = ?").run(id);
|
|
315
|
+
db.prepare("DELETE FROM symbols WHERE kb_id = ?").run(id);
|
|
267
316
|
db.prepare("DELETE FROM chunks WHERE kb_id = ?").run(id);
|
|
268
317
|
db.prepare("DELETE FROM knowledge_bases WHERE id = ?").run(id);
|
|
269
318
|
}
|
|
@@ -331,28 +380,32 @@ export function getIndexingJob(db, kbId) {
|
|
|
331
380
|
}
|
|
332
381
|
// --- CRUD: Chunks ---
|
|
333
382
|
export function insertChunks(db, kbId, chunks) {
|
|
383
|
+
const ids = [];
|
|
334
384
|
const stmt = db.prepare(`INSERT INTO chunks (id, kb_id, content_hash, content, content_tokenized, file_path, file_type, start_line, end_line, metadata_json, indexed_at)
|
|
335
385
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
336
386
|
const now = Date.now();
|
|
337
387
|
const insertMany = db.transaction((items) => {
|
|
338
388
|
for (const c of items) {
|
|
339
|
-
|
|
389
|
+
const id = randomUUID();
|
|
390
|
+
ids.push(id);
|
|
391
|
+
stmt.run(id, kbId, c.content_hash, c.content, c.content_tokenized, c.file_path, c.file_type, c.start_line, c.end_line, c.metadata_json, now);
|
|
340
392
|
}
|
|
341
393
|
});
|
|
342
394
|
insertMany(chunks);
|
|
395
|
+
return ids;
|
|
343
396
|
}
|
|
344
397
|
export function getChunksByKB(db, kbId) {
|
|
345
398
|
return db.prepare("SELECT * FROM chunks WHERE kb_id = ? ORDER BY rowid").all(kbId);
|
|
346
399
|
}
|
|
347
400
|
export function iterateChunksByKB(db, kbId) {
|
|
348
|
-
return db
|
|
401
|
+
return iterateChunkRowsByKB(db, kbId);
|
|
349
402
|
}
|
|
350
403
|
export function getChunkIdsByKB(db, kbId) {
|
|
351
404
|
const rows = db.prepare("SELECT id FROM chunks WHERE kb_id = ? ORDER BY rowid").all(kbId);
|
|
352
405
|
return rows.map((r) => r.id);
|
|
353
406
|
}
|
|
354
407
|
export function iterateChunkIdsByKB(db, kbId) {
|
|
355
|
-
return db
|
|
408
|
+
return iterateChunkIdRowsByKB(db, kbId);
|
|
356
409
|
}
|
|
357
410
|
export function getChunkById(db, id) {
|
|
358
411
|
return db.prepare("SELECT * FROM chunks WHERE id = ?").get(id);
|
|
@@ -387,9 +440,56 @@ export function getChunkHashesByKB(db, kbId) {
|
|
|
387
440
|
return new Map(rows.map((r) => [r.content_hash, r.id]));
|
|
388
441
|
}
|
|
389
442
|
export function iterateChunkHashesByKB(db, kbId) {
|
|
390
|
-
return db
|
|
391
|
-
|
|
392
|
-
|
|
443
|
+
return iterateChunkHashRowsByKB(db, kbId);
|
|
444
|
+
}
|
|
445
|
+
function* iterateChunkRowsByKB(db, kbId) {
|
|
446
|
+
let lastRowid = 0;
|
|
447
|
+
const statement = db.prepare(`SELECT rowid as rowid, * FROM chunks
|
|
448
|
+
WHERE kb_id = ? AND rowid > ?
|
|
449
|
+
ORDER BY rowid
|
|
450
|
+
LIMIT ?`);
|
|
451
|
+
while (true) {
|
|
452
|
+
const rows = statement.all(kbId, lastRowid, ITERATION_BATCH_SIZE);
|
|
453
|
+
if (rows.length === 0)
|
|
454
|
+
return;
|
|
455
|
+
for (const row of rows) {
|
|
456
|
+
lastRowid = row.rowid;
|
|
457
|
+
const { rowid: _rowid, ...chunk } = row;
|
|
458
|
+
yield chunk;
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
function* iterateChunkIdRowsByKB(db, kbId) {
|
|
463
|
+
let lastRowid = 0;
|
|
464
|
+
const statement = db.prepare(`SELECT rowid, id FROM chunks
|
|
465
|
+
WHERE kb_id = ? AND rowid > ?
|
|
466
|
+
ORDER BY rowid
|
|
467
|
+
LIMIT ?`);
|
|
468
|
+
while (true) {
|
|
469
|
+
const rows = statement.all(kbId, lastRowid, ITERATION_BATCH_SIZE);
|
|
470
|
+
if (rows.length === 0)
|
|
471
|
+
return;
|
|
472
|
+
for (const row of rows) {
|
|
473
|
+
lastRowid = row.rowid;
|
|
474
|
+
yield { id: row.id };
|
|
475
|
+
}
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
function* iterateChunkHashRowsByKB(db, kbId) {
|
|
479
|
+
let lastRowid = 0;
|
|
480
|
+
const statement = db.prepare(`SELECT rowid, id, content_hash FROM chunks
|
|
481
|
+
WHERE kb_id = ? AND rowid > ?
|
|
482
|
+
ORDER BY rowid
|
|
483
|
+
LIMIT ?`);
|
|
484
|
+
while (true) {
|
|
485
|
+
const rows = statement.all(kbId, lastRowid, ITERATION_BATCH_SIZE);
|
|
486
|
+
if (rows.length === 0)
|
|
487
|
+
return;
|
|
488
|
+
for (const row of rows) {
|
|
489
|
+
lastRowid = row.rowid;
|
|
490
|
+
yield { id: row.id, content_hash: row.content_hash };
|
|
491
|
+
}
|
|
492
|
+
}
|
|
393
493
|
}
|
|
394
494
|
export function getChunkCount(db, kbId) {
|
|
395
495
|
const row = db.prepare("SELECT COUNT(*) as count FROM chunks WHERE kb_id = ?").get(kbId);
|
|
@@ -399,3 +499,108 @@ export function getFileCount(db, kbId) {
|
|
|
399
499
|
const row = db.prepare("SELECT COUNT(DISTINCT file_path) as count FROM chunks WHERE kb_id = ?").get(kbId);
|
|
400
500
|
return row.count;
|
|
401
501
|
}
|
|
502
|
+
// --- CRUD: Symbols ---
|
|
503
|
+
function normalizeSymbolName(name) {
|
|
504
|
+
return name.trim().toLowerCase();
|
|
505
|
+
}
|
|
506
|
+
function escapeLike(value) {
|
|
507
|
+
return value.replaceAll("\\", "\\\\").replaceAll("%", "\\%").replaceAll("_", "\\_");
|
|
508
|
+
}
|
|
509
|
+
export function insertSymbols(db, kbId, symbols) {
|
|
510
|
+
if (symbols.length === 0)
|
|
511
|
+
return;
|
|
512
|
+
const stmt = db.prepare(`INSERT INTO symbols (
|
|
513
|
+
id, kb_id, name, normalized_name, kind, file_path, file_type, start_line, end_line,
|
|
514
|
+
container_name, signature, text, metadata_json, indexed_at
|
|
515
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
516
|
+
const now = Date.now();
|
|
517
|
+
const insertMany = db.transaction((items) => {
|
|
518
|
+
for (const symbol of items) {
|
|
519
|
+
stmt.run(randomUUID(), kbId, symbol.name, normalizeSymbolName(symbol.name), symbol.kind, symbol.file_path, symbol.file_type, symbol.start_line, symbol.end_line, symbol.container_name ?? null, symbol.signature ?? null, symbol.text, symbol.metadata_json, now);
|
|
520
|
+
}
|
|
521
|
+
});
|
|
522
|
+
insertMany(symbols);
|
|
523
|
+
}
|
|
524
|
+
export function deleteSymbolsByKB(db, kbId) {
|
|
525
|
+
db.prepare("DELETE FROM symbols WHERE kb_id = ?").run(kbId);
|
|
526
|
+
}
|
|
527
|
+
export function deleteSymbolsByFile(db, kbId, filePath) {
|
|
528
|
+
db.prepare("DELETE FROM symbols WHERE kb_id = ? AND file_path = ?").run(kbId, filePath);
|
|
529
|
+
}
|
|
530
|
+
export function searchSymbols(db, query, options = {}) {
|
|
531
|
+
const clauses = ["kb.status IN ('ready', 'stale')"];
|
|
532
|
+
const params = [];
|
|
533
|
+
const normalizedQuery = normalizeSymbolName(query);
|
|
534
|
+
if (options.kbId) {
|
|
535
|
+
clauses.push("s.kb_id = ?");
|
|
536
|
+
params.push(options.kbId);
|
|
537
|
+
}
|
|
538
|
+
if (options.kind) {
|
|
539
|
+
clauses.push("s.kind = ?");
|
|
540
|
+
params.push(options.kind);
|
|
541
|
+
}
|
|
542
|
+
if (options.filePattern) {
|
|
543
|
+
clauses.push("s.file_path LIKE ? ESCAPE '\\'");
|
|
544
|
+
params.push(`%${escapeLike(options.filePattern)}%`);
|
|
545
|
+
}
|
|
546
|
+
if (normalizedQuery) {
|
|
547
|
+
if (options.exact) {
|
|
548
|
+
clauses.push("s.normalized_name = ?");
|
|
549
|
+
params.push(normalizedQuery);
|
|
550
|
+
}
|
|
551
|
+
else {
|
|
552
|
+
clauses.push("(s.normalized_name LIKE ? ESCAPE '\\' OR s.file_path LIKE ? ESCAPE '\\' OR s.text LIKE ? ESCAPE '\\')");
|
|
553
|
+
const pattern = `%${escapeLike(normalizedQuery)}%`;
|
|
554
|
+
params.push(pattern, pattern, pattern);
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
const where = `WHERE ${clauses.join(" AND ")}`;
|
|
558
|
+
const limit = Math.max(1, Math.min(200, options.limit ?? 20));
|
|
559
|
+
const offset = Math.max(0, options.offset ?? 0);
|
|
560
|
+
return db
|
|
561
|
+
.prepare(`SELECT s.* FROM symbols s
|
|
562
|
+
JOIN knowledge_bases kb ON kb.id = s.kb_id
|
|
563
|
+
${where}
|
|
564
|
+
ORDER BY
|
|
565
|
+
CASE WHEN s.normalized_name = ? THEN 0 WHEN s.normalized_name LIKE ? ESCAPE '\\' THEN 1 ELSE 2 END,
|
|
566
|
+
s.file_path, s.start_line
|
|
567
|
+
LIMIT ? OFFSET ?`)
|
|
568
|
+
.all(...params, normalizedQuery, `${escapeLike(normalizedQuery)}%`, limit, offset);
|
|
569
|
+
}
|
|
570
|
+
export function countSymbols(db, query, options = {}) {
|
|
571
|
+
const clauses = ["kb.status IN ('ready', 'stale')"];
|
|
572
|
+
const params = [];
|
|
573
|
+
const normalizedQuery = normalizeSymbolName(query);
|
|
574
|
+
if (options.kbId) {
|
|
575
|
+
clauses.push("s.kb_id = ?");
|
|
576
|
+
params.push(options.kbId);
|
|
577
|
+
}
|
|
578
|
+
if (options.kind) {
|
|
579
|
+
clauses.push("s.kind = ?");
|
|
580
|
+
params.push(options.kind);
|
|
581
|
+
}
|
|
582
|
+
if (options.filePattern) {
|
|
583
|
+
clauses.push("s.file_path LIKE ? ESCAPE '\\'");
|
|
584
|
+
params.push(`%${escapeLike(options.filePattern)}%`);
|
|
585
|
+
}
|
|
586
|
+
if (normalizedQuery) {
|
|
587
|
+
if (options.exact) {
|
|
588
|
+
clauses.push("s.normalized_name = ?");
|
|
589
|
+
params.push(normalizedQuery);
|
|
590
|
+
}
|
|
591
|
+
else {
|
|
592
|
+
clauses.push("(s.normalized_name LIKE ? ESCAPE '\\' OR s.file_path LIKE ? ESCAPE '\\' OR s.text LIKE ? ESCAPE '\\')");
|
|
593
|
+
const pattern = `%${escapeLike(normalizedQuery)}%`;
|
|
594
|
+
params.push(pattern, pattern, pattern);
|
|
595
|
+
}
|
|
596
|
+
}
|
|
597
|
+
const where = `WHERE ${clauses.join(" AND ")}`;
|
|
598
|
+
const row = db
|
|
599
|
+
.prepare(`SELECT COUNT(*) as count FROM symbols s JOIN knowledge_bases kb ON kb.id = s.kb_id ${where}`)
|
|
600
|
+
.get(...params);
|
|
601
|
+
return row.count;
|
|
602
|
+
}
|
|
603
|
+
export function getSymbolCount(db, kbId) {
|
|
604
|
+
const row = db.prepare("SELECT COUNT(*) as count FROM symbols WHERE kb_id = ?").get(kbId);
|
|
605
|
+
return row.count;
|
|
606
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
|
|
1
|
+
import { type ScanOptions } from "../indexer/chunker.ts";
|
|
2
|
+
export declare function startWatcher(kbId: string, dirPath: string, onUpdate: (kbId: string) => void, options?: ScanOptions): void;
|
|
2
3
|
export declare function stopWatcher(kbId: string): void;
|
|
3
4
|
export declare function stopAllWatchers(): void;
|
|
4
5
|
export declare function getActiveWatcherCount(): number;
|