pi-knowledge 0.4.7 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/README.md +9 -2
- package/dist/index.js +129 -20
- package/dist/src/diagnostics/health.d.ts +1 -1
- package/dist/src/diagnostics/health.js +12 -1
- package/dist/src/embedding/provider.d.ts +1 -1
- package/dist/src/embedding/provider.js +12 -4
- package/dist/src/engine.d.ts +61 -10
- package/dist/src/engine.js +519 -165
- package/dist/src/indexer/symbols.d.ts +2 -0
- package/dist/src/indexer/symbols.js +175 -0
- package/dist/src/model-worker-client.d.ts +1 -1
- package/dist/src/model-worker-client.js +7 -7
- package/dist/src/search/reranker.d.ts +1 -1
- package/dist/src/search/reranker.js +2 -2
- package/dist/src/search/vector.d.ts +1 -1
- package/dist/src/search/vector.js +5 -1
- package/dist/src/storage/sqlite.d.ts +36 -1
- package/dist/src/storage/sqlite.js +159 -2
- package/dist/src/watcher/file-watcher.d.ts +2 -1
- package/dist/src/watcher/file-watcher.js +20 -30
- package/docs/configuration.md +8 -2
- package/package.json +1 -1
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
import { extname } from "node:path";
|
|
2
|
+
const CODE_EXTENSIONS = new Set([".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".py", ".go", ".rs", ".java"]);
|
|
3
|
+
const CONFIG_EXTENSIONS = new Set([".json", ".yaml", ".yml", ".toml", ".env"]);
|
|
4
|
+
function lineNumberAt(content, offset) {
|
|
5
|
+
let line = 1;
|
|
6
|
+
for (let i = 0; i < offset; i++) {
|
|
7
|
+
if (content.charCodeAt(i) === 10)
|
|
8
|
+
line++;
|
|
9
|
+
}
|
|
10
|
+
return line;
|
|
11
|
+
}
|
|
12
|
+
function lineEnd(content, startLine, maxLines = 12) {
|
|
13
|
+
return Math.min(startLine + maxLines, content.split("\n").length);
|
|
14
|
+
}
|
|
15
|
+
function makeSymbol(name, kind, filePath, fileType, startLine, options = {}) {
|
|
16
|
+
const trimmed = name.trim();
|
|
17
|
+
if (!trimmed)
|
|
18
|
+
return undefined;
|
|
19
|
+
return {
|
|
20
|
+
name: trimmed,
|
|
21
|
+
kind,
|
|
22
|
+
file_path: filePath,
|
|
23
|
+
file_type: fileType,
|
|
24
|
+
start_line: startLine,
|
|
25
|
+
end_line: options.endLine ?? startLine,
|
|
26
|
+
container_name: options.containerName ?? null,
|
|
27
|
+
signature: options.signature ?? null,
|
|
28
|
+
text: options.text ?? trimmed,
|
|
29
|
+
metadata_json: JSON.stringify(options.metadata ?? {}),
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
function pushSymbol(symbols, symbol) {
|
|
33
|
+
if (symbol)
|
|
34
|
+
symbols.push(symbol);
|
|
35
|
+
}
|
|
36
|
+
function extractMarkdownSymbols(content, filePath, fileType) {
|
|
37
|
+
const symbols = [];
|
|
38
|
+
const lines = content.split("\n");
|
|
39
|
+
for (let index = 0; index < lines.length; index++) {
|
|
40
|
+
const match = /^(#{1,6})\s+(.+?)\s*$/.exec(lines[index]);
|
|
41
|
+
if (!match)
|
|
42
|
+
continue;
|
|
43
|
+
pushSymbol(symbols, makeSymbol(match[2].replace(/\s+#*$/, ""), "heading", filePath, fileType, index + 1, {
|
|
44
|
+
text: lines[index],
|
|
45
|
+
metadata: { depth: match[1].length },
|
|
46
|
+
}));
|
|
47
|
+
}
|
|
48
|
+
return symbols;
|
|
49
|
+
}
|
|
50
|
+
function extractConfigSymbols(content, filePath, fileType) {
|
|
51
|
+
const symbols = [];
|
|
52
|
+
const lines = content.split("\n");
|
|
53
|
+
const ext = extname(filePath).toLowerCase();
|
|
54
|
+
for (let index = 0; index < lines.length; index++) {
|
|
55
|
+
const line = lines[index];
|
|
56
|
+
if (ext === ".json") {
|
|
57
|
+
const match = /^\s*"([^"]+)"\s*:/.exec(line);
|
|
58
|
+
pushSymbol(symbols, match ? makeSymbol(match[1], "config_key", filePath, fileType, index + 1, { text: line.trim() }) : undefined);
|
|
59
|
+
continue;
|
|
60
|
+
}
|
|
61
|
+
if (ext === ".yaml" || ext === ".yml") {
|
|
62
|
+
const match = /^\s*([A-Za-z0-9_.-]+)\s*:/.exec(line);
|
|
63
|
+
pushSymbol(symbols, match ? makeSymbol(match[1], "config_key", filePath, fileType, index + 1, { text: line.trim() }) : undefined);
|
|
64
|
+
continue;
|
|
65
|
+
}
|
|
66
|
+
if (ext === ".toml") {
|
|
67
|
+
const section = /^\s*\[+([A-Za-z0-9_.-]+)\]+/.exec(line);
|
|
68
|
+
const key = /^\s*([A-Za-z0-9_.-]+)\s*=/.exec(line);
|
|
69
|
+
const name = section?.[1] ?? key?.[1];
|
|
70
|
+
pushSymbol(symbols, name ? makeSymbol(name, "config_key", filePath, fileType, index + 1, { text: line.trim() }) : undefined);
|
|
71
|
+
continue;
|
|
72
|
+
}
|
|
73
|
+
const env = /^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=/.exec(line);
|
|
74
|
+
pushSymbol(symbols, env ? makeSymbol(env[1], "env_var", filePath, fileType, index + 1, { text: line.trim() }) : undefined);
|
|
75
|
+
}
|
|
76
|
+
return symbols;
|
|
77
|
+
}
|
|
78
|
+
function extractCodeSymbols(content, filePath, fileType) {
|
|
79
|
+
const symbols = [];
|
|
80
|
+
const patterns = [
|
|
81
|
+
{
|
|
82
|
+
regex: /\b(?:export\s+)?(?:async\s+)?function\s+([A-Za-z_$][\w$]*)\s*\([^)]*\)/g,
|
|
83
|
+
kind: "function",
|
|
84
|
+
nameIndex: 1,
|
|
85
|
+
},
|
|
86
|
+
{ regex: /\b(?:export\s+)?class\s+([A-Za-z_$][\w$]*)\b/g, kind: "class", nameIndex: 1 },
|
|
87
|
+
{ regex: /\b(?:export\s+)?interface\s+([A-Za-z_$][\w$]*)\b/g, kind: "interface", nameIndex: 1 },
|
|
88
|
+
{ regex: /\b(?:export\s+)?type\s+([A-Za-z_$][\w$]*)\b/g, kind: "type", nameIndex: 1 },
|
|
89
|
+
{
|
|
90
|
+
regex: /\b(?:export\s+)?(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*(?:async\s*)?(?:\([^)]*\)|[A-Za-z_$][\w$]*)\s*=>/g,
|
|
91
|
+
kind: "function",
|
|
92
|
+
nameIndex: 1,
|
|
93
|
+
},
|
|
94
|
+
{ regex: /\b(?:export\s+)?(?:const|let|var)\s+([A-Za-z_$][\w$]*)\b/g, kind: "variable", nameIndex: 1 },
|
|
95
|
+
{ regex: /\bfunc\s+(?:\([^)]+\)\s*)?([A-Za-z_][\w]*)\s*\([^)]*\)/g, kind: "function", nameIndex: 1 },
|
|
96
|
+
{ regex: /\btype\s+([A-Za-z_][\w]*)\s+(?:struct|interface)\b/g, kind: "type", nameIndex: 1 },
|
|
97
|
+
{ regex: /\bdef\s+([A-Za-z_][\w]*)\s*\([^)]*\)\s*:/g, kind: "function", nameIndex: 1 },
|
|
98
|
+
{ regex: /\bclass\s+([A-Za-z_][\w]*)\b/g, kind: "class", nameIndex: 1 },
|
|
99
|
+
{ regex: /\bfn\s+([A-Za-z_][\w]*)\s*\([^)]*\)/g, kind: "function", nameIndex: 1 },
|
|
100
|
+
{ regex: /\b(?:struct|enum|trait)\s+([A-Za-z_][\w]*)\b/g, kind: "type", nameIndex: 1 },
|
|
101
|
+
];
|
|
102
|
+
const seen = new Set();
|
|
103
|
+
for (const pattern of patterns) {
|
|
104
|
+
for (const match of content.matchAll(pattern.regex)) {
|
|
105
|
+
const name = match[pattern.nameIndex];
|
|
106
|
+
const index = match.index ?? 0;
|
|
107
|
+
const startLine = lineNumberAt(content, index);
|
|
108
|
+
const key = `${pattern.kind}:${name}:${startLine}`;
|
|
109
|
+
if (seen.has(key))
|
|
110
|
+
continue;
|
|
111
|
+
seen.add(key);
|
|
112
|
+
pushSymbol(symbols, makeSymbol(name, pattern.kind, filePath, fileType, startLine, {
|
|
113
|
+
endLine: lineEnd(content, startLine),
|
|
114
|
+
signature: match[0].split("\n")[0].trim(),
|
|
115
|
+
text: match[0].split("\n")[0].trim(),
|
|
116
|
+
}));
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
return symbols;
|
|
120
|
+
}
|
|
121
|
+
function extractRouteSymbols(content, filePath, fileType) {
|
|
122
|
+
const symbols = [];
|
|
123
|
+
const patterns = [
|
|
124
|
+
{
|
|
125
|
+
regex: /\b(?:app|router|route|server|api|hono)\s*\.\s*(get|post|put|patch|delete|del|options|head|all|use)\s*\(\s*["'`]([^"'`]+)["'`]/g,
|
|
126
|
+
methodIndex: 1,
|
|
127
|
+
pathIndex: 2,
|
|
128
|
+
},
|
|
129
|
+
{
|
|
130
|
+
regex: /\b(?:http\.)?HandleFunc\s*\(\s*["'`]([^"'`]+)["'`]/g,
|
|
131
|
+
pathIndex: 1,
|
|
132
|
+
defaultMethod: "ANY",
|
|
133
|
+
},
|
|
134
|
+
{
|
|
135
|
+
regex: /\b(?:http\.)?Handle\s*\(\s*["'`]([^"'`]+)["'`]/g,
|
|
136
|
+
pathIndex: 1,
|
|
137
|
+
defaultMethod: "ANY",
|
|
138
|
+
},
|
|
139
|
+
];
|
|
140
|
+
const seen = new Set();
|
|
141
|
+
for (const pattern of patterns) {
|
|
142
|
+
for (const match of content.matchAll(pattern.regex)) {
|
|
143
|
+
const rawMethod = pattern.methodIndex ? match[pattern.methodIndex] : pattern.defaultMethod;
|
|
144
|
+
const method = rawMethod === "del" ? "DELETE" : (rawMethod ?? "ANY").toUpperCase();
|
|
145
|
+
const path = match[pattern.pathIndex];
|
|
146
|
+
const index = match.index ?? 0;
|
|
147
|
+
const startLine = lineNumberAt(content, index);
|
|
148
|
+
const name = `${method} ${path}`;
|
|
149
|
+
const key = `${name}:${startLine}`;
|
|
150
|
+
if (seen.has(key))
|
|
151
|
+
continue;
|
|
152
|
+
seen.add(key);
|
|
153
|
+
pushSymbol(symbols, makeSymbol(name, "route", filePath, fileType, startLine, {
|
|
154
|
+
signature: match[0].split("\n")[0].trim(),
|
|
155
|
+
text: match[0].split("\n")[0].trim(),
|
|
156
|
+
metadata: { method, path },
|
|
157
|
+
}));
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
return symbols;
|
|
161
|
+
}
|
|
162
|
+
export function extractSymbols(content, filePath, fileType) {
|
|
163
|
+
const ext = extname(filePath).toLowerCase();
|
|
164
|
+
const symbols = [];
|
|
165
|
+
if (fileType === "markdown" || ext === ".md" || ext === ".mdx")
|
|
166
|
+
symbols.push(...extractMarkdownSymbols(content, filePath, fileType));
|
|
167
|
+
if (CONFIG_EXTENSIONS.has(ext) || filePath.split("/").pop()?.startsWith(".env")) {
|
|
168
|
+
symbols.push(...extractConfigSymbols(content, filePath, fileType));
|
|
169
|
+
}
|
|
170
|
+
if (CODE_EXTENSIONS.has(ext) || ["typescript", "javascript", "python", "go", "rust", "java"].includes(fileType)) {
|
|
171
|
+
symbols.push(...extractCodeSymbols(content, filePath, fileType));
|
|
172
|
+
symbols.push(...extractRouteSymbols(content, filePath, fileType));
|
|
173
|
+
}
|
|
174
|
+
return symbols;
|
|
175
|
+
}
|
|
@@ -3,7 +3,7 @@ export interface RerankWorkerCandidate {
|
|
|
3
3
|
chunkId: string;
|
|
4
4
|
content: string;
|
|
5
5
|
}
|
|
6
|
-
export declare function rerankInModelWorker(query: string, candidates: RerankWorkerCandidate[], topK: number): Promise<Array<{
|
|
6
|
+
export declare function rerankInModelWorker(query: string, candidates: RerankWorkerCandidate[], topK: number, signal?: AbortSignal): Promise<Array<{
|
|
7
7
|
chunkId: string;
|
|
8
8
|
score: number;
|
|
9
9
|
}>>;
|
|
@@ -86,19 +86,19 @@ async function requestModelWorker(message, signal) {
|
|
|
86
86
|
const id = nextRequestId++;
|
|
87
87
|
return new Promise((resolve, reject) => {
|
|
88
88
|
let abortHandler;
|
|
89
|
+
const cleanup = () => {
|
|
90
|
+
if (abortHandler)
|
|
91
|
+
signal?.removeEventListener("abort", abortHandler);
|
|
92
|
+
};
|
|
89
93
|
pending.set(id, { resolve, reject });
|
|
90
94
|
if (signal) {
|
|
91
95
|
abortHandler = () => {
|
|
92
96
|
pending.delete(id);
|
|
97
|
+
cleanup();
|
|
93
98
|
reject(new Error("Cancelled"));
|
|
94
|
-
shutdownModelWorker();
|
|
95
99
|
};
|
|
96
100
|
signal.addEventListener("abort", abortHandler, { once: true });
|
|
97
101
|
}
|
|
98
|
-
const cleanup = () => {
|
|
99
|
-
if (abortHandler)
|
|
100
|
-
signal?.removeEventListener("abort", abortHandler);
|
|
101
|
-
};
|
|
102
102
|
const originalResolve = resolve;
|
|
103
103
|
const originalReject = reject;
|
|
104
104
|
pending.set(id, {
|
|
@@ -130,8 +130,8 @@ export async function embedInModelWorker(texts, prefix, signal) {
|
|
|
130
130
|
return new Float32Array(vector);
|
|
131
131
|
});
|
|
132
132
|
}
|
|
133
|
-
export async function rerankInModelWorker(query, candidates, topK) {
|
|
134
|
-
const result = await requestModelWorker({ type: "rerank", query, candidates, topK });
|
|
133
|
+
export async function rerankInModelWorker(query, candidates, topK, signal) {
|
|
134
|
+
const result = await requestModelWorker({ type: "rerank", query, candidates, topK }, signal);
|
|
135
135
|
if (!Array.isArray(result))
|
|
136
136
|
throw new Error("Invalid reranker worker response");
|
|
137
137
|
return result.map((item) => {
|
|
@@ -4,7 +4,7 @@ export interface RerankCandidate {
|
|
|
4
4
|
chunkId: string;
|
|
5
5
|
content: string;
|
|
6
6
|
}
|
|
7
|
-
export declare function rerank(query: string, candidates: RerankCandidate[], topK: number): Promise<Array<{
|
|
7
|
+
export declare function rerank(query: string, candidates: RerankCandidate[], topK: number, signal?: AbortSignal): Promise<Array<{
|
|
8
8
|
chunkId: string;
|
|
9
9
|
score: number;
|
|
10
10
|
}>>;
|
|
@@ -53,12 +53,12 @@ export async function prepareRerankerForShutdown() {
|
|
|
53
53
|
clearTimer();
|
|
54
54
|
await waitForNoActiveRuns();
|
|
55
55
|
}
|
|
56
|
-
export async function rerank(query, candidates, topK) {
|
|
56
|
+
export async function rerank(query, candidates, topK, signal) {
|
|
57
57
|
if (candidates.length === 0)
|
|
58
58
|
return [];
|
|
59
59
|
beginRun();
|
|
60
60
|
try {
|
|
61
|
-
return await rerankInModelWorker(query, candidates, topK);
|
|
61
|
+
return await rerankInModelWorker(query, candidates, topK, signal);
|
|
62
62
|
}
|
|
63
63
|
finally {
|
|
64
64
|
endRun();
|
|
@@ -10,5 +10,5 @@ type ChunkIdSource = Iterable<string | {
|
|
|
10
10
|
id: string;
|
|
11
11
|
}>;
|
|
12
12
|
export declare function searchVector(queryVec: Float32Array, vectors: Float32Array[], chunkIds: string[], limit?: number): VectorResult[];
|
|
13
|
-
export declare function searchVectorFile(queryVec: Float32Array, vectorPath: string, chunkIds: ChunkIdSource, limit?: number): VectorFileSearchResult;
|
|
13
|
+
export declare function searchVectorFile(queryVec: Float32Array, vectorPath: string, chunkIds: ChunkIdSource, limit?: number, signal?: AbortSignal): VectorFileSearchResult;
|
|
14
14
|
export {};
|
|
@@ -25,7 +25,9 @@ export function searchVector(queryVec, vectors, chunkIds, limit = 50) {
|
|
|
25
25
|
scores.sort((a, b) => b.score - a.score);
|
|
26
26
|
return scores.slice(0, limit);
|
|
27
27
|
}
|
|
28
|
-
export function searchVectorFile(queryVec, vectorPath, chunkIds, limit = 50) {
|
|
28
|
+
export function searchVectorFile(queryVec, vectorPath, chunkIds, limit = 50, signal) {
|
|
29
|
+
if (signal?.aborted)
|
|
30
|
+
throw new Error("Cancelled");
|
|
29
31
|
const reader = openVectorReader(vectorPath);
|
|
30
32
|
if (!reader)
|
|
31
33
|
return { results: [], vectorsByChunkId: new Map() };
|
|
@@ -36,6 +38,8 @@ export function searchVectorFile(queryVec, vectorPath, chunkIds, limit = 50) {
|
|
|
36
38
|
const top = [];
|
|
37
39
|
let i = 0;
|
|
38
40
|
for (const chunkIdRef of chunkIds) {
|
|
41
|
+
if (signal?.aborted)
|
|
42
|
+
throw new Error("Cancelled");
|
|
39
43
|
if (i >= reader.count)
|
|
40
44
|
break;
|
|
41
45
|
const vectorIndex = i;
|
|
@@ -45,6 +45,23 @@ export interface Chunk {
|
|
|
45
45
|
indexed_at: number;
|
|
46
46
|
}
|
|
47
47
|
export type ChunkInsert = Omit<Chunk, "id" | "kb_id" | "indexed_at">;
|
|
48
|
+
export interface KnowledgeSymbol {
|
|
49
|
+
id: string;
|
|
50
|
+
kb_id: string;
|
|
51
|
+
name: string;
|
|
52
|
+
normalized_name: string;
|
|
53
|
+
kind: "function" | "class" | "interface" | "type" | "variable" | "heading" | "config_key" | "env_var" | "route";
|
|
54
|
+
file_path: string;
|
|
55
|
+
file_type: string;
|
|
56
|
+
start_line: number;
|
|
57
|
+
end_line: number;
|
|
58
|
+
container_name: string | null;
|
|
59
|
+
signature: string | null;
|
|
60
|
+
text: string;
|
|
61
|
+
metadata_json: string;
|
|
62
|
+
indexed_at: number;
|
|
63
|
+
}
|
|
64
|
+
export type KnowledgeSymbolInsert = Omit<KnowledgeSymbol, "id" | "kb_id" | "indexed_at" | "normalized_name">;
|
|
48
65
|
export declare function getDefaultKnowledgeDir(): string;
|
|
49
66
|
export declare function resolveHostKnowledgeDir(hostRoot: string, options?: {
|
|
50
67
|
legacyPiDir?: string;
|
|
@@ -78,7 +95,7 @@ export declare function updateIndexingJob(db: Database.Database, kbId: string, p
|
|
|
78
95
|
}): void;
|
|
79
96
|
export declare function finishIndexingJob(db: Database.Database, kbId: string, status: Extract<IndexingJob["status"], "succeeded" | "failed" | "cancelled">, message: string, errorMessage?: string): void;
|
|
80
97
|
export declare function getIndexingJob(db: Database.Database, kbId: string): IndexingJob | undefined;
|
|
81
|
-
export declare function insertChunks(db: Database.Database, kbId: string, chunks: ChunkInsert[]):
|
|
98
|
+
export declare function insertChunks(db: Database.Database, kbId: string, chunks: ChunkInsert[]): string[];
|
|
82
99
|
export declare function getChunksByKB(db: Database.Database, kbId: string): Chunk[];
|
|
83
100
|
export declare function iterateChunksByKB(db: Database.Database, kbId: string): IterableIterator<Chunk>;
|
|
84
101
|
export declare function getChunkIdsByKB(db: Database.Database, kbId: string): string[];
|
|
@@ -97,3 +114,21 @@ export declare function iterateChunkHashesByKB(db: Database.Database, kbId: stri
|
|
|
97
114
|
}>;
|
|
98
115
|
export declare function getChunkCount(db: Database.Database, kbId: string): number;
|
|
99
116
|
export declare function getFileCount(db: Database.Database, kbId: string): number;
|
|
117
|
+
export declare function insertSymbols(db: Database.Database, kbId: string, symbols: KnowledgeSymbolInsert[]): void;
|
|
118
|
+
export declare function deleteSymbolsByKB(db: Database.Database, kbId: string): void;
|
|
119
|
+
export declare function deleteSymbolsByFile(db: Database.Database, kbId: string, filePath: string): void;
|
|
120
|
+
export declare function searchSymbols(db: Database.Database, query: string, options?: {
|
|
121
|
+
kbId?: string;
|
|
122
|
+
kind?: KnowledgeSymbol["kind"];
|
|
123
|
+
filePattern?: string;
|
|
124
|
+
limit?: number;
|
|
125
|
+
offset?: number;
|
|
126
|
+
exact?: boolean;
|
|
127
|
+
}): KnowledgeSymbol[];
|
|
128
|
+
export declare function countSymbols(db: Database.Database, query: string, options?: {
|
|
129
|
+
kbId?: string;
|
|
130
|
+
kind?: KnowledgeSymbol["kind"];
|
|
131
|
+
filePattern?: string;
|
|
132
|
+
exact?: boolean;
|
|
133
|
+
}): number;
|
|
134
|
+
export declare function getSymbolCount(db: Database.Database, kbId: string): number;
|
|
@@ -4,7 +4,7 @@ import { createRequire } from "node:module";
|
|
|
4
4
|
import { homedir } from "node:os";
|
|
5
5
|
import { basename, dirname, join } from "node:path";
|
|
6
6
|
import { fileURLToPath } from "node:url";
|
|
7
|
-
const SCHEMA_VERSION =
|
|
7
|
+
const SCHEMA_VERSION = 4;
|
|
8
8
|
const ITERATION_BATCH_SIZE = 500;
|
|
9
9
|
const SCHEMA_SQL = `
|
|
10
10
|
CREATE TABLE IF NOT EXISTS schema_version (
|
|
@@ -64,6 +64,28 @@ CREATE TABLE IF NOT EXISTS indexing_jobs (
|
|
|
64
64
|
CREATE INDEX IF NOT EXISTS idx_chunks_kb_id ON chunks(kb_id);
|
|
65
65
|
CREATE INDEX IF NOT EXISTS idx_chunks_content_hash ON chunks(content_hash);
|
|
66
66
|
|
|
67
|
+
CREATE TABLE IF NOT EXISTS symbols (
|
|
68
|
+
id TEXT PRIMARY KEY,
|
|
69
|
+
kb_id TEXT NOT NULL,
|
|
70
|
+
name TEXT NOT NULL,
|
|
71
|
+
normalized_name TEXT NOT NULL,
|
|
72
|
+
kind TEXT NOT NULL,
|
|
73
|
+
file_path TEXT NOT NULL,
|
|
74
|
+
file_type TEXT NOT NULL DEFAULT 'text',
|
|
75
|
+
start_line INTEGER NOT NULL DEFAULT 0,
|
|
76
|
+
end_line INTEGER NOT NULL DEFAULT 0,
|
|
77
|
+
container_name TEXT,
|
|
78
|
+
signature TEXT,
|
|
79
|
+
text TEXT NOT NULL DEFAULT '',
|
|
80
|
+
metadata_json TEXT NOT NULL DEFAULT '{}',
|
|
81
|
+
indexed_at INTEGER NOT NULL,
|
|
82
|
+
FOREIGN KEY (kb_id) REFERENCES knowledge_bases(id) ON DELETE CASCADE
|
|
83
|
+
);
|
|
84
|
+
|
|
85
|
+
CREATE INDEX IF NOT EXISTS idx_symbols_kb_name ON symbols(kb_id, normalized_name);
|
|
86
|
+
CREATE INDEX IF NOT EXISTS idx_symbols_kb_kind ON symbols(kb_id, kind);
|
|
87
|
+
CREATE INDEX IF NOT EXISTS idx_symbols_file ON symbols(kb_id, file_path);
|
|
88
|
+
|
|
67
89
|
CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
|
|
68
90
|
content_tokenized,
|
|
69
91
|
content=chunks,
|
|
@@ -239,6 +261,31 @@ CREATE TABLE IF NOT EXISTS indexing_jobs (
|
|
|
239
261
|
}
|
|
240
262
|
continue;
|
|
241
263
|
}
|
|
264
|
+
if (v === 4) {
|
|
265
|
+
db.exec(`
|
|
266
|
+
CREATE TABLE IF NOT EXISTS symbols (
|
|
267
|
+
id TEXT PRIMARY KEY,
|
|
268
|
+
kb_id TEXT NOT NULL,
|
|
269
|
+
name TEXT NOT NULL,
|
|
270
|
+
normalized_name TEXT NOT NULL,
|
|
271
|
+
kind TEXT NOT NULL,
|
|
272
|
+
file_path TEXT NOT NULL,
|
|
273
|
+
file_type TEXT NOT NULL DEFAULT 'text',
|
|
274
|
+
start_line INTEGER NOT NULL DEFAULT 0,
|
|
275
|
+
end_line INTEGER NOT NULL DEFAULT 0,
|
|
276
|
+
container_name TEXT,
|
|
277
|
+
signature TEXT,
|
|
278
|
+
text TEXT NOT NULL DEFAULT '',
|
|
279
|
+
metadata_json TEXT NOT NULL DEFAULT '{}',
|
|
280
|
+
indexed_at INTEGER NOT NULL,
|
|
281
|
+
FOREIGN KEY (kb_id) REFERENCES knowledge_bases(id) ON DELETE CASCADE
|
|
282
|
+
);
|
|
283
|
+
CREATE INDEX IF NOT EXISTS idx_symbols_kb_name ON symbols(kb_id, normalized_name);
|
|
284
|
+
CREATE INDEX IF NOT EXISTS idx_symbols_kb_kind ON symbols(kb_id, kind);
|
|
285
|
+
CREATE INDEX IF NOT EXISTS idx_symbols_file ON symbols(kb_id, file_path);
|
|
286
|
+
`);
|
|
287
|
+
continue;
|
|
288
|
+
}
|
|
242
289
|
if (migrations[v])
|
|
243
290
|
db.exec(migrations[v]);
|
|
244
291
|
}
|
|
@@ -265,6 +312,7 @@ export function listKBs(db) {
|
|
|
265
312
|
}
|
|
266
313
|
export function deleteKB(db, id) {
|
|
267
314
|
db.prepare("DELETE FROM indexing_jobs WHERE kb_id = ?").run(id);
|
|
315
|
+
db.prepare("DELETE FROM symbols WHERE kb_id = ?").run(id);
|
|
268
316
|
db.prepare("DELETE FROM chunks WHERE kb_id = ?").run(id);
|
|
269
317
|
db.prepare("DELETE FROM knowledge_bases WHERE id = ?").run(id);
|
|
270
318
|
}
|
|
@@ -332,15 +380,19 @@ export function getIndexingJob(db, kbId) {
|
|
|
332
380
|
}
|
|
333
381
|
// --- CRUD: Chunks ---
|
|
334
382
|
export function insertChunks(db, kbId, chunks) {
|
|
383
|
+
const ids = [];
|
|
335
384
|
const stmt = db.prepare(`INSERT INTO chunks (id, kb_id, content_hash, content, content_tokenized, file_path, file_type, start_line, end_line, metadata_json, indexed_at)
|
|
336
385
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
337
386
|
const now = Date.now();
|
|
338
387
|
const insertMany = db.transaction((items) => {
|
|
339
388
|
for (const c of items) {
|
|
340
|
-
|
|
389
|
+
const id = randomUUID();
|
|
390
|
+
ids.push(id);
|
|
391
|
+
stmt.run(id, kbId, c.content_hash, c.content, c.content_tokenized, c.file_path, c.file_type, c.start_line, c.end_line, c.metadata_json, now);
|
|
341
392
|
}
|
|
342
393
|
});
|
|
343
394
|
insertMany(chunks);
|
|
395
|
+
return ids;
|
|
344
396
|
}
|
|
345
397
|
export function getChunksByKB(db, kbId) {
|
|
346
398
|
return db.prepare("SELECT * FROM chunks WHERE kb_id = ? ORDER BY rowid").all(kbId);
|
|
@@ -447,3 +499,108 @@ export function getFileCount(db, kbId) {
|
|
|
447
499
|
const row = db.prepare("SELECT COUNT(DISTINCT file_path) as count FROM chunks WHERE kb_id = ?").get(kbId);
|
|
448
500
|
return row.count;
|
|
449
501
|
}
|
|
502
|
+
// --- CRUD: Symbols ---
|
|
503
|
+
function normalizeSymbolName(name) {
|
|
504
|
+
return name.trim().toLowerCase();
|
|
505
|
+
}
|
|
506
|
+
function escapeLike(value) {
|
|
507
|
+
return value.replaceAll("\\", "\\\\").replaceAll("%", "\\%").replaceAll("_", "\\_");
|
|
508
|
+
}
|
|
509
|
+
export function insertSymbols(db, kbId, symbols) {
|
|
510
|
+
if (symbols.length === 0)
|
|
511
|
+
return;
|
|
512
|
+
const stmt = db.prepare(`INSERT INTO symbols (
|
|
513
|
+
id, kb_id, name, normalized_name, kind, file_path, file_type, start_line, end_line,
|
|
514
|
+
container_name, signature, text, metadata_json, indexed_at
|
|
515
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
516
|
+
const now = Date.now();
|
|
517
|
+
const insertMany = db.transaction((items) => {
|
|
518
|
+
for (const symbol of items) {
|
|
519
|
+
stmt.run(randomUUID(), kbId, symbol.name, normalizeSymbolName(symbol.name), symbol.kind, symbol.file_path, symbol.file_type, symbol.start_line, symbol.end_line, symbol.container_name ?? null, symbol.signature ?? null, symbol.text, symbol.metadata_json, now);
|
|
520
|
+
}
|
|
521
|
+
});
|
|
522
|
+
insertMany(symbols);
|
|
523
|
+
}
|
|
524
|
+
export function deleteSymbolsByKB(db, kbId) {
|
|
525
|
+
db.prepare("DELETE FROM symbols WHERE kb_id = ?").run(kbId);
|
|
526
|
+
}
|
|
527
|
+
export function deleteSymbolsByFile(db, kbId, filePath) {
|
|
528
|
+
db.prepare("DELETE FROM symbols WHERE kb_id = ? AND file_path = ?").run(kbId, filePath);
|
|
529
|
+
}
|
|
530
|
+
export function searchSymbols(db, query, options = {}) {
|
|
531
|
+
const clauses = ["kb.status IN ('ready', 'stale')"];
|
|
532
|
+
const params = [];
|
|
533
|
+
const normalizedQuery = normalizeSymbolName(query);
|
|
534
|
+
if (options.kbId) {
|
|
535
|
+
clauses.push("s.kb_id = ?");
|
|
536
|
+
params.push(options.kbId);
|
|
537
|
+
}
|
|
538
|
+
if (options.kind) {
|
|
539
|
+
clauses.push("s.kind = ?");
|
|
540
|
+
params.push(options.kind);
|
|
541
|
+
}
|
|
542
|
+
if (options.filePattern) {
|
|
543
|
+
clauses.push("s.file_path LIKE ? ESCAPE '\\'");
|
|
544
|
+
params.push(`%${escapeLike(options.filePattern)}%`);
|
|
545
|
+
}
|
|
546
|
+
if (normalizedQuery) {
|
|
547
|
+
if (options.exact) {
|
|
548
|
+
clauses.push("s.normalized_name = ?");
|
|
549
|
+
params.push(normalizedQuery);
|
|
550
|
+
}
|
|
551
|
+
else {
|
|
552
|
+
clauses.push("(s.normalized_name LIKE ? ESCAPE '\\' OR s.file_path LIKE ? ESCAPE '\\' OR s.text LIKE ? ESCAPE '\\')");
|
|
553
|
+
const pattern = `%${escapeLike(normalizedQuery)}%`;
|
|
554
|
+
params.push(pattern, pattern, pattern);
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
const where = `WHERE ${clauses.join(" AND ")}`;
|
|
558
|
+
const limit = Math.max(1, Math.min(200, options.limit ?? 20));
|
|
559
|
+
const offset = Math.max(0, options.offset ?? 0);
|
|
560
|
+
return db
|
|
561
|
+
.prepare(`SELECT s.* FROM symbols s
|
|
562
|
+
JOIN knowledge_bases kb ON kb.id = s.kb_id
|
|
563
|
+
${where}
|
|
564
|
+
ORDER BY
|
|
565
|
+
CASE WHEN s.normalized_name = ? THEN 0 WHEN s.normalized_name LIKE ? ESCAPE '\\' THEN 1 ELSE 2 END,
|
|
566
|
+
s.file_path, s.start_line
|
|
567
|
+
LIMIT ? OFFSET ?`)
|
|
568
|
+
.all(...params, normalizedQuery, `${escapeLike(normalizedQuery)}%`, limit, offset);
|
|
569
|
+
}
|
|
570
|
+
export function countSymbols(db, query, options = {}) {
|
|
571
|
+
const clauses = ["kb.status IN ('ready', 'stale')"];
|
|
572
|
+
const params = [];
|
|
573
|
+
const normalizedQuery = normalizeSymbolName(query);
|
|
574
|
+
if (options.kbId) {
|
|
575
|
+
clauses.push("s.kb_id = ?");
|
|
576
|
+
params.push(options.kbId);
|
|
577
|
+
}
|
|
578
|
+
if (options.kind) {
|
|
579
|
+
clauses.push("s.kind = ?");
|
|
580
|
+
params.push(options.kind);
|
|
581
|
+
}
|
|
582
|
+
if (options.filePattern) {
|
|
583
|
+
clauses.push("s.file_path LIKE ? ESCAPE '\\'");
|
|
584
|
+
params.push(`%${escapeLike(options.filePattern)}%`);
|
|
585
|
+
}
|
|
586
|
+
if (normalizedQuery) {
|
|
587
|
+
if (options.exact) {
|
|
588
|
+
clauses.push("s.normalized_name = ?");
|
|
589
|
+
params.push(normalizedQuery);
|
|
590
|
+
}
|
|
591
|
+
else {
|
|
592
|
+
clauses.push("(s.normalized_name LIKE ? ESCAPE '\\' OR s.file_path LIKE ? ESCAPE '\\' OR s.text LIKE ? ESCAPE '\\')");
|
|
593
|
+
const pattern = `%${escapeLike(normalizedQuery)}%`;
|
|
594
|
+
params.push(pattern, pattern, pattern);
|
|
595
|
+
}
|
|
596
|
+
}
|
|
597
|
+
const where = `WHERE ${clauses.join(" AND ")}`;
|
|
598
|
+
const row = db
|
|
599
|
+
.prepare(`SELECT COUNT(*) as count FROM symbols s JOIN knowledge_bases kb ON kb.id = s.kb_id ${where}`)
|
|
600
|
+
.get(...params);
|
|
601
|
+
return row.count;
|
|
602
|
+
}
|
|
603
|
+
export function getSymbolCount(db, kbId) {
|
|
604
|
+
const row = db.prepare("SELECT COUNT(*) as count FROM symbols WHERE kb_id = ?").get(kbId);
|
|
605
|
+
return row.count;
|
|
606
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
|
|
1
|
+
import { type ScanOptions } from "../indexer/chunker.ts";
|
|
2
|
+
export declare function startWatcher(kbId: string, dirPath: string, onUpdate: (kbId: string) => void, options?: ScanOptions): void;
|
|
2
3
|
export declare function stopWatcher(kbId: string): void;
|
|
3
4
|
export declare function stopAllWatchers(): void;
|
|
4
5
|
export declare function getActiveWatcherCount(): number;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { existsSync,
|
|
2
|
-
import {
|
|
1
|
+
import { existsSync, statSync, watch } from "node:fs";
|
|
2
|
+
import { createSkippedScanStats, iterateScannableFiles } from "../indexer/chunker.js";
|
|
3
3
|
const watchers = new Map();
|
|
4
4
|
const pollers = new Map();
|
|
5
5
|
const snapshots = new Map();
|
|
@@ -15,34 +15,20 @@ function scheduleUpdate(kbId, onUpdate) {
|
|
|
15
15
|
onUpdate(kbId);
|
|
16
16
|
}, DEBOUNCE_MS));
|
|
17
17
|
}
|
|
18
|
-
function scanSnapshot(dirPath) {
|
|
18
|
+
function scanSnapshot(dirPath, options = {}) {
|
|
19
19
|
const snapshot = new Map();
|
|
20
|
-
|
|
21
|
-
|
|
20
|
+
if (!existsSync(dirPath))
|
|
21
|
+
return snapshot;
|
|
22
|
+
const skipped = createSkippedScanStats();
|
|
23
|
+
for (const file of iterateScannableFiles(dirPath, skipped, options)) {
|
|
22
24
|
try {
|
|
23
|
-
|
|
25
|
+
const stat = statSync(file.path);
|
|
26
|
+
snapshot.set(file.path, `${stat.mtimeMs}:${stat.size}`);
|
|
24
27
|
}
|
|
25
28
|
catch {
|
|
26
|
-
|
|
27
|
-
}
|
|
28
|
-
for (const entry of entries) {
|
|
29
|
-
const fullPath = join(dir, entry.name);
|
|
30
|
-
try {
|
|
31
|
-
const stat = statSync(fullPath);
|
|
32
|
-
if (entry.isDirectory()) {
|
|
33
|
-
scan(fullPath);
|
|
34
|
-
}
|
|
35
|
-
else if (entry.isFile()) {
|
|
36
|
-
snapshot.set(fullPath, `${stat.mtimeMs}:${stat.size}`);
|
|
37
|
-
}
|
|
38
|
-
}
|
|
39
|
-
catch {
|
|
40
|
-
/* file disappeared or is unreadable */
|
|
41
|
-
}
|
|
29
|
+
/* file disappeared or is unreadable */
|
|
42
30
|
}
|
|
43
31
|
}
|
|
44
|
-
if (existsSync(dirPath))
|
|
45
|
-
scan(dirPath);
|
|
46
32
|
return snapshot;
|
|
47
33
|
}
|
|
48
34
|
function snapshotsDiffer(a, b) {
|
|
@@ -54,23 +40,27 @@ function snapshotsDiffer(a, b) {
|
|
|
54
40
|
}
|
|
55
41
|
return false;
|
|
56
42
|
}
|
|
57
|
-
function startPoller(kbId, dirPath, onUpdate) {
|
|
58
|
-
snapshots.set(kbId, scanSnapshot(dirPath));
|
|
43
|
+
function startPoller(kbId, dirPath, onUpdate, options = {}) {
|
|
44
|
+
snapshots.set(kbId, scanSnapshot(dirPath, options));
|
|
59
45
|
pollers.set(kbId, setInterval(() => {
|
|
60
46
|
const previous = snapshots.get(kbId) ?? new Map();
|
|
61
|
-
const next = scanSnapshot(dirPath);
|
|
47
|
+
const next = scanSnapshot(dirPath, options);
|
|
62
48
|
if (snapshotsDiffer(previous, next)) {
|
|
63
49
|
snapshots.set(kbId, next);
|
|
64
50
|
scheduleUpdate(kbId, onUpdate);
|
|
65
51
|
}
|
|
66
52
|
}, POLL_MS));
|
|
67
53
|
}
|
|
68
|
-
export function startWatcher(kbId, dirPath, onUpdate) {
|
|
54
|
+
export function startWatcher(kbId, dirPath, onUpdate, options = {}) {
|
|
69
55
|
stopWatcher(kbId);
|
|
70
|
-
startPoller(kbId, dirPath, onUpdate);
|
|
56
|
+
startPoller(kbId, dirPath, onUpdate, options);
|
|
71
57
|
try {
|
|
72
58
|
const watcher = watch(dirPath, { recursive: true }, () => {
|
|
73
|
-
snapshots.
|
|
59
|
+
const previous = snapshots.get(kbId) ?? new Map();
|
|
60
|
+
const next = scanSnapshot(dirPath, options);
|
|
61
|
+
if (!snapshotsDiffer(previous, next))
|
|
62
|
+
return;
|
|
63
|
+
snapshots.set(kbId, next);
|
|
74
64
|
scheduleUpdate(kbId, onUpdate);
|
|
75
65
|
});
|
|
76
66
|
watcher.on("error", () => {
|
package/docs/configuration.md
CHANGED
|
@@ -26,8 +26,8 @@ Default data storage is `~/.pi/knowledge` under Pi and `~/.omp/knowledge` under
|
|
|
26
26
|
| `OPENAI_BASE_URL` | `https://api.openai.com/v1` | Common OpenAI-compatible API root fallback. |
|
|
27
27
|
| `PI_KNOWLEDGE_EMBEDDING_MAX_CHARS` | `20000` | Final per-input API embedding safety cap for OpenAI-compatible servers with smaller context windows. This does not replace chunker bounds. |
|
|
28
28
|
| `PI_KNOWLEDGE_EMBEDDING_API_FALLBACK` | unset | Set to `local` to explicitly fall back to local embeddings after API failures. Without this, API failures are surfaced. |
|
|
29
|
-
| `PI_KNOWLEDGE_ENABLE_NATIVE_IDLE_DISPOSE` | unset |
|
|
30
|
-
| `PI_KNOWLEDGE_EMBEDDING_IDLE_MS` | `30000` | Idle
|
|
29
|
+
| `PI_KNOWLEDGE_ENABLE_NATIVE_IDLE_DISPOSE` | unset | Enables idle timers for embedding/reranker run coordination. It does not guarantee immediate worker memory release; model-worker shutdown remains tied to engine/session shutdown. Disabled by default for stable shutdown. |
|
|
30
|
+
| `PI_KNOWLEDGE_EMBEDDING_IDLE_MS` | `30000` | Idle timer used only when native idle disposal coordination is enabled. Mainly for lifecycle stress tests. |
|
|
31
31
|
| `PI_KNOWLEDGE_OFFLINE` | unset | Use with a pre-populated model cache for offline local model operation. See `docs/offline-mode.md`. |
|
|
32
32
|
|
|
33
33
|
API embedding failures intentionally surface by default. Silent fallback can hide bad API keys, wrong base URLs, unsupported model names, or context-window errors and can produce a KB with a different embedding model than intended.
|
|
@@ -40,6 +40,12 @@ API embedding failures intentionally surface by default. Silent fallback can hid
|
|
|
40
40
|
| `PI_KNOWLEDGE_AUTO_INJECT` | unset | Set to `true` to auto-search KB context before model calls. This is opt-in. |
|
|
41
41
|
| `PI_KNOWLEDGE_STALE_INDEXING_MS` | built-in stale threshold | Override the stale indexing threshold used by diagnostics and `knowledge_doctor`. |
|
|
42
42
|
|
|
43
|
+
Search diagnostics include result provenance in both text output and structured `details`: chunk id, chunk hash, match reason, indexed timestamp, stale flag, and source mtime when the original source file is available. This provenance is stored in SQLite and derived from indexed source metadata; it does not create or modify files in the indexed project.
|
|
44
|
+
|
|
45
|
+
`knowledge_doctor` emits both human-readable issues and machine-readable actions in structured `details`. Action codes include `run_update`, `rebuild_kb`, `wait_for_indexing`, `review_skipped_scope`, `rebuild_vectors`, `check_source`, and `none`. Agents should prefer these action codes over parsing the English action text.
|
|
46
|
+
|
|
47
|
+
`knowledge_symbol_search` uses a lightweight symbol index stored in `knowledge.db`. It is rebuilt during `knowledge_add`, `knowledge_update`, and `knowledge_import`, and can find code symbols, route-like handlers, Markdown headings, config keys, and environment variables. Older file/directory/URL KBs without symbol metadata should be updated before relying on symbol lookup. Imported portable KBs and inline text KBs do not retain an active source path; rebuild their symbols by re-importing or re-adding the content.
|
|
48
|
+
|
|
43
49
|
## Test and Release Fixtures
|
|
44
50
|
|
|
45
51
|
| Variable | Default | Purpose |
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-knowledge",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.5.0",
|
|
4
4
|
"description": "Local-first RAG knowledge base for Pi and OMP coding agents with hybrid code/doc search, reranking, diagnostics, and large-project indexing",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|