pi-knowledge 0.4.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +4 -2
- package/dist/index.d.ts +38 -0
- package/dist/index.js +435 -0
- package/dist/src/diagnostics/health.d.ts +19 -0
- package/dist/src/diagnostics/health.js +110 -0
- package/dist/src/embedding/provider.d.ts +5 -0
- package/dist/src/embedding/provider.js +99 -0
- package/dist/src/embedding/vectors.d.ts +15 -0
- package/dist/src/embedding/vectors.js +111 -0
- package/dist/src/engine.d.ts +90 -0
- package/dist/src/engine.js +1370 -0
- package/dist/src/indexer/chunker.d.ts +53 -0
- package/dist/src/indexer/chunker.js +498 -0
- package/dist/src/indexer/chunkers/code-ast.d.ts +3 -0
- package/dist/src/indexer/chunkers/code-ast.js +179 -0
- package/dist/src/model-worker-client.d.ts +10 -0
- package/dist/src/model-worker-client.js +138 -0
- package/dist/src/model-worker.d.ts +1 -0
- package/dist/src/model-worker.js +61 -0
- package/dist/src/search/bm25.d.ts +8 -0
- package/dist/src/search/bm25.js +38 -0
- package/dist/src/search/fusion.d.ts +19 -0
- package/dist/src/search/fusion.js +34 -0
- package/dist/src/search/query.d.ts +5 -0
- package/dist/src/search/query.js +57 -0
- package/dist/src/search/ranking.d.ts +27 -0
- package/dist/src/search/ranking.js +180 -0
- package/dist/src/search/reranker.d.ts +10 -0
- package/dist/src/search/reranker.js +66 -0
- package/dist/src/search/vector.d.ts +14 -0
- package/dist/src/search/vector.js +58 -0
- package/dist/src/storage/sqlite.d.ts +99 -0
- package/dist/src/storage/sqlite.js +330 -0
- package/dist/src/watcher/file-watcher.d.ts +4 -0
- package/dist/src/watcher/file-watcher.js +108 -0
- package/extension.js +9 -0
- package/package.json +10 -5
- package/index.ts +0 -500
- package/src/diagnostics/health.ts +0 -145
- package/src/embedding/provider.ts +0 -106
- package/src/embedding/vectors.ts +0 -119
- package/src/engine.ts +0 -1592
- package/src/indexer/chunker.ts +0 -612
- package/src/indexer/chunkers/code-ast.ts +0 -209
- package/src/model-worker-client.ts +0 -146
- package/src/model-worker.ts +0 -88
- package/src/search/bm25.ts +0 -55
- package/src/search/fusion.ts +0 -48
- package/src/search/query.ts +0 -59
- package/src/search/ranking.ts +0 -201
- package/src/search/reranker.ts +0 -74
- package/src/search/vector.ts +0 -81
- package/src/storage/sqlite.ts +0 -489
- package/src/watcher/file-watcher.ts +0 -116
package/index.ts
DELETED
|
@@ -1,500 +0,0 @@
|
|
|
1
|
-
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
2
|
-
import { KnowledgeEngine } from "./src/engine.ts";
|
|
3
|
-
import { getDefaultKnowledgeDir } from "./src/storage/sqlite.ts";
|
|
4
|
-
import { getActiveWatcherCount, startWatcher, stopAllWatchers } from "./src/watcher/file-watcher.ts";
|
|
5
|
-
|
|
6
|
-
type Schema = Record<string, unknown> & { optional?: true };
|
|
7
|
-
type ContextMessage = { role: string; content: string };
|
|
8
|
-
|
|
9
|
-
const Type = {
|
|
10
|
-
Object(properties: Record<string, Schema>): Schema {
|
|
11
|
-
const required = Object.entries(properties)
|
|
12
|
-
.filter(([, schema]) => !schema.optional)
|
|
13
|
-
.map(([name]) => name);
|
|
14
|
-
const normalized = Object.fromEntries(
|
|
15
|
-
Object.entries(properties).map(([name, schema]) => {
|
|
16
|
-
const { optional: _optional, ...rest } = schema;
|
|
17
|
-
return [name, rest];
|
|
18
|
-
}),
|
|
19
|
-
);
|
|
20
|
-
return { type: "object", properties: normalized, required, additionalProperties: false };
|
|
21
|
-
},
|
|
22
|
-
String(options: Record<string, unknown> = {}): Schema {
|
|
23
|
-
return { type: "string", ...options };
|
|
24
|
-
},
|
|
25
|
-
Number(options: Record<string, unknown> = {}): Schema {
|
|
26
|
-
return { type: "number", ...options };
|
|
27
|
-
},
|
|
28
|
-
Boolean(options: Record<string, unknown> = {}): Schema {
|
|
29
|
-
return { type: "boolean", ...options };
|
|
30
|
-
},
|
|
31
|
-
Literal(value: string): Schema {
|
|
32
|
-
return { const: value };
|
|
33
|
-
},
|
|
34
|
-
Union(items: Schema[]): Schema {
|
|
35
|
-
return { anyOf: items };
|
|
36
|
-
},
|
|
37
|
-
Array(item: Schema): Schema {
|
|
38
|
-
return { type: "array", items: item };
|
|
39
|
-
},
|
|
40
|
-
Optional(schema: Schema): Schema {
|
|
41
|
-
return { ...schema, optional: true };
|
|
42
|
-
},
|
|
43
|
-
};
|
|
44
|
-
|
|
45
|
-
const engine = new KnowledgeEngine();
|
|
46
|
-
const WATCH_ENABLED = process.env.PI_KNOWLEDGE_WATCH === "true";
|
|
47
|
-
const AUTO_INJECT = process.env.PI_KNOWLEDGE_AUTO_INJECT === "true";
|
|
48
|
-
|
|
49
|
-
export default function (pi: ExtensionAPI) {
|
|
50
|
-
pi.on("session_start", async () => {
|
|
51
|
-
await engine.initialize(getDefaultKnowledgeDir());
|
|
52
|
-
if (WATCH_ENABLED) {
|
|
53
|
-
for (const kb of engine.list()) {
|
|
54
|
-
if (kb.source_path && kb.source_type === "directory") {
|
|
55
|
-
startWatcher(kb.id, kb.source_path, (kbId) => {
|
|
56
|
-
engine.update(kbId).catch(() => {});
|
|
57
|
-
});
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
}
|
|
61
|
-
});
|
|
62
|
-
|
|
63
|
-
pi.on("session_shutdown", async () => {
|
|
64
|
-
stopAllWatchers();
|
|
65
|
-
await engine.dispose({ disposeModels: false });
|
|
66
|
-
});
|
|
67
|
-
|
|
68
|
-
// Auto-inject: search KB for relevant context before each LLM call (opt-in)
|
|
69
|
-
if (AUTO_INJECT) {
|
|
70
|
-
pi.on("context", async (event) => {
|
|
71
|
-
const kbs = engine.list();
|
|
72
|
-
if (kbs.length === 0) return;
|
|
73
|
-
// Find last user message
|
|
74
|
-
const lastUser = [...event.messages].reverse().find((m) => m.role === "user");
|
|
75
|
-
if (!lastUser) return;
|
|
76
|
-
const text = "content" in lastUser && typeof lastUser.content === "string" ? lastUser.content : "";
|
|
77
|
-
if (!text || text.length < 5) return;
|
|
78
|
-
try {
|
|
79
|
-
const results = await engine.search(text, { mode: "fast", limit: 3 });
|
|
80
|
-
if (results.results.length === 0) return;
|
|
81
|
-
const context = results.results.map((r) => `[${r.file_path}]: ${r.snippet}`).join("\n\n");
|
|
82
|
-
const messages = event.messages as ContextMessage[];
|
|
83
|
-
messages.unshift({ role: "user", content: `[Knowledge context]\n${context}` });
|
|
84
|
-
} catch {
|
|
85
|
-
/* silent fail */
|
|
86
|
-
}
|
|
87
|
-
});
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
pi.on("before_agent_start", (event) => {
|
|
91
|
-
const kbs = engine.list();
|
|
92
|
-
if (kbs.length > 0) {
|
|
93
|
-
const desc = kbs.map((kb) => `"${kb.name}" (${kb.chunk_count} chunks, ${kb.file_count} files)`).join(", ");
|
|
94
|
-
event.systemPromptOptions.promptGuidelines?.push(
|
|
95
|
-
`Available knowledge bases: ${desc}. Use knowledge_search before answering domain questions.`,
|
|
96
|
-
);
|
|
97
|
-
}
|
|
98
|
-
});
|
|
99
|
-
|
|
100
|
-
pi.registerTool({
|
|
101
|
-
name: "knowledge_plan",
|
|
102
|
-
label: "Knowledge Plan",
|
|
103
|
-
description:
|
|
104
|
-
"Inspect an indexing source without writing a KB, showing scannable files, suggested exclusions, and technical skips",
|
|
105
|
-
promptSnippet: "Plan a knowledge-base indexing scope before calling knowledge_add",
|
|
106
|
-
promptGuidelines: [
|
|
107
|
-
"Use knowledge_plan before knowledge_add for broad directories, large repositories, or sources that may contain private or low-signal text",
|
|
108
|
-
"Show the user suggested exclusions and technical skips before asking whether to include risky or low-signal text",
|
|
109
|
-
"After the user confirms scope, call knowledge_add with matching include_suggested_text, include_paths, or exclude_paths",
|
|
110
|
-
"Do not use knowledge_plan as a substitute for knowledge_search; it only plans indexing scope",
|
|
111
|
-
],
|
|
112
|
-
parameters: Type.Object({
|
|
113
|
-
source: Type.String({ description: "File path, directory path, URL, or inline text to inspect before indexing" }),
|
|
114
|
-
include_suggested_text: Type.Optional(
|
|
115
|
-
Type.Boolean({
|
|
116
|
-
description:
|
|
117
|
-
"Preview the scope if suggested-excluded text such as vendor/build/runtime/cache/secret-named text is included",
|
|
118
|
-
}),
|
|
119
|
-
),
|
|
120
|
-
include_paths: Type.Optional(
|
|
121
|
-
Type.Array(
|
|
122
|
-
Type.String({
|
|
123
|
-
description:
|
|
124
|
-
"Relative paths under a directory source to include even when they match suggested-exclude patterns",
|
|
125
|
-
}),
|
|
126
|
-
),
|
|
127
|
-
),
|
|
128
|
-
exclude_paths: Type.Optional(
|
|
129
|
-
Type.Array(Type.String({ description: "Relative paths under a directory source to exclude from this plan" })),
|
|
130
|
-
),
|
|
131
|
-
}),
|
|
132
|
-
execute(_id, params) {
|
|
133
|
-
const { source, include_suggested_text, include_paths, exclude_paths } = params;
|
|
134
|
-
const plan = engine.plan(source, {
|
|
135
|
-
include_suggested_text: include_suggested_text === true,
|
|
136
|
-
include_paths: Array.isArray(include_paths)
|
|
137
|
-
? include_paths.filter((item) => typeof item === "string")
|
|
138
|
-
: undefined,
|
|
139
|
-
exclude_paths: Array.isArray(exclude_paths)
|
|
140
|
-
? exclude_paths.filter((item) => typeof item === "string")
|
|
141
|
-
: undefined,
|
|
142
|
-
});
|
|
143
|
-
const samples = plan.skipped.samples
|
|
144
|
-
.map((sample) => `- ${sample.reason}: ${sample.path}${sample.size ? ` (${sample.size} bytes)` : ""}`)
|
|
145
|
-
.join("\n");
|
|
146
|
-
return {
|
|
147
|
-
content: [
|
|
148
|
-
{
|
|
149
|
-
type: "text",
|
|
150
|
-
text: [
|
|
151
|
-
plan.summary,
|
|
152
|
-
`Source type: ${plan.source_type}`,
|
|
153
|
-
`Skipped summary: ${JSON.stringify(plan.skipped.by_reason)}`,
|
|
154
|
-
samples ? `Skipped samples:\n${samples}` : "Skipped samples: none",
|
|
155
|
-
].join("\n"),
|
|
156
|
-
},
|
|
157
|
-
],
|
|
158
|
-
};
|
|
159
|
-
},
|
|
160
|
-
});
|
|
161
|
-
|
|
162
|
-
pi.registerTool({
|
|
163
|
-
name: "knowledge_add",
|
|
164
|
-
label: "Knowledge Add",
|
|
165
|
-
description: "Index files, directories, or text into a named knowledge base for semantic search",
|
|
166
|
-
promptSnippet: "Index files/dirs/text into a searchable knowledge base",
|
|
167
|
-
promptGuidelines: [
|
|
168
|
-
"Use knowledge_plan first for broad directories, large repositories, or sources that may contain private or low-signal text",
|
|
169
|
-
"Use knowledge_add when the user asks to index, remember, or learn from files or documentation",
|
|
170
|
-
"Prefer one knowledge_add call for the project root or relevant directory with include_paths/exclude_paths; do not call it once per file",
|
|
171
|
-
"If a knowledge base with the same name already exists, use knowledge_update or ask before replacing it",
|
|
172
|
-
"Index source, documentation, and ordinary configuration files that explain how the project works",
|
|
173
|
-
"Default directory indexing suggests excluding generated, vendor, browser runtime, obvious secret, and low-signal text artifacts; these are not permanent blocks",
|
|
174
|
-
"For ambiguous or risky text such as .env, private keys, certificates, credential-named files, settings.json, appsettings.json, cloud config, editor config, generated reports, lockfiles, or vendor text, explain the tradeoff and ask the user before including it",
|
|
175
|
-
"If the user confirms a suggested-excluded text file or directory should be indexed, call knowledge_add with include_suggested_text or a focused include_paths value so the tool follows the confirmed scope",
|
|
176
|
-
"Non-text, unsupported binary, oversized, unreadable, and inaccessible files remain technical skips even when the user wants broad indexing",
|
|
177
|
-
"Provide a descriptive name for this single knowledge base",
|
|
178
|
-
],
|
|
179
|
-
parameters: Type.Object({
|
|
180
|
-
source: Type.String({ description: "File path, directory path, or inline text to index" }),
|
|
181
|
-
name: Type.String({ description: "Display name for this knowledge base" }),
|
|
182
|
-
include_suggested_text: Type.Optional(
|
|
183
|
-
Type.Boolean({
|
|
184
|
-
description:
|
|
185
|
-
"Include text files that are normally suggested for exclusion, such as vendor/build/runtime/cache/secret-named text, after user confirmation",
|
|
186
|
-
}),
|
|
187
|
-
),
|
|
188
|
-
include_paths: Type.Optional(
|
|
189
|
-
Type.Array(
|
|
190
|
-
Type.String({
|
|
191
|
-
description:
|
|
192
|
-
"Relative paths under a directory source to include even when they match suggested-exclude patterns",
|
|
193
|
-
}),
|
|
194
|
-
),
|
|
195
|
-
),
|
|
196
|
-
exclude_paths: Type.Optional(
|
|
197
|
-
Type.Array(Type.String({ description: "Relative paths under a directory source to exclude from this KB" })),
|
|
198
|
-
),
|
|
199
|
-
}),
|
|
200
|
-
async execute(_id, params, _signal, onUpdate) {
|
|
201
|
-
const { source, name, include_suggested_text, include_paths, exclude_paths } = params;
|
|
202
|
-
const { kb, chunkCount } = await engine.add(
|
|
203
|
-
source,
|
|
204
|
-
name,
|
|
205
|
-
(msg) => {
|
|
206
|
-
onUpdate?.({ content: [{ type: "text", text: msg }] });
|
|
207
|
-
},
|
|
208
|
-
_signal,
|
|
209
|
-
{
|
|
210
|
-
include_suggested_text: include_suggested_text === true,
|
|
211
|
-
include_paths: Array.isArray(include_paths)
|
|
212
|
-
? include_paths.filter((item) => typeof item === "string")
|
|
213
|
-
: undefined,
|
|
214
|
-
exclude_paths: Array.isArray(exclude_paths)
|
|
215
|
-
? exclude_paths.filter((item) => typeof item === "string")
|
|
216
|
-
: undefined,
|
|
217
|
-
},
|
|
218
|
-
);
|
|
219
|
-
// Start watcher for new directory KB
|
|
220
|
-
if (WATCH_ENABLED && kb.source_path && kb.source_type === "directory") {
|
|
221
|
-
startWatcher(kb.id, kb.source_path, (kbId) => {
|
|
222
|
-
engine.update(kbId).catch(() => {});
|
|
223
|
-
});
|
|
224
|
-
}
|
|
225
|
-
return {
|
|
226
|
-
content: [
|
|
227
|
-
{
|
|
228
|
-
type: "text",
|
|
229
|
-
text: `Indexed "${kb.name}": ${chunkCount} chunks from ${kb.file_count} files. KB ID: ${kb.id}`,
|
|
230
|
-
},
|
|
231
|
-
],
|
|
232
|
-
};
|
|
233
|
-
},
|
|
234
|
-
});
|
|
235
|
-
|
|
236
|
-
pi.registerTool({
|
|
237
|
-
name: "knowledge_search",
|
|
238
|
-
label: "Knowledge Search",
|
|
239
|
-
description: "Search indexed knowledge bases using hybrid semantic + keyword search",
|
|
240
|
-
promptSnippet: "Search knowledge bases (hybrid BM25 + semantic + weighted score fusion)",
|
|
241
|
-
promptGuidelines: [
|
|
242
|
-
"Use knowledge_search to find relevant context before answering domain questions",
|
|
243
|
-
"Default to mode 'hybrid' for most project questions because it combines lexical anchors with semantic recall",
|
|
244
|
-
"Use mode 'fast' for exact symbols, filenames, commands, error codes, API names, config keys, or quoted strings",
|
|
245
|
-
"Use mode 'semantic' for broad conceptual questions when exact terms may differ from the indexed wording",
|
|
246
|
-
"Use mode 'adaptive' when the user needs surrounding implementation context, related nearby sections, or enough context to make a code change",
|
|
247
|
-
"Use mode 'deep' for high-stakes answers, ambiguous top results, or final verification where slower cross-encoder reranking is acceptable",
|
|
248
|
-
"If a search returns no results or obviously weak results, retry once with a different mode before concluding the KB lacks the answer",
|
|
249
|
-
"If top results are repetitive, retry with diversity 'strong' or mode 'adaptive' instead of increasing the limit first",
|
|
250
|
-
],
|
|
251
|
-
parameters: Type.Object({
|
|
252
|
-
query: Type.String({ description: "Search query" }),
|
|
253
|
-
mode: Type.Optional(
|
|
254
|
-
Type.Union([
|
|
255
|
-
Type.Literal("auto"),
|
|
256
|
-
Type.Literal("fast"),
|
|
257
|
-
Type.Literal("semantic"),
|
|
258
|
-
Type.Literal("hybrid"),
|
|
259
|
-
Type.Literal("deep"),
|
|
260
|
-
Type.Literal("adaptive"),
|
|
261
|
-
]),
|
|
262
|
-
),
|
|
263
|
-
limit: Type.Optional(Type.Number({ description: "Max results (default 10)" })),
|
|
264
|
-
kb_id: Type.Optional(Type.String({ description: "Limit search to a specific KB by ID or exact name" })),
|
|
265
|
-
offset: Type.Optional(Type.Number({ description: "Pagination offset" })),
|
|
266
|
-
file_type: Type.Optional(Type.String({ description: "Filter by file type (e.g. typescript, markdown, python)" })),
|
|
267
|
-
diversity: Type.Optional(Type.Union([Type.Literal("off"), Type.Literal("balanced"), Type.Literal("strong")])),
|
|
268
|
-
diagnostics: Type.Optional(
|
|
269
|
-
Type.Boolean({ description: "Include ranking diagnostics and mode/fallback details in the result" }),
|
|
270
|
-
),
|
|
271
|
-
}),
|
|
272
|
-
async execute(_id, params) {
|
|
273
|
-
const { query, mode, limit, kb_id, offset, file_type, diversity, diagnostics } = params;
|
|
274
|
-
const filters = file_type ? { file_type } : undefined;
|
|
275
|
-
const response = await engine.search(query, { mode, limit, kb_id, offset, filters, diversity });
|
|
276
|
-
if (response.results.length === 0) {
|
|
277
|
-
const details = [
|
|
278
|
-
"No results found.",
|
|
279
|
-
response.mode_used ? `Mode used: ${response.mode_used}` : "",
|
|
280
|
-
response.retry_modes?.length ? `Retried: ${response.retry_modes.join(", ")}` : "",
|
|
281
|
-
response.warnings?.length ? `Warnings: ${response.warnings.join(" | ")}` : "",
|
|
282
|
-
response.suggestions?.length ? `Suggestions: ${response.suggestions.join(" | ")}` : "",
|
|
283
|
-
]
|
|
284
|
-
.filter(Boolean)
|
|
285
|
-
.join("\n");
|
|
286
|
-
return { content: [{ type: "text", text: details }] };
|
|
287
|
-
}
|
|
288
|
-
let output = `${response.total_count} results (showing ${response.results.length})`;
|
|
289
|
-
if (response.mode_used) output += ` — mode: ${response.mode_used}`;
|
|
290
|
-
if (response.retry_modes?.length) output += ` — retried: ${response.retry_modes.join(", ")}`;
|
|
291
|
-
output += ":\n\n";
|
|
292
|
-
if (response.warnings?.length) {
|
|
293
|
-
output = `Warnings:\n- ${response.warnings.join("\n- ")}\n\n${output}`;
|
|
294
|
-
}
|
|
295
|
-
output += response.results
|
|
296
|
-
.map((r, i) => {
|
|
297
|
-
const diag =
|
|
298
|
-
diagnostics && r.ranking
|
|
299
|
-
? `\nDiagnostics: base=${r.ranking.base_score.toFixed(3)}, adjusted=${r.ranking.adjusted_score.toFixed(
|
|
300
|
-
3,
|
|
301
|
-
)}, coverage=${r.ranking.coverage.toFixed(2)}, path_boost=${r.ranking.path_boost.toFixed(
|
|
302
|
-
2,
|
|
303
|
-
)}, source_boost=${r.ranking.source_boost.toFixed(2)}, test=${r.ranking.is_test}`
|
|
304
|
-
: "";
|
|
305
|
-
return `[${i + 1}] ${r.file_path} (${r.kb_name}, score: ${r.score.toFixed(3)})\n${r.snippet}${diag}`;
|
|
306
|
-
})
|
|
307
|
-
.join("\n\n");
|
|
308
|
-
return { content: [{ type: "text", text: output }] };
|
|
309
|
-
},
|
|
310
|
-
});
|
|
311
|
-
|
|
312
|
-
pi.registerTool({
|
|
313
|
-
name: "knowledge_update",
|
|
314
|
-
label: "Knowledge Update",
|
|
315
|
-
description: "Incrementally re-index a knowledge base (only re-embeds changed content)",
|
|
316
|
-
promptSnippet: "Incrementally update a knowledge base (only changed files re-embedded)",
|
|
317
|
-
parameters: Type.Object({
|
|
318
|
-
target: Type.String({ description: "KB name or ID to update" }),
|
|
319
|
-
}),
|
|
320
|
-
async execute(_id, params, _signal, onUpdate) {
|
|
321
|
-
const { added, removed, unchanged } = await engine.update(
|
|
322
|
-
params.target,
|
|
323
|
-
(msg) => {
|
|
324
|
-
onUpdate?.({ content: [{ type: "text", text: msg }] });
|
|
325
|
-
},
|
|
326
|
-
_signal,
|
|
327
|
-
);
|
|
328
|
-
return {
|
|
329
|
-
content: [{ type: "text", text: `Updated: +${added} added, -${removed} removed, ${unchanged} unchanged.` }],
|
|
330
|
-
};
|
|
331
|
-
},
|
|
332
|
-
});
|
|
333
|
-
|
|
334
|
-
pi.registerTool({
|
|
335
|
-
name: "knowledge_status",
|
|
336
|
-
label: "Knowledge Status",
|
|
337
|
-
description: "Show knowledge engine status with health diagnostics: staleness, orphans, and coverage",
|
|
338
|
-
parameters: Type.Object({}),
|
|
339
|
-
async execute() {
|
|
340
|
-
const kbs = engine.list();
|
|
341
|
-
const watchCount = getActiveWatcherCount();
|
|
342
|
-
const diagnostics = engine.diagnose();
|
|
343
|
-
const lines = [
|
|
344
|
-
`Storage: ${getDefaultKnowledgeDir()}`,
|
|
345
|
-
`Knowledge bases: ${kbs.length}`,
|
|
346
|
-
`Active watchers: ${watchCount}`,
|
|
347
|
-
"",
|
|
348
|
-
];
|
|
349
|
-
for (const kb of kbs) {
|
|
350
|
-
const age = Math.round((Date.now() - kb.updated_at) / 60000);
|
|
351
|
-
const diag = diagnostics.find((d) => d.kb_id === kb.id);
|
|
352
|
-
lines.push(
|
|
353
|
-
` "${kb.name}" — ${kb.status} — ${kb.chunk_count} chunks, ${kb.file_count} files — updated ${age}m ago`,
|
|
354
|
-
);
|
|
355
|
-
if (kb.source_path) lines.push(` source: ${kb.source_path}`);
|
|
356
|
-
if (diag) {
|
|
357
|
-
if (diag.job) {
|
|
358
|
-
const jobAge = Math.round((Date.now() - diag.job.last_progress_at) / 1000);
|
|
359
|
-
const progress = [
|
|
360
|
-
`${diag.job.processed_files} files`,
|
|
361
|
-
`${diag.job.processed_chunks} chunks`,
|
|
362
|
-
diag.job.skipped_total > 0 ? `${diag.job.skipped_total} skipped` : "",
|
|
363
|
-
diag.job.added_chunks > 0 ? `+${diag.job.added_chunks}` : "",
|
|
364
|
-
diag.job.removed_chunks > 0 ? `-${diag.job.removed_chunks}` : "",
|
|
365
|
-
diag.job.unchanged_chunks > 0 ? `=${diag.job.unchanged_chunks}` : "",
|
|
366
|
-
]
|
|
367
|
-
.filter(Boolean)
|
|
368
|
-
.join(", ");
|
|
369
|
-
lines.push(
|
|
370
|
-
` job: ${diag.job.status}/${diag.job.phase} — ${progress || "no processed items yet"} — last progress ${jobAge}s ago`,
|
|
371
|
-
);
|
|
372
|
-
if (diag.job.message) lines.push(` last: ${diag.job.message}`);
|
|
373
|
-
if (diag.job.error_message) lines.push(` error: ${diag.job.error_message}`);
|
|
374
|
-
}
|
|
375
|
-
lines.push(
|
|
376
|
-
` coverage: ${diag.coverage_percent}% (${diag.indexed_files}/${diag.total_source_files} files)`,
|
|
377
|
-
);
|
|
378
|
-
if (diag.stale_files.length > 0)
|
|
379
|
-
lines.push(` ⚠️ stale: ${diag.stale_files.length} files modified since last index`);
|
|
380
|
-
if (diag.orphan_files.length > 0)
|
|
381
|
-
lines.push(` ⚠️ orphans: ${diag.orphan_files.length} chunks reference deleted files`);
|
|
382
|
-
if (diag.stuck_indexing)
|
|
383
|
-
lines.push(
|
|
384
|
-
` ⚠️ indexing appears stuck for ${Math.round(diag.last_progress_age_ms / 60000)}m; remove and rebuild if no pi process is actively indexing it`,
|
|
385
|
-
);
|
|
386
|
-
if (diag.skipped_files.total > 0)
|
|
387
|
-
lines.push(
|
|
388
|
-
` skipped: ${diag.skipped_files.total} files (${Object.entries(diag.skipped_files.by_reason)
|
|
389
|
-
.filter(([, count]) => count > 0)
|
|
390
|
-
.map(([reason, count]) => `${reason}: ${count}`)
|
|
391
|
-
.join(", ")})`,
|
|
392
|
-
);
|
|
393
|
-
}
|
|
394
|
-
}
|
|
395
|
-
const totalStale = diagnostics.reduce((n, d) => n + d.stale_files.length, 0);
|
|
396
|
-
const totalOrphans = diagnostics.reduce((n, d) => n + d.orphan_files.length, 0);
|
|
397
|
-
const totalStuck = diagnostics.filter((d) => d.stuck_indexing).length;
|
|
398
|
-
if (totalStale === 0 && totalOrphans === 0 && totalStuck === 0 && kbs.length > 0)
|
|
399
|
-
lines.push("", "Health: ✓ all indexes up to date");
|
|
400
|
-
else if (totalStale > 0 || totalOrphans > 0 || totalStuck > 0)
|
|
401
|
-
lines.push(
|
|
402
|
-
"",
|
|
403
|
-
`Health: ⚠️ ${totalStale} stale, ${totalOrphans} orphans, ${totalStuck} stuck indexing — run knowledge_update or rebuild affected KBs`,
|
|
404
|
-
);
|
|
405
|
-
return { content: [{ type: "text", text: lines.join("\n") }] };
|
|
406
|
-
},
|
|
407
|
-
});
|
|
408
|
-
|
|
409
|
-
pi.registerTool({
|
|
410
|
-
name: "knowledge_doctor",
|
|
411
|
-
label: "Knowledge Doctor",
|
|
412
|
-
description: "Diagnose knowledge base health, skipped files, stale indexes, stuck jobs, and recommended fixes",
|
|
413
|
-
promptSnippet: "Diagnose knowledge base health and recommended actions",
|
|
414
|
-
parameters: Type.Object({}),
|
|
415
|
-
async execute() {
|
|
416
|
-
const report = engine.doctor();
|
|
417
|
-
const lines = [`Health score: ${report.health_score}/100`, report.summary, ""];
|
|
418
|
-
if (report.issues.length === 0) {
|
|
419
|
-
lines.push("No issues found.");
|
|
420
|
-
} else {
|
|
421
|
-
for (const issue of report.issues) {
|
|
422
|
-
const scope = issue.kb_name ? ` [${issue.kb_name}]` : "";
|
|
423
|
-
lines.push(`- ${issue.severity.toUpperCase()}${scope}: ${issue.message}`);
|
|
424
|
-
lines.push(` Action: ${issue.action}`);
|
|
425
|
-
}
|
|
426
|
-
}
|
|
427
|
-
return { content: [{ type: "text", text: lines.join("\n") }] };
|
|
428
|
-
},
|
|
429
|
-
});
|
|
430
|
-
|
|
431
|
-
pi.registerTool({
|
|
432
|
-
name: "knowledge_show",
|
|
433
|
-
label: "Knowledge Show",
|
|
434
|
-
description: "List all indexed knowledge bases",
|
|
435
|
-
parameters: Type.Object({}),
|
|
436
|
-
async execute() {
|
|
437
|
-
const kbs = engine.list();
|
|
438
|
-
if (kbs.length === 0) return { content: [{ type: "text", text: "No knowledge bases." }] };
|
|
439
|
-
const lines = kbs.map((kb) => `• ${kb.name} — ${kb.chunk_count} chunks, ${kb.file_count} files (${kb.status})`);
|
|
440
|
-
return { content: [{ type: "text", text: lines.join("\n") }] };
|
|
441
|
-
},
|
|
442
|
-
});
|
|
443
|
-
|
|
444
|
-
pi.registerTool({
|
|
445
|
-
name: "knowledge_remove",
|
|
446
|
-
label: "Knowledge Remove",
|
|
447
|
-
description: "Remove a knowledge base by name or ID",
|
|
448
|
-
parameters: Type.Object({
|
|
449
|
-
target: Type.String({ description: "KB name or ID to remove" }),
|
|
450
|
-
}),
|
|
451
|
-
async execute(_id, params) {
|
|
452
|
-
const ok = engine.remove(params.target);
|
|
453
|
-
return { content: [{ type: "text", text: ok ? "Removed." : "Not found." }] };
|
|
454
|
-
},
|
|
455
|
-
});
|
|
456
|
-
|
|
457
|
-
pi.registerTool({
|
|
458
|
-
name: "knowledge_export",
|
|
459
|
-
label: "Knowledge Export",
|
|
460
|
-
description: "Export a knowledge base to a JSONL file (shareable, git-friendly)",
|
|
461
|
-
parameters: Type.Object({
|
|
462
|
-
target: Type.String({ description: "KB name or ID to export" }),
|
|
463
|
-
output: Type.String({ description: "Output file path (.jsonl)" }),
|
|
464
|
-
}),
|
|
465
|
-
async execute(_id, params) {
|
|
466
|
-
const count = await engine.exportKB(params.target, params.output);
|
|
467
|
-
return { content: [{ type: "text", text: `Exported ${count} chunks to ${params.output}` }] };
|
|
468
|
-
},
|
|
469
|
-
});
|
|
470
|
-
|
|
471
|
-
pi.registerTool({
|
|
472
|
-
name: "knowledge_import",
|
|
473
|
-
label: "Knowledge Import",
|
|
474
|
-
description: "Import a knowledge base from a JSONL file (re-embeds content)",
|
|
475
|
-
parameters: Type.Object({
|
|
476
|
-
input: Type.String({ description: "Input JSONL file path" }),
|
|
477
|
-
}),
|
|
478
|
-
async execute(_id, params, _signal, onUpdate) {
|
|
479
|
-
const { kb, chunkCount } = await engine.importKB(
|
|
480
|
-
params.input,
|
|
481
|
-
(msg) => {
|
|
482
|
-
onUpdate?.({ content: [{ type: "text", text: msg }] });
|
|
483
|
-
},
|
|
484
|
-
_signal,
|
|
485
|
-
);
|
|
486
|
-
return { content: [{ type: "text", text: `Imported "${kb.name}": ${chunkCount} chunks (re-embedded)` }] };
|
|
487
|
-
},
|
|
488
|
-
});
|
|
489
|
-
|
|
490
|
-
pi.registerTool({
|
|
491
|
-
name: "knowledge_clear",
|
|
492
|
-
label: "Knowledge Clear",
|
|
493
|
-
description: "Remove all knowledge bases",
|
|
494
|
-
parameters: Type.Object({}),
|
|
495
|
-
async execute() {
|
|
496
|
-
engine.clear();
|
|
497
|
-
return { content: [{ type: "text", text: "All knowledge bases cleared." }] };
|
|
498
|
-
},
|
|
499
|
-
});
|
|
500
|
-
}
|
|
@@ -1,145 +0,0 @@
|
|
|
1
|
-
import { existsSync, statSync } from "node:fs";
|
|
2
|
-
import { join } from "node:path";
|
|
3
|
-
import type Database from "better-sqlite3";
|
|
4
|
-
import {
|
|
5
|
-
createSkippedScanStats,
|
|
6
|
-
iterateScannableFiles,
|
|
7
|
-
type ScanOptions,
|
|
8
|
-
type ScanResult,
|
|
9
|
-
} from "../indexer/chunker.ts";
|
|
10
|
-
import { getIndexingJob, type IndexingJob, iterateChunksByKB, type KnowledgeBase } from "../storage/sqlite.ts";
|
|
11
|
-
|
|
12
|
-
export interface DiagnosticResult {
|
|
13
|
-
kb_id: string;
|
|
14
|
-
kb_name: string;
|
|
15
|
-
status: KnowledgeBase["status"];
|
|
16
|
-
status_age_ms: number;
|
|
17
|
-
last_progress_age_ms: number;
|
|
18
|
-
stuck_indexing: boolean;
|
|
19
|
-
stale_files: string[]; // files modified after indexing
|
|
20
|
-
orphan_files: string[]; // chunks referencing deleted files
|
|
21
|
-
coverage_percent: number; // indexed files / total scannable files
|
|
22
|
-
total_source_files: number;
|
|
23
|
-
indexed_files: number;
|
|
24
|
-
skipped_files: ScanResult["skipped"];
|
|
25
|
-
job?: IndexingJob;
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
const DEFAULT_STALE_INDEXING_MS = 10 * 60 * 1000;
|
|
29
|
-
|
|
30
|
-
function staleIndexingMs(): number {
|
|
31
|
-
const configured = Number(process.env.PI_KNOWLEDGE_STALE_INDEXING_MS);
|
|
32
|
-
return Number.isFinite(configured) && configured > 0 ? configured : DEFAULT_STALE_INDEXING_MS;
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
function scanOptionsFromSourceOptions(raw: string | null): ScanOptions {
|
|
36
|
-
if (!raw) return {};
|
|
37
|
-
try {
|
|
38
|
-
const parsed = JSON.parse(raw) as {
|
|
39
|
-
include_suggested_text?: unknown;
|
|
40
|
-
include_paths?: unknown;
|
|
41
|
-
exclude_paths?: unknown;
|
|
42
|
-
};
|
|
43
|
-
return {
|
|
44
|
-
includeSuggestedText: parsed.include_suggested_text === true,
|
|
45
|
-
includePaths: Array.isArray(parsed.include_paths)
|
|
46
|
-
? parsed.include_paths.filter((item) => typeof item === "string")
|
|
47
|
-
: undefined,
|
|
48
|
-
excludePaths: Array.isArray(parsed.exclude_paths)
|
|
49
|
-
? parsed.exclude_paths.filter((item) => typeof item === "string")
|
|
50
|
-
: undefined,
|
|
51
|
-
};
|
|
52
|
-
} catch {
|
|
53
|
-
return {};
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
export function diagnoseKB(db: Database.Database, kb: KnowledgeBase): DiagnosticResult {
|
|
58
|
-
const statusAgeMs = Date.now() - kb.updated_at;
|
|
59
|
-
const job = getIndexingJob(db, kb.id);
|
|
60
|
-
const lastProgressAgeMs = job?.status === "running" ? Date.now() - job.last_progress_at : statusAgeMs;
|
|
61
|
-
const result: DiagnosticResult = {
|
|
62
|
-
kb_id: kb.id,
|
|
63
|
-
kb_name: kb.name,
|
|
64
|
-
status: kb.status,
|
|
65
|
-
status_age_ms: statusAgeMs,
|
|
66
|
-
last_progress_age_ms: lastProgressAgeMs,
|
|
67
|
-
stuck_indexing: kb.status === "indexing" && lastProgressAgeMs > staleIndexingMs(),
|
|
68
|
-
stale_files: [],
|
|
69
|
-
orphan_files: [],
|
|
70
|
-
coverage_percent: 100,
|
|
71
|
-
total_source_files: 0,
|
|
72
|
-
indexed_files: kb.file_count,
|
|
73
|
-
skipped_files: {
|
|
74
|
-
total: 0,
|
|
75
|
-
by_reason: {
|
|
76
|
-
suggested_excluded: 0,
|
|
77
|
-
oversized: 0,
|
|
78
|
-
binary: 0,
|
|
79
|
-
unreadable: 0,
|
|
80
|
-
inaccessible: 0,
|
|
81
|
-
},
|
|
82
|
-
samples: [],
|
|
83
|
-
},
|
|
84
|
-
job,
|
|
85
|
-
};
|
|
86
|
-
|
|
87
|
-
if (!kb.source_path || kb.source_type === "url" || !existsSync(kb.source_path)) {
|
|
88
|
-
return result; // text KBs or missing source — no diagnostics possible
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
// Scan current source files
|
|
92
|
-
const currentFiles = new Set<string>();
|
|
93
|
-
const isDirectory = statSync(kb.source_path).isDirectory();
|
|
94
|
-
try {
|
|
95
|
-
if (isDirectory) {
|
|
96
|
-
const skipped = createSkippedScanStats();
|
|
97
|
-
for (const file of iterateScannableFiles(
|
|
98
|
-
kb.source_path,
|
|
99
|
-
skipped,
|
|
100
|
-
scanOptionsFromSourceOptions(kb.source_options),
|
|
101
|
-
)) {
|
|
102
|
-
currentFiles.add(file.relPath);
|
|
103
|
-
}
|
|
104
|
-
result.skipped_files = skipped;
|
|
105
|
-
} else {
|
|
106
|
-
currentFiles.add(kb.source_path);
|
|
107
|
-
}
|
|
108
|
-
} catch {
|
|
109
|
-
return result;
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
result.total_source_files = currentFiles.size;
|
|
113
|
-
result.coverage_percent = currentFiles.size > 0 ? Math.round((result.indexed_files / currentFiles.size) * 100) : 100;
|
|
114
|
-
|
|
115
|
-
const indexedFilePaths = new Set<string>();
|
|
116
|
-
const latestIndexedByFile = new Map<string, number>();
|
|
117
|
-
for (const chunk of iterateChunksByKB(db, kb.id)) {
|
|
118
|
-
indexedFilePaths.add(chunk.file_path);
|
|
119
|
-
const currentLatest = latestIndexedByFile.get(chunk.file_path) ?? 0;
|
|
120
|
-
if (chunk.indexed_at > currentLatest) latestIndexedByFile.set(chunk.file_path, chunk.indexed_at);
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
// Orphan detection: chunks referencing files no longer in source
|
|
124
|
-
for (const filePath of indexedFilePaths) {
|
|
125
|
-
if (!currentFiles.has(filePath)) {
|
|
126
|
-
result.orphan_files.push(filePath);
|
|
127
|
-
}
|
|
128
|
-
}
|
|
129
|
-
|
|
130
|
-
// Staleness detection: source files modified after last indexing
|
|
131
|
-
for (const relPath of currentFiles) {
|
|
132
|
-
const absPath = isDirectory ? join(kb.source_path, relPath) : relPath;
|
|
133
|
-
try {
|
|
134
|
-
const mtime = statSync(absPath).mtimeMs;
|
|
135
|
-
const latestIndexed = latestIndexedByFile.get(relPath);
|
|
136
|
-
if (latestIndexed !== undefined && mtime > latestIndexed) {
|
|
137
|
-
result.stale_files.push(relPath);
|
|
138
|
-
}
|
|
139
|
-
} catch {
|
|
140
|
-
/* file unreadable — skip */
|
|
141
|
-
}
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
return result;
|
|
145
|
-
}
|