pi-knowledge 0.3.6 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.pi/skills/search-docs.md +24 -4
- package/CHANGELOG.md +9 -2
- package/README.md +40 -2
- package/index.ts +200 -14
- package/package.json +1 -1
- package/src/diagnostics/health.ts +84 -14
- package/src/embedding/vectors.ts +95 -12
- package/src/engine.ts +880 -140
- package/src/indexer/chunker.ts +232 -24
- package/src/search/bm25.ts +15 -4
- package/src/search/ranking.ts +4 -0
- package/src/search/vector.ts +60 -0
- package/src/storage/sqlite.ts +224 -9
|
@@ -10,10 +10,10 @@ Search the indexed knowledge bases for: $ARGUMENTS
|
|
|
10
10
|
## Steps
|
|
11
11
|
|
|
12
12
|
1. Use `knowledge_show` to see available knowledge bases
|
|
13
|
-
2.
|
|
14
|
-
3.
|
|
15
|
-
4.
|
|
16
|
-
5.
|
|
13
|
+
2. Choose the search mode from the decision rules below instead of always using the default
|
|
14
|
+
3. Present results with file paths and relevant snippets
|
|
15
|
+
4. If results are insufficient, retry once with a different mode before concluding the KB lacks the answer
|
|
16
|
+
5. If search behavior looks unhealthy, run `knowledge_doctor` and follow its actions
|
|
17
17
|
|
|
18
18
|
## Tips
|
|
19
19
|
|
|
@@ -21,3 +21,23 @@ Search the indexed knowledge bases for: $ARGUMENTS
|
|
|
21
21
|
- Concept queries: use natural language descriptions
|
|
22
22
|
- Filter: `file_type: "typescript"` or `file_type: "python"`
|
|
23
23
|
- No results? Check `knowledge_show` to confirm content is indexed
|
|
24
|
+
|
|
25
|
+
## Mode Selection
|
|
26
|
+
|
|
27
|
+
- Use `hybrid` by default for most project questions.
|
|
28
|
+
- Use `fast` for exact symbols, filenames, commands, error codes, API names, config keys, or quoted strings.
|
|
29
|
+
- Use `semantic` for broad conceptual questions when exact terms may differ from the indexed wording.
|
|
30
|
+
- Use `adaptive` when the user needs nearby implementation context, related sections, or enough context to safely edit code.
|
|
31
|
+
- Use `deep` for high-stakes answers, ambiguous top results, or final verification when slower reranking is acceptable.
|
|
32
|
+
- Use `auto` when you need the tool to choose and retry modes based on the query shape.
|
|
33
|
+
- If results are repetitive, retry with `diversity: "strong"` or `adaptive` before raising `limit`.
|
|
34
|
+
- If results are empty but the KB should contain the answer, retry once with `fast` for exact terms or `semantic` for conceptual wording.
|
|
35
|
+
- Request diagnostics when you need to explain why a result ranked highly or why a fallback mode was used.
|
|
36
|
+
|
|
37
|
+
## Indexing Safety
|
|
38
|
+
|
|
39
|
+
- When creating a KB, prefer one `knowledge_add` call for the project root or relevant source/docs directory.
|
|
40
|
+
- Do not create one KB per small file unless the user explicitly asks for separate KBs.
|
|
41
|
+
- Avoid indexing generated or runtime artifacts such as `bin/`, `obj/`, `dist/`, `build/`, `node_modules/`, Playwright browser caches, `.app` bundles, `.pak`, and `.asar` files.
|
|
42
|
+
- Do not exclude browser-domain source directories merely because they are named `chromium`, `chrome`, `firefox`, `webkit`, or `browsers`; those names may be first-party source in Playwright or browser-tooling projects.
|
|
43
|
+
- If a project contains large embedded browsers or vendor bundles, index the source/docs/config directories directly or rely on the extension's default ignore rules.
|
package/CHANGELOG.md
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
-
## [0.
|
|
3
|
+
## [0.4.0] - 2026-06-16
|
|
4
4
|
|
|
5
5
|
### Added
|
|
6
|
+
- Added persisted indexing job state for long-running `knowledge_add`, `knowledge_update`, and `knowledge_import` operations. `knowledge_status` now reports operation, phase, last progress message, last progress age, processed files/chunks, skipped count, and add/remove/unchanged counts so large indexing runs do not look frozen after transient tool updates disappear.
|
|
7
|
+
- Added metadata-only directory planning and chunk throughput in indexing progress so large repositories show total scannable files, skipped counts, chunks/sec, and file ETA before expensive embedding starts.
|
|
8
|
+
- Added `knowledge_plan` as a no-write indexing scope inspection tool so agents can show scannable files, suggested exclusions, and technical skips before asking the user to confirm risky or low-signal text.
|
|
6
9
|
- Added contextual indexing for rebuilds: embeddings and FTS now include file path, file type, heading breadcrumbs, and code symbols while keeping returned chunk content readable.
|
|
7
10
|
- Added more focused rebuild-time chunking for Markdown and plain text, with reduced overlap to avoid near-duplicate retrieval units.
|
|
8
11
|
- Added adaptive search mode with query-time contextual window expansion around relevant seed chunks.
|
|
@@ -14,7 +17,11 @@
|
|
|
14
17
|
- Added confidence gating for hybrid search so low-evidence garbage queries return no results instead of unrelated matches.
|
|
15
18
|
- Strengthened source-file intent scoring so named modules and core implementation files outrank overview and test files when appropriate.
|
|
16
19
|
- Demoted localization catalogs for implementation-oriented queries while preserving them for explicit translation or locale intent.
|
|
17
|
-
-
|
|
20
|
+
- Suggested excluding generated knowledge-base evaluation reports from default directory indexing to prevent self-referential retrieval pollution, while allowing confirmed inclusion through scope overrides.
|
|
21
|
+
|
|
22
|
+
### Changed
|
|
23
|
+
- Reworked directory indexing policy from hard text-file blocking to suggested exclusions plus confirmed scope overrides. Risky or low-signal text such as `.env`, secret/credential-named text, generated reports, lockfiles, vendor text, build output text, and runtime/cache text is skipped by default but can be included after user confirmation with `include_suggested_text` or focused `include_paths`. Unsupported binary/non-text, oversized, unreadable, inaccessible, and unextractable files remain technical skips.
|
|
24
|
+
- Persisted confirmed include/exclude scope options so `knowledge_update` preserves the same directory indexing scope instead of silently dropping user-confirmed text files.
|
|
18
25
|
|
|
19
26
|
## [0.3.5] - 2026-06-15
|
|
20
27
|
|
package/README.md
CHANGED
|
@@ -28,13 +28,29 @@ Unlike `pi-memory` (which manages the agent's own notes), `pi-knowledge` indexes
|
|
|
28
28
|
| **Local embeddings (zero API)** | ✅ | ❌ | ✅ (qmd) |
|
|
29
29
|
| **Index quality diagnostics** | ✅ | ❌ | ❌ |
|
|
30
30
|
| **Metadata filters in search** | ✅ | ❌ | ❌ |
|
|
31
|
-
| **Progress reporting** | ✅ | partial | ❌ |
|
|
31
|
+
| **Progress reporting + stuck indexing diagnostics** | ✅ | partial | ❌ |
|
|
32
32
|
| Cross-session persistence | ✅ | ✅ | ✅ |
|
|
33
33
|
| Pi extension native | ✅ | N/A | ✅ |
|
|
34
34
|
| Context injection per turn | ✅ | ❌ | ✅ |
|
|
35
35
|
| TUI custom rendering | planned | N/A | ❌ |
|
|
36
36
|
| RPC mode support | ✅ | N/A | N/A |
|
|
37
37
|
|
|
38
|
+
## Research-Backed Retrieval
|
|
39
|
+
|
|
40
|
+
`pi-knowledge` turns retrieval research into product behavior that agents can actually use:
|
|
41
|
+
|
|
42
|
+
- **RAG-native project memory**: follows the Retrieval-Augmented Generation pattern from Lewis et al. 2020: keep source truth outside the model, retrieve it at answer time, and inject only relevant context.
|
|
43
|
+
- **Dense semantic recall**: uses multilingual dense embeddings in the spirit of Dense Passage Retrieval (Karpukhin et al. 2020), so conceptual queries can find code/docs even when wording differs.
|
|
44
|
+
- **Contextual Retrieval without remote chunk rewriting**: applies Anthropic's Contextual Retrieval insight locally by embedding file path, file type, Markdown breadcrumbs, and code symbols with each chunk. This improves standalone chunk meaning without sending private source chunks to an LLM for context generation.
|
|
45
|
+
- **Hybrid retrieval with diagnosable scores**: combines BM25 and vectors with normalized weighted score fusion. RRF (Cormack et al. 2009) remains the baseline reference, but weighted fusion is used by default because project dogfood showed RRF compressed scores too much for ranking diagnostics.
|
|
46
|
+
- **MMR-style diversity**: uses Maximal Marginal Relevance ideas (Goldstein and Carbonell 1998), file interleaving, vector redundancy checks, and adaptive-window overlap collapse so repeated README or same-file chunks do not dominate top results.
|
|
47
|
+
- **Intent-aware and self-correcting agent UX**: mode selection (`auto`, `fast`, `semantic`, `hybrid`, `adaptive`, `deep`), ranking diagnostics, and `knowledge_doctor` turn retrieval failures into concrete next actions instead of silent bad answers.
|
|
48
|
+
- **Confidence gating**: low-evidence hybrid matches can return zero results instead of unrelated chunks, reducing false confidence when the KB does not contain the answer.
|
|
49
|
+
|
|
50
|
+
This is intentionally not a heavy ColBERT-style late-interaction index yet (Khattab and Zaharia 2020). The current product chooses lightweight local embeddings, BM25, query-aware ranking, optional cross-encoder reranking, streamed vector scans, and health diagnostics for commercial usefulness with low setup cost.
|
|
51
|
+
|
|
52
|
+
In project-level dogfood, these changes improved a real codebase evaluation from early 3.x/5 quality to above 4.5/5 after rebuilds, with fixes for score compression, README repetition, garbage-query false positives, small-module discoverability, source-vs-test ranking, indexing stability, and auto-mode false positives. Existing KBs should be rebuilt or updated after upgrades that change indexing text.
|
|
53
|
+
|
|
38
54
|
## Quick Start
|
|
39
55
|
|
|
40
56
|
```bash
|
|
@@ -58,12 +74,14 @@ pi install ./pi-knowledge
|
|
|
58
74
|
|
|
59
75
|
| Tool | Description |
|
|
60
76
|
|------|-------------|
|
|
77
|
+
| `knowledge_plan` | Inspect an indexing source before writing a KB; reports scannable files, suggested exclusions, and technical skips |
|
|
61
78
|
| `knowledge_add` | Index files, directories, URLs, PDFs, DOCX, or inline text |
|
|
62
79
|
| `knowledge_search` | Hybrid, deep, or adaptive search across one or all knowledge bases |
|
|
63
80
|
| `knowledge_remove` | Remove a knowledge base by name or ID |
|
|
64
81
|
| `knowledge_update` | Incrementally re-index changed files in a knowledge base |
|
|
65
82
|
| `knowledge_show` | List all knowledge bases with stats |
|
|
66
83
|
| `knowledge_status` | Show engine status with health diagnostics (stale, orphans, coverage) |
|
|
84
|
+
| `knowledge_doctor` | Diagnose health score, skipped files, stuck jobs, stale data, and recommended fixes |
|
|
67
85
|
| `knowledge_clear` | Remove all knowledge bases |
|
|
68
86
|
| `knowledge_export` | Export a KB to shareable JSONL file |
|
|
69
87
|
| `knowledge_import` | Import a KB from JSONL (re-embeds content) |
|
|
@@ -75,11 +93,31 @@ pi install ./pi-knowledge
|
|
|
75
93
|
- `hybrid`: BM25 + vector search with normalized weighted score fusion.
|
|
76
94
|
- `deep`: hybrid retrieval followed by cross-encoder reranking.
|
|
77
95
|
- `adaptive`: hybrid retrieval followed by query-time contextual window expansion around seed chunks. It keeps the matched seed, prefers nearby/query-relevant neighboring chunks, and collapses overlapping windows from the same file.
|
|
96
|
+
- `auto`: selects a primary mode from the query shape and retries alternate modes when results are empty or weak.
|
|
97
|
+
|
|
98
|
+
Mode selection contract:
|
|
78
99
|
|
|
79
|
-
|
|
100
|
+
- Start with `hybrid` for most project questions.
|
|
101
|
+
- Use `fast` for exact symbols, filenames, commands, error codes, API names, config keys, or quoted strings.
|
|
102
|
+
- Use `semantic` when the query is conceptual and exact terms may differ from the indexed wording.
|
|
103
|
+
- Use `adaptive` when the answer needs nearby code, neighboring documentation sections, or enough context to make a safe edit.
|
|
104
|
+
- Use `deep` for high-stakes answers, ambiguous top results, or final verification when slower reranking is acceptable.
|
|
105
|
+
- If results are empty or weak but the KB should contain the answer, retry once with a different mode before concluding no answer exists.
|
|
106
|
+
|
|
107
|
+
Search results use balanced diversity reranking by default so near-duplicate chunks from the same file do not dominate the top results. Diversity scoring considers lexical overlap, same-file line proximity, overlapping adaptive windows, available embedding-vector similarity, and file-level interleaving. Use `diversity: "off"` only when raw ranking order is needed for diagnostics. Agents can request search diagnostics to inspect mode fallback, ranking coverage, path/source/test boosts, and adjusted scores.
|
|
80
108
|
|
|
81
109
|
For best search quality, rebuild or update existing knowledge bases after upgrading. New indexes use contextual retrieval units: embeddings and FTS include file path, file type, Markdown heading breadcrumbs, and code symbol names while returned results keep the original chunk text readable. This improves queries that mention project structure, filenames, sections, or functions, and reduces duplicate-looking chunk hits.
|
|
82
110
|
|
|
111
|
+
## Large Project Indexing
|
|
112
|
+
|
|
113
|
+
Indexing is designed as a stable long-running operation, not a quick background trick. `knowledge_add`, `knowledge_update`, and `knowledge_import` scan directories incrementally, embed and store chunks in hard-capped batches, stream vector files to disk, and report progress with file/chunk counts, chunks/sec, skipped file counts, elapsed time, and file ETA where available. Directory indexing starts with a metadata-only planning scan so large repositories can show total scannable files and skipped counts before expensive embedding starts.
|
|
114
|
+
|
|
115
|
+
Directory indexing separates technical skips from user-confirmable suggestions. `knowledge_plan` inspects a source without writing a KB, so agents can show scannable files, suggested exclusions, and technical skips before asking the user to confirm scope. Unsupported binary/non-text files, oversized files, unreadable files, inaccessible paths, and documents that cannot be extracted are skipped for stability. Text files that may be private or low-signal, such as `.env`, credential-named files, generated reports, lockfiles, vendor text, build output text, and runtime/cache text, are suggested exclusions by default rather than permanent blocks. Agents should explain the privacy and search-precision tradeoff, ask the user when the choice is ambiguous, and then use `include_suggested_text` or focused `include_paths` when the user confirms those text files belong in the KB. Ordinary project configuration files such as `settings.json` or `appsettings.json` remain indexable because they often describe real system behavior.
|
|
116
|
+
|
|
117
|
+
Indexing progress is persisted in SQLite, not only printed as transient tool updates. `knowledge_status` shows the current or last indexing operation, phase, last progress message, last progress age, processed file/chunk counts, skipped count, and add/remove/unchanged counts. This makes long indexing runs distinguishable from stuck jobs even if the user checks status from a later prompt.
|
|
118
|
+
|
|
119
|
+
Update and diagnostics paths are also streaming-oriented: changed chunks are embedded in batches, newly produced vectors are written to temporary vector files, deleted rows are removed in batches, and final vector rebuilds iterate SQLite rows instead of loading the whole KB. Search also avoids loading a full KB vector file or all chunk IDs into memory. Semantic and hybrid modes scan vectors from disk and retain only the top candidate vectors needed for ranking/diversity. `knowledge_status` reports stale files, orphaned chunks, coverage, skipped files, and indexing jobs that appear stuck after an interrupted or crashed Pi process. `knowledge_doctor` summarizes the same signals as a health score with concrete actions. KBs still marked `indexing` or `error` are visible in status but skipped by search, so interrupted work is not treated as a healthy searchable KB. A stuck `indexing` KB should be removed and rebuilt after confirming no active Pi process is still building it.
|
|
120
|
+
|
|
83
121
|
## Architecture
|
|
84
122
|
|
|
85
123
|
See [DESIGN.md](DESIGN.md) for the full technical design document.
|
package/index.ts
CHANGED
|
@@ -111,12 +111,18 @@ const Type = {
|
|
|
111
111
|
Number(options: Record<string, unknown> = {}): Schema {
|
|
112
112
|
return { type: "number", ...options };
|
|
113
113
|
},
|
|
114
|
+
Boolean(options: Record<string, unknown> = {}): Schema {
|
|
115
|
+
return { type: "boolean", ...options };
|
|
116
|
+
},
|
|
114
117
|
Literal(value: string): Schema {
|
|
115
118
|
return { const: value };
|
|
116
119
|
},
|
|
117
120
|
Union(items: Schema[]): Schema {
|
|
118
121
|
return { anyOf: items };
|
|
119
122
|
},
|
|
123
|
+
Array(item: Schema): Schema {
|
|
124
|
+
return { type: "array", items: item };
|
|
125
|
+
},
|
|
120
126
|
Optional(schema: Schema): Schema {
|
|
121
127
|
return { ...schema, optional: true };
|
|
122
128
|
},
|
|
@@ -187,24 +193,108 @@ export default function (pi: ExtensionAPI) {
|
|
|
187
193
|
}
|
|
188
194
|
});
|
|
189
195
|
|
|
196
|
+
pi.registerTool({
|
|
197
|
+
name: "knowledge_plan",
|
|
198
|
+
label: "Knowledge Plan",
|
|
199
|
+
description:
|
|
200
|
+
"Inspect an indexing source without writing a KB, showing scannable files, suggested exclusions, and technical skips",
|
|
201
|
+
promptSnippet: "Plan a knowledge-base indexing scope before calling knowledge_add",
|
|
202
|
+
promptGuidelines: [
|
|
203
|
+
"Use knowledge_plan before knowledge_add for broad directories, large repositories, or sources that may contain private or low-signal text",
|
|
204
|
+
"Show the user suggested exclusions and technical skips before asking whether to include risky or low-signal text",
|
|
205
|
+
"After the user confirms scope, call knowledge_add with matching include_suggested_text, include_paths, or exclude_paths",
|
|
206
|
+
"Do not use knowledge_plan as a substitute for knowledge_search; it only plans indexing scope",
|
|
207
|
+
],
|
|
208
|
+
parameters: Type.Object({
|
|
209
|
+
source: Type.String({ description: "File path, directory path, URL, or inline text to inspect before indexing" }),
|
|
210
|
+
include_suggested_text: Type.Optional(
|
|
211
|
+
Type.Boolean({
|
|
212
|
+
description:
|
|
213
|
+
"Preview the scope if suggested-excluded text such as vendor/build/runtime/cache/secret-named text is included",
|
|
214
|
+
}),
|
|
215
|
+
),
|
|
216
|
+
include_paths: Type.Optional(
|
|
217
|
+
Type.Array(
|
|
218
|
+
Type.String({
|
|
219
|
+
description:
|
|
220
|
+
"Relative paths under a directory source to include even when they match suggested-exclude patterns",
|
|
221
|
+
}),
|
|
222
|
+
),
|
|
223
|
+
),
|
|
224
|
+
exclude_paths: Type.Optional(
|
|
225
|
+
Type.Array(Type.String({ description: "Relative paths under a directory source to exclude from this plan" })),
|
|
226
|
+
),
|
|
227
|
+
}),
|
|
228
|
+
execute(_id, params) {
|
|
229
|
+
const { source, include_suggested_text, include_paths, exclude_paths } = params;
|
|
230
|
+
const plan = engine.plan(source, {
|
|
231
|
+
include_suggested_text: include_suggested_text === true,
|
|
232
|
+
include_paths: Array.isArray(include_paths)
|
|
233
|
+
? include_paths.filter((item) => typeof item === "string")
|
|
234
|
+
: undefined,
|
|
235
|
+
exclude_paths: Array.isArray(exclude_paths)
|
|
236
|
+
? exclude_paths.filter((item) => typeof item === "string")
|
|
237
|
+
: undefined,
|
|
238
|
+
});
|
|
239
|
+
const samples = plan.skipped.samples
|
|
240
|
+
.map((sample) => `- ${sample.reason}: ${sample.path}${sample.size ? ` (${sample.size} bytes)` : ""}`)
|
|
241
|
+
.join("\n");
|
|
242
|
+
return {
|
|
243
|
+
content: [
|
|
244
|
+
{
|
|
245
|
+
type: "text",
|
|
246
|
+
text: [
|
|
247
|
+
plan.summary,
|
|
248
|
+
`Source type: ${plan.source_type}`,
|
|
249
|
+
`Skipped summary: ${JSON.stringify(plan.skipped.by_reason)}`,
|
|
250
|
+
samples ? `Skipped samples:\n${samples}` : "Skipped samples: none",
|
|
251
|
+
].join("\n"),
|
|
252
|
+
},
|
|
253
|
+
],
|
|
254
|
+
};
|
|
255
|
+
},
|
|
256
|
+
});
|
|
257
|
+
|
|
190
258
|
pi.registerTool({
|
|
191
259
|
name: "knowledge_add",
|
|
192
260
|
label: "Knowledge Add",
|
|
193
261
|
description: "Index files, directories, or text into a named knowledge base for semantic search",
|
|
194
262
|
promptSnippet: "Index files/dirs/text into a searchable knowledge base",
|
|
195
263
|
promptGuidelines: [
|
|
264
|
+
"Use knowledge_plan first for broad directories, large repositories, or sources that may contain private or low-signal text",
|
|
196
265
|
"Use knowledge_add when the user asks to index, remember, or learn from files or documentation",
|
|
197
|
-
"Prefer one knowledge_add call for the project root or relevant directory; do not call it once per file",
|
|
266
|
+
"Prefer one knowledge_add call for the project root or relevant directory with include_paths/exclude_paths; do not call it once per file",
|
|
198
267
|
"If a knowledge base with the same name already exists, use knowledge_update or ask before replacing it",
|
|
199
|
-
"
|
|
268
|
+
"Index source, documentation, and ordinary configuration files that explain how the project works",
|
|
269
|
+
"Default directory indexing suggests excluding generated, vendor, browser runtime, obvious secret, and low-signal text artifacts; these are not permanent blocks",
|
|
270
|
+
"For ambiguous or risky text such as .env, private keys, certificates, credential-named files, settings.json, appsettings.json, cloud config, editor config, generated reports, lockfiles, or vendor text, explain the tradeoff and ask the user before including it",
|
|
271
|
+
"If the user confirms a suggested-excluded text file or directory should be indexed, call knowledge_add with include_suggested_text or a focused include_paths value so the tool follows the confirmed scope",
|
|
272
|
+
"Non-text, unsupported binary, oversized, unreadable, and inaccessible files remain technical skips even when the user wants broad indexing",
|
|
200
273
|
"Provide a descriptive name for this single knowledge base",
|
|
201
274
|
],
|
|
202
275
|
parameters: Type.Object({
|
|
203
276
|
source: Type.String({ description: "File path, directory path, or inline text to index" }),
|
|
204
277
|
name: Type.String({ description: "Display name for this knowledge base" }),
|
|
278
|
+
include_suggested_text: Type.Optional(
|
|
279
|
+
Type.Boolean({
|
|
280
|
+
description:
|
|
281
|
+
"Include text files that are normally suggested for exclusion, such as vendor/build/runtime/cache/secret-named text, after user confirmation",
|
|
282
|
+
}),
|
|
283
|
+
),
|
|
284
|
+
include_paths: Type.Optional(
|
|
285
|
+
Type.Array(
|
|
286
|
+
Type.String({
|
|
287
|
+
description:
|
|
288
|
+
"Relative paths under a directory source to include even when they match suggested-exclude patterns",
|
|
289
|
+
}),
|
|
290
|
+
),
|
|
291
|
+
),
|
|
292
|
+
exclude_paths: Type.Optional(
|
|
293
|
+
Type.Array(Type.String({ description: "Relative paths under a directory source to exclude from this KB" })),
|
|
294
|
+
),
|
|
205
295
|
}),
|
|
206
296
|
async execute(_id, params, _signal, onUpdate) {
|
|
207
|
-
const { source, name } = params;
|
|
297
|
+
const { source, name, include_suggested_text, include_paths, exclude_paths } = params;
|
|
208
298
|
const { kb, chunkCount } = await engine.add(
|
|
209
299
|
source,
|
|
210
300
|
name,
|
|
@@ -212,6 +302,15 @@ export default function (pi: ExtensionAPI) {
|
|
|
212
302
|
onUpdate?.({ content: [{ type: "text", text: msg }] });
|
|
213
303
|
},
|
|
214
304
|
_signal,
|
|
305
|
+
{
|
|
306
|
+
include_suggested_text: include_suggested_text === true,
|
|
307
|
+
include_paths: Array.isArray(include_paths)
|
|
308
|
+
? include_paths.filter((item) => typeof item === "string")
|
|
309
|
+
: undefined,
|
|
310
|
+
exclude_paths: Array.isArray(exclude_paths)
|
|
311
|
+
? exclude_paths.filter((item) => typeof item === "string")
|
|
312
|
+
: undefined,
|
|
313
|
+
},
|
|
215
314
|
);
|
|
216
315
|
// Start watcher for new directory KB
|
|
217
316
|
if (WATCH_ENABLED && kb.source_path && kb.source_type === "directory") {
|
|
@@ -234,16 +333,22 @@ export default function (pi: ExtensionAPI) {
|
|
|
234
333
|
name: "knowledge_search",
|
|
235
334
|
label: "Knowledge Search",
|
|
236
335
|
description: "Search indexed knowledge bases using hybrid semantic + keyword search",
|
|
237
|
-
promptSnippet: "Search knowledge bases (hybrid BM25 + semantic +
|
|
336
|
+
promptSnippet: "Search knowledge bases (hybrid BM25 + semantic + weighted score fusion)",
|
|
238
337
|
promptGuidelines: [
|
|
239
338
|
"Use knowledge_search to find relevant context before answering domain questions",
|
|
240
|
-
"Default mode 'hybrid'
|
|
241
|
-
"Use mode 'fast' for exact
|
|
339
|
+
"Default to mode 'hybrid' for most project questions because it combines lexical anchors with semantic recall",
|
|
340
|
+
"Use mode 'fast' for exact symbols, filenames, commands, error codes, API names, config keys, or quoted strings",
|
|
341
|
+
"Use mode 'semantic' for broad conceptual questions when exact terms may differ from the indexed wording",
|
|
342
|
+
"Use mode 'adaptive' when the user needs surrounding implementation context, related nearby sections, or enough context to make a code change",
|
|
343
|
+
"Use mode 'deep' for high-stakes answers, ambiguous top results, or final verification where slower cross-encoder reranking is acceptable",
|
|
344
|
+
"If a search returns no results or obviously weak results, retry once with a different mode before concluding the KB lacks the answer",
|
|
345
|
+
"If top results are repetitive, retry with diversity 'strong' or mode 'adaptive' instead of increasing the limit first",
|
|
242
346
|
],
|
|
243
347
|
parameters: Type.Object({
|
|
244
348
|
query: Type.String({ description: "Search query" }),
|
|
245
349
|
mode: Type.Optional(
|
|
246
350
|
Type.Union([
|
|
351
|
+
Type.Literal("auto"),
|
|
247
352
|
Type.Literal("fast"),
|
|
248
353
|
Type.Literal("semantic"),
|
|
249
354
|
Type.Literal("hybrid"),
|
|
@@ -256,6 +361,9 @@ export default function (pi: ExtensionAPI) {
|
|
|
256
361
|
offset: Type.Optional(Type.Number({ description: "Pagination offset" })),
|
|
257
362
|
file_type: Type.Optional(Type.String({ description: "Filter by file type (e.g. typescript, markdown, python)" })),
|
|
258
363
|
diversity: Type.Optional(Type.Union([Type.Literal("off"), Type.Literal("balanced"), Type.Literal("strong")])),
|
|
364
|
+
diagnostics: Type.Optional(
|
|
365
|
+
Type.Boolean({ description: "Include ranking diagnostics and mode/fallback details in the result" }),
|
|
366
|
+
),
|
|
259
367
|
}),
|
|
260
368
|
renderCall(args, theme: ThemeLike, context: RenderContextLike) {
|
|
261
369
|
const query = typeof args.query === "string" ? args.query : "";
|
|
@@ -271,18 +379,40 @@ export default function (pi: ExtensionAPI) {
|
|
|
271
379
|
return renderText(context, lines);
|
|
272
380
|
},
|
|
273
381
|
async execute(_id, params) {
|
|
274
|
-
const { query, mode, limit, kb_id, offset, file_type } = params;
|
|
382
|
+
const { query, mode, limit, kb_id, offset, file_type, diversity, diagnostics } = params;
|
|
275
383
|
const filters = file_type ? { file_type } : undefined;
|
|
276
|
-
const response = await engine.search(query, { mode, limit, kb_id, offset, filters });
|
|
384
|
+
const response = await engine.search(query, { mode, limit, kb_id, offset, filters, diversity });
|
|
277
385
|
if (response.results.length === 0) {
|
|
278
|
-
|
|
386
|
+
const details = [
|
|
387
|
+
"No results found.",
|
|
388
|
+
response.mode_used ? `Mode used: ${response.mode_used}` : "",
|
|
389
|
+
response.retry_modes?.length ? `Retried: ${response.retry_modes.join(", ")}` : "",
|
|
390
|
+
response.warnings?.length ? `Warnings: ${response.warnings.join(" | ")}` : "",
|
|
391
|
+
response.suggestions?.length ? `Suggestions: ${response.suggestions.join(" | ")}` : "",
|
|
392
|
+
]
|
|
393
|
+
.filter(Boolean)
|
|
394
|
+
.join("\n");
|
|
395
|
+
return { content: [{ type: "text", text: details }] };
|
|
279
396
|
}
|
|
280
|
-
let output = `${response.total_count} results (showing ${response.results.length})
|
|
397
|
+
let output = `${response.total_count} results (showing ${response.results.length})`;
|
|
398
|
+
if (response.mode_used) output += ` — mode: ${response.mode_used}`;
|
|
399
|
+
if (response.retry_modes?.length) output += ` — retried: ${response.retry_modes.join(", ")}`;
|
|
400
|
+
output += ":\n\n";
|
|
281
401
|
if (response.warnings?.length) {
|
|
282
402
|
output = `⚠️ ${response.warnings.join("\n⚠️ ")}\n\n${output}`;
|
|
283
403
|
}
|
|
284
404
|
output += response.results
|
|
285
|
-
.map((r, i) =>
|
|
405
|
+
.map((r, i) => {
|
|
406
|
+
const diag =
|
|
407
|
+
diagnostics && r.ranking
|
|
408
|
+
? `\nDiagnostics: base=${r.ranking.base_score.toFixed(3)}, adjusted=${r.ranking.adjusted_score.toFixed(
|
|
409
|
+
3,
|
|
410
|
+
)}, coverage=${r.ranking.coverage.toFixed(2)}, path_boost=${r.ranking.path_boost.toFixed(
|
|
411
|
+
2,
|
|
412
|
+
)}, source_boost=${r.ranking.source_boost.toFixed(2)}, test=${r.ranking.is_test}`
|
|
413
|
+
: "";
|
|
414
|
+
return `[${i + 1}] ${r.file_path} (${r.kb_name}, score: ${r.score.toFixed(3)})\n${r.snippet}${diag}`;
|
|
415
|
+
})
|
|
286
416
|
.join("\n\n");
|
|
287
417
|
return { content: [{ type: "text", text: output }] };
|
|
288
418
|
},
|
|
@@ -333,6 +463,24 @@ export default function (pi: ExtensionAPI) {
|
|
|
333
463
|
);
|
|
334
464
|
if (kb.source_path) lines.push(` source: ${kb.source_path}`);
|
|
335
465
|
if (diag) {
|
|
466
|
+
if (diag.job) {
|
|
467
|
+
const jobAge = Math.round((Date.now() - diag.job.last_progress_at) / 1000);
|
|
468
|
+
const progress = [
|
|
469
|
+
`${diag.job.processed_files} files`,
|
|
470
|
+
`${diag.job.processed_chunks} chunks`,
|
|
471
|
+
diag.job.skipped_total > 0 ? `${diag.job.skipped_total} skipped` : "",
|
|
472
|
+
diag.job.added_chunks > 0 ? `+${diag.job.added_chunks}` : "",
|
|
473
|
+
diag.job.removed_chunks > 0 ? `-${diag.job.removed_chunks}` : "",
|
|
474
|
+
diag.job.unchanged_chunks > 0 ? `=${diag.job.unchanged_chunks}` : "",
|
|
475
|
+
]
|
|
476
|
+
.filter(Boolean)
|
|
477
|
+
.join(", ");
|
|
478
|
+
lines.push(
|
|
479
|
+
` job: ${diag.job.status}/${diag.job.phase} — ${progress || "no processed items yet"} — last progress ${jobAge}s ago`,
|
|
480
|
+
);
|
|
481
|
+
if (diag.job.message) lines.push(` last: ${diag.job.message}`);
|
|
482
|
+
if (diag.job.error_message) lines.push(` error: ${diag.job.error_message}`);
|
|
483
|
+
}
|
|
336
484
|
lines.push(
|
|
337
485
|
` coverage: ${diag.coverage_percent}% (${diag.indexed_files}/${diag.total_source_files} files)`,
|
|
338
486
|
);
|
|
@@ -340,13 +488,51 @@ export default function (pi: ExtensionAPI) {
|
|
|
340
488
|
lines.push(` ⚠️ stale: ${diag.stale_files.length} files modified since last index`);
|
|
341
489
|
if (diag.orphan_files.length > 0)
|
|
342
490
|
lines.push(` ⚠️ orphans: ${diag.orphan_files.length} chunks reference deleted files`);
|
|
491
|
+
if (diag.stuck_indexing)
|
|
492
|
+
lines.push(
|
|
493
|
+
` ⚠️ indexing appears stuck for ${Math.round(diag.last_progress_age_ms / 60000)}m; remove and rebuild if no pi process is actively indexing it`,
|
|
494
|
+
);
|
|
495
|
+
if (diag.skipped_files.total > 0)
|
|
496
|
+
lines.push(
|
|
497
|
+
` skipped: ${diag.skipped_files.total} files (${Object.entries(diag.skipped_files.by_reason)
|
|
498
|
+
.filter(([, count]) => count > 0)
|
|
499
|
+
.map(([reason, count]) => `${reason}: ${count}`)
|
|
500
|
+
.join(", ")})`,
|
|
501
|
+
);
|
|
343
502
|
}
|
|
344
503
|
}
|
|
345
504
|
const totalStale = diagnostics.reduce((n, d) => n + d.stale_files.length, 0);
|
|
346
505
|
const totalOrphans = diagnostics.reduce((n, d) => n + d.orphan_files.length, 0);
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
lines.push("",
|
|
506
|
+
const totalStuck = diagnostics.filter((d) => d.stuck_indexing).length;
|
|
507
|
+
if (totalStale === 0 && totalOrphans === 0 && totalStuck === 0 && kbs.length > 0)
|
|
508
|
+
lines.push("", "Health: ✓ all indexes up to date");
|
|
509
|
+
else if (totalStale > 0 || totalOrphans > 0 || totalStuck > 0)
|
|
510
|
+
lines.push(
|
|
511
|
+
"",
|
|
512
|
+
`Health: ⚠️ ${totalStale} stale, ${totalOrphans} orphans, ${totalStuck} stuck indexing — run knowledge_update or rebuild affected KBs`,
|
|
513
|
+
);
|
|
514
|
+
return { content: [{ type: "text", text: lines.join("\n") }] };
|
|
515
|
+
},
|
|
516
|
+
});
|
|
517
|
+
|
|
518
|
+
pi.registerTool({
|
|
519
|
+
name: "knowledge_doctor",
|
|
520
|
+
label: "Knowledge Doctor",
|
|
521
|
+
description: "Diagnose knowledge base health, skipped files, stale indexes, stuck jobs, and recommended fixes",
|
|
522
|
+
promptSnippet: "Diagnose knowledge base health and recommended actions",
|
|
523
|
+
parameters: Type.Object({}),
|
|
524
|
+
async execute() {
|
|
525
|
+
const report = engine.doctor();
|
|
526
|
+
const lines = [`Health score: ${report.health_score}/100`, report.summary, ""];
|
|
527
|
+
if (report.issues.length === 0) {
|
|
528
|
+
lines.push("No issues found.");
|
|
529
|
+
} else {
|
|
530
|
+
for (const issue of report.issues) {
|
|
531
|
+
const scope = issue.kb_name ? ` [${issue.kb_name}]` : "";
|
|
532
|
+
lines.push(`- ${issue.severity.toUpperCase()}${scope}: ${issue.message}`);
|
|
533
|
+
lines.push(` Action: ${issue.action}`);
|
|
534
|
+
}
|
|
535
|
+
}
|
|
350
536
|
return { content: [{ type: "text", text: lines.join("\n") }] };
|
|
351
537
|
},
|
|
352
538
|
});
|
package/package.json
CHANGED
|
@@ -1,28 +1,87 @@
|
|
|
1
1
|
import { existsSync, statSync } from "node:fs";
|
|
2
2
|
import { join } from "node:path";
|
|
3
3
|
import type Database from "better-sqlite3";
|
|
4
|
-
import {
|
|
5
|
-
|
|
4
|
+
import {
|
|
5
|
+
createSkippedScanStats,
|
|
6
|
+
iterateScannableFiles,
|
|
7
|
+
type ScanOptions,
|
|
8
|
+
type ScanResult,
|
|
9
|
+
} from "../indexer/chunker.ts";
|
|
10
|
+
import { getIndexingJob, type IndexingJob, iterateChunksByKB, type KnowledgeBase } from "../storage/sqlite.ts";
|
|
6
11
|
|
|
7
12
|
export interface DiagnosticResult {
|
|
8
13
|
kb_id: string;
|
|
9
14
|
kb_name: string;
|
|
15
|
+
status: KnowledgeBase["status"];
|
|
16
|
+
status_age_ms: number;
|
|
17
|
+
last_progress_age_ms: number;
|
|
18
|
+
stuck_indexing: boolean;
|
|
10
19
|
stale_files: string[]; // files modified after indexing
|
|
11
20
|
orphan_files: string[]; // chunks referencing deleted files
|
|
12
21
|
coverage_percent: number; // indexed files / total scannable files
|
|
13
22
|
total_source_files: number;
|
|
14
23
|
indexed_files: number;
|
|
24
|
+
skipped_files: ScanResult["skipped"];
|
|
25
|
+
job?: IndexingJob;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
const DEFAULT_STALE_INDEXING_MS = 10 * 60 * 1000;
|
|
29
|
+
|
|
30
|
+
function staleIndexingMs(): number {
|
|
31
|
+
const configured = Number(process.env.PI_KNOWLEDGE_STALE_INDEXING_MS);
|
|
32
|
+
return Number.isFinite(configured) && configured > 0 ? configured : DEFAULT_STALE_INDEXING_MS;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function scanOptionsFromSourceOptions(raw: string | null): ScanOptions {
|
|
36
|
+
if (!raw) return {};
|
|
37
|
+
try {
|
|
38
|
+
const parsed = JSON.parse(raw) as {
|
|
39
|
+
include_suggested_text?: unknown;
|
|
40
|
+
include_paths?: unknown;
|
|
41
|
+
exclude_paths?: unknown;
|
|
42
|
+
};
|
|
43
|
+
return {
|
|
44
|
+
includeSuggestedText: parsed.include_suggested_text === true,
|
|
45
|
+
includePaths: Array.isArray(parsed.include_paths)
|
|
46
|
+
? parsed.include_paths.filter((item) => typeof item === "string")
|
|
47
|
+
: undefined,
|
|
48
|
+
excludePaths: Array.isArray(parsed.exclude_paths)
|
|
49
|
+
? parsed.exclude_paths.filter((item) => typeof item === "string")
|
|
50
|
+
: undefined,
|
|
51
|
+
};
|
|
52
|
+
} catch {
|
|
53
|
+
return {};
|
|
54
|
+
}
|
|
15
55
|
}
|
|
16
56
|
|
|
17
57
|
export function diagnoseKB(db: Database.Database, kb: KnowledgeBase): DiagnosticResult {
|
|
58
|
+
const statusAgeMs = Date.now() - kb.updated_at;
|
|
59
|
+
const job = getIndexingJob(db, kb.id);
|
|
60
|
+
const lastProgressAgeMs = job?.status === "running" ? Date.now() - job.last_progress_at : statusAgeMs;
|
|
18
61
|
const result: DiagnosticResult = {
|
|
19
62
|
kb_id: kb.id,
|
|
20
63
|
kb_name: kb.name,
|
|
64
|
+
status: kb.status,
|
|
65
|
+
status_age_ms: statusAgeMs,
|
|
66
|
+
last_progress_age_ms: lastProgressAgeMs,
|
|
67
|
+
stuck_indexing: kb.status === "indexing" && lastProgressAgeMs > staleIndexingMs(),
|
|
21
68
|
stale_files: [],
|
|
22
69
|
orphan_files: [],
|
|
23
70
|
coverage_percent: 100,
|
|
24
71
|
total_source_files: 0,
|
|
25
72
|
indexed_files: kb.file_count,
|
|
73
|
+
skipped_files: {
|
|
74
|
+
total: 0,
|
|
75
|
+
by_reason: {
|
|
76
|
+
suggested_excluded: 0,
|
|
77
|
+
oversized: 0,
|
|
78
|
+
binary: 0,
|
|
79
|
+
unreadable: 0,
|
|
80
|
+
inaccessible: 0,
|
|
81
|
+
},
|
|
82
|
+
samples: [],
|
|
83
|
+
},
|
|
84
|
+
job,
|
|
26
85
|
};
|
|
27
86
|
|
|
28
87
|
if (!kb.source_path || kb.source_type === "url" || !existsSync(kb.source_path)) {
|
|
@@ -33,8 +92,19 @@ export function diagnoseKB(db: Database.Database, kb: KnowledgeBase): Diagnostic
|
|
|
33
92
|
const currentFiles = new Set<string>();
|
|
34
93
|
const isDirectory = statSync(kb.source_path).isDirectory();
|
|
35
94
|
try {
|
|
36
|
-
|
|
37
|
-
|
|
95
|
+
if (isDirectory) {
|
|
96
|
+
const skipped = createSkippedScanStats();
|
|
97
|
+
for (const file of iterateScannableFiles(
|
|
98
|
+
kb.source_path,
|
|
99
|
+
skipped,
|
|
100
|
+
scanOptionsFromSourceOptions(kb.source_options),
|
|
101
|
+
)) {
|
|
102
|
+
currentFiles.add(file.relPath);
|
|
103
|
+
}
|
|
104
|
+
result.skipped_files = skipped;
|
|
105
|
+
} else {
|
|
106
|
+
currentFiles.add(kb.source_path);
|
|
107
|
+
}
|
|
38
108
|
} catch {
|
|
39
109
|
return result;
|
|
40
110
|
}
|
|
@@ -42,9 +112,13 @@ export function diagnoseKB(db: Database.Database, kb: KnowledgeBase): Diagnostic
|
|
|
42
112
|
result.total_source_files = currentFiles.size;
|
|
43
113
|
result.coverage_percent = currentFiles.size > 0 ? Math.round((result.indexed_files / currentFiles.size) * 100) : 100;
|
|
44
114
|
|
|
45
|
-
|
|
46
|
-
const
|
|
47
|
-
const
|
|
115
|
+
const indexedFilePaths = new Set<string>();
|
|
116
|
+
const latestIndexedByFile = new Map<string, number>();
|
|
117
|
+
for (const chunk of iterateChunksByKB(db, kb.id)) {
|
|
118
|
+
indexedFilePaths.add(chunk.file_path);
|
|
119
|
+
const currentLatest = latestIndexedByFile.get(chunk.file_path) ?? 0;
|
|
120
|
+
if (chunk.indexed_at > currentLatest) latestIndexedByFile.set(chunk.file_path, chunk.indexed_at);
|
|
121
|
+
}
|
|
48
122
|
|
|
49
123
|
// Orphan detection: chunks referencing files no longer in source
|
|
50
124
|
for (const filePath of indexedFilePaths) {
|
|
@@ -58,13 +132,9 @@ export function diagnoseKB(db: Database.Database, kb: KnowledgeBase): Diagnostic
|
|
|
58
132
|
const absPath = isDirectory ? join(kb.source_path, relPath) : relPath;
|
|
59
133
|
try {
|
|
60
134
|
const mtime = statSync(absPath).mtimeMs;
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
const latestIndexed = Math.max(...fileChunks.map((c) => c.indexed_at));
|
|
65
|
-
if (mtime > latestIndexed) {
|
|
66
|
-
result.stale_files.push(relPath);
|
|
67
|
-
}
|
|
135
|
+
const latestIndexed = latestIndexedByFile.get(relPath);
|
|
136
|
+
if (latestIndexed !== undefined && mtime > latestIndexed) {
|
|
137
|
+
result.stale_files.push(relPath);
|
|
68
138
|
}
|
|
69
139
|
} catch {
|
|
70
140
|
/* file unreadable — skip */
|