pi-knowledge 0.4.6 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/README.md +19 -8
- package/dist/index.js +129 -20
- package/dist/src/diagnostics/health.d.ts +1 -1
- package/dist/src/diagnostics/health.js +12 -1
- package/dist/src/embedding/provider.d.ts +1 -1
- package/dist/src/embedding/provider.js +12 -4
- package/dist/src/engine.d.ts +63 -10
- package/dist/src/engine.js +538 -164
- package/dist/src/indexer/symbols.d.ts +2 -0
- package/dist/src/indexer/symbols.js +175 -0
- package/dist/src/model-worker-client.d.ts +1 -1
- package/dist/src/model-worker-client.js +7 -7
- package/dist/src/search/reranker.d.ts +1 -1
- package/dist/src/search/reranker.js +2 -2
- package/dist/src/search/vector.d.ts +1 -1
- package/dist/src/search/vector.js +5 -1
- package/dist/src/storage/sqlite.d.ts +36 -1
- package/dist/src/storage/sqlite.js +213 -8
- package/dist/src/watcher/file-watcher.d.ts +2 -1
- package/dist/src/watcher/file-watcher.js +20 -30
- package/docs/configuration.md +8 -2
- package/package.json +4 -2
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,32 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [0.5.0] - 2026-07-07
|
|
4
|
+
|
|
5
|
+
### Added
|
|
6
|
+
- Added route symbol extraction and exact route lookup support through `knowledge_symbol_search`.
|
|
7
|
+
- Added structured `details` payloads for search diagnostics and doctor actions so agents can consume provenance and remediation codes without parsing text.
|
|
8
|
+
|
|
9
|
+
### Fixed
|
|
10
|
+
- Deleted vector files when knowledge bases are removed or cleared, including orphan detection in diagnostics.
|
|
11
|
+
- Guarded destructive and long-running operations with stronger mutation and shutdown lifecycle checks.
|
|
12
|
+
- Preserved ready knowledge bases when update cancellation happens before mutation starts, and kept partial update state from polluting symbol search.
|
|
13
|
+
- Propagated cancellation through search, export, import, diagnostics, and external embedding API requests.
|
|
14
|
+
|
|
15
|
+
### Changed
|
|
16
|
+
- Reworked JSONL import/export around streaming I/O and bounded batches for large portable knowledge bases.
|
|
17
|
+
- Improved watcher exclusions so generated, vendor, and runtime artifacts do not trigger unnecessary scans.
|
|
18
|
+
- Expanded Pi/OMP release coverage with lifecycle, cancellation, watcher, symbol, tool-contract, and model-worker regressions.
|
|
19
|
+
|
|
20
|
+
## [0.4.7] - 2026-07-04
|
|
21
|
+
|
|
22
|
+
### Fixed
|
|
23
|
+
- Avoided live SQLite statement iterators in chunk row scans so search, diagnostics, and vector rebuild paths cannot leave the database connection busy before `knowledge_update`.
|
|
24
|
+
- Coalesced overlapping `knowledge_update` calls per knowledge base so watcher-triggered updates, manual updates, retries, and shutdown do not re-enter the same update flow.
|
|
25
|
+
|
|
26
|
+
### Changed
|
|
27
|
+
- Increased SQLite busy timeout to tolerate short-lived external contention during long indexing/update work.
|
|
28
|
+
- Updated npm-facing README and package metadata to show Pi and OMP (`omp.sh`) coding-agent support.
|
|
29
|
+
|
|
3
30
|
## [0.4.6] - 2026-06-26
|
|
4
31
|
|
|
5
32
|
### Fixed
|
package/README.md
CHANGED
|
@@ -1,16 +1,17 @@
|
|
|
1
1
|
# pi-knowledge
|
|
2
2
|
|
|
3
|
-
Local-first RAG knowledge base for Pi agents.
|
|
3
|
+
Local-first RAG knowledge base for Pi and OMP coding agents.
|
|
4
4
|
|
|
5
|
-
Index your codebase, docs, PDFs, URLs, and notes into persistent knowledge bases that Pi can search across sessions. `pi-knowledge` combines semantic embeddings, BM25 keyword search, code-aware chunking, reranking, diagnostics, and stable large-project indexing so agents can answer from your actual project knowledge instead of guessing.
|
|
5
|
+
Index your codebase, docs, PDFs, URLs, and notes into persistent knowledge bases that Pi and [OMP](https://omp.sh/) can search across sessions. `pi-knowledge` combines semantic embeddings, BM25 keyword search, code-aware chunking, reranking, diagnostics, and stable large-project indexing so agents can answer from your actual project knowledge instead of guessing.
|
|
6
6
|
|
|
7
|
-
Built as a native [Pi extension](https://pi.dev/docs/latest/extensions)
|
|
7
|
+
Built as a native [Pi extension](https://pi.dev/docs/latest/extensions) with verified [OMP](https://omp.sh/) compatibility through the packaged extension entry. Designed for local-first project memory, agentic code search, and retrieval-augmented development workflows.
|
|
8
8
|
|
|
9
9
|
## Why It Matters
|
|
10
10
|
|
|
11
|
-
Agents lose project context between sessions and cannot fit large repositories into one prompt. `pi-knowledge` gives Pi durable, searchable project memory:
|
|
11
|
+
Agents lose project context between sessions and cannot fit large repositories into one prompt. `pi-knowledge` gives Pi and OMP durable, searchable project memory:
|
|
12
12
|
|
|
13
13
|
- Search code and documentation by meaning, exact symbols, or both.
|
|
14
|
+
- Look up lightweight code symbols, Markdown headings, config keys, and environment variables before broader search.
|
|
14
15
|
- Keep private source data local by default.
|
|
15
16
|
- Re-index changed files incrementally.
|
|
16
17
|
- Diagnose stale, stuck, or low-quality indexes.
|
|
@@ -23,6 +24,7 @@ Unlike `pi-memory` (which manages the agent's own notes), `pi-knowledge` indexes
|
|
|
23
24
|
- **Local-first project memory**: stores indexes under `~/.pi/knowledge/`; no project files are modified.
|
|
24
25
|
- **Hybrid retrieval**: semantic vectors + BM25 keyword search + normalized weighted score fusion.
|
|
25
26
|
- **Code-aware indexing**: AST-aware chunking for TypeScript/JavaScript, Python, Go, Rust, and Java.
|
|
27
|
+
- **Lightweight symbol lookup**: indexes functions, classes, interfaces, types, variables, route-like handlers, Markdown headings, config keys, and env vars for exact agent lookup.
|
|
26
28
|
- **Better agent answers**: adaptive context windows, diversity reranking, optional cross-encoder reranking, and diagnostics.
|
|
27
29
|
- **Large-project stability**: persisted indexing progress, capped batches, streaming vector scans, and stuck-job detection.
|
|
28
30
|
- **Private by default**: local embeddings run without an API key.
|
|
@@ -42,6 +44,7 @@ Unlike `pi-memory` (which manages the agent's own notes), `pi-knowledge` indexes
|
|
|
42
44
|
| **Incremental re-indexing** | ✅ | ❌ | ❌ |
|
|
43
45
|
| **File watcher (auto-update)** | ✅ | ❌ | ❌ |
|
|
44
46
|
| **Code-aware chunking** | ✅ (TS/JS/Py/Go/Rust/Java) | ❌ | ❌ |
|
|
47
|
+
| **Symbol/config/heading lookup** | ✅ | ❌ | ❌ |
|
|
45
48
|
| **Local embeddings (zero API)** | ✅ | ❌ | ✅ (qmd) |
|
|
46
49
|
| **Index quality diagnostics** | ✅ | ❌ | ❌ |
|
|
47
50
|
| **Metadata filters in search** | ✅ | ❌ | ❌ |
|
|
@@ -74,11 +77,15 @@ In project-level dogfood, these changes improved a real codebase evaluation from
|
|
|
74
77
|
## Quick Start
|
|
75
78
|
|
|
76
79
|
```bash
|
|
77
|
-
# Install
|
|
80
|
+
# Install for Pi
|
|
78
81
|
pi install npm:pi-knowledge
|
|
79
82
|
|
|
83
|
+
# Install for OMP
|
|
84
|
+
omp install npm:pi-knowledge
|
|
85
|
+
|
|
80
86
|
# Or from source
|
|
81
87
|
pi install ./pi-knowledge
|
|
88
|
+
omp install ./pi-knowledge
|
|
82
89
|
|
|
83
90
|
# Index a directory
|
|
84
91
|
# (agent will call knowledge_add automatically, or you can ask it)
|
|
@@ -97,6 +104,7 @@ pi install ./pi-knowledge
|
|
|
97
104
|
| `knowledge_plan` | Inspect an indexing source before writing a KB; reports scannable files, suggested exclusions, and technical skips |
|
|
98
105
|
| `knowledge_add` | Index files, directories, URLs, PDFs, DOCX, or inline text |
|
|
99
106
|
| `knowledge_search` | Hybrid, deep, or adaptive search across one or all knowledge bases |
|
|
107
|
+
| `knowledge_symbol_search` | Exact or substring lookup for code symbols, route-like handlers, Markdown headings, config keys, and env vars |
|
|
100
108
|
| `knowledge_remove` | Remove a knowledge base by name or ID |
|
|
101
109
|
| `knowledge_update` | Incrementally re-index changed files in a knowledge base |
|
|
102
110
|
| `knowledge_show` | List all knowledge bases with stats |
|
|
@@ -114,6 +122,7 @@ pi install ./pi-knowledge
|
|
|
114
122
|
- `deep`: hybrid retrieval followed by cross-encoder reranking.
|
|
115
123
|
- `adaptive`: hybrid retrieval followed by query-time contextual window expansion around seed chunks. It keeps the matched seed, prefers nearby/query-relevant neighboring chunks, and collapses overlapping windows from the same file.
|
|
116
124
|
- `auto`: selects a primary mode from the query shape and retries alternate modes when results are empty or weak.
|
|
125
|
+
- `code`, `config`, `errors`, `docs`, `decision`: intent aliases for agents. `code`, `config`, and `errors` bias toward exact lexical/symbol evidence; `docs` and `decision` keep hybrid recall while making the intended source type explicit.
|
|
117
126
|
|
|
118
127
|
Mode selection contract:
|
|
119
128
|
|
|
@@ -124,9 +133,10 @@ Mode selection contract:
|
|
|
124
133
|
- Use `deep` for high-stakes answers, ambiguous top results, or final verification when slower reranking is acceptable.
|
|
125
134
|
- If results are empty or weak but the KB should contain the answer, retry once with a different mode before concluding no answer exists.
|
|
126
135
|
|
|
127
|
-
Search results use balanced diversity reranking by default so near-duplicate chunks from the same file do not dominate the top results. Diversity scoring considers lexical overlap, same-file line proximity, overlapping adaptive windows, available embedding-vector similarity,
|
|
136
|
+
Search results use balanced diversity reranking by default so near-duplicate chunks from the same file do not dominate the top results. Diversity scoring considers lexical overlap, same-file line proximity, overlapping adaptive windows, available embedding-vector similarity, file-level interleaving, and a small KB trust multiplier that favors ready file/directory sources over stale, URL, or imported text sources. Use `diversity: "off"` only when raw ranking order is needed for diagnostics. Agents can request search diagnostics to inspect mode fallback, ranking coverage, path/source/test boosts, adjusted scores, and provenance such as chunk id, chunk hash, match reason, stale flag, and source freshness where available.
|
|
128
137
|
|
|
129
138
|
For best search quality, rebuild or update existing knowledge bases after upgrading. New indexes use contextual retrieval units: embeddings and FTS include file path, file type, Markdown heading breadcrumbs, and code symbol names while returned results keep the original chunk text readable. This improves queries that mention project structure, filenames, sections, or functions, and reduces duplicate-looking chunk hits.
|
|
139
|
+
Symbol/config/heading metadata is also rebuilt during `knowledge_update`; older KBs created before this feature may show a doctor action recommending update.
|
|
130
140
|
|
|
131
141
|
## Embedding Configuration
|
|
132
142
|
|
|
@@ -170,7 +180,7 @@ Full configuration details are in [docs/configuration.md](docs/configuration.md)
|
|
|
170
180
|
|
|
171
181
|
## Pi and OMP Support
|
|
172
182
|
|
|
173
|
-
`pi-knowledge` supports Pi and OMP
|
|
183
|
+
`pi-knowledge` supports Pi and [OMP](https://omp.sh/) extension loading through the packaged `extension.js` entry shim. The entry stays startup-light: install-time validation can inspect the extension without resolving native runtime dependencies, and runtime modules load lazily only when tools or lifecycle hooks need them.
|
|
174
184
|
|
|
175
185
|
Default storage is `~/.pi/knowledge` for Pi and `~/.omp/knowledge` for OMP. Explicit overrides are available with `PI_KNOWLEDGE_DIR` and `OMP_KNOWLEDGE_DIR`. Under the default home OMP root, existing legacy `~/.pi/knowledge` data remains visible when `~/.omp/knowledge` has not been created yet.
|
|
176
186
|
|
|
@@ -188,7 +198,7 @@ Update and diagnostics paths are also streaming-oriented: changed chunks are emb
|
|
|
188
198
|
|
|
189
199
|
## Architecture
|
|
190
200
|
|
|
191
|
-
|
|
201
|
+
[DESIGN.md](DESIGN.md) is the historical technical design. For current behavior contracts, prefer this README, [AGENTS.md](AGENTS.md), [docs/configuration.md](docs/configuration.md), and the ADR/pitfall notes under `docs/`.
|
|
192
202
|
|
|
193
203
|
## Data Storage
|
|
194
204
|
|
|
@@ -206,6 +216,7 @@ All data is stored globally at `~/.pi/knowledge/` under Pi or `~/.omp/knowledge/
|
|
|
206
216
|
- **Override**: set `PI_KNOWLEDGE_DIR` or `OMP_KNOWLEDGE_DIR`
|
|
207
217
|
- **Project safety**: pi-knowledge is read-only on indexed directories — no files are created or modified in your project
|
|
208
218
|
- **Updates**: extension updates do not affect existing indexed data. Schema migrations run automatically if needed.
|
|
219
|
+
- **Symbol index**: `knowledge.db` also stores lightweight symbol/config/heading metadata used by `knowledge_symbol_search`; it is derived from indexed source. File, directory, and URL KBs with retained source paths can rebuild it with `knowledge_update`; imported portable KBs and inline text KBs should be re-imported or re-added because they intentionally do not store an active local source manifest.
|
|
209
220
|
|
|
210
221
|
## Development
|
|
211
222
|
|
package/dist/index.js
CHANGED
|
@@ -47,6 +47,25 @@ let runtimePromise;
|
|
|
47
47
|
let initializePromise;
|
|
48
48
|
let initialized = false;
|
|
49
49
|
let disposePromise;
|
|
50
|
+
function scanOptionsFromSourceOptions(raw) {
|
|
51
|
+
if (!raw)
|
|
52
|
+
return {};
|
|
53
|
+
try {
|
|
54
|
+
const parsed = JSON.parse(raw);
|
|
55
|
+
return {
|
|
56
|
+
includeSuggestedText: parsed.include_suggested_text === true,
|
|
57
|
+
includePaths: Array.isArray(parsed.include_paths)
|
|
58
|
+
? parsed.include_paths.filter((item) => typeof item === "string")
|
|
59
|
+
: undefined,
|
|
60
|
+
excludePaths: Array.isArray(parsed.exclude_paths)
|
|
61
|
+
? parsed.exclude_paths.filter((item) => typeof item === "string")
|
|
62
|
+
: undefined,
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
catch {
|
|
66
|
+
return {};
|
|
67
|
+
}
|
|
68
|
+
}
|
|
50
69
|
function runtimeModule(modulePath) {
|
|
51
70
|
return `${modulePath}${RUNTIME_EXTENSION}`;
|
|
52
71
|
}
|
|
@@ -131,7 +150,7 @@ export default function (pi) {
|
|
|
131
150
|
if (kb.source_path && kb.source_type === "directory") {
|
|
132
151
|
watcher.startWatcher(kb.id, kb.source_path, (kbId) => {
|
|
133
152
|
engine.update(kbId).catch(() => { });
|
|
134
|
-
});
|
|
153
|
+
}, scanOptionsFromSourceOptions(kb.source_options));
|
|
135
154
|
}
|
|
136
155
|
}
|
|
137
156
|
}
|
|
@@ -199,7 +218,7 @@ export default function (pi) {
|
|
|
199
218
|
}))),
|
|
200
219
|
exclude_paths: Type.Optional(Type.Array(Type.String({ description: "Relative paths under a directory source to exclude from this plan" }))),
|
|
201
220
|
}),
|
|
202
|
-
async execute(_id, params) {
|
|
221
|
+
async execute(_id, params, _signal) {
|
|
203
222
|
const { engine } = await ensureInitialized();
|
|
204
223
|
const { source, include_suggested_text, include_paths, exclude_paths } = params;
|
|
205
224
|
const plan = engine.plan(source, {
|
|
@@ -210,7 +229,7 @@ export default function (pi) {
|
|
|
210
229
|
exclude_paths: Array.isArray(exclude_paths)
|
|
211
230
|
? exclude_paths.filter((item) => typeof item === "string")
|
|
212
231
|
: undefined,
|
|
213
|
-
});
|
|
232
|
+
}, _signal);
|
|
214
233
|
const samples = plan.skipped.samples
|
|
215
234
|
.map((sample) => `- ${sample.reason}: ${sample.path}${sample.size ? ` (${sample.size} bytes)` : ""}`)
|
|
216
235
|
.join("\n");
|
|
@@ -275,6 +294,14 @@ export default function (pi) {
|
|
|
275
294
|
if (WATCH_ENABLED && kb.source_path && kb.source_type === "directory") {
|
|
276
295
|
watcher.startWatcher(kb.id, kb.source_path, (kbId) => {
|
|
277
296
|
engine.update(kbId).catch(() => { });
|
|
297
|
+
}, {
|
|
298
|
+
includeSuggestedText: include_suggested_text === true,
|
|
299
|
+
includePaths: Array.isArray(include_paths)
|
|
300
|
+
? include_paths.filter((item) => typeof item === "string")
|
|
301
|
+
: undefined,
|
|
302
|
+
excludePaths: Array.isArray(exclude_paths)
|
|
303
|
+
? exclude_paths.filter((item) => typeof item === "string")
|
|
304
|
+
: undefined,
|
|
278
305
|
});
|
|
279
306
|
}
|
|
280
307
|
return {
|
|
@@ -311,6 +338,11 @@ export default function (pi) {
|
|
|
311
338
|
Type.Literal("hybrid"),
|
|
312
339
|
Type.Literal("deep"),
|
|
313
340
|
Type.Literal("adaptive"),
|
|
341
|
+
Type.Literal("code"),
|
|
342
|
+
Type.Literal("config"),
|
|
343
|
+
Type.Literal("docs"),
|
|
344
|
+
Type.Literal("errors"),
|
|
345
|
+
Type.Literal("decision"),
|
|
314
346
|
])),
|
|
315
347
|
limit: Type.Optional(Type.Number({ description: "Max results (default 10)" })),
|
|
316
348
|
kb_id: Type.Optional(Type.String({ description: "Limit search to a specific KB by ID or exact name" })),
|
|
@@ -319,11 +351,11 @@ export default function (pi) {
|
|
|
319
351
|
diversity: Type.Optional(Type.Union([Type.Literal("off"), Type.Literal("balanced"), Type.Literal("strong")])),
|
|
320
352
|
diagnostics: Type.Optional(Type.Boolean({ description: "Include ranking diagnostics and mode/fallback details in the result" })),
|
|
321
353
|
}),
|
|
322
|
-
async execute(_id, params) {
|
|
354
|
+
async execute(_id, params, _signal) {
|
|
323
355
|
const { engine } = await ensureInitialized();
|
|
324
356
|
const { query, mode, limit, kb_id, offset, file_type, diversity, diagnostics } = params;
|
|
325
357
|
const filters = file_type ? { file_type } : undefined;
|
|
326
|
-
const response = await engine.search(query, { mode, limit, kb_id, offset, filters, diversity });
|
|
358
|
+
const response = await engine.search(query, { mode, limit, kb_id, offset, filters, diversity }, _signal);
|
|
327
359
|
if (response.results.length === 0) {
|
|
328
360
|
const details = [
|
|
329
361
|
"No results found.",
|
|
@@ -334,7 +366,7 @@ export default function (pi) {
|
|
|
334
366
|
]
|
|
335
367
|
.filter(Boolean)
|
|
336
368
|
.join("\n");
|
|
337
|
-
return { content: [{ type: "text", text: details }] };
|
|
369
|
+
return { content: [{ type: "text", text: details }], details: diagnostics ? response : undefined };
|
|
338
370
|
}
|
|
339
371
|
let output = `${response.total_count} results (showing ${response.results.length})`;
|
|
340
372
|
if (response.mode_used)
|
|
@@ -350,10 +382,73 @@ export default function (pi) {
|
|
|
350
382
|
const diag = diagnostics && r.ranking
|
|
351
383
|
? `\nDiagnostics: base=${r.ranking.base_score.toFixed(3)}, adjusted=${r.ranking.adjusted_score.toFixed(3)}, coverage=${r.ranking.coverage.toFixed(2)}, path_boost=${r.ranking.path_boost.toFixed(2)}, source_boost=${r.ranking.source_boost.toFixed(2)}, test=${r.ranking.is_test}`
|
|
352
384
|
: "";
|
|
353
|
-
|
|
385
|
+
const provenance = diagnostics && r.provenance
|
|
386
|
+
? `\nProvenance: chunk=${r.provenance.chunk_id}, hash=${r.provenance.chunk_hash.slice(0, 12)}, reason=${r.provenance.match_reason}, indexed_at=${r.provenance.indexed_at}, source_mtime=${r.provenance.source_mtime ?? "unknown"}, stale=${r.provenance.stale}`
|
|
387
|
+
: "";
|
|
388
|
+
return `[${i + 1}] ${r.file_path} (${r.kb_name}, score: ${r.score.toFixed(3)})\n${r.snippet}${diag}${provenance}`;
|
|
354
389
|
})
|
|
355
390
|
.join("\n\n");
|
|
356
|
-
return { content: [{ type: "text", text: output }] };
|
|
391
|
+
return { content: [{ type: "text", text: output }], details: diagnostics ? response : undefined };
|
|
392
|
+
},
|
|
393
|
+
});
|
|
394
|
+
pi.registerTool({
|
|
395
|
+
name: "knowledge_symbol_search",
|
|
396
|
+
label: "Knowledge Symbol Search",
|
|
397
|
+
description: "Search indexed code symbols, Markdown headings, config keys, and environment variables exactly or by substring",
|
|
398
|
+
promptSnippet: "Find exact symbols/config keys/headings before broad semantic search",
|
|
399
|
+
promptGuidelines: [
|
|
400
|
+
"Use knowledge_symbol_search before knowledge_search when looking for a function, class, env var, config key, route, or Markdown heading",
|
|
401
|
+
"Use exact=true when the user provides a precise symbol or key",
|
|
402
|
+
"After finding a symbol, use knowledge_search mode 'adaptive' with the file path or symbol name when surrounding implementation context is needed",
|
|
403
|
+
],
|
|
404
|
+
parameters: Type.Object({
|
|
405
|
+
query: Type.String({ description: "Symbol, heading, config key, env var, route, or file text to find" }),
|
|
406
|
+
kind: Type.Optional(Type.Union([
|
|
407
|
+
Type.Literal("function"),
|
|
408
|
+
Type.Literal("class"),
|
|
409
|
+
Type.Literal("interface"),
|
|
410
|
+
Type.Literal("type"),
|
|
411
|
+
Type.Literal("variable"),
|
|
412
|
+
Type.Literal("heading"),
|
|
413
|
+
Type.Literal("config_key"),
|
|
414
|
+
Type.Literal("env_var"),
|
|
415
|
+
Type.Literal("route"),
|
|
416
|
+
])),
|
|
417
|
+
kb_id: Type.Optional(Type.String({ description: "Limit lookup to a specific KB by ID or exact name" })),
|
|
418
|
+
file_pattern: Type.Optional(Type.String({ description: "Limit lookup to paths containing this text" })),
|
|
419
|
+
limit: Type.Optional(Type.Number({ description: "Max results (default 20)" })),
|
|
420
|
+
offset: Type.Optional(Type.Number({ description: "Pagination offset" })),
|
|
421
|
+
exact: Type.Optional(Type.Boolean({ description: "Require exact normalized symbol/key match" })),
|
|
422
|
+
}),
|
|
423
|
+
async execute(_id, params, _signal) {
|
|
424
|
+
if (_signal?.aborted)
|
|
425
|
+
throw new Error("Cancelled");
|
|
426
|
+
const { engine } = await ensureInitialized();
|
|
427
|
+
if (_signal?.aborted)
|
|
428
|
+
throw new Error("Cancelled");
|
|
429
|
+
const response = engine.symbolSearch(params.query, {
|
|
430
|
+
kind: params.kind,
|
|
431
|
+
kb_id: params.kb_id,
|
|
432
|
+
file_pattern: params.file_pattern,
|
|
433
|
+
limit: params.limit,
|
|
434
|
+
offset: params.offset,
|
|
435
|
+
exact: params.exact === true,
|
|
436
|
+
}, _signal);
|
|
437
|
+
if (response.results.length === 0)
|
|
438
|
+
return { content: [{ type: "text", text: "No symbols found." }] };
|
|
439
|
+
const lines = response.results.map((result, index) => {
|
|
440
|
+
const signature = result.signature ? ` — ${result.signature}` : "";
|
|
441
|
+
return `[${index + 1}] ${result.name} (${result.kind}) ${result.file_path}:${result.start_line}-${result.end_line} [${result.kb_name}]${signature}`;
|
|
442
|
+
});
|
|
443
|
+
const suffix = response.has_more ? "\nMore results available; increase offset to continue." : "";
|
|
444
|
+
return {
|
|
445
|
+
content: [
|
|
446
|
+
{
|
|
447
|
+
type: "text",
|
|
448
|
+
text: `${response.total_count} symbol result(s):\n\n${lines.join("\n")}${suffix}`,
|
|
449
|
+
},
|
|
450
|
+
],
|
|
451
|
+
};
|
|
357
452
|
},
|
|
358
453
|
});
|
|
359
454
|
pi.registerTool({
|
|
@@ -379,11 +474,13 @@ export default function (pi) {
|
|
|
379
474
|
label: "Knowledge Status",
|
|
380
475
|
description: "Show knowledge engine status with health diagnostics: staleness, orphans, and coverage",
|
|
381
476
|
parameters: Type.Object({}),
|
|
382
|
-
async execute() {
|
|
477
|
+
async execute(_id, _params, _signal) {
|
|
478
|
+
if (_signal?.aborted)
|
|
479
|
+
throw new Error("Cancelled");
|
|
383
480
|
const { engine, storage, watcher } = await ensureInitialized();
|
|
384
481
|
const kbs = engine.list();
|
|
385
482
|
const watchCount = watcher.getActiveWatcherCount();
|
|
386
|
-
const diagnostics = engine.diagnose();
|
|
483
|
+
const diagnostics = engine.diagnose(_signal);
|
|
387
484
|
const lines = [
|
|
388
485
|
`Storage: ${storage.getDefaultKnowledgeDir()}`,
|
|
389
486
|
`Knowledge bases: ${kbs.length}`,
|
|
@@ -445,9 +542,11 @@ export default function (pi) {
|
|
|
445
542
|
description: "Diagnose knowledge base health, skipped files, stale indexes, stuck jobs, and recommended fixes",
|
|
446
543
|
promptSnippet: "Diagnose knowledge base health and recommended actions",
|
|
447
544
|
parameters: Type.Object({}),
|
|
448
|
-
async execute() {
|
|
545
|
+
async execute(_id, _params, _signal) {
|
|
546
|
+
if (_signal?.aborted)
|
|
547
|
+
throw new Error("Cancelled");
|
|
449
548
|
const { engine } = await ensureInitialized();
|
|
450
|
-
const report = engine.doctor();
|
|
549
|
+
const report = engine.doctor(_signal);
|
|
451
550
|
const lines = [`Health score: ${report.health_score}/100`, report.summary, ""];
|
|
452
551
|
if (report.issues.length === 0) {
|
|
453
552
|
lines.push("No issues found.");
|
|
@@ -456,10 +555,10 @@ export default function (pi) {
|
|
|
456
555
|
for (const issue of report.issues) {
|
|
457
556
|
const scope = issue.kb_name ? ` [${issue.kb_name}]` : "";
|
|
458
557
|
lines.push(`- ${issue.severity.toUpperCase()}${scope}: ${issue.message}`);
|
|
459
|
-
lines.push(` Action: ${issue.action}`);
|
|
558
|
+
lines.push(` Action (${issue.action_code ?? "none"}): ${issue.action}`);
|
|
460
559
|
}
|
|
461
560
|
}
|
|
462
|
-
return { content: [{ type: "text", text: lines.join("\n") }] };
|
|
561
|
+
return { content: [{ type: "text", text: lines.join("\n") }], details: report };
|
|
463
562
|
},
|
|
464
563
|
});
|
|
465
564
|
pi.registerTool({
|
|
@@ -467,9 +566,13 @@ export default function (pi) {
|
|
|
467
566
|
label: "Knowledge Show",
|
|
468
567
|
description: "List all indexed knowledge bases",
|
|
469
568
|
parameters: Type.Object({}),
|
|
470
|
-
async execute() {
|
|
569
|
+
async execute(_id, _params, _signal) {
|
|
570
|
+
if (_signal?.aborted)
|
|
571
|
+
throw new Error("Cancelled");
|
|
471
572
|
const { engine } = await ensureInitialized();
|
|
472
|
-
|
|
573
|
+
if (_signal?.aborted)
|
|
574
|
+
throw new Error("Cancelled");
|
|
575
|
+
const kbs = engine.list(_signal);
|
|
473
576
|
if (kbs.length === 0)
|
|
474
577
|
return { content: [{ type: "text", text: "No knowledge bases." }] };
|
|
475
578
|
const lines = kbs.map((kb) => `• ${kb.name} — ${kb.chunk_count} chunks, ${kb.file_count} files (${kb.status})`);
|
|
@@ -483,7 +586,9 @@ export default function (pi) {
|
|
|
483
586
|
parameters: Type.Object({
|
|
484
587
|
target: Type.String({ description: "KB name or ID to remove" }),
|
|
485
588
|
}),
|
|
486
|
-
async execute(_id, params) {
|
|
589
|
+
async execute(_id, params, _signal) {
|
|
590
|
+
if (_signal?.aborted)
|
|
591
|
+
throw new Error("Cancelled");
|
|
487
592
|
const { engine } = await ensureInitialized();
|
|
488
593
|
const ok = engine.remove(params.target);
|
|
489
594
|
return { content: [{ type: "text", text: ok ? "Removed." : "Not found." }] };
|
|
@@ -497,10 +602,12 @@ export default function (pi) {
|
|
|
497
602
|
target: Type.String({ description: "KB name or ID to export" }),
|
|
498
603
|
output: Type.String({ description: "Output file path (.jsonl)" }),
|
|
499
604
|
}),
|
|
500
|
-
async execute(_id, params) {
|
|
605
|
+
async execute(_id, params, _signal, onUpdate) {
|
|
501
606
|
const { engine } = await ensureInitialized();
|
|
502
607
|
const { target, output } = params;
|
|
503
|
-
const count = await engine.exportKB(target, output)
|
|
608
|
+
const count = await engine.exportKB(target, output, _signal, (msg) => {
|
|
609
|
+
onUpdate?.({ content: [{ type: "text", text: msg }] });
|
|
610
|
+
});
|
|
504
611
|
return { content: [{ type: "text", text: `Exported ${count} chunks to ${output}` }] };
|
|
505
612
|
},
|
|
506
613
|
});
|
|
@@ -524,7 +631,9 @@ export default function (pi) {
|
|
|
524
631
|
label: "Knowledge Clear",
|
|
525
632
|
description: "Remove all knowledge bases",
|
|
526
633
|
parameters: Type.Object({}),
|
|
527
|
-
async execute() {
|
|
634
|
+
async execute(_id, _params, _signal) {
|
|
635
|
+
if (_signal?.aborted)
|
|
636
|
+
throw new Error("Cancelled");
|
|
528
637
|
const { engine } = await ensureInitialized();
|
|
529
638
|
engine.clear();
|
|
530
639
|
return { content: [{ type: "text", text: "All knowledge bases cleared." }] };
|
|
@@ -16,4 +16,4 @@ export interface DiagnosticResult {
|
|
|
16
16
|
skipped_files: ScanResult["skipped"];
|
|
17
17
|
job?: IndexingJob;
|
|
18
18
|
}
|
|
19
|
-
export declare function diagnoseKB(db: Database.Database, kb: KnowledgeBase): DiagnosticResult;
|
|
19
|
+
export declare function diagnoseKB(db: Database.Database, kb: KnowledgeBase, signal?: AbortSignal): DiagnosticResult;
|
|
@@ -26,7 +26,11 @@ function scanOptionsFromSourceOptions(raw) {
|
|
|
26
26
|
return {};
|
|
27
27
|
}
|
|
28
28
|
}
|
|
29
|
-
|
|
29
|
+
function throwIfAborted(signal) {
|
|
30
|
+
if (signal?.aborted)
|
|
31
|
+
throw new Error("Cancelled");
|
|
32
|
+
}
|
|
33
|
+
export function diagnoseKB(db, kb, signal) {
|
|
30
34
|
const statusAgeMs = Date.now() - kb.updated_at;
|
|
31
35
|
const job = getIndexingJob(db, kb.id);
|
|
32
36
|
const lastProgressAgeMs = job?.status === "running" ? Date.now() - job.last_progress_at : statusAgeMs;
|
|
@@ -55,6 +59,7 @@ export function diagnoseKB(db, kb) {
|
|
|
55
59
|
},
|
|
56
60
|
job,
|
|
57
61
|
};
|
|
62
|
+
throwIfAborted(signal);
|
|
58
63
|
if (!kb.source_path || kb.source_type === "url" || !existsSync(kb.source_path)) {
|
|
59
64
|
return result; // text KBs or missing source — no diagnostics possible
|
|
60
65
|
}
|
|
@@ -65,6 +70,7 @@ export function diagnoseKB(db, kb) {
|
|
|
65
70
|
if (isDirectory) {
|
|
66
71
|
const skipped = createSkippedScanStats();
|
|
67
72
|
for (const file of iterateScannableFiles(kb.source_path, skipped, scanOptionsFromSourceOptions(kb.source_options))) {
|
|
73
|
+
throwIfAborted(signal);
|
|
68
74
|
currentFiles.add(file.relPath);
|
|
69
75
|
}
|
|
70
76
|
result.skipped_files = skipped;
|
|
@@ -74,6 +80,8 @@ export function diagnoseKB(db, kb) {
|
|
|
74
80
|
}
|
|
75
81
|
}
|
|
76
82
|
catch {
|
|
83
|
+
if (signal?.aborted)
|
|
84
|
+
throw new Error("Cancelled");
|
|
77
85
|
return result;
|
|
78
86
|
}
|
|
79
87
|
result.total_source_files = currentFiles.size;
|
|
@@ -81,6 +89,7 @@ export function diagnoseKB(db, kb) {
|
|
|
81
89
|
const indexedFilePaths = new Set();
|
|
82
90
|
const latestIndexedByFile = new Map();
|
|
83
91
|
for (const chunk of iterateChunksByKB(db, kb.id)) {
|
|
92
|
+
throwIfAborted(signal);
|
|
84
93
|
indexedFilePaths.add(chunk.file_path);
|
|
85
94
|
const currentLatest = latestIndexedByFile.get(chunk.file_path) ?? 0;
|
|
86
95
|
if (chunk.indexed_at > currentLatest)
|
|
@@ -88,12 +97,14 @@ export function diagnoseKB(db, kb) {
|
|
|
88
97
|
}
|
|
89
98
|
// Orphan detection: chunks referencing files no longer in source
|
|
90
99
|
for (const filePath of indexedFilePaths) {
|
|
100
|
+
throwIfAborted(signal);
|
|
91
101
|
if (!currentFiles.has(filePath)) {
|
|
92
102
|
result.orphan_files.push(filePath);
|
|
93
103
|
}
|
|
94
104
|
}
|
|
95
105
|
// Staleness detection: source files modified after last indexing
|
|
96
106
|
for (const relPath of currentFiles) {
|
|
107
|
+
throwIfAborted(signal);
|
|
97
108
|
const absPath = isDirectory ? join(kb.source_path, relPath) : relPath;
|
|
98
109
|
try {
|
|
99
110
|
const mtime = statSync(absPath).mtimeMs;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export declare function dispose(): Promise<void>;
|
|
2
2
|
export declare function prepareForShutdown(): Promise<void>;
|
|
3
3
|
export declare function embedTexts(texts: string[], prefix: "query" | "passage", signal?: AbortSignal): Promise<Float32Array[]>;
|
|
4
|
-
export declare function embedQuery(text: string): Promise<Float32Array>;
|
|
4
|
+
export declare function embedQuery(text: string, signal?: AbortSignal): Promise<Float32Array>;
|
|
5
5
|
export declare function embedDocuments(texts: string[], signal?: AbortSignal): Promise<Float32Array[]>;
|
|
@@ -10,6 +10,9 @@ const ENABLE_NATIVE_IDLE_DISPOSE = process.env.PI_KNOWLEDGE_ENABLE_NATIVE_IDLE_D
|
|
|
10
10
|
const API_FALLBACK_TO_LOCAL = process.env.PI_KNOWLEDGE_EMBEDDING_API_FALLBACK === "local";
|
|
11
11
|
const DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1";
|
|
12
12
|
const DEFAULT_API_MAX_EMBED_CHARS = 20_000;
|
|
13
|
+
function isAbortError(error) {
|
|
14
|
+
return typeof error === "object" && error !== null && "name" in error && error.name === "AbortError";
|
|
15
|
+
}
|
|
13
16
|
function clearIdleTimer() {
|
|
14
17
|
if (disposeTimer)
|
|
15
18
|
clearTimeout(disposeTimer);
|
|
@@ -57,7 +60,7 @@ export async function prepareForShutdown() {
|
|
|
57
60
|
clearIdleTimer();
|
|
58
61
|
await waitForNoActiveRuns();
|
|
59
62
|
}
|
|
60
|
-
async function embedViaAPI(texts, prefix) {
|
|
63
|
+
async function embedViaAPI(texts, prefix, signal) {
|
|
61
64
|
const [provider, model] = EMBEDDING_CONFIG.split(":");
|
|
62
65
|
const configuredMaxChars = Number(process.env.PI_KNOWLEDGE_EMBEDDING_MAX_CHARS ?? DEFAULT_API_MAX_EMBED_CHARS);
|
|
63
66
|
const maxChars = Number.isFinite(configuredMaxChars) && configuredMaxChars > 0 ? configuredMaxChars : DEFAULT_API_MAX_EMBED_CHARS;
|
|
@@ -75,6 +78,7 @@ async function embedViaAPI(texts, prefix) {
|
|
|
75
78
|
method: "POST",
|
|
76
79
|
headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}` },
|
|
77
80
|
body: JSON.stringify({ input: safeTexts, model: model || "text-embedding-3-small" }),
|
|
81
|
+
signal,
|
|
78
82
|
});
|
|
79
83
|
if (!res.ok) {
|
|
80
84
|
const detail = await res.text().catch(() => "");
|
|
@@ -87,10 +91,14 @@ async function embedViaAPI(texts, prefix) {
|
|
|
87
91
|
}
|
|
88
92
|
export async function embedTexts(texts, prefix, signal) {
|
|
89
93
|
if (!EMBEDDING_CONFIG.startsWith("local")) {
|
|
94
|
+
if (signal?.aborted)
|
|
95
|
+
throw new Error("Cancelled");
|
|
90
96
|
try {
|
|
91
|
-
return await embedViaAPI(texts, prefix);
|
|
97
|
+
return await embedViaAPI(texts, prefix, signal);
|
|
92
98
|
}
|
|
93
99
|
catch (error) {
|
|
100
|
+
if (signal?.aborted || isAbortError(error))
|
|
101
|
+
throw new Error("Cancelled");
|
|
94
102
|
if (!API_FALLBACK_TO_LOCAL)
|
|
95
103
|
throw error;
|
|
96
104
|
console.warn(`pi-knowledge: embedding API failed; falling back to local model because PI_KNOWLEDGE_EMBEDDING_API_FALLBACK=local (${error instanceof Error ? error.message : String(error)})`);
|
|
@@ -104,8 +112,8 @@ export async function embedTexts(texts, prefix, signal) {
|
|
|
104
112
|
endRun();
|
|
105
113
|
}
|
|
106
114
|
}
|
|
107
|
-
export async function embedQuery(text) {
|
|
108
|
-
const [vec] = await embedTexts([text], "query");
|
|
115
|
+
export async function embedQuery(text, signal) {
|
|
116
|
+
const [vec] = await embedTexts([text], "query", signal);
|
|
109
117
|
return vec;
|
|
110
118
|
}
|
|
111
119
|
export async function embedDocuments(texts, signal) {
|