pi-knowledge 0.4.7 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/README.md +9 -2
- package/dist/index.js +129 -20
- package/dist/src/diagnostics/health.d.ts +1 -1
- package/dist/src/diagnostics/health.js +12 -1
- package/dist/src/embedding/provider.d.ts +1 -1
- package/dist/src/embedding/provider.js +12 -4
- package/dist/src/engine.d.ts +61 -10
- package/dist/src/engine.js +519 -165
- package/dist/src/indexer/symbols.d.ts +2 -0
- package/dist/src/indexer/symbols.js +175 -0
- package/dist/src/model-worker-client.d.ts +1 -1
- package/dist/src/model-worker-client.js +7 -7
- package/dist/src/search/reranker.d.ts +1 -1
- package/dist/src/search/reranker.js +2 -2
- package/dist/src/search/vector.d.ts +1 -1
- package/dist/src/search/vector.js +5 -1
- package/dist/src/storage/sqlite.d.ts +36 -1
- package/dist/src/storage/sqlite.js +159 -2
- package/dist/src/watcher/file-watcher.d.ts +2 -1
- package/dist/src/watcher/file-watcher.js +20 -30
- package/docs/configuration.md +8 -2
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,22 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [0.5.0] - 2026-07-07
|
|
4
|
+
|
|
5
|
+
### Added
|
|
6
|
+
- Added route symbol extraction and exact route lookup support through `knowledge_symbol_search`.
|
|
7
|
+
- Added structured `details` payloads for search diagnostics and doctor actions so agents can consume provenance and remediation codes without parsing text.
|
|
8
|
+
|
|
9
|
+
### Fixed
|
|
10
|
+
- Deleted vector files when knowledge bases are removed or cleared, including orphan detection in diagnostics.
|
|
11
|
+
- Guarded destructive and long-running operations with stronger mutation and shutdown lifecycle checks.
|
|
12
|
+
- Preserved ready knowledge bases when update cancellation happens before mutation starts, and kept partial update state from polluting symbol search.
|
|
13
|
+
- Propagated cancellation through search, export, import, diagnostics, and external embedding API requests.
|
|
14
|
+
|
|
15
|
+
### Changed
|
|
16
|
+
- Reworked JSONL import/export around streaming I/O and bounded batches for large portable knowledge bases.
|
|
17
|
+
- Improved watcher exclusions so generated, vendor, and runtime artifacts do not trigger unnecessary scans.
|
|
18
|
+
- Expanded Pi/OMP release coverage with lifecycle, cancellation, watcher, symbol, tool-contract, and model-worker regressions.
|
|
19
|
+
|
|
3
20
|
## [0.4.7] - 2026-07-04
|
|
4
21
|
|
|
5
22
|
### Fixed
|
package/README.md
CHANGED
|
@@ -11,6 +11,7 @@ Built as a native [Pi extension](https://pi.dev/docs/latest/extensions) with ver
|
|
|
11
11
|
Agents lose project context between sessions and cannot fit large repositories into one prompt. `pi-knowledge` gives Pi and OMP durable, searchable project memory:
|
|
12
12
|
|
|
13
13
|
- Search code and documentation by meaning, exact symbols, or both.
|
|
14
|
+
- Look up lightweight code symbols, Markdown headings, config keys, and environment variables before broader search.
|
|
14
15
|
- Keep private source data local by default.
|
|
15
16
|
- Re-index changed files incrementally.
|
|
16
17
|
- Diagnose stale, stuck, or low-quality indexes.
|
|
@@ -23,6 +24,7 @@ Unlike `pi-memory` (which manages the agent's own notes), `pi-knowledge` indexes
|
|
|
23
24
|
- **Local-first project memory**: stores indexes under `~/.pi/knowledge/`; no project files are modified.
|
|
24
25
|
- **Hybrid retrieval**: semantic vectors + BM25 keyword search + normalized weighted score fusion.
|
|
25
26
|
- **Code-aware indexing**: AST-aware chunking for TypeScript/JavaScript, Python, Go, Rust, and Java.
|
|
27
|
+
- **Lightweight symbol lookup**: indexes functions, classes, interfaces, types, variables, route-like handlers, Markdown headings, config keys, and env vars for exact agent lookup.
|
|
26
28
|
- **Better agent answers**: adaptive context windows, diversity reranking, optional cross-encoder reranking, and diagnostics.
|
|
27
29
|
- **Large-project stability**: persisted indexing progress, capped batches, streaming vector scans, and stuck-job detection.
|
|
28
30
|
- **Private by default**: local embeddings run without an API key.
|
|
@@ -42,6 +44,7 @@ Unlike `pi-memory` (which manages the agent's own notes), `pi-knowledge` indexes
|
|
|
42
44
|
| **Incremental re-indexing** | ✅ | ❌ | ❌ |
|
|
43
45
|
| **File watcher (auto-update)** | ✅ | ❌ | ❌ |
|
|
44
46
|
| **Code-aware chunking** | ✅ (TS/JS/Py/Go/Rust/Java) | ❌ | ❌ |
|
|
47
|
+
| **Symbol/config/heading lookup** | ✅ | ❌ | ❌ |
|
|
45
48
|
| **Local embeddings (zero API)** | ✅ | ❌ | ✅ (qmd) |
|
|
46
49
|
| **Index quality diagnostics** | ✅ | ❌ | ❌ |
|
|
47
50
|
| **Metadata filters in search** | ✅ | ❌ | ❌ |
|
|
@@ -101,6 +104,7 @@ omp install ./pi-knowledge
|
|
|
101
104
|
| `knowledge_plan` | Inspect an indexing source before writing a KB; reports scannable files, suggested exclusions, and technical skips |
|
|
102
105
|
| `knowledge_add` | Index files, directories, URLs, PDFs, DOCX, or inline text |
|
|
103
106
|
| `knowledge_search` | Hybrid, deep, or adaptive search across one or all knowledge bases |
|
|
107
|
+
| `knowledge_symbol_search` | Exact or substring lookup for code symbols, route-like handlers, Markdown headings, config keys, and env vars |
|
|
104
108
|
| `knowledge_remove` | Remove a knowledge base by name or ID |
|
|
105
109
|
| `knowledge_update` | Incrementally re-index changed files in a knowledge base |
|
|
106
110
|
| `knowledge_show` | List all knowledge bases with stats |
|
|
@@ -118,6 +122,7 @@ omp install ./pi-knowledge
|
|
|
118
122
|
- `deep`: hybrid retrieval followed by cross-encoder reranking.
|
|
119
123
|
- `adaptive`: hybrid retrieval followed by query-time contextual window expansion around seed chunks. It keeps the matched seed, prefers nearby/query-relevant neighboring chunks, and collapses overlapping windows from the same file.
|
|
120
124
|
- `auto`: selects a primary mode from the query shape and retries alternate modes when results are empty or weak.
|
|
125
|
+
- `code`, `config`, `errors`, `docs`, `decision`: intent aliases for agents. `code`, `config`, and `errors` bias toward exact lexical/symbol evidence; `docs` and `decision` keep hybrid recall while making the intended source type explicit.
|
|
121
126
|
|
|
122
127
|
Mode selection contract:
|
|
123
128
|
|
|
@@ -128,9 +133,10 @@ Mode selection contract:
|
|
|
128
133
|
- Use `deep` for high-stakes answers, ambiguous top results, or final verification when slower reranking is acceptable.
|
|
129
134
|
- If results are empty or weak but the KB should contain the answer, retry once with a different mode before concluding no answer exists.
|
|
130
135
|
|
|
131
|
-
Search results use balanced diversity reranking by default so near-duplicate chunks from the same file do not dominate the top results. Diversity scoring considers lexical overlap, same-file line proximity, overlapping adaptive windows, available embedding-vector similarity,
|
|
136
|
+
Search results use balanced diversity reranking by default so near-duplicate chunks from the same file do not dominate the top results. Diversity scoring considers lexical overlap, same-file line proximity, overlapping adaptive windows, available embedding-vector similarity, file-level interleaving, and a small KB trust multiplier that favors ready file/directory sources over stale, URL, or imported text sources. Use `diversity: "off"` only when raw ranking order is needed for diagnostics. Agents can request search diagnostics to inspect mode fallback, ranking coverage, path/source/test boosts, adjusted scores, and provenance such as chunk id, chunk hash, match reason, stale flag, and source freshness where available.
|
|
132
137
|
|
|
133
138
|
For best search quality, rebuild or update existing knowledge bases after upgrading. New indexes use contextual retrieval units: embeddings and FTS include file path, file type, Markdown heading breadcrumbs, and code symbol names while returned results keep the original chunk text readable. This improves queries that mention project structure, filenames, sections, or functions, and reduces duplicate-looking chunk hits.
|
|
139
|
+
Symbol/config/heading metadata is also rebuilt during `knowledge_update`; older KBs created before this feature may show a doctor action recommending update.
|
|
134
140
|
|
|
135
141
|
## Embedding Configuration
|
|
136
142
|
|
|
@@ -192,7 +198,7 @@ Update and diagnostics paths are also streaming-oriented: changed chunks are emb
|
|
|
192
198
|
|
|
193
199
|
## Architecture
|
|
194
200
|
|
|
195
|
-
|
|
201
|
+
[DESIGN.md](DESIGN.md) is the historical technical design. For current behavior contracts, prefer this README, [AGENTS.md](AGENTS.md), [docs/configuration.md](docs/configuration.md), and the ADR/pitfall notes under `docs/`.
|
|
196
202
|
|
|
197
203
|
## Data Storage
|
|
198
204
|
|
|
@@ -210,6 +216,7 @@ All data is stored globally at `~/.pi/knowledge/` under Pi or `~/.omp/knowledge/
|
|
|
210
216
|
- **Override**: set `PI_KNOWLEDGE_DIR` or `OMP_KNOWLEDGE_DIR`
|
|
211
217
|
- **Project safety**: pi-knowledge is read-only on indexed directories — no files are created or modified in your project
|
|
212
218
|
- **Updates**: extension updates do not affect existing indexed data. Schema migrations run automatically if needed.
|
|
219
|
+
- **Symbol index**: `knowledge.db` also stores lightweight symbol/config/heading metadata used by `knowledge_symbol_search`; it is derived from indexed source. File, directory, and URL KBs with retained source paths can rebuild it with `knowledge_update`; imported portable KBs and inline text KBs should be re-imported or re-added because they intentionally do not store an active local source manifest.
|
|
213
220
|
|
|
214
221
|
## Development
|
|
215
222
|
|
package/dist/index.js
CHANGED
|
@@ -47,6 +47,25 @@ let runtimePromise;
|
|
|
47
47
|
let initializePromise;
|
|
48
48
|
let initialized = false;
|
|
49
49
|
let disposePromise;
|
|
50
|
+
function scanOptionsFromSourceOptions(raw) {
|
|
51
|
+
if (!raw)
|
|
52
|
+
return {};
|
|
53
|
+
try {
|
|
54
|
+
const parsed = JSON.parse(raw);
|
|
55
|
+
return {
|
|
56
|
+
includeSuggestedText: parsed.include_suggested_text === true,
|
|
57
|
+
includePaths: Array.isArray(parsed.include_paths)
|
|
58
|
+
? parsed.include_paths.filter((item) => typeof item === "string")
|
|
59
|
+
: undefined,
|
|
60
|
+
excludePaths: Array.isArray(parsed.exclude_paths)
|
|
61
|
+
? parsed.exclude_paths.filter((item) => typeof item === "string")
|
|
62
|
+
: undefined,
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
catch {
|
|
66
|
+
return {};
|
|
67
|
+
}
|
|
68
|
+
}
|
|
50
69
|
function runtimeModule(modulePath) {
|
|
51
70
|
return `${modulePath}${RUNTIME_EXTENSION}`;
|
|
52
71
|
}
|
|
@@ -131,7 +150,7 @@ export default function (pi) {
|
|
|
131
150
|
if (kb.source_path && kb.source_type === "directory") {
|
|
132
151
|
watcher.startWatcher(kb.id, kb.source_path, (kbId) => {
|
|
133
152
|
engine.update(kbId).catch(() => { });
|
|
134
|
-
});
|
|
153
|
+
}, scanOptionsFromSourceOptions(kb.source_options));
|
|
135
154
|
}
|
|
136
155
|
}
|
|
137
156
|
}
|
|
@@ -199,7 +218,7 @@ export default function (pi) {
|
|
|
199
218
|
}))),
|
|
200
219
|
exclude_paths: Type.Optional(Type.Array(Type.String({ description: "Relative paths under a directory source to exclude from this plan" }))),
|
|
201
220
|
}),
|
|
202
|
-
async execute(_id, params) {
|
|
221
|
+
async execute(_id, params, _signal) {
|
|
203
222
|
const { engine } = await ensureInitialized();
|
|
204
223
|
const { source, include_suggested_text, include_paths, exclude_paths } = params;
|
|
205
224
|
const plan = engine.plan(source, {
|
|
@@ -210,7 +229,7 @@ export default function (pi) {
|
|
|
210
229
|
exclude_paths: Array.isArray(exclude_paths)
|
|
211
230
|
? exclude_paths.filter((item) => typeof item === "string")
|
|
212
231
|
: undefined,
|
|
213
|
-
});
|
|
232
|
+
}, _signal);
|
|
214
233
|
const samples = plan.skipped.samples
|
|
215
234
|
.map((sample) => `- ${sample.reason}: ${sample.path}${sample.size ? ` (${sample.size} bytes)` : ""}`)
|
|
216
235
|
.join("\n");
|
|
@@ -275,6 +294,14 @@ export default function (pi) {
|
|
|
275
294
|
if (WATCH_ENABLED && kb.source_path && kb.source_type === "directory") {
|
|
276
295
|
watcher.startWatcher(kb.id, kb.source_path, (kbId) => {
|
|
277
296
|
engine.update(kbId).catch(() => { });
|
|
297
|
+
}, {
|
|
298
|
+
includeSuggestedText: include_suggested_text === true,
|
|
299
|
+
includePaths: Array.isArray(include_paths)
|
|
300
|
+
? include_paths.filter((item) => typeof item === "string")
|
|
301
|
+
: undefined,
|
|
302
|
+
excludePaths: Array.isArray(exclude_paths)
|
|
303
|
+
? exclude_paths.filter((item) => typeof item === "string")
|
|
304
|
+
: undefined,
|
|
278
305
|
});
|
|
279
306
|
}
|
|
280
307
|
return {
|
|
@@ -311,6 +338,11 @@ export default function (pi) {
|
|
|
311
338
|
Type.Literal("hybrid"),
|
|
312
339
|
Type.Literal("deep"),
|
|
313
340
|
Type.Literal("adaptive"),
|
|
341
|
+
Type.Literal("code"),
|
|
342
|
+
Type.Literal("config"),
|
|
343
|
+
Type.Literal("docs"),
|
|
344
|
+
Type.Literal("errors"),
|
|
345
|
+
Type.Literal("decision"),
|
|
314
346
|
])),
|
|
315
347
|
limit: Type.Optional(Type.Number({ description: "Max results (default 10)" })),
|
|
316
348
|
kb_id: Type.Optional(Type.String({ description: "Limit search to a specific KB by ID or exact name" })),
|
|
@@ -319,11 +351,11 @@ export default function (pi) {
|
|
|
319
351
|
diversity: Type.Optional(Type.Union([Type.Literal("off"), Type.Literal("balanced"), Type.Literal("strong")])),
|
|
320
352
|
diagnostics: Type.Optional(Type.Boolean({ description: "Include ranking diagnostics and mode/fallback details in the result" })),
|
|
321
353
|
}),
|
|
322
|
-
async execute(_id, params) {
|
|
354
|
+
async execute(_id, params, _signal) {
|
|
323
355
|
const { engine } = await ensureInitialized();
|
|
324
356
|
const { query, mode, limit, kb_id, offset, file_type, diversity, diagnostics } = params;
|
|
325
357
|
const filters = file_type ? { file_type } : undefined;
|
|
326
|
-
const response = await engine.search(query, { mode, limit, kb_id, offset, filters, diversity });
|
|
358
|
+
const response = await engine.search(query, { mode, limit, kb_id, offset, filters, diversity }, _signal);
|
|
327
359
|
if (response.results.length === 0) {
|
|
328
360
|
const details = [
|
|
329
361
|
"No results found.",
|
|
@@ -334,7 +366,7 @@ export default function (pi) {
|
|
|
334
366
|
]
|
|
335
367
|
.filter(Boolean)
|
|
336
368
|
.join("\n");
|
|
337
|
-
return { content: [{ type: "text", text: details }] };
|
|
369
|
+
return { content: [{ type: "text", text: details }], details: diagnostics ? response : undefined };
|
|
338
370
|
}
|
|
339
371
|
let output = `${response.total_count} results (showing ${response.results.length})`;
|
|
340
372
|
if (response.mode_used)
|
|
@@ -350,10 +382,73 @@ export default function (pi) {
|
|
|
350
382
|
const diag = diagnostics && r.ranking
|
|
351
383
|
? `\nDiagnostics: base=${r.ranking.base_score.toFixed(3)}, adjusted=${r.ranking.adjusted_score.toFixed(3)}, coverage=${r.ranking.coverage.toFixed(2)}, path_boost=${r.ranking.path_boost.toFixed(2)}, source_boost=${r.ranking.source_boost.toFixed(2)}, test=${r.ranking.is_test}`
|
|
352
384
|
: "";
|
|
353
|
-
|
|
385
|
+
const provenance = diagnostics && r.provenance
|
|
386
|
+
? `\nProvenance: chunk=${r.provenance.chunk_id}, hash=${r.provenance.chunk_hash.slice(0, 12)}, reason=${r.provenance.match_reason}, indexed_at=${r.provenance.indexed_at}, source_mtime=${r.provenance.source_mtime ?? "unknown"}, stale=${r.provenance.stale}`
|
|
387
|
+
: "";
|
|
388
|
+
return `[${i + 1}] ${r.file_path} (${r.kb_name}, score: ${r.score.toFixed(3)})\n${r.snippet}${diag}${provenance}`;
|
|
354
389
|
})
|
|
355
390
|
.join("\n\n");
|
|
356
|
-
return { content: [{ type: "text", text: output }] };
|
|
391
|
+
return { content: [{ type: "text", text: output }], details: diagnostics ? response : undefined };
|
|
392
|
+
},
|
|
393
|
+
});
|
|
394
|
+
pi.registerTool({
|
|
395
|
+
name: "knowledge_symbol_search",
|
|
396
|
+
label: "Knowledge Symbol Search",
|
|
397
|
+
description: "Search indexed code symbols, Markdown headings, config keys, and environment variables exactly or by substring",
|
|
398
|
+
promptSnippet: "Find exact symbols/config keys/headings before broad semantic search",
|
|
399
|
+
promptGuidelines: [
|
|
400
|
+
"Use knowledge_symbol_search before knowledge_search when looking for a function, class, env var, config key, route, or Markdown heading",
|
|
401
|
+
"Use exact=true when the user provides a precise symbol or key",
|
|
402
|
+
"After finding a symbol, use knowledge_search mode 'adaptive' with the file path or symbol name when surrounding implementation context is needed",
|
|
403
|
+
],
|
|
404
|
+
parameters: Type.Object({
|
|
405
|
+
query: Type.String({ description: "Symbol, heading, config key, env var, route, or file text to find" }),
|
|
406
|
+
kind: Type.Optional(Type.Union([
|
|
407
|
+
Type.Literal("function"),
|
|
408
|
+
Type.Literal("class"),
|
|
409
|
+
Type.Literal("interface"),
|
|
410
|
+
Type.Literal("type"),
|
|
411
|
+
Type.Literal("variable"),
|
|
412
|
+
Type.Literal("heading"),
|
|
413
|
+
Type.Literal("config_key"),
|
|
414
|
+
Type.Literal("env_var"),
|
|
415
|
+
Type.Literal("route"),
|
|
416
|
+
])),
|
|
417
|
+
kb_id: Type.Optional(Type.String({ description: "Limit lookup to a specific KB by ID or exact name" })),
|
|
418
|
+
file_pattern: Type.Optional(Type.String({ description: "Limit lookup to paths containing this text" })),
|
|
419
|
+
limit: Type.Optional(Type.Number({ description: "Max results (default 20)" })),
|
|
420
|
+
offset: Type.Optional(Type.Number({ description: "Pagination offset" })),
|
|
421
|
+
exact: Type.Optional(Type.Boolean({ description: "Require exact normalized symbol/key match" })),
|
|
422
|
+
}),
|
|
423
|
+
async execute(_id, params, _signal) {
|
|
424
|
+
if (_signal?.aborted)
|
|
425
|
+
throw new Error("Cancelled");
|
|
426
|
+
const { engine } = await ensureInitialized();
|
|
427
|
+
if (_signal?.aborted)
|
|
428
|
+
throw new Error("Cancelled");
|
|
429
|
+
const response = engine.symbolSearch(params.query, {
|
|
430
|
+
kind: params.kind,
|
|
431
|
+
kb_id: params.kb_id,
|
|
432
|
+
file_pattern: params.file_pattern,
|
|
433
|
+
limit: params.limit,
|
|
434
|
+
offset: params.offset,
|
|
435
|
+
exact: params.exact === true,
|
|
436
|
+
}, _signal);
|
|
437
|
+
if (response.results.length === 0)
|
|
438
|
+
return { content: [{ type: "text", text: "No symbols found." }] };
|
|
439
|
+
const lines = response.results.map((result, index) => {
|
|
440
|
+
const signature = result.signature ? ` — ${result.signature}` : "";
|
|
441
|
+
return `[${index + 1}] ${result.name} (${result.kind}) ${result.file_path}:${result.start_line}-${result.end_line} [${result.kb_name}]${signature}`;
|
|
442
|
+
});
|
|
443
|
+
const suffix = response.has_more ? "\nMore results available; increase offset to continue." : "";
|
|
444
|
+
return {
|
|
445
|
+
content: [
|
|
446
|
+
{
|
|
447
|
+
type: "text",
|
|
448
|
+
text: `${response.total_count} symbol result(s):\n\n${lines.join("\n")}${suffix}`,
|
|
449
|
+
},
|
|
450
|
+
],
|
|
451
|
+
};
|
|
357
452
|
},
|
|
358
453
|
});
|
|
359
454
|
pi.registerTool({
|
|
@@ -379,11 +474,13 @@ export default function (pi) {
|
|
|
379
474
|
label: "Knowledge Status",
|
|
380
475
|
description: "Show knowledge engine status with health diagnostics: staleness, orphans, and coverage",
|
|
381
476
|
parameters: Type.Object({}),
|
|
382
|
-
async execute() {
|
|
477
|
+
async execute(_id, _params, _signal) {
|
|
478
|
+
if (_signal?.aborted)
|
|
479
|
+
throw new Error("Cancelled");
|
|
383
480
|
const { engine, storage, watcher } = await ensureInitialized();
|
|
384
481
|
const kbs = engine.list();
|
|
385
482
|
const watchCount = watcher.getActiveWatcherCount();
|
|
386
|
-
const diagnostics = engine.diagnose();
|
|
483
|
+
const diagnostics = engine.diagnose(_signal);
|
|
387
484
|
const lines = [
|
|
388
485
|
`Storage: ${storage.getDefaultKnowledgeDir()}`,
|
|
389
486
|
`Knowledge bases: ${kbs.length}`,
|
|
@@ -445,9 +542,11 @@ export default function (pi) {
|
|
|
445
542
|
description: "Diagnose knowledge base health, skipped files, stale indexes, stuck jobs, and recommended fixes",
|
|
446
543
|
promptSnippet: "Diagnose knowledge base health and recommended actions",
|
|
447
544
|
parameters: Type.Object({}),
|
|
448
|
-
async execute() {
|
|
545
|
+
async execute(_id, _params, _signal) {
|
|
546
|
+
if (_signal?.aborted)
|
|
547
|
+
throw new Error("Cancelled");
|
|
449
548
|
const { engine } = await ensureInitialized();
|
|
450
|
-
const report = engine.doctor();
|
|
549
|
+
const report = engine.doctor(_signal);
|
|
451
550
|
const lines = [`Health score: ${report.health_score}/100`, report.summary, ""];
|
|
452
551
|
if (report.issues.length === 0) {
|
|
453
552
|
lines.push("No issues found.");
|
|
@@ -456,10 +555,10 @@ export default function (pi) {
|
|
|
456
555
|
for (const issue of report.issues) {
|
|
457
556
|
const scope = issue.kb_name ? ` [${issue.kb_name}]` : "";
|
|
458
557
|
lines.push(`- ${issue.severity.toUpperCase()}${scope}: ${issue.message}`);
|
|
459
|
-
lines.push(` Action: ${issue.action}`);
|
|
558
|
+
lines.push(` Action (${issue.action_code ?? "none"}): ${issue.action}`);
|
|
460
559
|
}
|
|
461
560
|
}
|
|
462
|
-
return { content: [{ type: "text", text: lines.join("\n") }] };
|
|
561
|
+
return { content: [{ type: "text", text: lines.join("\n") }], details: report };
|
|
463
562
|
},
|
|
464
563
|
});
|
|
465
564
|
pi.registerTool({
|
|
@@ -467,9 +566,13 @@ export default function (pi) {
|
|
|
467
566
|
label: "Knowledge Show",
|
|
468
567
|
description: "List all indexed knowledge bases",
|
|
469
568
|
parameters: Type.Object({}),
|
|
470
|
-
async execute() {
|
|
569
|
+
async execute(_id, _params, _signal) {
|
|
570
|
+
if (_signal?.aborted)
|
|
571
|
+
throw new Error("Cancelled");
|
|
471
572
|
const { engine } = await ensureInitialized();
|
|
472
|
-
|
|
573
|
+
if (_signal?.aborted)
|
|
574
|
+
throw new Error("Cancelled");
|
|
575
|
+
const kbs = engine.list(_signal);
|
|
473
576
|
if (kbs.length === 0)
|
|
474
577
|
return { content: [{ type: "text", text: "No knowledge bases." }] };
|
|
475
578
|
const lines = kbs.map((kb) => `• ${kb.name} — ${kb.chunk_count} chunks, ${kb.file_count} files (${kb.status})`);
|
|
@@ -483,7 +586,9 @@ export default function (pi) {
|
|
|
483
586
|
parameters: Type.Object({
|
|
484
587
|
target: Type.String({ description: "KB name or ID to remove" }),
|
|
485
588
|
}),
|
|
486
|
-
async execute(_id, params) {
|
|
589
|
+
async execute(_id, params, _signal) {
|
|
590
|
+
if (_signal?.aborted)
|
|
591
|
+
throw new Error("Cancelled");
|
|
487
592
|
const { engine } = await ensureInitialized();
|
|
488
593
|
const ok = engine.remove(params.target);
|
|
489
594
|
return { content: [{ type: "text", text: ok ? "Removed." : "Not found." }] };
|
|
@@ -497,10 +602,12 @@ export default function (pi) {
|
|
|
497
602
|
target: Type.String({ description: "KB name or ID to export" }),
|
|
498
603
|
output: Type.String({ description: "Output file path (.jsonl)" }),
|
|
499
604
|
}),
|
|
500
|
-
async execute(_id, params) {
|
|
605
|
+
async execute(_id, params, _signal, onUpdate) {
|
|
501
606
|
const { engine } = await ensureInitialized();
|
|
502
607
|
const { target, output } = params;
|
|
503
|
-
const count = await engine.exportKB(target, output)
|
|
608
|
+
const count = await engine.exportKB(target, output, _signal, (msg) => {
|
|
609
|
+
onUpdate?.({ content: [{ type: "text", text: msg }] });
|
|
610
|
+
});
|
|
504
611
|
return { content: [{ type: "text", text: `Exported ${count} chunks to ${output}` }] };
|
|
505
612
|
},
|
|
506
613
|
});
|
|
@@ -524,7 +631,9 @@ export default function (pi) {
|
|
|
524
631
|
label: "Knowledge Clear",
|
|
525
632
|
description: "Remove all knowledge bases",
|
|
526
633
|
parameters: Type.Object({}),
|
|
527
|
-
async execute() {
|
|
634
|
+
async execute(_id, _params, _signal) {
|
|
635
|
+
if (_signal?.aborted)
|
|
636
|
+
throw new Error("Cancelled");
|
|
528
637
|
const { engine } = await ensureInitialized();
|
|
529
638
|
engine.clear();
|
|
530
639
|
return { content: [{ type: "text", text: "All knowledge bases cleared." }] };
|
|
@@ -16,4 +16,4 @@ export interface DiagnosticResult {
|
|
|
16
16
|
skipped_files: ScanResult["skipped"];
|
|
17
17
|
job?: IndexingJob;
|
|
18
18
|
}
|
|
19
|
-
export declare function diagnoseKB(db: Database.Database, kb: KnowledgeBase): DiagnosticResult;
|
|
19
|
+
export declare function diagnoseKB(db: Database.Database, kb: KnowledgeBase, signal?: AbortSignal): DiagnosticResult;
|
|
@@ -26,7 +26,11 @@ function scanOptionsFromSourceOptions(raw) {
|
|
|
26
26
|
return {};
|
|
27
27
|
}
|
|
28
28
|
}
|
|
29
|
-
|
|
29
|
+
function throwIfAborted(signal) {
|
|
30
|
+
if (signal?.aborted)
|
|
31
|
+
throw new Error("Cancelled");
|
|
32
|
+
}
|
|
33
|
+
export function diagnoseKB(db, kb, signal) {
|
|
30
34
|
const statusAgeMs = Date.now() - kb.updated_at;
|
|
31
35
|
const job = getIndexingJob(db, kb.id);
|
|
32
36
|
const lastProgressAgeMs = job?.status === "running" ? Date.now() - job.last_progress_at : statusAgeMs;
|
|
@@ -55,6 +59,7 @@ export function diagnoseKB(db, kb) {
|
|
|
55
59
|
},
|
|
56
60
|
job,
|
|
57
61
|
};
|
|
62
|
+
throwIfAborted(signal);
|
|
58
63
|
if (!kb.source_path || kb.source_type === "url" || !existsSync(kb.source_path)) {
|
|
59
64
|
return result; // text KBs or missing source — no diagnostics possible
|
|
60
65
|
}
|
|
@@ -65,6 +70,7 @@ export function diagnoseKB(db, kb) {
|
|
|
65
70
|
if (isDirectory) {
|
|
66
71
|
const skipped = createSkippedScanStats();
|
|
67
72
|
for (const file of iterateScannableFiles(kb.source_path, skipped, scanOptionsFromSourceOptions(kb.source_options))) {
|
|
73
|
+
throwIfAborted(signal);
|
|
68
74
|
currentFiles.add(file.relPath);
|
|
69
75
|
}
|
|
70
76
|
result.skipped_files = skipped;
|
|
@@ -74,6 +80,8 @@ export function diagnoseKB(db, kb) {
|
|
|
74
80
|
}
|
|
75
81
|
}
|
|
76
82
|
catch {
|
|
83
|
+
if (signal?.aborted)
|
|
84
|
+
throw new Error("Cancelled");
|
|
77
85
|
return result;
|
|
78
86
|
}
|
|
79
87
|
result.total_source_files = currentFiles.size;
|
|
@@ -81,6 +89,7 @@ export function diagnoseKB(db, kb) {
|
|
|
81
89
|
const indexedFilePaths = new Set();
|
|
82
90
|
const latestIndexedByFile = new Map();
|
|
83
91
|
for (const chunk of iterateChunksByKB(db, kb.id)) {
|
|
92
|
+
throwIfAborted(signal);
|
|
84
93
|
indexedFilePaths.add(chunk.file_path);
|
|
85
94
|
const currentLatest = latestIndexedByFile.get(chunk.file_path) ?? 0;
|
|
86
95
|
if (chunk.indexed_at > currentLatest)
|
|
@@ -88,12 +97,14 @@ export function diagnoseKB(db, kb) {
|
|
|
88
97
|
}
|
|
89
98
|
// Orphan detection: chunks referencing files no longer in source
|
|
90
99
|
for (const filePath of indexedFilePaths) {
|
|
100
|
+
throwIfAborted(signal);
|
|
91
101
|
if (!currentFiles.has(filePath)) {
|
|
92
102
|
result.orphan_files.push(filePath);
|
|
93
103
|
}
|
|
94
104
|
}
|
|
95
105
|
// Staleness detection: source files modified after last indexing
|
|
96
106
|
for (const relPath of currentFiles) {
|
|
107
|
+
throwIfAborted(signal);
|
|
97
108
|
const absPath = isDirectory ? join(kb.source_path, relPath) : relPath;
|
|
98
109
|
try {
|
|
99
110
|
const mtime = statSync(absPath).mtimeMs;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export declare function dispose(): Promise<void>;
|
|
2
2
|
export declare function prepareForShutdown(): Promise<void>;
|
|
3
3
|
export declare function embedTexts(texts: string[], prefix: "query" | "passage", signal?: AbortSignal): Promise<Float32Array[]>;
|
|
4
|
-
export declare function embedQuery(text: string): Promise<Float32Array>;
|
|
4
|
+
export declare function embedQuery(text: string, signal?: AbortSignal): Promise<Float32Array>;
|
|
5
5
|
export declare function embedDocuments(texts: string[], signal?: AbortSignal): Promise<Float32Array[]>;
|
|
@@ -10,6 +10,9 @@ const ENABLE_NATIVE_IDLE_DISPOSE = process.env.PI_KNOWLEDGE_ENABLE_NATIVE_IDLE_D
|
|
|
10
10
|
const API_FALLBACK_TO_LOCAL = process.env.PI_KNOWLEDGE_EMBEDDING_API_FALLBACK === "local";
|
|
11
11
|
const DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1";
|
|
12
12
|
const DEFAULT_API_MAX_EMBED_CHARS = 20_000;
|
|
13
|
+
function isAbortError(error) {
|
|
14
|
+
return typeof error === "object" && error !== null && "name" in error && error.name === "AbortError";
|
|
15
|
+
}
|
|
13
16
|
function clearIdleTimer() {
|
|
14
17
|
if (disposeTimer)
|
|
15
18
|
clearTimeout(disposeTimer);
|
|
@@ -57,7 +60,7 @@ export async function prepareForShutdown() {
|
|
|
57
60
|
clearIdleTimer();
|
|
58
61
|
await waitForNoActiveRuns();
|
|
59
62
|
}
|
|
60
|
-
async function embedViaAPI(texts, prefix) {
|
|
63
|
+
async function embedViaAPI(texts, prefix, signal) {
|
|
61
64
|
const [provider, model] = EMBEDDING_CONFIG.split(":");
|
|
62
65
|
const configuredMaxChars = Number(process.env.PI_KNOWLEDGE_EMBEDDING_MAX_CHARS ?? DEFAULT_API_MAX_EMBED_CHARS);
|
|
63
66
|
const maxChars = Number.isFinite(configuredMaxChars) && configuredMaxChars > 0 ? configuredMaxChars : DEFAULT_API_MAX_EMBED_CHARS;
|
|
@@ -75,6 +78,7 @@ async function embedViaAPI(texts, prefix) {
|
|
|
75
78
|
method: "POST",
|
|
76
79
|
headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}` },
|
|
77
80
|
body: JSON.stringify({ input: safeTexts, model: model || "text-embedding-3-small" }),
|
|
81
|
+
signal,
|
|
78
82
|
});
|
|
79
83
|
if (!res.ok) {
|
|
80
84
|
const detail = await res.text().catch(() => "");
|
|
@@ -87,10 +91,14 @@ async function embedViaAPI(texts, prefix) {
|
|
|
87
91
|
}
|
|
88
92
|
export async function embedTexts(texts, prefix, signal) {
|
|
89
93
|
if (!EMBEDDING_CONFIG.startsWith("local")) {
|
|
94
|
+
if (signal?.aborted)
|
|
95
|
+
throw new Error("Cancelled");
|
|
90
96
|
try {
|
|
91
|
-
return await embedViaAPI(texts, prefix);
|
|
97
|
+
return await embedViaAPI(texts, prefix, signal);
|
|
92
98
|
}
|
|
93
99
|
catch (error) {
|
|
100
|
+
if (signal?.aborted || isAbortError(error))
|
|
101
|
+
throw new Error("Cancelled");
|
|
94
102
|
if (!API_FALLBACK_TO_LOCAL)
|
|
95
103
|
throw error;
|
|
96
104
|
console.warn(`pi-knowledge: embedding API failed; falling back to local model because PI_KNOWLEDGE_EMBEDDING_API_FALLBACK=local (${error instanceof Error ? error.message : String(error)})`);
|
|
@@ -104,8 +112,8 @@ export async function embedTexts(texts, prefix, signal) {
|
|
|
104
112
|
endRun();
|
|
105
113
|
}
|
|
106
114
|
}
|
|
107
|
-
export async function embedQuery(text) {
|
|
108
|
-
const [vec] = await embedTexts([text], "query");
|
|
115
|
+
export async function embedQuery(text, signal) {
|
|
116
|
+
const [vec] = await embedTexts([text], "query", signal);
|
|
109
117
|
return vec;
|
|
110
118
|
}
|
|
111
119
|
export async function embedDocuments(texts, signal) {
|
package/dist/src/engine.d.ts
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { type DiagnosticResult } from "./diagnostics/health.ts";
|
|
2
2
|
import { createSkippedScanStats } from "./indexer/chunker.ts";
|
|
3
3
|
import { type RankingDiagnostics } from "./search/ranking.ts";
|
|
4
|
-
import { type KnowledgeBase } from "./storage/sqlite.ts";
|
|
4
|
+
import { type KnowledgeBase, type KnowledgeSymbol } from "./storage/sqlite.ts";
|
|
5
|
+
export type SearchMode = "auto" | "fast" | "semantic" | "hybrid" | "deep" | "adaptive" | "code" | "config" | "docs" | "errors" | "decision";
|
|
5
6
|
export interface SearchOptions {
|
|
6
|
-
mode?:
|
|
7
|
+
mode?: SearchMode;
|
|
7
8
|
limit?: number;
|
|
8
9
|
offset?: number;
|
|
9
10
|
kb_id?: string;
|
|
@@ -23,6 +24,15 @@ export interface SearchResult {
|
|
|
23
24
|
start_line: number;
|
|
24
25
|
end_line: number;
|
|
25
26
|
ranking?: RankingDiagnostics;
|
|
27
|
+
provenance?: {
|
|
28
|
+
chunk_id: string;
|
|
29
|
+
chunk_hash: string;
|
|
30
|
+
indexed_at: number;
|
|
31
|
+
source_mtime?: number;
|
|
32
|
+
stale: boolean;
|
|
33
|
+
match_reason: "bm25" | "vector" | "hybrid" | "rerank" | "symbol" | "adaptive";
|
|
34
|
+
source_chunk_ids?: string[];
|
|
35
|
+
};
|
|
26
36
|
}
|
|
27
37
|
export declare const CURRENT_EMBEDDING_MODEL = "multilingual-e5-small";
|
|
28
38
|
export interface SearchResponse {
|
|
@@ -30,8 +40,8 @@ export interface SearchResponse {
|
|
|
30
40
|
total_count: number;
|
|
31
41
|
has_more: boolean;
|
|
32
42
|
warnings?: string[];
|
|
33
|
-
mode_used?:
|
|
34
|
-
retry_modes?:
|
|
43
|
+
mode_used?: SearchMode;
|
|
44
|
+
retry_modes?: SearchMode[];
|
|
35
45
|
suggestions?: string[];
|
|
36
46
|
}
|
|
37
47
|
export type ProgressCallback = (msg: string) => void;
|
|
@@ -45,12 +55,44 @@ export interface DoctorIssue {
|
|
|
45
55
|
kb_name?: string;
|
|
46
56
|
message: string;
|
|
47
57
|
action: string;
|
|
58
|
+
action_code?: "run_update" | "rebuild_kb" | "wait_for_indexing" | "review_skipped_scope" | "rebuild_vectors" | "check_source" | "none";
|
|
48
59
|
}
|
|
49
60
|
export interface DoctorReport {
|
|
50
61
|
health_score: number;
|
|
51
62
|
summary: string;
|
|
52
63
|
issues: DoctorIssue[];
|
|
53
64
|
diagnostics: DiagnosticResult[];
|
|
65
|
+
actions: Array<{
|
|
66
|
+
code: NonNullable<DoctorIssue["action_code"]>;
|
|
67
|
+
kb_name?: string;
|
|
68
|
+
target?: string;
|
|
69
|
+
description: string;
|
|
70
|
+
}>;
|
|
71
|
+
}
|
|
72
|
+
export interface SymbolSearchOptions {
|
|
73
|
+
kb_id?: string;
|
|
74
|
+
kind?: KnowledgeSymbol["kind"];
|
|
75
|
+
file_pattern?: string;
|
|
76
|
+
limit?: number;
|
|
77
|
+
offset?: number;
|
|
78
|
+
exact?: boolean;
|
|
79
|
+
}
|
|
80
|
+
export interface SymbolSearchResponse {
|
|
81
|
+
results: Array<{
|
|
82
|
+
name: string;
|
|
83
|
+
kind: KnowledgeSymbol["kind"];
|
|
84
|
+
file_path: string;
|
|
85
|
+
file_type: string;
|
|
86
|
+
kb_name: string;
|
|
87
|
+
start_line: number;
|
|
88
|
+
end_line: number;
|
|
89
|
+
signature?: string;
|
|
90
|
+
container_name?: string;
|
|
91
|
+
text: string;
|
|
92
|
+
indexed_at: number;
|
|
93
|
+
}>;
|
|
94
|
+
total_count: number;
|
|
95
|
+
has_more: boolean;
|
|
54
96
|
}
|
|
55
97
|
export interface IndexPlan {
|
|
56
98
|
source_type: "file" | "directory" | "text" | "url";
|
|
@@ -63,29 +105,38 @@ export declare class KnowledgeEngine {
|
|
|
63
105
|
private db;
|
|
64
106
|
private knowledgeDir;
|
|
65
107
|
private activeUpdates;
|
|
108
|
+
private activeMutations;
|
|
109
|
+
private disposing;
|
|
110
|
+
private runExclusive;
|
|
66
111
|
initialize(knowledgeDir: string): Promise<void>;
|
|
67
|
-
|
|
112
|
+
private vectorPathFor;
|
|
113
|
+
private deleteVectorFile;
|
|
114
|
+
plan(source: string, options?: AddOptions, signal?: AbortSignal): IndexPlan;
|
|
68
115
|
add(source: string, name: string, onProgress?: ProgressCallback, signal?: AbortSignal, options?: AddOptions): Promise<{
|
|
69
116
|
kb: KnowledgeBase;
|
|
70
117
|
chunkCount: number;
|
|
71
118
|
}>;
|
|
119
|
+
private addUnlocked;
|
|
72
120
|
update(nameOrId: string, onProgress?: ProgressCallback, signal?: AbortSignal): Promise<{
|
|
73
121
|
added: number;
|
|
74
122
|
removed: number;
|
|
75
123
|
unchanged: number;
|
|
76
124
|
}>;
|
|
77
125
|
private runUpdate;
|
|
78
|
-
search(query: string, options?: SearchOptions): Promise<SearchResponse>;
|
|
126
|
+
search(query: string, options?: SearchOptions, signal?: AbortSignal): Promise<SearchResponse>;
|
|
79
127
|
remove(nameOrId: string): boolean;
|
|
80
|
-
list(): KnowledgeBase[];
|
|
128
|
+
list(signal?: AbortSignal): KnowledgeBase[];
|
|
81
129
|
clear(): void;
|
|
82
|
-
diagnose(): DiagnosticResult[];
|
|
83
|
-
|
|
84
|
-
|
|
130
|
+
diagnose(signal?: AbortSignal): DiagnosticResult[];
|
|
131
|
+
symbolSearch(query: string, options?: SymbolSearchOptions, signal?: AbortSignal): SymbolSearchResponse;
|
|
132
|
+
doctor(signal?: AbortSignal): DoctorReport;
|
|
133
|
+
exportKB(nameOrId: string, outputPath: string, signal?: AbortSignal, onProgress?: ProgressCallback): Promise<number>;
|
|
134
|
+
private exportKBUnlocked;
|
|
85
135
|
importKB(inputPath: string, onProgress?: ProgressCallback, signal?: AbortSignal): Promise<{
|
|
86
136
|
kb: KnowledgeBase;
|
|
87
137
|
chunkCount: number;
|
|
88
138
|
}>;
|
|
139
|
+
private importKBUnlocked;
|
|
89
140
|
dispose(options?: {
|
|
90
141
|
disposeModels?: boolean;
|
|
91
142
|
}): Promise<void>;
|