pi-knowledge 0.3.4 → 0.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/README.md +16 -2
- package/index.ts +63 -6
- package/package.json +1 -1
- package/src/embedding/provider.ts +50 -12
- package/src/engine.ts +402 -67
- package/src/indexer/chunker.ts +100 -77
- package/src/indexer/chunkers/code-ast.ts +14 -11
- package/src/search/bm25.ts +26 -22
- package/src/search/fusion.ts +33 -0
- package/src/search/query.ts +59 -0
- package/src/search/ranking.ts +197 -0
- package/src/search/reranker.ts +49 -11
- package/src/storage/sqlite.ts +16 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,32 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [0.3.6] - 2026-06-16
|
|
4
|
+
|
|
5
|
+
### Added
|
|
6
|
+
- Added contextual indexing for rebuilds: embeddings and FTS now include file path, file type, heading breadcrumbs, and code symbols while keeping returned chunk content readable.
|
|
7
|
+
- Added more focused rebuild-time chunking for Markdown and plain text, with reduced overlap to avoid near-duplicate retrieval units.
|
|
8
|
+
- Added adaptive search mode with query-time contextual window expansion around relevant seed chunks.
|
|
9
|
+
- Added balanced/strong/off diversity controls for search result reranking to reduce near-duplicate chunk clusters.
|
|
10
|
+
- Added query-aware snippets so search results show the matched context instead of always showing the chunk prefix.
|
|
11
|
+
- Added vector-aware redundancy scoring and overlapping adaptive window collapse for higher-diversity top results.
|
|
12
|
+
- Replaced hybrid RRF scoring with normalized weighted score fusion to preserve meaningful score spread.
|
|
13
|
+
- Added file-level result interleaving so README-style overview files cannot dominate top results with repeated chunks.
|
|
14
|
+
- Added confidence gating for hybrid search so low-evidence garbage queries return no results instead of unrelated matches.
|
|
15
|
+
- Strengthened source-file intent scoring so named modules and core implementation files outrank overview and test files when appropriate.
|
|
16
|
+
- Demoted localization catalogs for implementation-oriented queries while preserving them for explicit translation or locale intent.
|
|
17
|
+
- Excluded generated knowledge-base evaluation reports from default directory indexing to prevent self-referential retrieval pollution.
|
|
18
|
+
|
|
19
|
+
## [0.3.5] - 2026-06-15
|
|
20
|
+
|
|
21
|
+
### Fixed
|
|
22
|
+
- Prevented idle model disposal from running during active embedding or reranking batches, fixing `Session already disposed` during large `knowledge_add` operations.
|
|
23
|
+
- Avoided ONNX native disposal during Pi `session_shutdown`, preventing macOS onnxruntime mutex crashes on session exit.
|
|
24
|
+
- Allowed `knowledge_search` `kb_id` to accept either a KB UUID or exact KB name.
|
|
25
|
+
- Truncated custom TUI render lines to prevent Pi crashes when search result snippets exceed terminal width.
|
|
26
|
+
|
|
27
|
+
### Changed
|
|
28
|
+
- Added `PI_KNOWLEDGE_EMBEDDING_IDLE_MS` for lifecycle stress testing of embedding idle disposal.
|
|
29
|
+
|
|
3
30
|
## [0.3.4] - 2026-06-15
|
|
4
31
|
|
|
5
32
|
### Fixed
|
package/README.md
CHANGED
|
@@ -18,8 +18,10 @@ Unlike `pi-memory` (which manages the agent's own notes), `pi-knowledge` indexes
|
|
|
18
18
|
| Multiple named knowledge bases | ✅ | ✅ | ❌ |
|
|
19
19
|
| Semantic (vector) search | ✅ | ✅ | ✅ (via qmd) |
|
|
20
20
|
| BM25 keyword search | ✅ | ✅ | ✅ (via qmd) |
|
|
21
|
-
| **Hybrid search +
|
|
21
|
+
| **Hybrid search + weighted score fusion** | ✅ | ❌ | partial |
|
|
22
22
|
| **Cross-encoder reranking** | ✅ | ❌ | ❌ |
|
|
23
|
+
| **Adaptive contextual search** | ✅ | ❌ | ❌ |
|
|
24
|
+
| **Diversity reranking** | ✅ | ❌ | ❌ |
|
|
23
25
|
| **Incremental re-indexing** | ✅ | ❌ | ❌ |
|
|
24
26
|
| **File watcher (auto-update)** | ✅ | ❌ | ❌ |
|
|
25
27
|
| **Code-aware chunking** | ✅ (TS/JS/Py/Go/Rust/Java) | ❌ | ❌ |
|
|
@@ -57,7 +59,7 @@ pi install ./pi-knowledge
|
|
|
57
59
|
| Tool | Description |
|
|
58
60
|
|------|-------------|
|
|
59
61
|
| `knowledge_add` | Index files, directories, URLs, PDFs, DOCX, or inline text |
|
|
60
|
-
| `knowledge_search` | Hybrid search across one or all knowledge bases |
|
|
62
|
+
| `knowledge_search` | Hybrid, deep, or adaptive search across one or all knowledge bases |
|
|
61
63
|
| `knowledge_remove` | Remove a knowledge base by name or ID |
|
|
62
64
|
| `knowledge_update` | Incrementally re-index changed files in a knowledge base |
|
|
63
65
|
| `knowledge_show` | List all knowledge bases with stats |
|
|
@@ -66,6 +68,18 @@ pi install ./pi-knowledge
|
|
|
66
68
|
| `knowledge_export` | Export a KB to shareable JSONL file |
|
|
67
69
|
| `knowledge_import` | Import a KB from JSONL (re-embeds content) |
|
|
68
70
|
|
|
71
|
+
### Search Modes
|
|
72
|
+
|
|
73
|
+
- `fast`: BM25 keyword search for exact symbols, commands, and identifiers.
|
|
74
|
+
- `semantic`: vector search for conceptual matches.
|
|
75
|
+
- `hybrid`: BM25 + vector search with normalized weighted score fusion.
|
|
76
|
+
- `deep`: hybrid retrieval followed by cross-encoder reranking.
|
|
77
|
+
- `adaptive`: hybrid retrieval followed by query-time contextual window expansion around seed chunks. It keeps the matched seed, prefers nearby/query-relevant neighboring chunks, and collapses overlapping windows from the same file.
|
|
78
|
+
|
|
79
|
+
Search results use balanced diversity reranking by default so near-duplicate chunks from the same file do not dominate the top results. Diversity scoring considers lexical overlap, same-file line proximity, overlapping adaptive windows, available embedding-vector similarity, and file-level interleaving. Use `diversity: "off"` only when raw ranking order is needed for diagnostics.
|
|
80
|
+
|
|
81
|
+
For best search quality, rebuild or update existing knowledge bases after upgrading. New indexes use contextual retrieval units: embeddings and FTS include file path, file type, Markdown heading breadcrumbs, and code symbol names while returned results keep the original chunk text readable. This improves queries that mention project structure, filenames, sections, or functions, and reduces duplicate-looking chunk hits.
|
|
82
|
+
|
|
69
83
|
## Architecture
|
|
70
84
|
|
|
71
85
|
See [DESIGN.md](DESIGN.md) for the full technical design document.
|
package/index.ts
CHANGED
|
@@ -18,6 +18,58 @@ type ToolResultLike = {
|
|
|
18
18
|
content?: Array<{ type: string; text?: string }>;
|
|
19
19
|
};
|
|
20
20
|
|
|
21
|
+
const MAX_RENDER_LINE_WIDTH = 80;
|
|
22
|
+
const ESCAPE = "\u001B";
|
|
23
|
+
|
|
24
|
+
function ansiSequenceLength(line: string, offset: number): number {
|
|
25
|
+
if (line[offset] !== ESCAPE || line[offset + 1] !== "[") return 0;
|
|
26
|
+
for (let i = offset + 2; i < line.length; i++) {
|
|
27
|
+
const codePoint = line.charCodeAt(i);
|
|
28
|
+
if (codePoint >= 0x40 && codePoint <= 0x7e) return i - offset + 1;
|
|
29
|
+
}
|
|
30
|
+
return 0;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function charRenderWidth(char: string): number {
|
|
34
|
+
const codePoint = char.codePointAt(0);
|
|
35
|
+
if (codePoint === undefined) return 0;
|
|
36
|
+
if (
|
|
37
|
+
(codePoint >= 0x1100 && codePoint <= 0x115f) ||
|
|
38
|
+
(codePoint >= 0x2e80 && codePoint <= 0xa4cf) ||
|
|
39
|
+
(codePoint >= 0xac00 && codePoint <= 0xd7a3) ||
|
|
40
|
+
(codePoint >= 0xf900 && codePoint <= 0xfaff) ||
|
|
41
|
+
(codePoint >= 0xfe10 && codePoint <= 0xfe19) ||
|
|
42
|
+
(codePoint >= 0xfe30 && codePoint <= 0xfe6f) ||
|
|
43
|
+
(codePoint >= 0xff00 && codePoint <= 0xff60) ||
|
|
44
|
+
(codePoint >= 0xffe0 && codePoint <= 0xffe6)
|
|
45
|
+
) {
|
|
46
|
+
return 2;
|
|
47
|
+
}
|
|
48
|
+
return 1;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function truncateRenderLine(line: string): string {
|
|
52
|
+
let visibleWidth = 0;
|
|
53
|
+
let output = "";
|
|
54
|
+
for (let i = 0; i < line.length; ) {
|
|
55
|
+
const ansiLength = ansiSequenceLength(line, i);
|
|
56
|
+
if (ansiLength > 0) {
|
|
57
|
+
output += line.slice(i, i + ansiLength);
|
|
58
|
+
i += ansiLength;
|
|
59
|
+
continue;
|
|
60
|
+
}
|
|
61
|
+
const codePoint = line.codePointAt(i);
|
|
62
|
+
if (codePoint === undefined) break;
|
|
63
|
+
const char = String.fromCodePoint(codePoint);
|
|
64
|
+
const width = charRenderWidth(char);
|
|
65
|
+
if (visibleWidth + width > MAX_RENDER_LINE_WIDTH - 1) return `${output}…`;
|
|
66
|
+
output += char;
|
|
67
|
+
visibleWidth += width;
|
|
68
|
+
i += char.length;
|
|
69
|
+
}
|
|
70
|
+
return output;
|
|
71
|
+
}
|
|
72
|
+
|
|
21
73
|
/**
|
|
22
74
|
* Minimal duck-type shim for @earendil-works/pi-tui Text component.
|
|
23
75
|
* Allows startup without Pi virtual module while implementing the render host methods this extension needs.
|
|
@@ -36,7 +88,7 @@ class RenderableText {
|
|
|
36
88
|
invalidate(): void {}
|
|
37
89
|
|
|
38
90
|
render(): string[] {
|
|
39
|
-
return this.text.split("\n");
|
|
91
|
+
return this.text.split("\n").map(truncateRenderLine);
|
|
40
92
|
}
|
|
41
93
|
}
|
|
42
94
|
|
|
@@ -100,9 +152,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
100
152
|
|
|
101
153
|
pi.on("session_shutdown", async () => {
|
|
102
154
|
stopAllWatchers();
|
|
103
|
-
await engine.dispose();
|
|
104
|
-
// Brief delay to let native onnxruntime threads complete cleanup before process exit
|
|
105
|
-
await new Promise((r) => setTimeout(r, 500));
|
|
155
|
+
await engine.dispose({ disposeModels: false });
|
|
106
156
|
});
|
|
107
157
|
|
|
108
158
|
// Auto-inject: search KB for relevant context before each LLM call (opt-in)
|
|
@@ -193,12 +243,19 @@ export default function (pi: ExtensionAPI) {
|
|
|
193
243
|
parameters: Type.Object({
|
|
194
244
|
query: Type.String({ description: "Search query" }),
|
|
195
245
|
mode: Type.Optional(
|
|
196
|
-
Type.Union([
|
|
246
|
+
Type.Union([
|
|
247
|
+
Type.Literal("fast"),
|
|
248
|
+
Type.Literal("semantic"),
|
|
249
|
+
Type.Literal("hybrid"),
|
|
250
|
+
Type.Literal("deep"),
|
|
251
|
+
Type.Literal("adaptive"),
|
|
252
|
+
]),
|
|
197
253
|
),
|
|
198
254
|
limit: Type.Optional(Type.Number({ description: "Max results (default 10)" })),
|
|
199
|
-
kb_id: Type.Optional(Type.String({ description: "Limit search to specific KB" })),
|
|
255
|
+
kb_id: Type.Optional(Type.String({ description: "Limit search to a specific KB by ID or exact name" })),
|
|
200
256
|
offset: Type.Optional(Type.Number({ description: "Pagination offset" })),
|
|
201
257
|
file_type: Type.Optional(Type.String({ description: "Filter by file type (e.g. typescript, markdown, python)" })),
|
|
258
|
+
diversity: Type.Optional(Type.Union([Type.Literal("off"), Type.Literal("balanced"), Type.Literal("strong")])),
|
|
202
259
|
}),
|
|
203
260
|
renderCall(args, theme: ThemeLike, context: RenderContextLike) {
|
|
204
261
|
const query = typeof args.query === "string" ? args.query : "";
|
package/package.json
CHANGED
|
@@ -9,8 +9,11 @@ type FeatureExtractionPipeline = {
|
|
|
9
9
|
let pipelineInstance: FeatureExtractionPipeline | null = null;
|
|
10
10
|
let disposeTimer: ReturnType<typeof setTimeout> | null = null;
|
|
11
11
|
let disposePromise: Promise<void> | null = null;
|
|
12
|
+
let activeRuns = 0;
|
|
13
|
+
let disposeRequested = false;
|
|
14
|
+
const idleWaiters: Array<() => void> = [];
|
|
12
15
|
|
|
13
|
-
const IDLE_TIMEOUT_MS = 30_000;
|
|
16
|
+
const IDLE_TIMEOUT_MS = Number(process.env.PI_KNOWLEDGE_EMBEDDING_IDLE_MS ?? 30_000);
|
|
14
17
|
const EMBEDDING_CONFIG = process.env.PI_KNOWLEDGE_EMBEDDING ?? "local:multilingual-e5-small";
|
|
15
18
|
|
|
16
19
|
function getModelCacheDir(): string {
|
|
@@ -29,26 +32,57 @@ async function loadPipeline(): Promise<FeatureExtractionPipeline> {
|
|
|
29
32
|
return pipelineInstance;
|
|
30
33
|
}
|
|
31
34
|
|
|
32
|
-
function
|
|
35
|
+
function clearIdleTimer(): void {
|
|
33
36
|
if (disposeTimer) clearTimeout(disposeTimer);
|
|
37
|
+
disposeTimer = null;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function scheduleIdleDispose(): void {
|
|
41
|
+
if (activeRuns > 0 || disposeRequested) return;
|
|
42
|
+
clearIdleTimer();
|
|
34
43
|
disposeTimer = setTimeout(() => dispose(), IDLE_TIMEOUT_MS);
|
|
35
44
|
}
|
|
36
45
|
|
|
46
|
+
function beginRun(): void {
|
|
47
|
+
activeRuns++;
|
|
48
|
+
clearIdleTimer();
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function endRun(): void {
|
|
52
|
+
activeRuns--;
|
|
53
|
+
if (activeRuns > 0) return;
|
|
54
|
+
for (const resolve of idleWaiters.splice(0)) resolve();
|
|
55
|
+
if (!disposeRequested) scheduleIdleDispose();
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function waitForNoActiveRuns(): Promise<void> {
|
|
59
|
+
if (activeRuns === 0) return Promise.resolve();
|
|
60
|
+
return new Promise((resolve) => idleWaiters.push(resolve));
|
|
61
|
+
}
|
|
62
|
+
|
|
37
63
|
export async function dispose(): Promise<void> {
|
|
38
|
-
|
|
39
|
-
clearTimeout(disposeTimer);
|
|
40
|
-
disposeTimer = null;
|
|
41
|
-
}
|
|
64
|
+
clearIdleTimer();
|
|
42
65
|
if (disposePromise) return disposePromise;
|
|
66
|
+
disposeRequested = true;
|
|
67
|
+
await waitForNoActiveRuns();
|
|
43
68
|
const instance = pipelineInstance;
|
|
44
69
|
pipelineInstance = null;
|
|
45
|
-
if (!instance)
|
|
70
|
+
if (!instance) {
|
|
71
|
+
disposeRequested = false;
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
46
74
|
disposePromise = Promise.resolve(instance.dispose()).finally(() => {
|
|
47
75
|
disposePromise = null;
|
|
76
|
+
disposeRequested = false;
|
|
48
77
|
});
|
|
49
78
|
return disposePromise;
|
|
50
79
|
}
|
|
51
80
|
|
|
81
|
+
export async function prepareForShutdown(): Promise<void> {
|
|
82
|
+
clearIdleTimer();
|
|
83
|
+
await waitForNoActiveRuns();
|
|
84
|
+
}
|
|
85
|
+
|
|
52
86
|
async function embedViaAPI(texts: string[], prefix: "query" | "passage"): Promise<Float32Array[]> {
|
|
53
87
|
const [provider, model] = EMBEDDING_CONFIG.split(":");
|
|
54
88
|
const prefixedTexts = texts.map((t) => `${prefix}: ${t}`);
|
|
@@ -82,12 +116,16 @@ export async function embedTexts(
|
|
|
82
116
|
}
|
|
83
117
|
}
|
|
84
118
|
const pipe = await loadPipeline();
|
|
85
|
-
|
|
119
|
+
beginRun();
|
|
86
120
|
const results: Float32Array[] = [];
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
121
|
+
try {
|
|
122
|
+
for (const text of texts) {
|
|
123
|
+
if (signal?.aborted) throw new Error("Cancelled");
|
|
124
|
+
const output = await pipe(`${prefix}: ${text}`, { pooling: "mean", normalize: true });
|
|
125
|
+
results.push(new Float32Array(output.data));
|
|
126
|
+
}
|
|
127
|
+
} finally {
|
|
128
|
+
endRun();
|
|
91
129
|
}
|
|
92
130
|
return results;
|
|
93
131
|
}
|