pi-knowledge 0.3.3 → 0.3.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,41 @@
1
1
  # Changelog
2
2
 
3
+ ## [0.3.6] - 2026-06-16
4
+
5
+ ### Added
6
+ - Added contextual indexing for rebuilds: embeddings and FTS now include file path, file type, heading breadcrumbs, and code symbols while keeping returned chunk content readable.
7
+ - Added more focused rebuild-time chunking for Markdown and plain text, with reduced overlap to avoid near-duplicate retrieval units.
8
+ - Added adaptive search mode with query-time contextual window expansion around relevant seed chunks.
9
+ - Added balanced/strong/off diversity controls for search result reranking to reduce near-duplicate chunk clusters.
10
+ - Added query-aware snippets so search results show the matched context instead of always showing the chunk prefix.
11
+ - Added vector-aware redundancy scoring and overlapping adaptive window collapse for higher-diversity top results.
12
+ - Replaced hybrid RRF scoring with normalized weighted score fusion to preserve meaningful score spread.
13
+ - Added file-level result interleaving so README-style overview files cannot dominate top results with repeated chunks.
14
+ - Added confidence gating for hybrid search so low-evidence garbage queries return no results instead of unrelated matches.
15
+ - Strengthened source-file intent scoring so named modules and core implementation files outrank overview and test files when appropriate.
16
+ - Demoted localization catalogs for implementation-oriented queries while preserving them for explicit translation or locale intent.
17
+ - Excluded generated knowledge-base evaluation reports from default directory indexing to prevent self-referential retrieval pollution.
18
+
19
+ ## [0.3.5] - 2026-06-15
20
+
21
+ ### Fixed
22
+ - Prevented idle model disposal from running during active embedding or reranking batches, fixing `Session already disposed` during large `knowledge_add` operations.
23
+ - Avoided ONNX native disposal during Pi `session_shutdown`, preventing macOS onnxruntime mutex crashes on session exit.
24
+ - Allowed `knowledge_search` `kb_id` to accept either a KB UUID or exact KB name.
25
+ - Truncated custom TUI render lines to prevent Pi crashes when search result snippets exceed terminal width.
26
+
27
+ ### Changed
28
+ - Added `PI_KNOWLEDGE_EMBEDDING_IDLE_MS` for lifecycle stress testing of embedding idle disposal.
29
+
30
+ ## [0.3.4] - 2026-06-15
31
+
32
+ ### Fixed
33
+ - Reject duplicate `knowledge_add` names with an actionable message instead of silently creating multiple same-name knowledge bases.
34
+ - Strengthened default directory indexing ignores for build outputs and common secret/config files.
35
+
36
+ ### Changed
37
+ - Updated `knowledge_add` tool guidance to prefer one directory-level indexing call and avoid per-file indexing loops.
38
+
3
39
  ## [0.3.3] - 2026-06-15
4
40
 
5
41
  ### Changed
package/README.md CHANGED
@@ -18,8 +18,10 @@ Unlike `pi-memory` (which manages the agent's own notes), `pi-knowledge` indexes
18
18
  | Multiple named knowledge bases | ✅ | ✅ | ❌ |
19
19
  | Semantic (vector) search | ✅ | ✅ | ✅ (via qmd) |
20
20
  | BM25 keyword search | ✅ | ✅ | ✅ (via qmd) |
21
- | **Hybrid search + RRF fusion** | ✅ | ❌ | partial |
21
+ | **Hybrid search + weighted score fusion** | ✅ | ❌ | partial |
22
22
  | **Cross-encoder reranking** | ✅ | ❌ | ❌ |
23
+ | **Adaptive contextual search** | ✅ | ❌ | ❌ |
24
+ | **Diversity reranking** | ✅ | ❌ | ❌ |
23
25
  | **Incremental re-indexing** | ✅ | ❌ | ❌ |
24
26
  | **File watcher (auto-update)** | ✅ | ❌ | ❌ |
25
27
  | **Code-aware chunking** | ✅ (TS/JS/Py/Go/Rust/Java) | ❌ | ❌ |
@@ -57,7 +59,7 @@ pi install ./pi-knowledge
57
59
  | Tool | Description |
58
60
  |------|-------------|
59
61
  | `knowledge_add` | Index files, directories, URLs, PDFs, DOCX, or inline text |
60
- | `knowledge_search` | Hybrid search across one or all knowledge bases |
62
+ | `knowledge_search` | Hybrid, deep, or adaptive search across one or all knowledge bases |
61
63
  | `knowledge_remove` | Remove a knowledge base by name or ID |
62
64
  | `knowledge_update` | Incrementally re-index changed files in a knowledge base |
63
65
  | `knowledge_show` | List all knowledge bases with stats |
@@ -66,6 +68,18 @@ pi install ./pi-knowledge
66
68
  | `knowledge_export` | Export a KB to shareable JSONL file |
67
69
  | `knowledge_import` | Import a KB from JSONL (re-embeds content) |
68
70
 
71
+ ### Search Modes
72
+
73
+ - `fast`: BM25 keyword search for exact symbols, commands, and identifiers.
74
+ - `semantic`: vector search for conceptual matches.
75
+ - `hybrid`: BM25 + vector search with normalized weighted score fusion.
76
+ - `deep`: hybrid retrieval followed by cross-encoder reranking.
77
+ - `adaptive`: hybrid retrieval followed by query-time contextual window expansion around seed chunks. It keeps the matched seed, prefers nearby/query-relevant neighboring chunks, and collapses overlapping windows from the same file.
78
+
79
+ Search results use balanced diversity reranking by default so near-duplicate chunks from the same file do not dominate the top results. Diversity scoring considers lexical overlap, same-file line proximity, overlapping adaptive windows, available embedding-vector similarity, and file-level interleaving. Use `diversity: "off"` only when raw ranking order is needed for diagnostics.
80
+
81
+ For best search quality, rebuild or update existing knowledge bases after upgrading. New indexes use contextual retrieval units: embeddings and FTS include file path, file type, Markdown heading breadcrumbs, and code symbol names while returned results keep the original chunk text readable. This improves queries that mention project structure, filenames, sections, or functions, and reduces duplicate-looking chunk hits.
82
+
69
83
  ## Architecture
70
84
 
71
85
  See [DESIGN.md](DESIGN.md) for the full technical design document.
package/index.ts CHANGED
@@ -18,6 +18,58 @@ type ToolResultLike = {
18
18
  content?: Array<{ type: string; text?: string }>;
19
19
  };
20
20
 
21
+ const MAX_RENDER_LINE_WIDTH = 80;
22
+ const ESCAPE = "\u001B";
23
+
24
+ function ansiSequenceLength(line: string, offset: number): number {
25
+ if (line[offset] !== ESCAPE || line[offset + 1] !== "[") return 0;
26
+ for (let i = offset + 2; i < line.length; i++) {
27
+ const codePoint = line.charCodeAt(i);
28
+ if (codePoint >= 0x40 && codePoint <= 0x7e) return i - offset + 1;
29
+ }
30
+ return 0;
31
+ }
32
+
33
+ function charRenderWidth(char: string): number {
34
+ const codePoint = char.codePointAt(0);
35
+ if (codePoint === undefined) return 0;
36
+ if (
37
+ (codePoint >= 0x1100 && codePoint <= 0x115f) ||
38
+ (codePoint >= 0x2e80 && codePoint <= 0xa4cf) ||
39
+ (codePoint >= 0xac00 && codePoint <= 0xd7a3) ||
40
+ (codePoint >= 0xf900 && codePoint <= 0xfaff) ||
41
+ (codePoint >= 0xfe10 && codePoint <= 0xfe19) ||
42
+ (codePoint >= 0xfe30 && codePoint <= 0xfe6f) ||
43
+ (codePoint >= 0xff00 && codePoint <= 0xff60) ||
44
+ (codePoint >= 0xffe0 && codePoint <= 0xffe6)
45
+ ) {
46
+ return 2;
47
+ }
48
+ return 1;
49
+ }
50
+
51
+ function truncateRenderLine(line: string): string {
52
+ let visibleWidth = 0;
53
+ let output = "";
54
+ for (let i = 0; i < line.length; ) {
55
+ const ansiLength = ansiSequenceLength(line, i);
56
+ if (ansiLength > 0) {
57
+ output += line.slice(i, i + ansiLength);
58
+ i += ansiLength;
59
+ continue;
60
+ }
61
+ const codePoint = line.codePointAt(i);
62
+ if (codePoint === undefined) break;
63
+ const char = String.fromCodePoint(codePoint);
64
+ const width = charRenderWidth(char);
65
+ if (visibleWidth + width > MAX_RENDER_LINE_WIDTH - 1) return `${output}…`;
66
+ output += char;
67
+ visibleWidth += width;
68
+ i += char.length;
69
+ }
70
+ return output;
71
+ }
72
+
21
73
  /**
22
74
  * Minimal duck-type shim for @earendil-works/pi-tui Text component.
23
75
  * Allows startup without Pi virtual module while implementing the render host methods this extension needs.
@@ -36,7 +88,7 @@ class RenderableText {
36
88
  invalidate(): void {}
37
89
 
38
90
  render(): string[] {
39
- return this.text.split("\n");
91
+ return this.text.split("\n").map(truncateRenderLine);
40
92
  }
41
93
  }
42
94
 
@@ -100,9 +152,7 @@ export default function (pi: ExtensionAPI) {
100
152
 
101
153
  pi.on("session_shutdown", async () => {
102
154
  stopAllWatchers();
103
- await engine.dispose();
104
- // Brief delay to let native onnxruntime threads complete cleanup before process exit
105
- await new Promise((r) => setTimeout(r, 500));
155
+ await engine.dispose({ disposeModels: false });
106
156
  });
107
157
 
108
158
  // Auto-inject: search KB for relevant context before each LLM call (opt-in)
@@ -144,7 +194,10 @@ export default function (pi: ExtensionAPI) {
144
194
  promptSnippet: "Index files/dirs/text into a searchable knowledge base",
145
195
  promptGuidelines: [
146
196
  "Use knowledge_add when the user asks to index, remember, or learn from files or documentation",
147
- "Provide a descriptive name for the knowledge base",
197
+ "Prefer one knowledge_add call for the project root or relevant directory; do not call it once per file",
198
+ "If a knowledge base with the same name already exists, use knowledge_update or ask before replacing it",
199
+ "Do not index secrets or environment-specific config files unless the user explicitly asks for that exact file",
200
+ "Provide a descriptive name for this single knowledge base",
148
201
  ],
149
202
  parameters: Type.Object({
150
203
  source: Type.String({ description: "File path, directory path, or inline text to index" }),
@@ -190,12 +243,19 @@ export default function (pi: ExtensionAPI) {
190
243
  parameters: Type.Object({
191
244
  query: Type.String({ description: "Search query" }),
192
245
  mode: Type.Optional(
193
- Type.Union([Type.Literal("fast"), Type.Literal("semantic"), Type.Literal("hybrid"), Type.Literal("deep")]),
246
+ Type.Union([
247
+ Type.Literal("fast"),
248
+ Type.Literal("semantic"),
249
+ Type.Literal("hybrid"),
250
+ Type.Literal("deep"),
251
+ Type.Literal("adaptive"),
252
+ ]),
194
253
  ),
195
254
  limit: Type.Optional(Type.Number({ description: "Max results (default 10)" })),
196
- kb_id: Type.Optional(Type.String({ description: "Limit search to specific KB" })),
255
+ kb_id: Type.Optional(Type.String({ description: "Limit search to a specific KB by ID or exact name" })),
197
256
  offset: Type.Optional(Type.Number({ description: "Pagination offset" })),
198
257
  file_type: Type.Optional(Type.String({ description: "Filter by file type (e.g. typescript, markdown, python)" })),
258
+ diversity: Type.Optional(Type.Union([Type.Literal("off"), Type.Literal("balanced"), Type.Literal("strong")])),
199
259
  }),
200
260
  renderCall(args, theme: ThemeLike, context: RenderContextLike) {
201
261
  const query = typeof args.query === "string" ? args.query : "";
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "pi-knowledge",
3
- "version": "0.3.3",
3
+ "version": "0.3.6",
4
4
  "description": "Vector knowledge base extension for Pi — index files, hybrid search, local embeddings",
5
5
  "type": "module",
6
6
  "main": "index.ts",
@@ -9,8 +9,11 @@ type FeatureExtractionPipeline = {
9
9
  let pipelineInstance: FeatureExtractionPipeline | null = null;
10
10
  let disposeTimer: ReturnType<typeof setTimeout> | null = null;
11
11
  let disposePromise: Promise<void> | null = null;
12
+ let activeRuns = 0;
13
+ let disposeRequested = false;
14
+ const idleWaiters: Array<() => void> = [];
12
15
 
13
- const IDLE_TIMEOUT_MS = 30_000;
16
+ const IDLE_TIMEOUT_MS = Number(process.env.PI_KNOWLEDGE_EMBEDDING_IDLE_MS ?? 30_000);
14
17
  const EMBEDDING_CONFIG = process.env.PI_KNOWLEDGE_EMBEDDING ?? "local:multilingual-e5-small";
15
18
 
16
19
  function getModelCacheDir(): string {
@@ -29,26 +32,57 @@ async function loadPipeline(): Promise<FeatureExtractionPipeline> {
29
32
  return pipelineInstance;
30
33
  }
31
34
 
32
- function resetIdleTimer(): void {
35
+ function clearIdleTimer(): void {
33
36
  if (disposeTimer) clearTimeout(disposeTimer);
37
+ disposeTimer = null;
38
+ }
39
+
40
+ function scheduleIdleDispose(): void {
41
+ if (activeRuns > 0 || disposeRequested) return;
42
+ clearIdleTimer();
34
43
  disposeTimer = setTimeout(() => dispose(), IDLE_TIMEOUT_MS);
35
44
  }
36
45
 
46
+ function beginRun(): void {
47
+ activeRuns++;
48
+ clearIdleTimer();
49
+ }
50
+
51
+ function endRun(): void {
52
+ activeRuns--;
53
+ if (activeRuns > 0) return;
54
+ for (const resolve of idleWaiters.splice(0)) resolve();
55
+ if (!disposeRequested) scheduleIdleDispose();
56
+ }
57
+
58
+ function waitForNoActiveRuns(): Promise<void> {
59
+ if (activeRuns === 0) return Promise.resolve();
60
+ return new Promise((resolve) => idleWaiters.push(resolve));
61
+ }
62
+
37
63
  export async function dispose(): Promise<void> {
38
- if (disposeTimer) {
39
- clearTimeout(disposeTimer);
40
- disposeTimer = null;
41
- }
64
+ clearIdleTimer();
42
65
  if (disposePromise) return disposePromise;
66
+ disposeRequested = true;
67
+ await waitForNoActiveRuns();
43
68
  const instance = pipelineInstance;
44
69
  pipelineInstance = null;
45
- if (!instance) return;
70
+ if (!instance) {
71
+ disposeRequested = false;
72
+ return;
73
+ }
46
74
  disposePromise = Promise.resolve(instance.dispose()).finally(() => {
47
75
  disposePromise = null;
76
+ disposeRequested = false;
48
77
  });
49
78
  return disposePromise;
50
79
  }
51
80
 
81
+ export async function prepareForShutdown(): Promise<void> {
82
+ clearIdleTimer();
83
+ await waitForNoActiveRuns();
84
+ }
85
+
52
86
  async function embedViaAPI(texts: string[], prefix: "query" | "passage"): Promise<Float32Array[]> {
53
87
  const [provider, model] = EMBEDDING_CONFIG.split(":");
54
88
  const prefixedTexts = texts.map((t) => `${prefix}: ${t}`);
@@ -82,12 +116,16 @@ export async function embedTexts(
82
116
  }
83
117
  }
84
118
  const pipe = await loadPipeline();
85
- resetIdleTimer();
119
+ beginRun();
86
120
  const results: Float32Array[] = [];
87
- for (const text of texts) {
88
- if (signal?.aborted) throw new Error("Cancelled");
89
- const output = await pipe(`${prefix}: ${text}`, { pooling: "mean", normalize: true });
90
- results.push(new Float32Array(output.data));
121
+ try {
122
+ for (const text of texts) {
123
+ if (signal?.aborted) throw new Error("Cancelled");
124
+ const output = await pipe(`${prefix}: ${text}`, { pooling: "mean", normalize: true });
125
+ results.push(new Float32Array(output.data));
126
+ }
127
+ } finally {
128
+ endRun();
91
129
  }
92
130
  return results;
93
131
  }