pi-knowledge 0.1.4 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.pi/skills/search-docs.md +23 -0
- package/CHANGELOG.md +30 -0
- package/README.md +3 -1
- package/index.ts +42 -2
- package/package.json +5 -2
- package/src/engine.ts +64 -8
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: search-docs
|
|
3
|
+
description: Search indexed knowledge bases for relevant documentation or code
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Search Knowledge Base
|
|
7
|
+
|
|
8
|
+
Search the indexed knowledge bases for: $ARGUMENTS
|
|
9
|
+
|
|
10
|
+
## Steps
|
|
11
|
+
|
|
12
|
+
1. Use `knowledge_show` to see available knowledge bases
|
|
13
|
+
2. Use `knowledge_search` with the user's query (hybrid mode by default)
|
|
14
|
+
3. If results are insufficient, try mode "deep" for cross-encoder reranking
|
|
15
|
+
4. For exact symbol lookups, use mode "fast" (BM25 only)
|
|
16
|
+
5. Present results with file paths and relevant snippets
|
|
17
|
+
|
|
18
|
+
## Tips
|
|
19
|
+
|
|
20
|
+
- Code queries: use function names, class names, or API signatures
|
|
21
|
+
- Concept queries: use natural language descriptions
|
|
22
|
+
- Filter: `file_type: "typescript"` or `file_type: "python"`
|
|
23
|
+
- No results? Check `knowledge_show` to confirm content is indexed
|
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,32 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [0.2.2] - 2026-06-15
|
|
4
|
+
|
|
5
|
+
### Fixed
|
|
6
|
+
- AbortSignal now threads end-to-end (tool → engine → embedding loop)
|
|
7
|
+
- Cancellation actually works during long embedding operations
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
- TUI custom rendering for knowledge_search (renderCall + renderResult)
|
|
11
|
+
- All DESIGN.md phases complete (30/30 checkboxes)
|
|
12
|
+
|
|
13
|
+
## [0.2.1] - 2026-06-15
|
|
14
|
+
|
|
15
|
+
### Fixed
|
|
16
|
+
- README lists all 9 tools (was missing export/import)
|
|
17
|
+
- npm package includes .pi/skills/ for skill distribution
|
|
18
|
+
|
|
19
|
+
## [0.2.0] - 2026-06-15
|
|
20
|
+
|
|
21
|
+
### Added
|
|
22
|
+
- URL indexing (fetch → HTML strip → chunk)
|
|
23
|
+
- PDF text extraction (via unpdf, pure JS)
|
|
24
|
+
- DOCX text extraction (via mammoth, pure JS)
|
|
25
|
+
- Import/export knowledge bases (JSONL format, git-friendly)
|
|
26
|
+
- Performance benchmarks (BM25: 0.05ms, hybrid: 2.1ms)
|
|
27
|
+
- Pi Skill: /skill:search-docs
|
|
28
|
+
- 9 tools total
|
|
29
|
+
|
|
3
30
|
## [0.1.4] - 2026-06-15
|
|
4
31
|
|
|
5
32
|
### Added
|
|
@@ -9,6 +36,9 @@
|
|
|
9
36
|
- Model mismatch warning on search (suggests re-index)
|
|
10
37
|
- Engine regression tests (+6, total 34)
|
|
11
38
|
- README Data Storage section
|
|
39
|
+
- URL indexing (knowledge_add with http/https URLs, auto HTML strip)
|
|
40
|
+
- Performance benchmarks (BM25: 0.05ms, hybrid: 2.1ms, semantic: 2.0ms)
|
|
41
|
+
- Pi Skill: `/skill:search-docs` for guided knowledge search
|
|
12
42
|
|
|
13
43
|
### Fixed
|
|
14
44
|
- walkDir skips permission-denied directories
|
package/README.md
CHANGED
|
@@ -56,13 +56,15 @@ pi install ./pi-knowledge
|
|
|
56
56
|
|
|
57
57
|
| Tool | Description |
|
|
58
58
|
|------|-------------|
|
|
59
|
-
| `knowledge_add` | Index files, directories, or inline text
|
|
59
|
+
| `knowledge_add` | Index files, directories, URLs, PDFs, DOCX, or inline text |
|
|
60
60
|
| `knowledge_search` | Hybrid search across one or all knowledge bases |
|
|
61
61
|
| `knowledge_remove` | Remove a knowledge base by name or ID |
|
|
62
62
|
| `knowledge_update` | Incrementally re-index changed files in a knowledge base |
|
|
63
63
|
| `knowledge_show` | List all knowledge bases with stats |
|
|
64
64
|
| `knowledge_status` | Show engine status with health diagnostics (stale, orphans, coverage) |
|
|
65
65
|
| `knowledge_clear` | Remove all knowledge bases |
|
|
66
|
+
| `knowledge_export` | Export a KB to shareable JSONL file |
|
|
67
|
+
| `knowledge_import` | Import a KB from JSONL (re-embeds content) |
|
|
66
68
|
|
|
67
69
|
## Architecture
|
|
68
70
|
|
package/index.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
2
|
+
import { Text } from "@earendil-works/pi-tui";
|
|
2
3
|
import { Type } from "typebox";
|
|
3
4
|
import { KnowledgeEngine } from "./src/engine.ts";
|
|
4
5
|
import { getDefaultKnowledgeDir } from "./src/storage/sqlite.ts";
|
|
@@ -71,7 +72,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
71
72
|
const { source, name } = params;
|
|
72
73
|
const { kb, chunkCount } = await engine.add(source, name, (msg) => {
|
|
73
74
|
onUpdate?.({ content: [{ type: "text", text: msg }] });
|
|
74
|
-
});
|
|
75
|
+
}, _signal);
|
|
75
76
|
// Start watcher for new directory KB
|
|
76
77
|
if (WATCH_ENABLED && kb.source_path && kb.source_type === "directory") {
|
|
77
78
|
startWatcher(kb.id, kb.source_path, (kbId) => { engine.update(kbId).catch(() => {}); });
|
|
@@ -100,6 +101,15 @@ export default function (pi: ExtensionAPI) {
|
|
|
100
101
|
offset: Type.Optional(Type.Number({ description: "Pagination offset" })),
|
|
101
102
|
file_type: Type.Optional(Type.String({ description: "Filter by file type (e.g. typescript, markdown, python)" })),
|
|
102
103
|
}),
|
|
104
|
+
renderCall(args, theme) {
|
|
105
|
+
const mode = args.mode || "hybrid";
|
|
106
|
+
return new Text(`${theme.fg("accent", "Search")}: "${args.query}" ${theme.fg("muted", `(${mode})`)}`, 0, 0);
|
|
107
|
+
},
|
|
108
|
+
renderResult(result, options, theme) {
|
|
109
|
+
const text = result.content?.[0]?.text ?? "";
|
|
110
|
+
const lines = options.expanded ? text : text.split("\n").slice(0, 5).join("\n");
|
|
111
|
+
return new Text(lines, 0, 0);
|
|
112
|
+
},
|
|
103
113
|
async execute(_id, params) {
|
|
104
114
|
const { query, mode, limit, kb_id, offset, file_type } = params;
|
|
105
115
|
const filters = file_type ? { file_type } : undefined;
|
|
@@ -129,13 +139,14 @@ export default function (pi: ExtensionAPI) {
|
|
|
129
139
|
async execute(_id, params, _signal, onUpdate) {
|
|
130
140
|
const { added, removed, unchanged } = await engine.update(params.target, (msg) => {
|
|
131
141
|
onUpdate?.({ content: [{ type: "text", text: msg }] });
|
|
132
|
-
});
|
|
142
|
+
}, _signal);
|
|
133
143
|
return {
|
|
134
144
|
content: [{ type: "text", text: `Updated: +${added} added, -${removed} removed, ${unchanged} unchanged.` }],
|
|
135
145
|
};
|
|
136
146
|
},
|
|
137
147
|
});
|
|
138
148
|
|
|
149
|
+
|
|
139
150
|
pi.registerTool({
|
|
140
151
|
name: "knowledge_status",
|
|
141
152
|
label: "Knowledge Status",
|
|
@@ -191,6 +202,35 @@ export default function (pi: ExtensionAPI) {
|
|
|
191
202
|
},
|
|
192
203
|
});
|
|
193
204
|
|
|
205
|
+
pi.registerTool({
|
|
206
|
+
name: "knowledge_export",
|
|
207
|
+
label: "Knowledge Export",
|
|
208
|
+
description: "Export a knowledge base to a JSONL file (shareable, git-friendly)",
|
|
209
|
+
parameters: Type.Object({
|
|
210
|
+
target: Type.String({ description: "KB name or ID to export" }),
|
|
211
|
+
output: Type.String({ description: "Output file path (.jsonl)" }),
|
|
212
|
+
}),
|
|
213
|
+
async execute(_id, params) {
|
|
214
|
+
const count = await engine.exportKB(params.target, params.output);
|
|
215
|
+
return { content: [{ type: "text", text: `Exported ${count} chunks to ${params.output}` }] };
|
|
216
|
+
},
|
|
217
|
+
});
|
|
218
|
+
|
|
219
|
+
pi.registerTool({
|
|
220
|
+
name: "knowledge_import",
|
|
221
|
+
label: "Knowledge Import",
|
|
222
|
+
description: "Import a knowledge base from a JSONL file (re-embeds content)",
|
|
223
|
+
parameters: Type.Object({
|
|
224
|
+
input: Type.String({ description: "Input JSONL file path" }),
|
|
225
|
+
}),
|
|
226
|
+
async execute(_id, params, _signal, onUpdate) {
|
|
227
|
+
const { kb, chunkCount } = await engine.importKB(params.input, (msg) => {
|
|
228
|
+
onUpdate?.({ content: [{ type: "text", text: msg }] });
|
|
229
|
+
}, _signal);
|
|
230
|
+
return { content: [{ type: "text", text: `Imported "${kb.name}": ${chunkCount} chunks (re-embedded)` }] };
|
|
231
|
+
},
|
|
232
|
+
});
|
|
233
|
+
|
|
194
234
|
pi.registerTool({
|
|
195
235
|
name: "knowledge_clear",
|
|
196
236
|
label: "Knowledge Clear",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-knowledge",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.2",
|
|
4
4
|
"description": "Vector knowledge base extension for Pi — index files, hybrid search, local embeddings",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "index.ts",
|
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
"files": [
|
|
13
13
|
"index.ts",
|
|
14
14
|
"src/",
|
|
15
|
+
".pi/",
|
|
15
16
|
"README.md",
|
|
16
17
|
"CHANGELOG.md",
|
|
17
18
|
"LICENSE"
|
|
@@ -35,12 +36,14 @@
|
|
|
35
36
|
"@huggingface/transformers": "3.8.1",
|
|
36
37
|
"better-sqlite3": "11.9.1",
|
|
37
38
|
"ignore": "7.0.5",
|
|
39
|
+
"mammoth": "1.12.0",
|
|
38
40
|
"tree-sitter": "0.22.4",
|
|
39
41
|
"tree-sitter-go": "0.23.4",
|
|
40
42
|
"tree-sitter-java": "0.23.5",
|
|
41
43
|
"tree-sitter-python": "0.23.6",
|
|
42
44
|
"tree-sitter-rust": "0.23.2",
|
|
43
|
-
"tree-sitter-typescript": "0.23.2"
|
|
45
|
+
"tree-sitter-typescript": "0.23.2",
|
|
46
|
+
"unpdf": "1.6.2"
|
|
44
47
|
},
|
|
45
48
|
"devDependencies": {
|
|
46
49
|
"@biomejs/biome": "2.3.5",
|
package/src/engine.ts
CHANGED
|
@@ -3,7 +3,7 @@ import { join, resolve } from "node:path";
|
|
|
3
3
|
import type Database from "better-sqlite3";
|
|
4
4
|
import { embedDocuments, embedQuery, dispose as disposeEmbedding } from "./embedding/provider.ts";
|
|
5
5
|
import { loadVectors, saveVectors } from "./embedding/vectors.ts";
|
|
6
|
-
import { chunkFile, walkDir } from "./indexer/chunker.ts";
|
|
6
|
+
import { chunkFile, walkDir, contentHash, preTokenizeForFTS } from "./indexer/chunker.ts";
|
|
7
7
|
import { searchBM25 } from "./search/bm25.ts";
|
|
8
8
|
import { reciprocalRankFusion } from "./search/fusion.ts";
|
|
9
9
|
import { rerank, disposeReranker } from "./search/reranker.ts";
|
|
@@ -68,20 +68,42 @@ export class KnowledgeEngine {
|
|
|
68
68
|
this.vectorCache.delete(kbId);
|
|
69
69
|
}
|
|
70
70
|
|
|
71
|
-
async add(source: string, name: string, onProgress?: ProgressCallback): Promise<{ kb: KnowledgeBase; chunkCount: number }> {
|
|
71
|
+
async add(source: string, name: string, onProgress?: ProgressCallback, signal?: AbortSignal): Promise<{ kb: KnowledgeBase; chunkCount: number }> {
|
|
72
72
|
if (!this.db) throw new Error("Engine not initialized");
|
|
73
73
|
const resolvedSource = resolve(source);
|
|
74
|
-
const
|
|
75
|
-
const
|
|
74
|
+
const isUrl = source.startsWith("http://") || source.startsWith("https://");
|
|
75
|
+
const isDir = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isDirectory();
|
|
76
|
+
const isFile = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isFile();
|
|
76
77
|
const sourceType = isDir ? "directory" : isFile ? "file" : "text";
|
|
77
78
|
|
|
78
|
-
const kb = createKB(this.db, { name, source_path: isDir || isFile ? resolvedSource : undefined, source_type: sourceType });
|
|
79
|
+
const kb = createKB(this.db, { name, source_path: isDir || isFile ? resolvedSource : isUrl ? source : undefined, source_type: sourceType });
|
|
79
80
|
updateKBStatus(this.db, kb.id, "indexing");
|
|
80
81
|
|
|
81
82
|
try {
|
|
82
83
|
let allChunks: Awaited<ReturnType<typeof chunkFile>> = [];
|
|
83
84
|
|
|
84
|
-
if (
|
|
85
|
+
if (isUrl) {
|
|
86
|
+
onProgress?.(`Fetching ${source}...`);
|
|
87
|
+
const res = await fetch(source);
|
|
88
|
+
if (!res.ok) throw new Error(`Fetch failed: ${res.status}`);
|
|
89
|
+
const html = await res.text();
|
|
90
|
+
const text = html.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "")
|
|
91
|
+
.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "")
|
|
92
|
+
.replace(/<[^>]+>/g, " ").replace(/&[a-z]+;/gi, " ")
|
|
93
|
+
.replace(/\s+/g, " ").trim();
|
|
94
|
+
allChunks = await chunkFile(text, source);
|
|
95
|
+
} else if (isFile && resolvedSource.endsWith(".pdf")) {
|
|
96
|
+
onProgress?.("Extracting text from PDF...");
|
|
97
|
+
const { extractText } = await import("unpdf");
|
|
98
|
+
const buf = (await import("node:fs")).readFileSync(resolvedSource);
|
|
99
|
+
const { text } = await extractText(new Uint8Array(buf));
|
|
100
|
+
allChunks = await chunkFile(text, resolvedSource);
|
|
101
|
+
} else if (isFile && (resolvedSource.endsWith(".docx") || resolvedSource.endsWith(".doc"))) {
|
|
102
|
+
onProgress?.("Extracting text from DOCX...");
|
|
103
|
+
const mammoth = await import("mammoth");
|
|
104
|
+
const result = await mammoth.extractRawText({ path: resolvedSource });
|
|
105
|
+
allChunks = await chunkFile(result.value, resolvedSource);
|
|
106
|
+
} else if (isDir) {
|
|
85
107
|
onProgress?.(`Scanning ${resolvedSource}...`);
|
|
86
108
|
const files = walkDir(resolvedSource);
|
|
87
109
|
onProgress?.(`Found ${files.length} files, chunking...`);
|
|
@@ -98,7 +120,7 @@ export class KnowledgeEngine {
|
|
|
98
120
|
|
|
99
121
|
onProgress?.(`${allChunks.length} chunks, embedding...`);
|
|
100
122
|
const texts = allChunks.map((c) => c.content);
|
|
101
|
-
const vectors = await embedDocuments(texts);
|
|
123
|
+
const vectors = await embedDocuments(texts, signal);
|
|
102
124
|
|
|
103
125
|
onProgress?.(`Storing...`);
|
|
104
126
|
insertChunks(this.db, kb.id, allChunks);
|
|
@@ -119,7 +141,7 @@ export class KnowledgeEngine {
|
|
|
119
141
|
}
|
|
120
142
|
}
|
|
121
143
|
|
|
122
|
-
async update(nameOrId: string, onProgress?: ProgressCallback): Promise<{ added: number; removed: number; unchanged: number }> {
|
|
144
|
+
async update(nameOrId: string, onProgress?: ProgressCallback, signal?: AbortSignal): Promise<{ added: number; removed: number; unchanged: number }> {
|
|
123
145
|
if (!this.db) throw new Error("Engine not initialized");
|
|
124
146
|
const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
|
|
125
147
|
if (!kb) throw new Error(`Knowledge base not found: ${nameOrId}`);
|
|
@@ -316,6 +338,40 @@ export class KnowledgeEngine {
|
|
|
316
338
|
return listKBs(this.db).map((kb) => diagnoseKB(this.db!, kb));
|
|
317
339
|
}
|
|
318
340
|
|
|
341
|
+
async exportKB(nameOrId: string, outputPath: string): Promise<number> {
|
|
342
|
+
if (!this.db) throw new Error("Engine not initialized");
|
|
343
|
+
const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
|
|
344
|
+
if (!kb) throw new Error(`Knowledge base not found: ${nameOrId}`);
|
|
345
|
+
const chunks = getChunksByKB(this.db, kb.id);
|
|
346
|
+
const { writeFileSync } = await import("node:fs");
|
|
347
|
+
const header = JSON.stringify({ name: kb.name, description: kb.description, source_path: kb.source_path, source_type: kb.source_type, chunk_count: chunks.length });
|
|
348
|
+
const lines = [header, ...chunks.map((c) => JSON.stringify({ content: c.content, file_path: c.file_path, file_type: c.file_type, start_line: c.start_line, end_line: c.end_line, metadata_json: c.metadata_json }))];
|
|
349
|
+
writeFileSync(outputPath, lines.join("\n") + "\n");
|
|
350
|
+
return chunks.length;
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
async importKB(inputPath: string, onProgress?: ProgressCallback, signal?: AbortSignal): Promise<{ kb: KnowledgeBase; chunkCount: number }> {
|
|
354
|
+
if (!this.db) throw new Error("Engine not initialized");
|
|
355
|
+
const { readFileSync } = await import("node:fs");
|
|
356
|
+
const lines = readFileSync(inputPath, "utf-8").trim().split("\n");
|
|
357
|
+
if (lines.length < 1) throw new Error("Empty import file");
|
|
358
|
+
const header = JSON.parse(lines[0]);
|
|
359
|
+
const kb = createKB(this.db, { name: header.name, description: header.description, source_path: header.source_path, source_type: header.source_type || "text" });
|
|
360
|
+
updateKBStatus(this.db, kb.id, "indexing");
|
|
361
|
+
const chunkData = lines.slice(1).map((l) => JSON.parse(l));
|
|
362
|
+
onProgress?.(`Importing ${chunkData.length} chunks, embedding...`);
|
|
363
|
+
const texts = chunkData.map((c: any) => c.content);
|
|
364
|
+
const vectors = await embedDocuments(texts, signal);
|
|
365
|
+
const chunks = chunkData.map((c: any) => ({ content_hash: contentHash(c.content), content: c.content, content_tokenized: preTokenizeForFTS(c.content), file_path: c.file_path, file_type: c.file_type, start_line: c.start_line, end_line: c.end_line, metadata_json: c.metadata_json || "{}" }));
|
|
366
|
+
insertChunks(this.db, kb.id, chunks);
|
|
367
|
+
const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
|
|
368
|
+
saveVectors(vectorPath, vectors);
|
|
369
|
+
this.vectorCache.set(kb.id, vectors);
|
|
370
|
+
updateKBCounts(this.db, kb.id, chunks.length, new Set(chunks.map((c: any) => c.file_path)).size);
|
|
371
|
+
updateKBStatus(this.db, kb.id, "ready");
|
|
372
|
+
return { kb: getKB(this.db, kb.id)!, chunkCount: chunks.length };
|
|
373
|
+
}
|
|
374
|
+
|
|
319
375
|
async dispose(): Promise<void> {
|
|
320
376
|
await disposeEmbedding();
|
|
321
377
|
await disposeReranker();
|