pi-knowledge 0.1.4 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,23 @@
1
+ ---
2
+ name: search-docs
3
+ description: Search indexed knowledge bases for relevant documentation or code
4
+ ---
5
+
6
+ # Search Knowledge Base
7
+
8
+ Search the indexed knowledge bases for: $ARGUMENTS
9
+
10
+ ## Steps
11
+
12
+ 1. Use `knowledge_show` to see available knowledge bases
13
+ 2. Use `knowledge_search` with the user's query (hybrid mode by default)
14
+ 3. If results are insufficient, try mode "deep" for cross-encoder reranking
15
+ 4. For exact symbol lookups, use mode "fast" (BM25 only)
16
+ 5. Present results with file paths and relevant snippets
17
+
18
+ ## Tips
19
+
20
+ - Code queries: use function names, class names, or API signatures
21
+ - Concept queries: use natural language descriptions
22
+ - Filter: `file_type: "typescript"` or `file_type: "python"`
23
+ - No results? Check `knowledge_show` to confirm content is indexed
package/CHANGELOG.md CHANGED
@@ -1,5 +1,32 @@
1
1
  # Changelog
2
2
 
3
+ ## [0.2.2] - 2026-06-15
4
+
5
+ ### Fixed
6
+ - AbortSignal now threads end-to-end (tool → engine → embedding loop)
7
+ - Cancellation actually works during long embedding operations
8
+
9
+ ### Added
10
+ - TUI custom rendering for knowledge_search (renderCall + renderResult)
11
+ - All DESIGN.md phases complete (30/30 checkboxes)
12
+
13
+ ## [0.2.1] - 2026-06-15
14
+
15
+ ### Fixed
16
+ - README lists all 9 tools (was missing export/import)
17
+ - npm package includes .pi/skills/ for skill distribution
18
+
19
+ ## [0.2.0] - 2026-06-15
20
+
21
+ ### Added
22
+ - URL indexing (fetch → HTML strip → chunk)
23
+ - PDF text extraction (via unpdf, pure JS)
24
+ - DOCX text extraction (via mammoth, pure JS)
25
+ - Import/export knowledge bases (JSONL format, git-friendly)
26
+ - Performance benchmarks (BM25: 0.05ms, hybrid: 2.1ms)
27
+ - Pi Skill: /skill:search-docs
28
+ - 9 tools total
29
+
3
30
  ## [0.1.4] - 2026-06-15
4
31
 
5
32
  ### Added
@@ -9,6 +36,9 @@
9
36
  - Model mismatch warning on search (suggests re-index)
10
37
  - Engine regression tests (+6, total 34)
11
38
  - README Data Storage section
39
+ - URL indexing (knowledge_add with http/https URLs, auto HTML strip)
40
+ - Performance benchmarks (BM25: 0.05ms, hybrid: 2.1ms, semantic: 2.0ms)
41
+ - Pi Skill: `/skill:search-docs` for guided knowledge search
12
42
 
13
43
  ### Fixed
14
44
  - walkDir skips permission-denied directories
package/README.md CHANGED
@@ -56,13 +56,15 @@ pi install ./pi-knowledge
56
56
 
57
57
  | Tool | Description |
58
58
  |------|-------------|
59
- | `knowledge_add` | Index files, directories, or inline text into a named knowledge base |
59
+ | `knowledge_add` | Index files, directories, URLs, PDFs, DOCX, or inline text |
60
60
  | `knowledge_search` | Hybrid search across one or all knowledge bases |
61
61
  | `knowledge_remove` | Remove a knowledge base by name or ID |
62
62
  | `knowledge_update` | Incrementally re-index changed files in a knowledge base |
63
63
  | `knowledge_show` | List all knowledge bases with stats |
64
64
  | `knowledge_status` | Show engine status with health diagnostics (stale, orphans, coverage) |
65
65
  | `knowledge_clear` | Remove all knowledge bases |
66
+ | `knowledge_export` | Export a KB to shareable JSONL file |
67
+ | `knowledge_import` | Import a KB from JSONL (re-embeds content) |
66
68
 
67
69
  ## Architecture
68
70
 
package/index.ts CHANGED
@@ -1,4 +1,5 @@
1
1
  import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
2
+ import { Text } from "@earendil-works/pi-tui";
2
3
  import { Type } from "typebox";
3
4
  import { KnowledgeEngine } from "./src/engine.ts";
4
5
  import { getDefaultKnowledgeDir } from "./src/storage/sqlite.ts";
@@ -71,7 +72,7 @@ export default function (pi: ExtensionAPI) {
71
72
  const { source, name } = params;
72
73
  const { kb, chunkCount } = await engine.add(source, name, (msg) => {
73
74
  onUpdate?.({ content: [{ type: "text", text: msg }] });
74
- });
75
+ }, _signal);
75
76
  // Start watcher for new directory KB
76
77
  if (WATCH_ENABLED && kb.source_path && kb.source_type === "directory") {
77
78
  startWatcher(kb.id, kb.source_path, (kbId) => { engine.update(kbId).catch(() => {}); });
@@ -100,6 +101,15 @@ export default function (pi: ExtensionAPI) {
100
101
  offset: Type.Optional(Type.Number({ description: "Pagination offset" })),
101
102
  file_type: Type.Optional(Type.String({ description: "Filter by file type (e.g. typescript, markdown, python)" })),
102
103
  }),
104
+ renderCall(args, theme) {
105
+ const mode = args.mode || "hybrid";
106
+ return new Text(`${theme.fg("accent", "Search")}: "${args.query}" ${theme.fg("muted", `(${mode})`)}`, 0, 0);
107
+ },
108
+ renderResult(result, options, theme) {
109
+ const text = result.content?.[0]?.text ?? "";
110
+ const lines = options.expanded ? text : text.split("\n").slice(0, 5).join("\n");
111
+ return new Text(lines, 0, 0);
112
+ },
103
113
  async execute(_id, params) {
104
114
  const { query, mode, limit, kb_id, offset, file_type } = params;
105
115
  const filters = file_type ? { file_type } : undefined;
@@ -129,13 +139,14 @@ export default function (pi: ExtensionAPI) {
129
139
  async execute(_id, params, _signal, onUpdate) {
130
140
  const { added, removed, unchanged } = await engine.update(params.target, (msg) => {
131
141
  onUpdate?.({ content: [{ type: "text", text: msg }] });
132
- });
142
+ }, _signal);
133
143
  return {
134
144
  content: [{ type: "text", text: `Updated: +${added} added, -${removed} removed, ${unchanged} unchanged.` }],
135
145
  };
136
146
  },
137
147
  });
138
148
 
149
+
139
150
  pi.registerTool({
140
151
  name: "knowledge_status",
141
152
  label: "Knowledge Status",
@@ -191,6 +202,35 @@ export default function (pi: ExtensionAPI) {
191
202
  },
192
203
  });
193
204
 
205
+ pi.registerTool({
206
+ name: "knowledge_export",
207
+ label: "Knowledge Export",
208
+ description: "Export a knowledge base to a JSONL file (shareable, git-friendly)",
209
+ parameters: Type.Object({
210
+ target: Type.String({ description: "KB name or ID to export" }),
211
+ output: Type.String({ description: "Output file path (.jsonl)" }),
212
+ }),
213
+ async execute(_id, params) {
214
+ const count = await engine.exportKB(params.target, params.output);
215
+ return { content: [{ type: "text", text: `Exported ${count} chunks to ${params.output}` }] };
216
+ },
217
+ });
218
+
219
+ pi.registerTool({
220
+ name: "knowledge_import",
221
+ label: "Knowledge Import",
222
+ description: "Import a knowledge base from a JSONL file (re-embeds content)",
223
+ parameters: Type.Object({
224
+ input: Type.String({ description: "Input JSONL file path" }),
225
+ }),
226
+ async execute(_id, params, _signal, onUpdate) {
227
+ const { kb, chunkCount } = await engine.importKB(params.input, (msg) => {
228
+ onUpdate?.({ content: [{ type: "text", text: msg }] });
229
+ }, _signal);
230
+ return { content: [{ type: "text", text: `Imported "${kb.name}": ${chunkCount} chunks (re-embedded)` }] };
231
+ },
232
+ });
233
+
194
234
  pi.registerTool({
195
235
  name: "knowledge_clear",
196
236
  label: "Knowledge Clear",
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "pi-knowledge",
3
- "version": "0.1.4",
3
+ "version": "0.2.2",
4
4
  "description": "Vector knowledge base extension for Pi — index files, hybrid search, local embeddings",
5
5
  "type": "module",
6
6
  "main": "index.ts",
@@ -12,6 +12,7 @@
12
12
  "files": [
13
13
  "index.ts",
14
14
  "src/",
15
+ ".pi/",
15
16
  "README.md",
16
17
  "CHANGELOG.md",
17
18
  "LICENSE"
@@ -35,12 +36,14 @@
35
36
  "@huggingface/transformers": "3.8.1",
36
37
  "better-sqlite3": "11.9.1",
37
38
  "ignore": "7.0.5",
39
+ "mammoth": "1.12.0",
38
40
  "tree-sitter": "0.22.4",
39
41
  "tree-sitter-go": "0.23.4",
40
42
  "tree-sitter-java": "0.23.5",
41
43
  "tree-sitter-python": "0.23.6",
42
44
  "tree-sitter-rust": "0.23.2",
43
- "tree-sitter-typescript": "0.23.2"
45
+ "tree-sitter-typescript": "0.23.2",
46
+ "unpdf": "1.6.2"
44
47
  },
45
48
  "devDependencies": {
46
49
  "@biomejs/biome": "2.3.5",
package/src/engine.ts CHANGED
@@ -3,7 +3,7 @@ import { join, resolve } from "node:path";
3
3
  import type Database from "better-sqlite3";
4
4
  import { embedDocuments, embedQuery, dispose as disposeEmbedding } from "./embedding/provider.ts";
5
5
  import { loadVectors, saveVectors } from "./embedding/vectors.ts";
6
- import { chunkFile, walkDir } from "./indexer/chunker.ts";
6
+ import { chunkFile, walkDir, contentHash, preTokenizeForFTS } from "./indexer/chunker.ts";
7
7
  import { searchBM25 } from "./search/bm25.ts";
8
8
  import { reciprocalRankFusion } from "./search/fusion.ts";
9
9
  import { rerank, disposeReranker } from "./search/reranker.ts";
@@ -68,20 +68,42 @@ export class KnowledgeEngine {
68
68
  this.vectorCache.delete(kbId);
69
69
  }
70
70
 
71
- async add(source: string, name: string, onProgress?: ProgressCallback): Promise<{ kb: KnowledgeBase; chunkCount: number }> {
71
+ async add(source: string, name: string, onProgress?: ProgressCallback, signal?: AbortSignal): Promise<{ kb: KnowledgeBase; chunkCount: number }> {
72
72
  if (!this.db) throw new Error("Engine not initialized");
73
73
  const resolvedSource = resolve(source);
74
- const isDir = existsSync(resolvedSource) && statSync(resolvedSource).isDirectory();
75
- const isFile = existsSync(resolvedSource) && statSync(resolvedSource).isFile();
74
+ const isUrl = source.startsWith("http://") || source.startsWith("https://");
75
+ const isDir = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isDirectory();
76
+ const isFile = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isFile();
76
77
  const sourceType = isDir ? "directory" : isFile ? "file" : "text";
77
78
 
78
- const kb = createKB(this.db, { name, source_path: isDir || isFile ? resolvedSource : undefined, source_type: sourceType });
79
+ const kb = createKB(this.db, { name, source_path: isDir || isFile ? resolvedSource : isUrl ? source : undefined, source_type: sourceType });
79
80
  updateKBStatus(this.db, kb.id, "indexing");
80
81
 
81
82
  try {
82
83
  let allChunks: Awaited<ReturnType<typeof chunkFile>> = [];
83
84
 
84
- if (isDir) {
85
+ if (isUrl) {
86
+ onProgress?.(`Fetching ${source}...`);
87
+ const res = await fetch(source);
88
+ if (!res.ok) throw new Error(`Fetch failed: ${res.status}`);
89
+ const html = await res.text();
90
+ const text = html.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "")
91
+ .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "")
92
+ .replace(/<[^>]+>/g, " ").replace(/&[a-z]+;/gi, " ")
93
+ .replace(/\s+/g, " ").trim();
94
+ allChunks = await chunkFile(text, source);
95
+ } else if (isFile && resolvedSource.endsWith(".pdf")) {
96
+ onProgress?.("Extracting text from PDF...");
97
+ const { extractText } = await import("unpdf");
98
+ const buf = (await import("node:fs")).readFileSync(resolvedSource);
99
+ const { text } = await extractText(new Uint8Array(buf));
100
+ allChunks = await chunkFile(text, resolvedSource);
101
+ } else if (isFile && (resolvedSource.endsWith(".docx") || resolvedSource.endsWith(".doc"))) {
102
+ onProgress?.("Extracting text from DOCX...");
103
+ const mammoth = await import("mammoth");
104
+ const result = await mammoth.extractRawText({ path: resolvedSource });
105
+ allChunks = await chunkFile(result.value, resolvedSource);
106
+ } else if (isDir) {
85
107
  onProgress?.(`Scanning ${resolvedSource}...`);
86
108
  const files = walkDir(resolvedSource);
87
109
  onProgress?.(`Found ${files.length} files, chunking...`);
@@ -98,7 +120,7 @@ export class KnowledgeEngine {
98
120
 
99
121
  onProgress?.(`${allChunks.length} chunks, embedding...`);
100
122
  const texts = allChunks.map((c) => c.content);
101
- const vectors = await embedDocuments(texts);
123
+ const vectors = await embedDocuments(texts, signal);
102
124
 
103
125
  onProgress?.(`Storing...`);
104
126
  insertChunks(this.db, kb.id, allChunks);
@@ -119,7 +141,7 @@ export class KnowledgeEngine {
119
141
  }
120
142
  }
121
143
 
122
- async update(nameOrId: string, onProgress?: ProgressCallback): Promise<{ added: number; removed: number; unchanged: number }> {
144
+ async update(nameOrId: string, onProgress?: ProgressCallback, signal?: AbortSignal): Promise<{ added: number; removed: number; unchanged: number }> {
123
145
  if (!this.db) throw new Error("Engine not initialized");
124
146
  const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
125
147
  if (!kb) throw new Error(`Knowledge base not found: ${nameOrId}`);
@@ -316,6 +338,40 @@ export class KnowledgeEngine {
316
338
  return listKBs(this.db).map((kb) => diagnoseKB(this.db!, kb));
317
339
  }
318
340
 
341
+ async exportKB(nameOrId: string, outputPath: string): Promise<number> {
342
+ if (!this.db) throw new Error("Engine not initialized");
343
+ const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
344
+ if (!kb) throw new Error(`Knowledge base not found: ${nameOrId}`);
345
+ const chunks = getChunksByKB(this.db, kb.id);
346
+ const { writeFileSync } = await import("node:fs");
347
+ const header = JSON.stringify({ name: kb.name, description: kb.description, source_path: kb.source_path, source_type: kb.source_type, chunk_count: chunks.length });
348
+ const lines = [header, ...chunks.map((c) => JSON.stringify({ content: c.content, file_path: c.file_path, file_type: c.file_type, start_line: c.start_line, end_line: c.end_line, metadata_json: c.metadata_json }))];
349
+ writeFileSync(outputPath, lines.join("\n") + "\n");
350
+ return chunks.length;
351
+ }
352
+
353
+ async importKB(inputPath: string, onProgress?: ProgressCallback, signal?: AbortSignal): Promise<{ kb: KnowledgeBase; chunkCount: number }> {
354
+ if (!this.db) throw new Error("Engine not initialized");
355
+ const { readFileSync } = await import("node:fs");
356
+ const lines = readFileSync(inputPath, "utf-8").trim().split("\n");
357
+ if (lines.length < 1) throw new Error("Empty import file");
358
+ const header = JSON.parse(lines[0]);
359
+ const kb = createKB(this.db, { name: header.name, description: header.description, source_path: header.source_path, source_type: header.source_type || "text" });
360
+ updateKBStatus(this.db, kb.id, "indexing");
361
+ const chunkData = lines.slice(1).map((l) => JSON.parse(l));
362
+ onProgress?.(`Importing ${chunkData.length} chunks, embedding...`);
363
+ const texts = chunkData.map((c: any) => c.content);
364
+ const vectors = await embedDocuments(texts, signal);
365
+ const chunks = chunkData.map((c: any) => ({ content_hash: contentHash(c.content), content: c.content, content_tokenized: preTokenizeForFTS(c.content), file_path: c.file_path, file_type: c.file_type, start_line: c.start_line, end_line: c.end_line, metadata_json: c.metadata_json || "{}" }));
366
+ insertChunks(this.db, kb.id, chunks);
367
+ const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
368
+ saveVectors(vectorPath, vectors);
369
+ this.vectorCache.set(kb.id, vectors);
370
+ updateKBCounts(this.db, kb.id, chunks.length, new Set(chunks.map((c: any) => c.file_path)).size);
371
+ updateKBStatus(this.db, kb.id, "ready");
372
+ return { kb: getKB(this.db, kb.id)!, chunkCount: chunks.length };
373
+ }
374
+
319
375
  async dispose(): Promise<void> {
320
376
  await disposeEmbedding();
321
377
  await disposeReranker();