pi-knowledge 0.1.4 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,16 @@
1
1
  # Changelog
2
2
 
3
+ ## [0.2.0] - 2026-06-15
4
+
5
+ ### Added
6
+ - URL indexing (fetch → HTML strip → chunk)
7
+ - PDF text extraction (via unpdf, pure JS)
8
+ - DOCX text extraction (via mammoth, pure JS)
9
+ - Import/export knowledge bases (JSONL format, git-friendly)
10
+ - Performance benchmarks (BM25: 0.05ms, hybrid: 2.1ms)
11
+ - Pi Skill: /skill:search-docs
12
+ - 9 tools total
13
+
3
14
  ## [0.1.4] - 2026-06-15
4
15
 
5
16
  ### Added
@@ -9,6 +20,9 @@
9
20
  - Model mismatch warning on search (suggests re-index)
10
21
  - Engine regression tests (+6, total 34)
11
22
  - README Data Storage section
23
+ - URL indexing (knowledge_add with http/https URLs, auto HTML strip)
24
+ - Performance benchmarks (BM25: 0.05ms, hybrid: 2.1ms, semantic: 2.0ms)
25
+ - Pi Skill: `/skill:search-docs` for guided knowledge search
12
26
 
13
27
  ### Fixed
14
28
  - walkDir skips permission-denied directories
package/README.md CHANGED
@@ -56,7 +56,7 @@ pi install ./pi-knowledge
56
56
 
57
57
  | Tool | Description |
58
58
  |------|-------------|
59
- | `knowledge_add` | Index files, directories, or inline text into a named knowledge base |
59
+ | `knowledge_add` | Index files, directories, URLs, PDFs, DOCX, or inline text |
60
60
  | `knowledge_search` | Hybrid search across one or all knowledge bases |
61
61
  | `knowledge_remove` | Remove a knowledge base by name or ID |
62
62
  | `knowledge_update` | Incrementally re-index changed files in a knowledge base |
package/index.ts CHANGED
@@ -191,6 +191,35 @@ export default function (pi: ExtensionAPI) {
191
191
  },
192
192
  });
193
193
 
194
+ pi.registerTool({
195
+ name: "knowledge_export",
196
+ label: "Knowledge Export",
197
+ description: "Export a knowledge base to a JSONL file (shareable, git-friendly)",
198
+ parameters: Type.Object({
199
+ target: Type.String({ description: "KB name or ID to export" }),
200
+ output: Type.String({ description: "Output file path (.jsonl)" }),
201
+ }),
202
+ async execute(_id, params) {
203
+ const count = await engine.exportKB(params.target, params.output);
204
+ return { content: [{ type: "text", text: `Exported ${count} chunks to ${params.output}` }] };
205
+ },
206
+ });
207
+
208
+ pi.registerTool({
209
+ name: "knowledge_import",
210
+ label: "Knowledge Import",
211
+ description: "Import a knowledge base from a JSONL file (re-embeds content)",
212
+ parameters: Type.Object({
213
+ input: Type.String({ description: "Input JSONL file path" }),
214
+ }),
215
+ async execute(_id, params, _signal, onUpdate) {
216
+ const { kb, chunkCount } = await engine.importKB(params.input, (msg) => {
217
+ onUpdate?.({ content: [{ type: "text", text: msg }] });
218
+ });
219
+ return { content: [{ type: "text", text: `Imported "${kb.name}": ${chunkCount} chunks (re-embedded)` }] };
220
+ },
221
+ });
222
+
194
223
  pi.registerTool({
195
224
  name: "knowledge_clear",
196
225
  label: "Knowledge Clear",
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "pi-knowledge",
3
- "version": "0.1.4",
3
+ "version": "0.2.0",
4
4
  "description": "Vector knowledge base extension for Pi — index files, hybrid search, local embeddings",
5
5
  "type": "module",
6
6
  "main": "index.ts",
@@ -35,12 +35,14 @@
35
35
  "@huggingface/transformers": "3.8.1",
36
36
  "better-sqlite3": "11.9.1",
37
37
  "ignore": "7.0.5",
38
+ "mammoth": "1.12.0",
38
39
  "tree-sitter": "0.22.4",
39
40
  "tree-sitter-go": "0.23.4",
40
41
  "tree-sitter-java": "0.23.5",
41
42
  "tree-sitter-python": "0.23.6",
42
43
  "tree-sitter-rust": "0.23.2",
43
- "tree-sitter-typescript": "0.23.2"
44
+ "tree-sitter-typescript": "0.23.2",
45
+ "unpdf": "1.6.2"
44
46
  },
45
47
  "devDependencies": {
46
48
  "@biomejs/biome": "2.3.5",
package/src/engine.ts CHANGED
@@ -3,7 +3,7 @@ import { join, resolve } from "node:path";
3
3
  import type Database from "better-sqlite3";
4
4
  import { embedDocuments, embedQuery, dispose as disposeEmbedding } from "./embedding/provider.ts";
5
5
  import { loadVectors, saveVectors } from "./embedding/vectors.ts";
6
- import { chunkFile, walkDir } from "./indexer/chunker.ts";
6
+ import { chunkFile, walkDir, contentHash, preTokenizeForFTS } from "./indexer/chunker.ts";
7
7
  import { searchBM25 } from "./search/bm25.ts";
8
8
  import { reciprocalRankFusion } from "./search/fusion.ts";
9
9
  import { rerank, disposeReranker } from "./search/reranker.ts";
@@ -71,17 +71,39 @@ export class KnowledgeEngine {
71
71
  async add(source: string, name: string, onProgress?: ProgressCallback): Promise<{ kb: KnowledgeBase; chunkCount: number }> {
72
72
  if (!this.db) throw new Error("Engine not initialized");
73
73
  const resolvedSource = resolve(source);
74
- const isDir = existsSync(resolvedSource) && statSync(resolvedSource).isDirectory();
75
- const isFile = existsSync(resolvedSource) && statSync(resolvedSource).isFile();
74
+ const isUrl = source.startsWith("http://") || source.startsWith("https://");
75
+ const isDir = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isDirectory();
76
+ const isFile = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isFile();
76
77
  const sourceType = isDir ? "directory" : isFile ? "file" : "text";
77
78
 
78
- const kb = createKB(this.db, { name, source_path: isDir || isFile ? resolvedSource : undefined, source_type: sourceType });
79
+ const kb = createKB(this.db, { name, source_path: isDir || isFile ? resolvedSource : isUrl ? source : undefined, source_type: sourceType });
79
80
  updateKBStatus(this.db, kb.id, "indexing");
80
81
 
81
82
  try {
82
83
  let allChunks: Awaited<ReturnType<typeof chunkFile>> = [];
83
84
 
84
- if (isDir) {
85
+ if (isUrl) {
86
+ onProgress?.(`Fetching ${source}...`);
87
+ const res = await fetch(source);
88
+ if (!res.ok) throw new Error(`Fetch failed: ${res.status}`);
89
+ const html = await res.text();
90
+ const text = html.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "")
91
+ .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "")
92
+ .replace(/<[^>]+>/g, " ").replace(/&[a-z]+;/gi, " ")
93
+ .replace(/\s+/g, " ").trim();
94
+ allChunks = await chunkFile(text, source);
95
+ } else if (isFile && resolvedSource.endsWith(".pdf")) {
96
+ onProgress?.("Extracting text from PDF...");
97
+ const { extractText } = await import("unpdf");
98
+ const buf = (await import("node:fs")).readFileSync(resolvedSource);
99
+ const { text } = await extractText(new Uint8Array(buf));
100
+ allChunks = await chunkFile(text, resolvedSource);
101
+ } else if (isFile && (resolvedSource.endsWith(".docx") || resolvedSource.endsWith(".doc"))) {
102
+ onProgress?.("Extracting text from DOCX...");
103
+ const mammoth = await import("mammoth");
104
+ const result = await mammoth.extractRawText({ path: resolvedSource });
105
+ allChunks = await chunkFile(result.value, resolvedSource);
106
+ } else if (isDir) {
85
107
  onProgress?.(`Scanning ${resolvedSource}...`);
86
108
  const files = walkDir(resolvedSource);
87
109
  onProgress?.(`Found ${files.length} files, chunking...`);
@@ -316,6 +338,40 @@ export class KnowledgeEngine {
316
338
  return listKBs(this.db).map((kb) => diagnoseKB(this.db!, kb));
317
339
  }
318
340
 
341
+ async exportKB(nameOrId: string, outputPath: string): Promise<number> {
342
+ if (!this.db) throw new Error("Engine not initialized");
343
+ const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
344
+ if (!kb) throw new Error(`Knowledge base not found: ${nameOrId}`);
345
+ const chunks = getChunksByKB(this.db, kb.id);
346
+ const { writeFileSync } = await import("node:fs");
347
+ const header = JSON.stringify({ name: kb.name, description: kb.description, source_path: kb.source_path, source_type: kb.source_type, chunk_count: chunks.length });
348
+ const lines = [header, ...chunks.map((c) => JSON.stringify({ content: c.content, file_path: c.file_path, file_type: c.file_type, start_line: c.start_line, end_line: c.end_line, metadata_json: c.metadata_json }))];
349
+ writeFileSync(outputPath, lines.join("\n") + "\n");
350
+ return chunks.length;
351
+ }
352
+
353
+ async importKB(inputPath: string, onProgress?: ProgressCallback): Promise<{ kb: KnowledgeBase; chunkCount: number }> {
354
+ if (!this.db) throw new Error("Engine not initialized");
355
+ const { readFileSync } = await import("node:fs");
356
+ const lines = readFileSync(inputPath, "utf-8").trim().split("\n");
357
+ if (lines.length < 1) throw new Error("Empty import file");
358
+ const header = JSON.parse(lines[0]);
359
+ const kb = createKB(this.db, { name: header.name, description: header.description, source_path: header.source_path, source_type: header.source_type || "text" });
360
+ updateKBStatus(this.db, kb.id, "indexing");
361
+ const chunkData = lines.slice(1).map((l) => JSON.parse(l));
362
+ onProgress?.(`Importing ${chunkData.length} chunks, embedding...`);
363
+ const texts = chunkData.map((c: any) => c.content);
364
+ const vectors = await embedDocuments(texts);
365
+ const chunks = chunkData.map((c: any) => ({ content_hash: contentHash(c.content), content: c.content, content_tokenized: preTokenizeForFTS(c.content), file_path: c.file_path, file_type: c.file_type, start_line: c.start_line, end_line: c.end_line, metadata_json: c.metadata_json || "{}" }));
366
+ insertChunks(this.db, kb.id, chunks);
367
+ const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
368
+ saveVectors(vectorPath, vectors);
369
+ this.vectorCache.set(kb.id, vectors);
370
+ updateKBCounts(this.db, kb.id, chunks.length, new Set(chunks.map((c: any) => c.file_path)).size);
371
+ updateKBStatus(this.db, kb.id, "ready");
372
+ return { kb: getKB(this.db, kb.id)!, chunkCount: chunks.length };
373
+ }
374
+
319
375
  async dispose(): Promise<void> {
320
376
  await disposeEmbedding();
321
377
  await disposeReranker();