pi-knowledge 0.1.0 → 0.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,6 +1,35 @@
1
1
  # Changelog
2
2
 
3
- ## [Unreleased]
3
+ ## [0.1.4] - 2026-06-15
4
+
5
+ ### Added
6
+ - Vector memory cache (search no longer re-reads disk per query)
7
+ - AbortSignal support in embedding (cancellable long operations)
8
+ - Schema migration infrastructure (future-proof DB upgrades)
9
+ - Model mismatch warning on search (suggests re-index)
10
+ - Engine regression tests (+6, total 34)
11
+ - README Data Storage section
12
+
13
+ ### Fixed
14
+ - walkDir skips permission-denied directories
15
+ - DESIGN.md phases corrected
16
+
17
+ ## [0.1.3] - 2026-06-14
18
+
19
+ ### Fixed
20
+ - Short files index correctly (single chunk fallback)
21
+
22
+ ## [0.1.2] - 2026-06-14
23
+
24
+ ### Fixed
25
+ - Short files (<50 chars) now index correctly as a single chunk
26
+
27
+ ## [0.1.1] - 2026-06-14
28
+
29
+ ### Added
30
+ - Java AST chunking (6 languages total: TS, JS, Python, Go, Rust, Java)
31
+
32
+ ## [0.1.0] - 2026-06-14
4
33
 
5
34
  ### Added
6
35
  - Project scaffold and design document
package/README.md CHANGED
@@ -22,7 +22,7 @@ Unlike `pi-memory` (which manages the agent's own notes), `pi-knowledge` indexes
22
22
  | **Cross-encoder reranking** | ✅ | ❌ | ❌ |
23
23
  | **Incremental re-indexing** | ✅ | ❌ | ❌ |
24
24
  | **File watcher (auto-update)** | ✅ | ❌ | ❌ |
25
- | **Code-aware chunking** | ✅ | ❌ | ❌ |
25
+ | **Code-aware chunking** | ✅ (TS/JS/Py/Go/Rust/Java) | ❌ | ❌ |
26
26
  | **Local embeddings (zero API)** | ✅ | ❌ | ✅ (qmd) |
27
27
  | **Index quality diagnostics** | ✅ | ❌ | ❌ |
28
28
  | **Metadata filters in search** | ✅ | ❌ | ❌ |
@@ -68,6 +68,22 @@ pi install ./pi-knowledge
68
68
 
69
69
  See [DESIGN.md](DESIGN.md) for the full technical design document.
70
70
 
71
+ ## Data Storage
72
+
73
+ All data is stored globally at `~/.pi/knowledge/` (never in your project directory):
74
+
75
+ ```
76
+ ~/.pi/knowledge/
77
+ ├── knowledge.db ← SQLite (metadata + chunks + FTS5 index)
78
+ ├── vectors/ ← Embedding vectors per KB (binary)
79
+ └── models/ ← Downloaded ONNX models (~32MB, cached)
80
+ ```
81
+
82
+ - **Backup**: copy `~/.pi/knowledge/` directory
83
+ - **Reset**: delete `~/.pi/knowledge/` to start fresh
84
+ - **Project safety**: pi-knowledge is read-only on indexed directories — no files are created or modified in your project
85
+ - **Updates**: extension updates do not affect existing indexed data. Schema migrations run automatically if needed.
86
+
71
87
  ## Development
72
88
 
73
89
  ```bash
package/index.ts CHANGED
@@ -107,12 +107,14 @@ export default function (pi: ExtensionAPI) {
107
107
  if (response.results.length === 0) {
108
108
  return { content: [{ type: "text", text: "No results found." }] };
109
109
  }
110
- const formatted = response.results.map((r, i) =>
110
+ let output = `${response.total_count} results (showing ${response.results.length}):\n\n`;
111
+ if (response.warnings?.length) {
112
+ output = `⚠️ ${response.warnings.join("\n⚠️ ")}\n\n${output}`;
113
+ }
114
+ output += response.results.map((r, i) =>
111
115
  `[${i + 1}] ${r.file_path} (${r.kb_name}, score: ${r.score.toFixed(3)})\n${r.snippet}`,
112
116
  ).join("\n\n");
113
- return {
114
- content: [{ type: "text", text: `${response.total_count} results (showing ${response.results.length}):\n\n${formatted}` }],
115
- };
117
+ return { content: [{ type: "text", text: output }] };
116
118
  },
117
119
  });
118
120
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "pi-knowledge",
3
- "version": "0.1.0",
3
+ "version": "0.1.4",
4
4
  "description": "Vector knowledge base extension for Pi — index files, hybrid search, local embeddings",
5
5
  "type": "module",
6
6
  "main": "index.ts",
@@ -37,6 +37,7 @@
37
37
  "ignore": "7.0.5",
38
38
  "tree-sitter": "0.22.4",
39
39
  "tree-sitter-go": "0.23.4",
40
+ "tree-sitter-java": "0.23.5",
40
41
  "tree-sitter-python": "0.23.6",
41
42
  "tree-sitter-rust": "0.23.2",
42
43
  "tree-sitter-typescript": "0.23.2"
@@ -49,7 +49,7 @@ async function embedViaAPI(texts: string[], prefix: "query" | "passage"): Promis
49
49
  throw new Error(`Unsupported embedding provider: ${provider}`);
50
50
  }
51
51
 
52
- export async function embedTexts(texts: string[], prefix: "query" | "passage"): Promise<Float32Array[]> {
52
+ export async function embedTexts(texts: string[], prefix: "query" | "passage", signal?: AbortSignal): Promise<Float32Array[]> {
53
53
  if (!EMBEDDING_CONFIG.startsWith("local")) {
54
54
  try { return await embedViaAPI(texts, prefix); }
55
55
  catch { /* fallback to local */ }
@@ -58,6 +58,7 @@ export async function embedTexts(texts: string[], prefix: "query" | "passage"):
58
58
  resetIdleTimer();
59
59
  const results: Float32Array[] = [];
60
60
  for (const text of texts) {
61
+ if (signal?.aborted) throw new Error("Cancelled");
61
62
  const output = await pipe(`${prefix}: ${text}`, { pooling: "mean", normalize: true });
62
63
  results.push(new Float32Array(output.data));
63
64
  }
@@ -69,6 +70,6 @@ export async function embedQuery(text: string): Promise<Float32Array> {
69
70
  return vec;
70
71
  }
71
72
 
72
- export async function embedDocuments(texts: string[]): Promise<Float32Array[]> {
73
- return embedTexts(texts, "passage");
73
+ export async function embedDocuments(texts: string[], signal?: AbortSignal): Promise<Float32Array[]> {
74
+ return embedTexts(texts, "passage", signal);
74
75
  }
package/src/engine.ts CHANGED
@@ -34,10 +34,13 @@ export interface SearchResult {
34
34
  end_line: number;
35
35
  }
36
36
 
37
+ export const CURRENT_EMBEDDING_MODEL = "multilingual-e5-small";
38
+
37
39
  export interface SearchResponse {
38
40
  results: SearchResult[];
39
41
  total_count: number;
40
42
  has_more: boolean;
43
+ warnings?: string[];
41
44
  }
42
45
 
43
46
  export type ProgressCallback = (msg: string) => void;
@@ -45,10 +48,24 @@ export type ProgressCallback = (msg: string) => void;
45
48
  export class KnowledgeEngine {
46
49
  private db: Database.Database | null = null;
47
50
  private knowledgeDir: string = "";
51
+ private vectorCache: Map<string, Float32Array[]> = new Map();
48
52
 
49
53
  async initialize(knowledgeDir: string): Promise<void> {
50
54
  this.knowledgeDir = knowledgeDir;
51
55
  this.db = openDatabase(knowledgeDir);
56
+ this.vectorCache.clear();
57
+ }
58
+
59
+ private getVectors(kbId: string): Float32Array[] {
60
+ if (this.vectorCache.has(kbId)) return this.vectorCache.get(kbId)!;
61
+ const vectorPath = join(this.knowledgeDir, "vectors", `${kbId}.bin`);
62
+ const vectors = loadVectors(vectorPath);
63
+ this.vectorCache.set(kbId, vectors);
64
+ return vectors;
65
+ }
66
+
67
+ private invalidateVectorCache(kbId: string): void {
68
+ this.vectorCache.delete(kbId);
52
69
  }
53
70
 
54
71
  async add(source: string, name: string, onProgress?: ProgressCallback): Promise<{ kb: KnowledgeBase; chunkCount: number }> {
@@ -88,6 +105,7 @@ export class KnowledgeEngine {
88
105
 
89
106
  const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
90
107
  saveVectors(vectorPath, vectors);
108
+ this.vectorCache.set(kb.id, vectors);
91
109
 
92
110
  const fileCount = isDir ? getFileCount(this.db, kb.id) : 1;
93
111
  updateKBCounts(this.db, kb.id, allChunks.length, fileCount);
@@ -165,6 +183,7 @@ export class KnowledgeEngine {
165
183
  const finalChunks = getChunksByKB(this.db, kb.id);
166
184
  const finalVectors = finalChunks.map((c) => vectorCache.get(c.content_hash)!).filter(Boolean);
167
185
  saveVectors(vectorPath, finalVectors);
186
+ this.vectorCache.set(kb.id, finalVectors);
168
187
 
169
188
  // 7. Update counts
170
189
  updateKBCounts(this.db, kb.id, finalChunks.length, getFileCount(this.db, kb.id));
@@ -184,17 +203,21 @@ export class KnowledgeEngine {
184
203
  const kbs = kb_id ? [getKB(this.db, kb_id)].filter(Boolean) as KnowledgeBase[] : listKBs(this.db);
185
204
  if (kbs.length === 0) return { results: [], total_count: 0, has_more: false };
186
205
 
206
+ const warnings: string[] = [];
187
207
  let allResults: { chunkId: string; score: number }[] = [];
188
208
 
189
209
  for (const kb of kbs) {
210
+ if (kb.embedding_model !== CURRENT_EMBEDDING_MODEL) {
211
+ warnings.push(`"${kb.name}" was indexed with ${kb.embedding_model} (current: ${CURRENT_EMBEDDING_MODEL}) — run knowledge_update for best results`);
212
+ }
190
213
  const chunkIds = getChunkIdsByKB(this.db, kb.id);
191
214
  if (chunkIds.length === 0) continue;
192
215
 
193
216
  if (mode === "fast") {
194
217
  allResults.push(...searchBM25(this.db, query, 50, kb.id));
195
218
  } else if (mode === "semantic") {
196
- const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
197
- const vectors = loadVectors(vectorPath);
219
+
220
+ const vectors = this.getVectors(kb.id);
198
221
  if (vectors.length > 0) {
199
222
  const queryVec = await embedQuery(query);
200
223
  allResults.push(...searchVector(queryVec, vectors, chunkIds));
@@ -202,8 +225,8 @@ export class KnowledgeEngine {
202
225
  } else {
203
226
  // hybrid: BM25 + vector + RRF (both scoped to this KB)
204
227
  const bm25Results = searchBM25(this.db, query, 50, kb.id);
205
- const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
206
- const vectors = loadVectors(vectorPath);
228
+
229
+ const vectors = this.getVectors(kb.id);
207
230
  let vecResults: { chunkId: string; score: number }[] = [];
208
231
  if (vectors.length > 0) {
209
232
  const queryVec = await embedQuery(query);
@@ -243,7 +266,7 @@ export class KnowledgeEngine {
243
266
  const kbObj = getKB(this.db!, chunk.kb_id);
244
267
  return { content: chunk.content, file_path: chunk.file_path, file_type: chunk.file_type, kb_name: kbObj?.name ?? "unknown", score: r.score, snippet: chunk.content.slice(0, 200), start_line: chunk.start_line, end_line: chunk.end_line };
245
268
  });
246
- return { results, total_count: results.length, has_more: false };
269
+ return { results, total_count: results.length, has_more: false, warnings: warnings.length > 0 ? warnings : undefined };
247
270
  }
248
271
 
249
272
  const total = filtered.length;
@@ -265,13 +288,14 @@ export class KnowledgeEngine {
265
288
  };
266
289
  }).filter(Boolean) as SearchResult[];
267
290
 
268
- return { results, total_count: total, has_more: offset + limit < total };
291
+ return { results, total_count: total, has_more: offset + limit < total, warnings: warnings.length > 0 ? warnings : undefined };
269
292
  }
270
293
 
271
294
  remove(nameOrId: string): boolean {
272
295
  if (!this.db) return false;
273
296
  const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
274
297
  if (!kb) return false;
298
+ this.invalidateVectorCache(kb.id);
275
299
  deleteKB(this.db, kb.id);
276
300
  return true;
277
301
  }
@@ -284,6 +308,7 @@ export class KnowledgeEngine {
284
308
  clear(): void {
285
309
  if (!this.db) return;
286
310
  for (const kb of listKBs(this.db)) deleteKB(this.db, kb.id);
311
+ this.vectorCache.clear();
287
312
  }
288
313
 
289
314
  diagnose(): DiagnosticResult[] {
@@ -70,7 +70,9 @@ export function walkDir(dirPath: string): ScannedFile[] {
70
70
  const results: ScannedFile[] = [];
71
71
 
72
72
  function walk(dir: string): void {
73
- const entries = readdirSync(dir, { withFileTypes: true });
73
+ let entries;
74
+ try { entries = readdirSync(dir, { withFileTypes: true }); }
75
+ catch { return; }
74
76
  for (const entry of entries) {
75
77
  const fullPath = join(dir, entry.name);
76
78
  const relPath = relative(dirPath, fullPath).split(sep).join("/");
@@ -251,13 +253,32 @@ export function chunkText(content: string, filePath: string): Omit<ChunkInsert,
251
253
 
252
254
  export async function chunkFile(content: string, filePath: string): Promise<Omit<ChunkInsert, "kb_id">[]> {
253
255
  const fileType = detectFileType(filePath);
254
- if (fileType === "markdown") return chunkMarkdown(content, filePath);
255
- if (["typescript", "javascript", "python", "go", "rust"].includes(fileType)) {
256
+ let chunks: Omit<ChunkInsert, "kb_id">[] = [];
257
+
258
+ if (fileType === "markdown") {
259
+ chunks = chunkMarkdown(content, filePath);
260
+ } else if (["typescript", "javascript", "python", "go", "rust", "java"].includes(fileType)) {
256
261
  try {
257
262
  const { chunkWithAST } = await import("./chunkers/code-ast.ts");
258
- const astChunks = await chunkWithAST(content, filePath, fileType);
259
- if (astChunks.length > 0) return astChunks;
260
- } catch { /* fallback to text chunker */ }
263
+ chunks = await chunkWithAST(content, filePath, fileType);
264
+ } catch { /* fallback below */ }
261
265
  }
262
- return chunkText(content, filePath);
266
+
267
+ if (chunks.length === 0) chunks = chunkText(content, filePath);
268
+
269
+ // Fallback: if file has content but no chunks (too short for splitting), keep as single chunk
270
+ if (chunks.length === 0 && content.trim().length > 10) {
271
+ chunks = [{
272
+ content_hash: contentHash(content),
273
+ content: content.trim(),
274
+ content_tokenized: preTokenizeForFTS(content),
275
+ file_path: filePath,
276
+ file_type: fileType,
277
+ start_line: 1,
278
+ end_line: content.split("\n").length,
279
+ metadata_json: "{}",
280
+ }];
281
+ }
282
+
283
+ return chunks;
263
284
  }
@@ -54,6 +54,13 @@ const LANGS: Record<string, LangConfig> = {
54
54
  methodContainer: "impl_item",
55
55
  fileType: "rust",
56
56
  },
57
+ java: {
58
+ grammar: async () => { const m = await import("tree-sitter-java"); return m.default ?? m; },
59
+ fnTypes: new Set(["method_declaration", "constructor_declaration"]),
60
+ classTypes: new Set(["class_declaration", "interface_declaration", "enum_declaration"]),
61
+ classBody: "class_body",
62
+ fileType: "java",
63
+ },
57
64
  };
58
65
 
59
66
  function getName(node: ASTNode): string {
@@ -113,11 +113,28 @@ export function openDatabase(knowledgeDir?: string): Database.Database {
113
113
  if (!hasVersion) {
114
114
  db.exec(SCHEMA_SQL);
115
115
  db.prepare("INSERT INTO schema_version (version) VALUES (?)").run(SCHEMA_VERSION);
116
+ } else {
117
+ const row = db.prepare("SELECT version FROM schema_version").get() as { version: number } | undefined;
118
+ const currentVersion = row?.version ?? 0;
119
+ if (currentVersion < SCHEMA_VERSION) {
120
+ runMigrations(db, currentVersion, SCHEMA_VERSION);
121
+ db.prepare("UPDATE schema_version SET version = ?").run(SCHEMA_VERSION);
122
+ }
116
123
  }
117
124
 
118
125
  return db;
119
126
  }
120
127
 
128
+ function runMigrations(db: Database.Database, from: number, to: number): void {
129
+ const migrations: Record<number, string> = {
130
+ // Future migrations go here, keyed by target version number:
131
+ // 2: "ALTER TABLE ...",
132
+ };
133
+ for (let v = from + 1; v <= to; v++) {
134
+ if (migrations[v]) db.exec(migrations[v]);
135
+ }
136
+ }
137
+
121
138
  // --- CRUD: Knowledge Bases ---
122
139
 
123
140
  export function createKB(