pi-knowledge 0.1.1 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -1
- package/README.md +17 -1
- package/index.ts +6 -4
- package/package.json +1 -1
- package/src/embedding/provider.ts +4 -3
- package/src/engine.ts +31 -6
- package/src/indexer/chunker.ts +28 -7
- package/src/storage/sqlite.ts +17 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,6 +1,35 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
-
## [
|
|
3
|
+
## [0.1.4] - 2026-06-15
|
|
4
|
+
|
|
5
|
+
### Added
|
|
6
|
+
- Vector memory cache (search no longer re-reads disk per query)
|
|
7
|
+
- AbortSignal support in embedding (cancellable long operations)
|
|
8
|
+
- Schema migration infrastructure (future-proof DB upgrades)
|
|
9
|
+
- Model mismatch warning on search (suggests re-index)
|
|
10
|
+
- Engine regression tests (+6, total 34)
|
|
11
|
+
- README Data Storage section
|
|
12
|
+
|
|
13
|
+
### Fixed
|
|
14
|
+
- walkDir skips permission-denied directories
|
|
15
|
+
- DESIGN.md phases corrected
|
|
16
|
+
|
|
17
|
+
## [0.1.3] - 2026-06-14
|
|
18
|
+
|
|
19
|
+
### Fixed
|
|
20
|
+
- Short files index correctly (single chunk fallback)
|
|
21
|
+
|
|
22
|
+
## [0.1.2] - 2026-06-14
|
|
23
|
+
|
|
24
|
+
### Fixed
|
|
25
|
+
- Short files (<50 chars) now index correctly as a single chunk
|
|
26
|
+
|
|
27
|
+
## [0.1.1] - 2026-06-14
|
|
28
|
+
|
|
29
|
+
### Added
|
|
30
|
+
- Java AST chunking (6 languages total: TS, JS, Python, Go, Rust, Java)
|
|
31
|
+
|
|
32
|
+
## [0.1.0] - 2026-06-14
|
|
4
33
|
|
|
5
34
|
### Added
|
|
6
35
|
- Project scaffold and design document
|
package/README.md
CHANGED
|
@@ -22,7 +22,7 @@ Unlike `pi-memory` (which manages the agent's own notes), `pi-knowledge` indexes
|
|
|
22
22
|
| **Cross-encoder reranking** | ✅ | ❌ | ❌ |
|
|
23
23
|
| **Incremental re-indexing** | ✅ | ❌ | ❌ |
|
|
24
24
|
| **File watcher (auto-update)** | ✅ | ❌ | ❌ |
|
|
25
|
-
| **Code-aware chunking** | ✅ | ❌ | ❌ |
|
|
25
|
+
| **Code-aware chunking** | ✅ (TS/JS/Py/Go/Rust/Java) | ❌ | ❌ |
|
|
26
26
|
| **Local embeddings (zero API)** | ✅ | ❌ | ✅ (qmd) |
|
|
27
27
|
| **Index quality diagnostics** | ✅ | ❌ | ❌ |
|
|
28
28
|
| **Metadata filters in search** | ✅ | ❌ | ❌ |
|
|
@@ -68,6 +68,22 @@ pi install ./pi-knowledge
|
|
|
68
68
|
|
|
69
69
|
See [DESIGN.md](DESIGN.md) for the full technical design document.
|
|
70
70
|
|
|
71
|
+
## Data Storage
|
|
72
|
+
|
|
73
|
+
All data is stored globally at `~/.pi/knowledge/` (never in your project directory):
|
|
74
|
+
|
|
75
|
+
```
|
|
76
|
+
~/.pi/knowledge/
|
|
77
|
+
├── knowledge.db ← SQLite (metadata + chunks + FTS5 index)
|
|
78
|
+
├── vectors/ ← Embedding vectors per KB (binary)
|
|
79
|
+
└── models/ ← Downloaded ONNX models (~32MB, cached)
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
- **Backup**: copy `~/.pi/knowledge/` directory
|
|
83
|
+
- **Reset**: delete `~/.pi/knowledge/` to start fresh
|
|
84
|
+
- **Project safety**: pi-knowledge is read-only on indexed directories — no files are created or modified in your project
|
|
85
|
+
- **Updates**: extension updates do not affect existing indexed data. Schema migrations run automatically if needed.
|
|
86
|
+
|
|
71
87
|
## Development
|
|
72
88
|
|
|
73
89
|
```bash
|
package/index.ts
CHANGED
|
@@ -107,12 +107,14 @@ export default function (pi: ExtensionAPI) {
|
|
|
107
107
|
if (response.results.length === 0) {
|
|
108
108
|
return { content: [{ type: "text", text: "No results found." }] };
|
|
109
109
|
}
|
|
110
|
-
|
|
110
|
+
let output = `${response.total_count} results (showing ${response.results.length}):\n\n`;
|
|
111
|
+
if (response.warnings?.length) {
|
|
112
|
+
output = `⚠️ ${response.warnings.join("\n⚠️ ")}\n\n${output}`;
|
|
113
|
+
}
|
|
114
|
+
output += response.results.map((r, i) =>
|
|
111
115
|
`[${i + 1}] ${r.file_path} (${r.kb_name}, score: ${r.score.toFixed(3)})\n${r.snippet}`,
|
|
112
116
|
).join("\n\n");
|
|
113
|
-
return {
|
|
114
|
-
content: [{ type: "text", text: `${response.total_count} results (showing ${response.results.length}):\n\n${formatted}` }],
|
|
115
|
-
};
|
|
117
|
+
return { content: [{ type: "text", text: output }] };
|
|
116
118
|
},
|
|
117
119
|
});
|
|
118
120
|
|
package/package.json
CHANGED
|
@@ -49,7 +49,7 @@ async function embedViaAPI(texts: string[], prefix: "query" | "passage"): Promis
|
|
|
49
49
|
throw new Error(`Unsupported embedding provider: ${provider}`);
|
|
50
50
|
}
|
|
51
51
|
|
|
52
|
-
export async function embedTexts(texts: string[], prefix: "query" | "passage"): Promise<Float32Array[]> {
|
|
52
|
+
export async function embedTexts(texts: string[], prefix: "query" | "passage", signal?: AbortSignal): Promise<Float32Array[]> {
|
|
53
53
|
if (!EMBEDDING_CONFIG.startsWith("local")) {
|
|
54
54
|
try { return await embedViaAPI(texts, prefix); }
|
|
55
55
|
catch { /* fallback to local */ }
|
|
@@ -58,6 +58,7 @@ export async function embedTexts(texts: string[], prefix: "query" | "passage"):
|
|
|
58
58
|
resetIdleTimer();
|
|
59
59
|
const results: Float32Array[] = [];
|
|
60
60
|
for (const text of texts) {
|
|
61
|
+
if (signal?.aborted) throw new Error("Cancelled");
|
|
61
62
|
const output = await pipe(`${prefix}: ${text}`, { pooling: "mean", normalize: true });
|
|
62
63
|
results.push(new Float32Array(output.data));
|
|
63
64
|
}
|
|
@@ -69,6 +70,6 @@ export async function embedQuery(text: string): Promise<Float32Array> {
|
|
|
69
70
|
return vec;
|
|
70
71
|
}
|
|
71
72
|
|
|
72
|
-
export async function embedDocuments(texts: string[]): Promise<Float32Array[]> {
|
|
73
|
-
return embedTexts(texts, "passage");
|
|
73
|
+
export async function embedDocuments(texts: string[], signal?: AbortSignal): Promise<Float32Array[]> {
|
|
74
|
+
return embedTexts(texts, "passage", signal);
|
|
74
75
|
}
|
package/src/engine.ts
CHANGED
|
@@ -34,10 +34,13 @@ export interface SearchResult {
|
|
|
34
34
|
end_line: number;
|
|
35
35
|
}
|
|
36
36
|
|
|
37
|
+
export const CURRENT_EMBEDDING_MODEL = "multilingual-e5-small";
|
|
38
|
+
|
|
37
39
|
export interface SearchResponse {
|
|
38
40
|
results: SearchResult[];
|
|
39
41
|
total_count: number;
|
|
40
42
|
has_more: boolean;
|
|
43
|
+
warnings?: string[];
|
|
41
44
|
}
|
|
42
45
|
|
|
43
46
|
export type ProgressCallback = (msg: string) => void;
|
|
@@ -45,10 +48,24 @@ export type ProgressCallback = (msg: string) => void;
|
|
|
45
48
|
export class KnowledgeEngine {
|
|
46
49
|
private db: Database.Database | null = null;
|
|
47
50
|
private knowledgeDir: string = "";
|
|
51
|
+
private vectorCache: Map<string, Float32Array[]> = new Map();
|
|
48
52
|
|
|
49
53
|
async initialize(knowledgeDir: string): Promise<void> {
|
|
50
54
|
this.knowledgeDir = knowledgeDir;
|
|
51
55
|
this.db = openDatabase(knowledgeDir);
|
|
56
|
+
this.vectorCache.clear();
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
private getVectors(kbId: string): Float32Array[] {
|
|
60
|
+
if (this.vectorCache.has(kbId)) return this.vectorCache.get(kbId)!;
|
|
61
|
+
const vectorPath = join(this.knowledgeDir, "vectors", `${kbId}.bin`);
|
|
62
|
+
const vectors = loadVectors(vectorPath);
|
|
63
|
+
this.vectorCache.set(kbId, vectors);
|
|
64
|
+
return vectors;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
private invalidateVectorCache(kbId: string): void {
|
|
68
|
+
this.vectorCache.delete(kbId);
|
|
52
69
|
}
|
|
53
70
|
|
|
54
71
|
async add(source: string, name: string, onProgress?: ProgressCallback): Promise<{ kb: KnowledgeBase; chunkCount: number }> {
|
|
@@ -88,6 +105,7 @@ export class KnowledgeEngine {
|
|
|
88
105
|
|
|
89
106
|
const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
|
|
90
107
|
saveVectors(vectorPath, vectors);
|
|
108
|
+
this.vectorCache.set(kb.id, vectors);
|
|
91
109
|
|
|
92
110
|
const fileCount = isDir ? getFileCount(this.db, kb.id) : 1;
|
|
93
111
|
updateKBCounts(this.db, kb.id, allChunks.length, fileCount);
|
|
@@ -165,6 +183,7 @@ export class KnowledgeEngine {
|
|
|
165
183
|
const finalChunks = getChunksByKB(this.db, kb.id);
|
|
166
184
|
const finalVectors = finalChunks.map((c) => vectorCache.get(c.content_hash)!).filter(Boolean);
|
|
167
185
|
saveVectors(vectorPath, finalVectors);
|
|
186
|
+
this.vectorCache.set(kb.id, finalVectors);
|
|
168
187
|
|
|
169
188
|
// 7. Update counts
|
|
170
189
|
updateKBCounts(this.db, kb.id, finalChunks.length, getFileCount(this.db, kb.id));
|
|
@@ -184,17 +203,21 @@ export class KnowledgeEngine {
|
|
|
184
203
|
const kbs = kb_id ? [getKB(this.db, kb_id)].filter(Boolean) as KnowledgeBase[] : listKBs(this.db);
|
|
185
204
|
if (kbs.length === 0) return { results: [], total_count: 0, has_more: false };
|
|
186
205
|
|
|
206
|
+
const warnings: string[] = [];
|
|
187
207
|
let allResults: { chunkId: string; score: number }[] = [];
|
|
188
208
|
|
|
189
209
|
for (const kb of kbs) {
|
|
210
|
+
if (kb.embedding_model !== CURRENT_EMBEDDING_MODEL) {
|
|
211
|
+
warnings.push(`"${kb.name}" was indexed with ${kb.embedding_model} (current: ${CURRENT_EMBEDDING_MODEL}) — run knowledge_update for best results`);
|
|
212
|
+
}
|
|
190
213
|
const chunkIds = getChunkIdsByKB(this.db, kb.id);
|
|
191
214
|
if (chunkIds.length === 0) continue;
|
|
192
215
|
|
|
193
216
|
if (mode === "fast") {
|
|
194
217
|
allResults.push(...searchBM25(this.db, query, 50, kb.id));
|
|
195
218
|
} else if (mode === "semantic") {
|
|
196
|
-
|
|
197
|
-
const vectors =
|
|
219
|
+
|
|
220
|
+
const vectors = this.getVectors(kb.id);
|
|
198
221
|
if (vectors.length > 0) {
|
|
199
222
|
const queryVec = await embedQuery(query);
|
|
200
223
|
allResults.push(...searchVector(queryVec, vectors, chunkIds));
|
|
@@ -202,8 +225,8 @@ export class KnowledgeEngine {
|
|
|
202
225
|
} else {
|
|
203
226
|
// hybrid: BM25 + vector + RRF (both scoped to this KB)
|
|
204
227
|
const bm25Results = searchBM25(this.db, query, 50, kb.id);
|
|
205
|
-
|
|
206
|
-
const vectors =
|
|
228
|
+
|
|
229
|
+
const vectors = this.getVectors(kb.id);
|
|
207
230
|
let vecResults: { chunkId: string; score: number }[] = [];
|
|
208
231
|
if (vectors.length > 0) {
|
|
209
232
|
const queryVec = await embedQuery(query);
|
|
@@ -243,7 +266,7 @@ export class KnowledgeEngine {
|
|
|
243
266
|
const kbObj = getKB(this.db!, chunk.kb_id);
|
|
244
267
|
return { content: chunk.content, file_path: chunk.file_path, file_type: chunk.file_type, kb_name: kbObj?.name ?? "unknown", score: r.score, snippet: chunk.content.slice(0, 200), start_line: chunk.start_line, end_line: chunk.end_line };
|
|
245
268
|
});
|
|
246
|
-
return { results, total_count: results.length, has_more: false };
|
|
269
|
+
return { results, total_count: results.length, has_more: false, warnings: warnings.length > 0 ? warnings : undefined };
|
|
247
270
|
}
|
|
248
271
|
|
|
249
272
|
const total = filtered.length;
|
|
@@ -265,13 +288,14 @@ export class KnowledgeEngine {
|
|
|
265
288
|
};
|
|
266
289
|
}).filter(Boolean) as SearchResult[];
|
|
267
290
|
|
|
268
|
-
return { results, total_count: total, has_more: offset + limit < total };
|
|
291
|
+
return { results, total_count: total, has_more: offset + limit < total, warnings: warnings.length > 0 ? warnings : undefined };
|
|
269
292
|
}
|
|
270
293
|
|
|
271
294
|
remove(nameOrId: string): boolean {
|
|
272
295
|
if (!this.db) return false;
|
|
273
296
|
const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
|
|
274
297
|
if (!kb) return false;
|
|
298
|
+
this.invalidateVectorCache(kb.id);
|
|
275
299
|
deleteKB(this.db, kb.id);
|
|
276
300
|
return true;
|
|
277
301
|
}
|
|
@@ -284,6 +308,7 @@ export class KnowledgeEngine {
|
|
|
284
308
|
clear(): void {
|
|
285
309
|
if (!this.db) return;
|
|
286
310
|
for (const kb of listKBs(this.db)) deleteKB(this.db, kb.id);
|
|
311
|
+
this.vectorCache.clear();
|
|
287
312
|
}
|
|
288
313
|
|
|
289
314
|
diagnose(): DiagnosticResult[] {
|
package/src/indexer/chunker.ts
CHANGED
|
@@ -70,7 +70,9 @@ export function walkDir(dirPath: string): ScannedFile[] {
|
|
|
70
70
|
const results: ScannedFile[] = [];
|
|
71
71
|
|
|
72
72
|
function walk(dir: string): void {
|
|
73
|
-
|
|
73
|
+
let entries;
|
|
74
|
+
try { entries = readdirSync(dir, { withFileTypes: true }); }
|
|
75
|
+
catch { return; }
|
|
74
76
|
for (const entry of entries) {
|
|
75
77
|
const fullPath = join(dir, entry.name);
|
|
76
78
|
const relPath = relative(dirPath, fullPath).split(sep).join("/");
|
|
@@ -251,13 +253,32 @@ export function chunkText(content: string, filePath: string): Omit<ChunkInsert,
|
|
|
251
253
|
|
|
252
254
|
export async function chunkFile(content: string, filePath: string): Promise<Omit<ChunkInsert, "kb_id">[]> {
|
|
253
255
|
const fileType = detectFileType(filePath);
|
|
254
|
-
|
|
255
|
-
|
|
256
|
+
let chunks: Omit<ChunkInsert, "kb_id">[] = [];
|
|
257
|
+
|
|
258
|
+
if (fileType === "markdown") {
|
|
259
|
+
chunks = chunkMarkdown(content, filePath);
|
|
260
|
+
} else if (["typescript", "javascript", "python", "go", "rust", "java"].includes(fileType)) {
|
|
256
261
|
try {
|
|
257
262
|
const { chunkWithAST } = await import("./chunkers/code-ast.ts");
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
} catch { /* fallback to text chunker */ }
|
|
263
|
+
chunks = await chunkWithAST(content, filePath, fileType);
|
|
264
|
+
} catch { /* fallback below */ }
|
|
261
265
|
}
|
|
262
|
-
|
|
266
|
+
|
|
267
|
+
if (chunks.length === 0) chunks = chunkText(content, filePath);
|
|
268
|
+
|
|
269
|
+
// Fallback: if file has content but no chunks (too short for splitting), keep as single chunk
|
|
270
|
+
if (chunks.length === 0 && content.trim().length > 10) {
|
|
271
|
+
chunks = [{
|
|
272
|
+
content_hash: contentHash(content),
|
|
273
|
+
content: content.trim(),
|
|
274
|
+
content_tokenized: preTokenizeForFTS(content),
|
|
275
|
+
file_path: filePath,
|
|
276
|
+
file_type: fileType,
|
|
277
|
+
start_line: 1,
|
|
278
|
+
end_line: content.split("\n").length,
|
|
279
|
+
metadata_json: "{}",
|
|
280
|
+
}];
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
return chunks;
|
|
263
284
|
}
|
package/src/storage/sqlite.ts
CHANGED
|
@@ -113,11 +113,28 @@ export function openDatabase(knowledgeDir?: string): Database.Database {
|
|
|
113
113
|
if (!hasVersion) {
|
|
114
114
|
db.exec(SCHEMA_SQL);
|
|
115
115
|
db.prepare("INSERT INTO schema_version (version) VALUES (?)").run(SCHEMA_VERSION);
|
|
116
|
+
} else {
|
|
117
|
+
const row = db.prepare("SELECT version FROM schema_version").get() as { version: number } | undefined;
|
|
118
|
+
const currentVersion = row?.version ?? 0;
|
|
119
|
+
if (currentVersion < SCHEMA_VERSION) {
|
|
120
|
+
runMigrations(db, currentVersion, SCHEMA_VERSION);
|
|
121
|
+
db.prepare("UPDATE schema_version SET version = ?").run(SCHEMA_VERSION);
|
|
122
|
+
}
|
|
116
123
|
}
|
|
117
124
|
|
|
118
125
|
return db;
|
|
119
126
|
}
|
|
120
127
|
|
|
128
|
+
function runMigrations(db: Database.Database, from: number, to: number): void {
|
|
129
|
+
const migrations: Record<number, string> = {
|
|
130
|
+
// Future migrations go here, keyed by target version number:
|
|
131
|
+
// 2: "ALTER TABLE ...",
|
|
132
|
+
};
|
|
133
|
+
for (let v = from + 1; v <= to; v++) {
|
|
134
|
+
if (migrations[v]) db.exec(migrations[v]);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
121
138
|
// --- CRUD: Knowledge Bases ---
|
|
122
139
|
|
|
123
140
|
export function createKB(
|