pi-knowledge 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +44 -1
- package/README.md +18 -2
- package/index.ts +35 -4
- package/package.json +4 -2
- package/src/embedding/provider.ts +4 -3
- package/src/engine.ts +92 -11
- package/src/indexer/chunker.ts +28 -7
- package/src/storage/sqlite.ts +17 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,6 +1,49 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
-
## [
|
|
3
|
+
## [0.2.0] - 2026-06-15
|
|
4
|
+
|
|
5
|
+
### Added
|
|
6
|
+
- URL indexing (fetch → HTML strip → chunk)
|
|
7
|
+
- PDF text extraction (via unpdf, pure JS)
|
|
8
|
+
- DOCX text extraction (via mammoth, pure JS)
|
|
9
|
+
- Import/export knowledge bases (JSONL format, git-friendly)
|
|
10
|
+
- Performance benchmarks (BM25: 0.05ms, hybrid: 2.1ms)
|
|
11
|
+
- Pi Skill: /skill:search-docs
|
|
12
|
+
- 9 tools total
|
|
13
|
+
|
|
14
|
+
## [0.1.4] - 2026-06-15
|
|
15
|
+
|
|
16
|
+
### Added
|
|
17
|
+
- Vector memory cache (search no longer re-reads disk per query)
|
|
18
|
+
- AbortSignal support in embedding (cancellable long operations)
|
|
19
|
+
- Schema migration infrastructure (future-proof DB upgrades)
|
|
20
|
+
- Model mismatch warning on search (suggests re-index)
|
|
21
|
+
- Engine regression tests (+6, total 34)
|
|
22
|
+
- README Data Storage section
|
|
23
|
+
- URL indexing (knowledge_add with http/https URLs, auto HTML strip)
|
|
24
|
+
- Performance benchmarks (BM25: 0.05ms, hybrid: 2.1ms, semantic: 2.0ms)
|
|
25
|
+
- Pi Skill: `/skill:search-docs` for guided knowledge search
|
|
26
|
+
|
|
27
|
+
### Fixed
|
|
28
|
+
- walkDir skips permission-denied directories
|
|
29
|
+
- DESIGN.md phases corrected
|
|
30
|
+
|
|
31
|
+
## [0.1.3] - 2026-06-14
|
|
32
|
+
|
|
33
|
+
### Fixed
|
|
34
|
+
- Short files index correctly (single chunk fallback)
|
|
35
|
+
|
|
36
|
+
## [0.1.2] - 2026-06-14
|
|
37
|
+
|
|
38
|
+
### Fixed
|
|
39
|
+
- Short files (<50 chars) now index correctly as a single chunk
|
|
40
|
+
|
|
41
|
+
## [0.1.1] - 2026-06-14
|
|
42
|
+
|
|
43
|
+
### Added
|
|
44
|
+
- Java AST chunking (6 languages total: TS, JS, Python, Go, Rust, Java)
|
|
45
|
+
|
|
46
|
+
## [0.1.0] - 2026-06-14
|
|
4
47
|
|
|
5
48
|
### Added
|
|
6
49
|
- Project scaffold and design document
|
package/README.md
CHANGED
|
@@ -22,7 +22,7 @@ Unlike `pi-memory` (which manages the agent's own notes), `pi-knowledge` indexes
|
|
|
22
22
|
| **Cross-encoder reranking** | ✅ | ❌ | ❌ |
|
|
23
23
|
| **Incremental re-indexing** | ✅ | ❌ | ❌ |
|
|
24
24
|
| **File watcher (auto-update)** | ✅ | ❌ | ❌ |
|
|
25
|
-
| **Code-aware chunking** | ✅ | ❌ | ❌ |
|
|
25
|
+
| **Code-aware chunking** | ✅ (TS/JS/Py/Go/Rust/Java) | ❌ | ❌ |
|
|
26
26
|
| **Local embeddings (zero API)** | ✅ | ❌ | ✅ (qmd) |
|
|
27
27
|
| **Index quality diagnostics** | ✅ | ❌ | ❌ |
|
|
28
28
|
| **Metadata filters in search** | ✅ | ❌ | ❌ |
|
|
@@ -56,7 +56,7 @@ pi install ./pi-knowledge
|
|
|
56
56
|
|
|
57
57
|
| Tool | Description |
|
|
58
58
|
|------|-------------|
|
|
59
|
-
| `knowledge_add` | Index files, directories, or inline text
|
|
59
|
+
| `knowledge_add` | Index files, directories, URLs, PDFs, DOCX, or inline text |
|
|
60
60
|
| `knowledge_search` | Hybrid search across one or all knowledge bases |
|
|
61
61
|
| `knowledge_remove` | Remove a knowledge base by name or ID |
|
|
62
62
|
| `knowledge_update` | Incrementally re-index changed files in a knowledge base |
|
|
@@ -68,6 +68,22 @@ pi install ./pi-knowledge
|
|
|
68
68
|
|
|
69
69
|
See [DESIGN.md](DESIGN.md) for the full technical design document.
|
|
70
70
|
|
|
71
|
+
## Data Storage
|
|
72
|
+
|
|
73
|
+
All data is stored globally at `~/.pi/knowledge/` (never in your project directory):
|
|
74
|
+
|
|
75
|
+
```
|
|
76
|
+
~/.pi/knowledge/
|
|
77
|
+
├── knowledge.db ← SQLite (metadata + chunks + FTS5 index)
|
|
78
|
+
├── vectors/ ← Embedding vectors per KB (binary)
|
|
79
|
+
└── models/ ← Downloaded ONNX models (~32MB, cached)
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
- **Backup**: copy `~/.pi/knowledge/` directory
|
|
83
|
+
- **Reset**: delete `~/.pi/knowledge/` to start fresh
|
|
84
|
+
- **Project safety**: pi-knowledge is read-only on indexed directories — no files are created or modified in your project
|
|
85
|
+
- **Updates**: extension updates do not affect existing indexed data. Schema migrations run automatically if needed.
|
|
86
|
+
|
|
71
87
|
## Development
|
|
72
88
|
|
|
73
89
|
```bash
|
package/index.ts
CHANGED
|
@@ -107,12 +107,14 @@ export default function (pi: ExtensionAPI) {
|
|
|
107
107
|
if (response.results.length === 0) {
|
|
108
108
|
return { content: [{ type: "text", text: "No results found." }] };
|
|
109
109
|
}
|
|
110
|
-
|
|
110
|
+
let output = `${response.total_count} results (showing ${response.results.length}):\n\n`;
|
|
111
|
+
if (response.warnings?.length) {
|
|
112
|
+
output = `⚠️ ${response.warnings.join("\n⚠️ ")}\n\n${output}`;
|
|
113
|
+
}
|
|
114
|
+
output += response.results.map((r, i) =>
|
|
111
115
|
`[${i + 1}] ${r.file_path} (${r.kb_name}, score: ${r.score.toFixed(3)})\n${r.snippet}`,
|
|
112
116
|
).join("\n\n");
|
|
113
|
-
return {
|
|
114
|
-
content: [{ type: "text", text: `${response.total_count} results (showing ${response.results.length}):\n\n${formatted}` }],
|
|
115
|
-
};
|
|
117
|
+
return { content: [{ type: "text", text: output }] };
|
|
116
118
|
},
|
|
117
119
|
});
|
|
118
120
|
|
|
@@ -189,6 +191,35 @@ export default function (pi: ExtensionAPI) {
|
|
|
189
191
|
},
|
|
190
192
|
});
|
|
191
193
|
|
|
194
|
+
pi.registerTool({
|
|
195
|
+
name: "knowledge_export",
|
|
196
|
+
label: "Knowledge Export",
|
|
197
|
+
description: "Export a knowledge base to a JSONL file (shareable, git-friendly)",
|
|
198
|
+
parameters: Type.Object({
|
|
199
|
+
target: Type.String({ description: "KB name or ID to export" }),
|
|
200
|
+
output: Type.String({ description: "Output file path (.jsonl)" }),
|
|
201
|
+
}),
|
|
202
|
+
async execute(_id, params) {
|
|
203
|
+
const count = await engine.exportKB(params.target, params.output);
|
|
204
|
+
return { content: [{ type: "text", text: `Exported ${count} chunks to ${params.output}` }] };
|
|
205
|
+
},
|
|
206
|
+
});
|
|
207
|
+
|
|
208
|
+
pi.registerTool({
|
|
209
|
+
name: "knowledge_import",
|
|
210
|
+
label: "Knowledge Import",
|
|
211
|
+
description: "Import a knowledge base from a JSONL file (re-embeds content)",
|
|
212
|
+
parameters: Type.Object({
|
|
213
|
+
input: Type.String({ description: "Input JSONL file path" }),
|
|
214
|
+
}),
|
|
215
|
+
async execute(_id, params, _signal, onUpdate) {
|
|
216
|
+
const { kb, chunkCount } = await engine.importKB(params.input, (msg) => {
|
|
217
|
+
onUpdate?.({ content: [{ type: "text", text: msg }] });
|
|
218
|
+
});
|
|
219
|
+
return { content: [{ type: "text", text: `Imported "${kb.name}": ${chunkCount} chunks (re-embedded)` }] };
|
|
220
|
+
},
|
|
221
|
+
});
|
|
222
|
+
|
|
192
223
|
pi.registerTool({
|
|
193
224
|
name: "knowledge_clear",
|
|
194
225
|
label: "Knowledge Clear",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-knowledge",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"description": "Vector knowledge base extension for Pi — index files, hybrid search, local embeddings",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "index.ts",
|
|
@@ -35,12 +35,14 @@
|
|
|
35
35
|
"@huggingface/transformers": "3.8.1",
|
|
36
36
|
"better-sqlite3": "11.9.1",
|
|
37
37
|
"ignore": "7.0.5",
|
|
38
|
+
"mammoth": "1.12.0",
|
|
38
39
|
"tree-sitter": "0.22.4",
|
|
39
40
|
"tree-sitter-go": "0.23.4",
|
|
40
41
|
"tree-sitter-java": "0.23.5",
|
|
41
42
|
"tree-sitter-python": "0.23.6",
|
|
42
43
|
"tree-sitter-rust": "0.23.2",
|
|
43
|
-
"tree-sitter-typescript": "0.23.2"
|
|
44
|
+
"tree-sitter-typescript": "0.23.2",
|
|
45
|
+
"unpdf": "1.6.2"
|
|
44
46
|
},
|
|
45
47
|
"devDependencies": {
|
|
46
48
|
"@biomejs/biome": "2.3.5",
|
|
@@ -49,7 +49,7 @@ async function embedViaAPI(texts: string[], prefix: "query" | "passage"): Promis
|
|
|
49
49
|
throw new Error(`Unsupported embedding provider: ${provider}`);
|
|
50
50
|
}
|
|
51
51
|
|
|
52
|
-
export async function embedTexts(texts: string[], prefix: "query" | "passage"): Promise<Float32Array[]> {
|
|
52
|
+
export async function embedTexts(texts: string[], prefix: "query" | "passage", signal?: AbortSignal): Promise<Float32Array[]> {
|
|
53
53
|
if (!EMBEDDING_CONFIG.startsWith("local")) {
|
|
54
54
|
try { return await embedViaAPI(texts, prefix); }
|
|
55
55
|
catch { /* fallback to local */ }
|
|
@@ -58,6 +58,7 @@ export async function embedTexts(texts: string[], prefix: "query" | "passage"):
|
|
|
58
58
|
resetIdleTimer();
|
|
59
59
|
const results: Float32Array[] = [];
|
|
60
60
|
for (const text of texts) {
|
|
61
|
+
if (signal?.aborted) throw new Error("Cancelled");
|
|
61
62
|
const output = await pipe(`${prefix}: ${text}`, { pooling: "mean", normalize: true });
|
|
62
63
|
results.push(new Float32Array(output.data));
|
|
63
64
|
}
|
|
@@ -69,6 +70,6 @@ export async function embedQuery(text: string): Promise<Float32Array> {
|
|
|
69
70
|
return vec;
|
|
70
71
|
}
|
|
71
72
|
|
|
72
|
-
export async function embedDocuments(texts: string[]): Promise<Float32Array[]> {
|
|
73
|
-
return embedTexts(texts, "passage");
|
|
73
|
+
export async function embedDocuments(texts: string[], signal?: AbortSignal): Promise<Float32Array[]> {
|
|
74
|
+
return embedTexts(texts, "passage", signal);
|
|
74
75
|
}
|
package/src/engine.ts
CHANGED
|
@@ -3,7 +3,7 @@ import { join, resolve } from "node:path";
|
|
|
3
3
|
import type Database from "better-sqlite3";
|
|
4
4
|
import { embedDocuments, embedQuery, dispose as disposeEmbedding } from "./embedding/provider.ts";
|
|
5
5
|
import { loadVectors, saveVectors } from "./embedding/vectors.ts";
|
|
6
|
-
import { chunkFile, walkDir } from "./indexer/chunker.ts";
|
|
6
|
+
import { chunkFile, walkDir, contentHash, preTokenizeForFTS } from "./indexer/chunker.ts";
|
|
7
7
|
import { searchBM25 } from "./search/bm25.ts";
|
|
8
8
|
import { reciprocalRankFusion } from "./search/fusion.ts";
|
|
9
9
|
import { rerank, disposeReranker } from "./search/reranker.ts";
|
|
@@ -34,10 +34,13 @@ export interface SearchResult {
|
|
|
34
34
|
end_line: number;
|
|
35
35
|
}
|
|
36
36
|
|
|
37
|
+
export const CURRENT_EMBEDDING_MODEL = "multilingual-e5-small";
|
|
38
|
+
|
|
37
39
|
export interface SearchResponse {
|
|
38
40
|
results: SearchResult[];
|
|
39
41
|
total_count: number;
|
|
40
42
|
has_more: boolean;
|
|
43
|
+
warnings?: string[];
|
|
41
44
|
}
|
|
42
45
|
|
|
43
46
|
export type ProgressCallback = (msg: string) => void;
|
|
@@ -45,26 +48,62 @@ export type ProgressCallback = (msg: string) => void;
|
|
|
45
48
|
export class KnowledgeEngine {
|
|
46
49
|
private db: Database.Database | null = null;
|
|
47
50
|
private knowledgeDir: string = "";
|
|
51
|
+
private vectorCache: Map<string, Float32Array[]> = new Map();
|
|
48
52
|
|
|
49
53
|
async initialize(knowledgeDir: string): Promise<void> {
|
|
50
54
|
this.knowledgeDir = knowledgeDir;
|
|
51
55
|
this.db = openDatabase(knowledgeDir);
|
|
56
|
+
this.vectorCache.clear();
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
private getVectors(kbId: string): Float32Array[] {
|
|
60
|
+
if (this.vectorCache.has(kbId)) return this.vectorCache.get(kbId)!;
|
|
61
|
+
const vectorPath = join(this.knowledgeDir, "vectors", `${kbId}.bin`);
|
|
62
|
+
const vectors = loadVectors(vectorPath);
|
|
63
|
+
this.vectorCache.set(kbId, vectors);
|
|
64
|
+
return vectors;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
private invalidateVectorCache(kbId: string): void {
|
|
68
|
+
this.vectorCache.delete(kbId);
|
|
52
69
|
}
|
|
53
70
|
|
|
54
71
|
async add(source: string, name: string, onProgress?: ProgressCallback): Promise<{ kb: KnowledgeBase; chunkCount: number }> {
|
|
55
72
|
if (!this.db) throw new Error("Engine not initialized");
|
|
56
73
|
const resolvedSource = resolve(source);
|
|
57
|
-
const
|
|
58
|
-
const
|
|
74
|
+
const isUrl = source.startsWith("http://") || source.startsWith("https://");
|
|
75
|
+
const isDir = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isDirectory();
|
|
76
|
+
const isFile = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isFile();
|
|
59
77
|
const sourceType = isDir ? "directory" : isFile ? "file" : "text";
|
|
60
78
|
|
|
61
|
-
const kb = createKB(this.db, { name, source_path: isDir || isFile ? resolvedSource : undefined, source_type: sourceType });
|
|
79
|
+
const kb = createKB(this.db, { name, source_path: isDir || isFile ? resolvedSource : isUrl ? source : undefined, source_type: sourceType });
|
|
62
80
|
updateKBStatus(this.db, kb.id, "indexing");
|
|
63
81
|
|
|
64
82
|
try {
|
|
65
83
|
let allChunks: Awaited<ReturnType<typeof chunkFile>> = [];
|
|
66
84
|
|
|
67
|
-
if (
|
|
85
|
+
if (isUrl) {
|
|
86
|
+
onProgress?.(`Fetching ${source}...`);
|
|
87
|
+
const res = await fetch(source);
|
|
88
|
+
if (!res.ok) throw new Error(`Fetch failed: ${res.status}`);
|
|
89
|
+
const html = await res.text();
|
|
90
|
+
const text = html.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "")
|
|
91
|
+
.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "")
|
|
92
|
+
.replace(/<[^>]+>/g, " ").replace(/&[a-z]+;/gi, " ")
|
|
93
|
+
.replace(/\s+/g, " ").trim();
|
|
94
|
+
allChunks = await chunkFile(text, source);
|
|
95
|
+
} else if (isFile && resolvedSource.endsWith(".pdf")) {
|
|
96
|
+
onProgress?.("Extracting text from PDF...");
|
|
97
|
+
const { extractText } = await import("unpdf");
|
|
98
|
+
const buf = (await import("node:fs")).readFileSync(resolvedSource);
|
|
99
|
+
const { text } = await extractText(new Uint8Array(buf));
|
|
100
|
+
allChunks = await chunkFile(text, resolvedSource);
|
|
101
|
+
} else if (isFile && (resolvedSource.endsWith(".docx") || resolvedSource.endsWith(".doc"))) {
|
|
102
|
+
onProgress?.("Extracting text from DOCX...");
|
|
103
|
+
const mammoth = await import("mammoth");
|
|
104
|
+
const result = await mammoth.extractRawText({ path: resolvedSource });
|
|
105
|
+
allChunks = await chunkFile(result.value, resolvedSource);
|
|
106
|
+
} else if (isDir) {
|
|
68
107
|
onProgress?.(`Scanning ${resolvedSource}...`);
|
|
69
108
|
const files = walkDir(resolvedSource);
|
|
70
109
|
onProgress?.(`Found ${files.length} files, chunking...`);
|
|
@@ -88,6 +127,7 @@ export class KnowledgeEngine {
|
|
|
88
127
|
|
|
89
128
|
const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
|
|
90
129
|
saveVectors(vectorPath, vectors);
|
|
130
|
+
this.vectorCache.set(kb.id, vectors);
|
|
91
131
|
|
|
92
132
|
const fileCount = isDir ? getFileCount(this.db, kb.id) : 1;
|
|
93
133
|
updateKBCounts(this.db, kb.id, allChunks.length, fileCount);
|
|
@@ -165,6 +205,7 @@ export class KnowledgeEngine {
|
|
|
165
205
|
const finalChunks = getChunksByKB(this.db, kb.id);
|
|
166
206
|
const finalVectors = finalChunks.map((c) => vectorCache.get(c.content_hash)!).filter(Boolean);
|
|
167
207
|
saveVectors(vectorPath, finalVectors);
|
|
208
|
+
this.vectorCache.set(kb.id, finalVectors);
|
|
168
209
|
|
|
169
210
|
// 7. Update counts
|
|
170
211
|
updateKBCounts(this.db, kb.id, finalChunks.length, getFileCount(this.db, kb.id));
|
|
@@ -184,17 +225,21 @@ export class KnowledgeEngine {
|
|
|
184
225
|
const kbs = kb_id ? [getKB(this.db, kb_id)].filter(Boolean) as KnowledgeBase[] : listKBs(this.db);
|
|
185
226
|
if (kbs.length === 0) return { results: [], total_count: 0, has_more: false };
|
|
186
227
|
|
|
228
|
+
const warnings: string[] = [];
|
|
187
229
|
let allResults: { chunkId: string; score: number }[] = [];
|
|
188
230
|
|
|
189
231
|
for (const kb of kbs) {
|
|
232
|
+
if (kb.embedding_model !== CURRENT_EMBEDDING_MODEL) {
|
|
233
|
+
warnings.push(`"${kb.name}" was indexed with ${kb.embedding_model} (current: ${CURRENT_EMBEDDING_MODEL}) — run knowledge_update for best results`);
|
|
234
|
+
}
|
|
190
235
|
const chunkIds = getChunkIdsByKB(this.db, kb.id);
|
|
191
236
|
if (chunkIds.length === 0) continue;
|
|
192
237
|
|
|
193
238
|
if (mode === "fast") {
|
|
194
239
|
allResults.push(...searchBM25(this.db, query, 50, kb.id));
|
|
195
240
|
} else if (mode === "semantic") {
|
|
196
|
-
|
|
197
|
-
const vectors =
|
|
241
|
+
|
|
242
|
+
const vectors = this.getVectors(kb.id);
|
|
198
243
|
if (vectors.length > 0) {
|
|
199
244
|
const queryVec = await embedQuery(query);
|
|
200
245
|
allResults.push(...searchVector(queryVec, vectors, chunkIds));
|
|
@@ -202,8 +247,8 @@ export class KnowledgeEngine {
|
|
|
202
247
|
} else {
|
|
203
248
|
// hybrid: BM25 + vector + RRF (both scoped to this KB)
|
|
204
249
|
const bm25Results = searchBM25(this.db, query, 50, kb.id);
|
|
205
|
-
|
|
206
|
-
const vectors =
|
|
250
|
+
|
|
251
|
+
const vectors = this.getVectors(kb.id);
|
|
207
252
|
let vecResults: { chunkId: string; score: number }[] = [];
|
|
208
253
|
if (vectors.length > 0) {
|
|
209
254
|
const queryVec = await embedQuery(query);
|
|
@@ -243,7 +288,7 @@ export class KnowledgeEngine {
|
|
|
243
288
|
const kbObj = getKB(this.db!, chunk.kb_id);
|
|
244
289
|
return { content: chunk.content, file_path: chunk.file_path, file_type: chunk.file_type, kb_name: kbObj?.name ?? "unknown", score: r.score, snippet: chunk.content.slice(0, 200), start_line: chunk.start_line, end_line: chunk.end_line };
|
|
245
290
|
});
|
|
246
|
-
return { results, total_count: results.length, has_more: false };
|
|
291
|
+
return { results, total_count: results.length, has_more: false, warnings: warnings.length > 0 ? warnings : undefined };
|
|
247
292
|
}
|
|
248
293
|
|
|
249
294
|
const total = filtered.length;
|
|
@@ -265,13 +310,14 @@ export class KnowledgeEngine {
|
|
|
265
310
|
};
|
|
266
311
|
}).filter(Boolean) as SearchResult[];
|
|
267
312
|
|
|
268
|
-
return { results, total_count: total, has_more: offset + limit < total };
|
|
313
|
+
return { results, total_count: total, has_more: offset + limit < total, warnings: warnings.length > 0 ? warnings : undefined };
|
|
269
314
|
}
|
|
270
315
|
|
|
271
316
|
remove(nameOrId: string): boolean {
|
|
272
317
|
if (!this.db) return false;
|
|
273
318
|
const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
|
|
274
319
|
if (!kb) return false;
|
|
320
|
+
this.invalidateVectorCache(kb.id);
|
|
275
321
|
deleteKB(this.db, kb.id);
|
|
276
322
|
return true;
|
|
277
323
|
}
|
|
@@ -284,6 +330,7 @@ export class KnowledgeEngine {
|
|
|
284
330
|
clear(): void {
|
|
285
331
|
if (!this.db) return;
|
|
286
332
|
for (const kb of listKBs(this.db)) deleteKB(this.db, kb.id);
|
|
333
|
+
this.vectorCache.clear();
|
|
287
334
|
}
|
|
288
335
|
|
|
289
336
|
diagnose(): DiagnosticResult[] {
|
|
@@ -291,6 +338,40 @@ export class KnowledgeEngine {
|
|
|
291
338
|
return listKBs(this.db).map((kb) => diagnoseKB(this.db!, kb));
|
|
292
339
|
}
|
|
293
340
|
|
|
341
|
+
async exportKB(nameOrId: string, outputPath: string): Promise<number> {
|
|
342
|
+
if (!this.db) throw new Error("Engine not initialized");
|
|
343
|
+
const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
|
|
344
|
+
if (!kb) throw new Error(`Knowledge base not found: ${nameOrId}`);
|
|
345
|
+
const chunks = getChunksByKB(this.db, kb.id);
|
|
346
|
+
const { writeFileSync } = await import("node:fs");
|
|
347
|
+
const header = JSON.stringify({ name: kb.name, description: kb.description, source_path: kb.source_path, source_type: kb.source_type, chunk_count: chunks.length });
|
|
348
|
+
const lines = [header, ...chunks.map((c) => JSON.stringify({ content: c.content, file_path: c.file_path, file_type: c.file_type, start_line: c.start_line, end_line: c.end_line, metadata_json: c.metadata_json }))];
|
|
349
|
+
writeFileSync(outputPath, lines.join("\n") + "\n");
|
|
350
|
+
return chunks.length;
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
async importKB(inputPath: string, onProgress?: ProgressCallback): Promise<{ kb: KnowledgeBase; chunkCount: number }> {
|
|
354
|
+
if (!this.db) throw new Error("Engine not initialized");
|
|
355
|
+
const { readFileSync } = await import("node:fs");
|
|
356
|
+
const lines = readFileSync(inputPath, "utf-8").trim().split("\n");
|
|
357
|
+
if (lines.length < 1) throw new Error("Empty import file");
|
|
358
|
+
const header = JSON.parse(lines[0]);
|
|
359
|
+
const kb = createKB(this.db, { name: header.name, description: header.description, source_path: header.source_path, source_type: header.source_type || "text" });
|
|
360
|
+
updateKBStatus(this.db, kb.id, "indexing");
|
|
361
|
+
const chunkData = lines.slice(1).map((l) => JSON.parse(l));
|
|
362
|
+
onProgress?.(`Importing ${chunkData.length} chunks, embedding...`);
|
|
363
|
+
const texts = chunkData.map((c: any) => c.content);
|
|
364
|
+
const vectors = await embedDocuments(texts);
|
|
365
|
+
const chunks = chunkData.map((c: any) => ({ content_hash: contentHash(c.content), content: c.content, content_tokenized: preTokenizeForFTS(c.content), file_path: c.file_path, file_type: c.file_type, start_line: c.start_line, end_line: c.end_line, metadata_json: c.metadata_json || "{}" }));
|
|
366
|
+
insertChunks(this.db, kb.id, chunks);
|
|
367
|
+
const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
|
|
368
|
+
saveVectors(vectorPath, vectors);
|
|
369
|
+
this.vectorCache.set(kb.id, vectors);
|
|
370
|
+
updateKBCounts(this.db, kb.id, chunks.length, new Set(chunks.map((c: any) => c.file_path)).size);
|
|
371
|
+
updateKBStatus(this.db, kb.id, "ready");
|
|
372
|
+
return { kb: getKB(this.db, kb.id)!, chunkCount: chunks.length };
|
|
373
|
+
}
|
|
374
|
+
|
|
294
375
|
async dispose(): Promise<void> {
|
|
295
376
|
await disposeEmbedding();
|
|
296
377
|
await disposeReranker();
|
package/src/indexer/chunker.ts
CHANGED
|
@@ -70,7 +70,9 @@ export function walkDir(dirPath: string): ScannedFile[] {
|
|
|
70
70
|
const results: ScannedFile[] = [];
|
|
71
71
|
|
|
72
72
|
function walk(dir: string): void {
|
|
73
|
-
|
|
73
|
+
let entries;
|
|
74
|
+
try { entries = readdirSync(dir, { withFileTypes: true }); }
|
|
75
|
+
catch { return; }
|
|
74
76
|
for (const entry of entries) {
|
|
75
77
|
const fullPath = join(dir, entry.name);
|
|
76
78
|
const relPath = relative(dirPath, fullPath).split(sep).join("/");
|
|
@@ -251,13 +253,32 @@ export function chunkText(content: string, filePath: string): Omit<ChunkInsert,
|
|
|
251
253
|
|
|
252
254
|
export async function chunkFile(content: string, filePath: string): Promise<Omit<ChunkInsert, "kb_id">[]> {
|
|
253
255
|
const fileType = detectFileType(filePath);
|
|
254
|
-
|
|
255
|
-
|
|
256
|
+
let chunks: Omit<ChunkInsert, "kb_id">[] = [];
|
|
257
|
+
|
|
258
|
+
if (fileType === "markdown") {
|
|
259
|
+
chunks = chunkMarkdown(content, filePath);
|
|
260
|
+
} else if (["typescript", "javascript", "python", "go", "rust", "java"].includes(fileType)) {
|
|
256
261
|
try {
|
|
257
262
|
const { chunkWithAST } = await import("./chunkers/code-ast.ts");
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
} catch { /* fallback to text chunker */ }
|
|
263
|
+
chunks = await chunkWithAST(content, filePath, fileType);
|
|
264
|
+
} catch { /* fallback below */ }
|
|
261
265
|
}
|
|
262
|
-
|
|
266
|
+
|
|
267
|
+
if (chunks.length === 0) chunks = chunkText(content, filePath);
|
|
268
|
+
|
|
269
|
+
// Fallback: if file has content but no chunks (too short for splitting), keep as single chunk
|
|
270
|
+
if (chunks.length === 0 && content.trim().length > 10) {
|
|
271
|
+
chunks = [{
|
|
272
|
+
content_hash: contentHash(content),
|
|
273
|
+
content: content.trim(),
|
|
274
|
+
content_tokenized: preTokenizeForFTS(content),
|
|
275
|
+
file_path: filePath,
|
|
276
|
+
file_type: fileType,
|
|
277
|
+
start_line: 1,
|
|
278
|
+
end_line: content.split("\n").length,
|
|
279
|
+
metadata_json: "{}",
|
|
280
|
+
}];
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
return chunks;
|
|
263
284
|
}
|
package/src/storage/sqlite.ts
CHANGED
|
@@ -113,11 +113,28 @@ export function openDatabase(knowledgeDir?: string): Database.Database {
|
|
|
113
113
|
if (!hasVersion) {
|
|
114
114
|
db.exec(SCHEMA_SQL);
|
|
115
115
|
db.prepare("INSERT INTO schema_version (version) VALUES (?)").run(SCHEMA_VERSION);
|
|
116
|
+
} else {
|
|
117
|
+
const row = db.prepare("SELECT version FROM schema_version").get() as { version: number } | undefined;
|
|
118
|
+
const currentVersion = row?.version ?? 0;
|
|
119
|
+
if (currentVersion < SCHEMA_VERSION) {
|
|
120
|
+
runMigrations(db, currentVersion, SCHEMA_VERSION);
|
|
121
|
+
db.prepare("UPDATE schema_version SET version = ?").run(SCHEMA_VERSION);
|
|
122
|
+
}
|
|
116
123
|
}
|
|
117
124
|
|
|
118
125
|
return db;
|
|
119
126
|
}
|
|
120
127
|
|
|
128
|
+
function runMigrations(db: Database.Database, from: number, to: number): void {
|
|
129
|
+
const migrations: Record<number, string> = {
|
|
130
|
+
// Future migrations go here, keyed by target version number:
|
|
131
|
+
// 2: "ALTER TABLE ...",
|
|
132
|
+
};
|
|
133
|
+
for (let v = from + 1; v <= to; v++) {
|
|
134
|
+
if (migrations[v]) db.exec(migrations[v]);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
121
138
|
// --- CRUD: Knowledge Bases ---
|
|
122
139
|
|
|
123
140
|
export function createKB(
|