pi-knowledge 0.1.1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,6 +1,49 @@
1
1
  # Changelog
2
2
 
3
- ## [Unreleased]
3
+ ## [0.2.0] - 2026-06-15
4
+
5
+ ### Added
6
+ - URL indexing (fetch → HTML strip → chunk)
7
+ - PDF text extraction (via unpdf, pure JS)
8
+ - DOCX text extraction (via mammoth, pure JS)
9
+ - Import/export knowledge bases (JSONL format, git-friendly)
10
+ - Performance benchmarks (BM25: 0.05ms, hybrid: 2.1ms)
11
+ - Pi Skill: /skill:search-docs
12
+ - 9 tools total
13
+
14
+ ## [0.1.4] - 2026-06-15
15
+
16
+ ### Added
17
+ - Vector memory cache (search no longer re-reads disk per query)
18
+ - AbortSignal support in embedding (cancellable long operations)
19
+ - Schema migration infrastructure (future-proof DB upgrades)
20
+ - Model mismatch warning on search (suggests re-index)
21
+ - Engine regression tests (+6, total 34)
22
+ - README Data Storage section
23
+ - URL indexing (knowledge_add with http/https URLs, auto HTML strip)
24
+ - Performance benchmarks (BM25: 0.05ms, hybrid: 2.1ms, semantic: 2.0ms)
25
+ - Pi Skill: `/skill:search-docs` for guided knowledge search
26
+
27
+ ### Fixed
28
+ - walkDir skips permission-denied directories
29
+ - DESIGN.md phases corrected
30
+
31
+ ## [0.1.3] - 2026-06-14
32
+
33
+ ### Fixed
34
+ - Short files index correctly (single chunk fallback)
35
+
36
+ ## [0.1.2] - 2026-06-14
37
+
38
+ ### Fixed
39
+ - Short files (<50 chars) now index correctly as a single chunk
40
+
41
+ ## [0.1.1] - 2026-06-14
42
+
43
+ ### Added
44
+ - Java AST chunking (6 languages total: TS, JS, Python, Go, Rust, Java)
45
+
46
+ ## [0.1.0] - 2026-06-14
4
47
 
5
48
  ### Added
6
49
  - Project scaffold and design document
package/README.md CHANGED
@@ -22,7 +22,7 @@ Unlike `pi-memory` (which manages the agent's own notes), `pi-knowledge` indexes
22
22
  | **Cross-encoder reranking** | ✅ | ❌ | ❌ |
23
23
  | **Incremental re-indexing** | ✅ | ❌ | ❌ |
24
24
  | **File watcher (auto-update)** | ✅ | ❌ | ❌ |
25
- | **Code-aware chunking** | ✅ | ❌ | ❌ |
25
+ | **Code-aware chunking** | ✅ (TS/JS/Py/Go/Rust/Java) | ❌ | ❌ |
26
26
  | **Local embeddings (zero API)** | ✅ | ❌ | ✅ (qmd) |
27
27
  | **Index quality diagnostics** | ✅ | ❌ | ❌ |
28
28
  | **Metadata filters in search** | ✅ | ❌ | ❌ |
@@ -56,7 +56,7 @@ pi install ./pi-knowledge
56
56
 
57
57
  | Tool | Description |
58
58
  |------|-------------|
59
- | `knowledge_add` | Index files, directories, or inline text into a named knowledge base |
59
+ | `knowledge_add` | Index files, directories, URLs, PDFs, DOCX, or inline text |
60
60
  | `knowledge_search` | Hybrid search across one or all knowledge bases |
61
61
  | `knowledge_remove` | Remove a knowledge base by name or ID |
62
62
  | `knowledge_update` | Incrementally re-index changed files in a knowledge base |
@@ -68,6 +68,22 @@ pi install ./pi-knowledge
68
68
 
69
69
  See [DESIGN.md](DESIGN.md) for the full technical design document.
70
70
 
71
+ ## Data Storage
72
+
73
+ All data is stored globally at `~/.pi/knowledge/` (never in your project directory):
74
+
75
+ ```
76
+ ~/.pi/knowledge/
77
+ ├── knowledge.db ← SQLite (metadata + chunks + FTS5 index)
78
+ ├── vectors/ ← Embedding vectors per KB (binary)
79
+ └── models/ ← Downloaded ONNX models (~32MB, cached)
80
+ ```
81
+
82
+ - **Backup**: copy `~/.pi/knowledge/` directory
83
+ - **Reset**: delete `~/.pi/knowledge/` to start fresh
84
+ - **Project safety**: pi-knowledge is read-only on indexed directories — no files are created or modified in your project
85
+ - **Updates**: extension updates do not affect existing indexed data. Schema migrations run automatically if needed.
86
+
71
87
  ## Development
72
88
 
73
89
  ```bash
package/index.ts CHANGED
@@ -107,12 +107,14 @@ export default function (pi: ExtensionAPI) {
107
107
  if (response.results.length === 0) {
108
108
  return { content: [{ type: "text", text: "No results found." }] };
109
109
  }
110
- const formatted = response.results.map((r, i) =>
110
+ let output = `${response.total_count} results (showing ${response.results.length}):\n\n`;
111
+ if (response.warnings?.length) {
112
+ output = `⚠️ ${response.warnings.join("\n⚠️ ")}\n\n${output}`;
113
+ }
114
+ output += response.results.map((r, i) =>
111
115
  `[${i + 1}] ${r.file_path} (${r.kb_name}, score: ${r.score.toFixed(3)})\n${r.snippet}`,
112
116
  ).join("\n\n");
113
- return {
114
- content: [{ type: "text", text: `${response.total_count} results (showing ${response.results.length}):\n\n${formatted}` }],
115
- };
117
+ return { content: [{ type: "text", text: output }] };
116
118
  },
117
119
  });
118
120
 
@@ -189,6 +191,35 @@ export default function (pi: ExtensionAPI) {
189
191
  },
190
192
  });
191
193
 
194
+ pi.registerTool({
195
+ name: "knowledge_export",
196
+ label: "Knowledge Export",
197
+ description: "Export a knowledge base to a JSONL file (shareable, git-friendly)",
198
+ parameters: Type.Object({
199
+ target: Type.String({ description: "KB name or ID to export" }),
200
+ output: Type.String({ description: "Output file path (.jsonl)" }),
201
+ }),
202
+ async execute(_id, params) {
203
+ const count = await engine.exportKB(params.target, params.output);
204
+ return { content: [{ type: "text", text: `Exported ${count} chunks to ${params.output}` }] };
205
+ },
206
+ });
207
+
208
+ pi.registerTool({
209
+ name: "knowledge_import",
210
+ label: "Knowledge Import",
211
+ description: "Import a knowledge base from a JSONL file (re-embeds content)",
212
+ parameters: Type.Object({
213
+ input: Type.String({ description: "Input JSONL file path" }),
214
+ }),
215
+ async execute(_id, params, _signal, onUpdate) {
216
+ const { kb, chunkCount } = await engine.importKB(params.input, (msg) => {
217
+ onUpdate?.({ content: [{ type: "text", text: msg }] });
218
+ });
219
+ return { content: [{ type: "text", text: `Imported "${kb.name}": ${chunkCount} chunks (re-embedded)` }] };
220
+ },
221
+ });
222
+
192
223
  pi.registerTool({
193
224
  name: "knowledge_clear",
194
225
  label: "Knowledge Clear",
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "pi-knowledge",
3
- "version": "0.1.1",
3
+ "version": "0.2.0",
4
4
  "description": "Vector knowledge base extension for Pi — index files, hybrid search, local embeddings",
5
5
  "type": "module",
6
6
  "main": "index.ts",
@@ -35,12 +35,14 @@
35
35
  "@huggingface/transformers": "3.8.1",
36
36
  "better-sqlite3": "11.9.1",
37
37
  "ignore": "7.0.5",
38
+ "mammoth": "1.12.0",
38
39
  "tree-sitter": "0.22.4",
39
40
  "tree-sitter-go": "0.23.4",
40
41
  "tree-sitter-java": "0.23.5",
41
42
  "tree-sitter-python": "0.23.6",
42
43
  "tree-sitter-rust": "0.23.2",
43
- "tree-sitter-typescript": "0.23.2"
44
+ "tree-sitter-typescript": "0.23.2",
45
+ "unpdf": "1.6.2"
44
46
  },
45
47
  "devDependencies": {
46
48
  "@biomejs/biome": "2.3.5",
@@ -49,7 +49,7 @@ async function embedViaAPI(texts: string[], prefix: "query" | "passage"): Promis
49
49
  throw new Error(`Unsupported embedding provider: ${provider}`);
50
50
  }
51
51
 
52
- export async function embedTexts(texts: string[], prefix: "query" | "passage"): Promise<Float32Array[]> {
52
+ export async function embedTexts(texts: string[], prefix: "query" | "passage", signal?: AbortSignal): Promise<Float32Array[]> {
53
53
  if (!EMBEDDING_CONFIG.startsWith("local")) {
54
54
  try { return await embedViaAPI(texts, prefix); }
55
55
  catch { /* fallback to local */ }
@@ -58,6 +58,7 @@ export async function embedTexts(texts: string[], prefix: "query" | "passage"):
58
58
  resetIdleTimer();
59
59
  const results: Float32Array[] = [];
60
60
  for (const text of texts) {
61
+ if (signal?.aborted) throw new Error("Cancelled");
61
62
  const output = await pipe(`${prefix}: ${text}`, { pooling: "mean", normalize: true });
62
63
  results.push(new Float32Array(output.data));
63
64
  }
@@ -69,6 +70,6 @@ export async function embedQuery(text: string): Promise<Float32Array> {
69
70
  return vec;
70
71
  }
71
72
 
72
- export async function embedDocuments(texts: string[]): Promise<Float32Array[]> {
73
- return embedTexts(texts, "passage");
73
+ export async function embedDocuments(texts: string[], signal?: AbortSignal): Promise<Float32Array[]> {
74
+ return embedTexts(texts, "passage", signal);
74
75
  }
package/src/engine.ts CHANGED
@@ -3,7 +3,7 @@ import { join, resolve } from "node:path";
3
3
  import type Database from "better-sqlite3";
4
4
  import { embedDocuments, embedQuery, dispose as disposeEmbedding } from "./embedding/provider.ts";
5
5
  import { loadVectors, saveVectors } from "./embedding/vectors.ts";
6
- import { chunkFile, walkDir } from "./indexer/chunker.ts";
6
+ import { chunkFile, walkDir, contentHash, preTokenizeForFTS } from "./indexer/chunker.ts";
7
7
  import { searchBM25 } from "./search/bm25.ts";
8
8
  import { reciprocalRankFusion } from "./search/fusion.ts";
9
9
  import { rerank, disposeReranker } from "./search/reranker.ts";
@@ -34,10 +34,13 @@ export interface SearchResult {
34
34
  end_line: number;
35
35
  }
36
36
 
37
+ export const CURRENT_EMBEDDING_MODEL = "multilingual-e5-small";
38
+
37
39
  export interface SearchResponse {
38
40
  results: SearchResult[];
39
41
  total_count: number;
40
42
  has_more: boolean;
43
+ warnings?: string[];
41
44
  }
42
45
 
43
46
  export type ProgressCallback = (msg: string) => void;
@@ -45,26 +48,62 @@ export type ProgressCallback = (msg: string) => void;
45
48
  export class KnowledgeEngine {
46
49
  private db: Database.Database | null = null;
47
50
  private knowledgeDir: string = "";
51
+ private vectorCache: Map<string, Float32Array[]> = new Map();
48
52
 
49
53
  async initialize(knowledgeDir: string): Promise<void> {
50
54
  this.knowledgeDir = knowledgeDir;
51
55
  this.db = openDatabase(knowledgeDir);
56
+ this.vectorCache.clear();
57
+ }
58
+
59
+ private getVectors(kbId: string): Float32Array[] {
60
+ if (this.vectorCache.has(kbId)) return this.vectorCache.get(kbId)!;
61
+ const vectorPath = join(this.knowledgeDir, "vectors", `${kbId}.bin`);
62
+ const vectors = loadVectors(vectorPath);
63
+ this.vectorCache.set(kbId, vectors);
64
+ return vectors;
65
+ }
66
+
67
+ private invalidateVectorCache(kbId: string): void {
68
+ this.vectorCache.delete(kbId);
52
69
  }
53
70
 
54
71
  async add(source: string, name: string, onProgress?: ProgressCallback): Promise<{ kb: KnowledgeBase; chunkCount: number }> {
55
72
  if (!this.db) throw new Error("Engine not initialized");
56
73
  const resolvedSource = resolve(source);
57
- const isDir = existsSync(resolvedSource) && statSync(resolvedSource).isDirectory();
58
- const isFile = existsSync(resolvedSource) && statSync(resolvedSource).isFile();
74
+ const isUrl = source.startsWith("http://") || source.startsWith("https://");
75
+ const isDir = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isDirectory();
76
+ const isFile = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isFile();
59
77
  const sourceType = isDir ? "directory" : isFile ? "file" : "text";
60
78
 
61
- const kb = createKB(this.db, { name, source_path: isDir || isFile ? resolvedSource : undefined, source_type: sourceType });
79
+ const kb = createKB(this.db, { name, source_path: isDir || isFile ? resolvedSource : isUrl ? source : undefined, source_type: sourceType });
62
80
  updateKBStatus(this.db, kb.id, "indexing");
63
81
 
64
82
  try {
65
83
  let allChunks: Awaited<ReturnType<typeof chunkFile>> = [];
66
84
 
67
- if (isDir) {
85
+ if (isUrl) {
86
+ onProgress?.(`Fetching ${source}...`);
87
+ const res = await fetch(source);
88
+ if (!res.ok) throw new Error(`Fetch failed: ${res.status}`);
89
+ const html = await res.text();
90
+ const text = html.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "")
91
+ .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "")
92
+ .replace(/<[^>]+>/g, " ").replace(/&[a-z]+;/gi, " ")
93
+ .replace(/\s+/g, " ").trim();
94
+ allChunks = await chunkFile(text, source);
95
+ } else if (isFile && resolvedSource.endsWith(".pdf")) {
96
+ onProgress?.("Extracting text from PDF...");
97
+ const { extractText } = await import("unpdf");
98
+ const buf = (await import("node:fs")).readFileSync(resolvedSource);
99
+ const { text } = await extractText(new Uint8Array(buf));
100
+ allChunks = await chunkFile(text, resolvedSource);
101
+ } else if (isFile && (resolvedSource.endsWith(".docx") || resolvedSource.endsWith(".doc"))) {
102
+ onProgress?.("Extracting text from DOCX...");
103
+ const mammoth = await import("mammoth");
104
+ const result = await mammoth.extractRawText({ path: resolvedSource });
105
+ allChunks = await chunkFile(result.value, resolvedSource);
106
+ } else if (isDir) {
68
107
  onProgress?.(`Scanning ${resolvedSource}...`);
69
108
  const files = walkDir(resolvedSource);
70
109
  onProgress?.(`Found ${files.length} files, chunking...`);
@@ -88,6 +127,7 @@ export class KnowledgeEngine {
88
127
 
89
128
  const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
90
129
  saveVectors(vectorPath, vectors);
130
+ this.vectorCache.set(kb.id, vectors);
91
131
 
92
132
  const fileCount = isDir ? getFileCount(this.db, kb.id) : 1;
93
133
  updateKBCounts(this.db, kb.id, allChunks.length, fileCount);
@@ -165,6 +205,7 @@ export class KnowledgeEngine {
165
205
  const finalChunks = getChunksByKB(this.db, kb.id);
166
206
  const finalVectors = finalChunks.map((c) => vectorCache.get(c.content_hash)!).filter(Boolean);
167
207
  saveVectors(vectorPath, finalVectors);
208
+ this.vectorCache.set(kb.id, finalVectors);
168
209
 
169
210
  // 7. Update counts
170
211
  updateKBCounts(this.db, kb.id, finalChunks.length, getFileCount(this.db, kb.id));
@@ -184,17 +225,21 @@ export class KnowledgeEngine {
184
225
  const kbs = kb_id ? [getKB(this.db, kb_id)].filter(Boolean) as KnowledgeBase[] : listKBs(this.db);
185
226
  if (kbs.length === 0) return { results: [], total_count: 0, has_more: false };
186
227
 
228
+ const warnings: string[] = [];
187
229
  let allResults: { chunkId: string; score: number }[] = [];
188
230
 
189
231
  for (const kb of kbs) {
232
+ if (kb.embedding_model !== CURRENT_EMBEDDING_MODEL) {
233
+ warnings.push(`"${kb.name}" was indexed with ${kb.embedding_model} (current: ${CURRENT_EMBEDDING_MODEL}) — run knowledge_update for best results`);
234
+ }
190
235
  const chunkIds = getChunkIdsByKB(this.db, kb.id);
191
236
  if (chunkIds.length === 0) continue;
192
237
 
193
238
  if (mode === "fast") {
194
239
  allResults.push(...searchBM25(this.db, query, 50, kb.id));
195
240
  } else if (mode === "semantic") {
196
- const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
197
- const vectors = loadVectors(vectorPath);
241
+
242
+ const vectors = this.getVectors(kb.id);
198
243
  if (vectors.length > 0) {
199
244
  const queryVec = await embedQuery(query);
200
245
  allResults.push(...searchVector(queryVec, vectors, chunkIds));
@@ -202,8 +247,8 @@ export class KnowledgeEngine {
202
247
  } else {
203
248
  // hybrid: BM25 + vector + RRF (both scoped to this KB)
204
249
  const bm25Results = searchBM25(this.db, query, 50, kb.id);
205
- const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
206
- const vectors = loadVectors(vectorPath);
250
+
251
+ const vectors = this.getVectors(kb.id);
207
252
  let vecResults: { chunkId: string; score: number }[] = [];
208
253
  if (vectors.length > 0) {
209
254
  const queryVec = await embedQuery(query);
@@ -243,7 +288,7 @@ export class KnowledgeEngine {
243
288
  const kbObj = getKB(this.db!, chunk.kb_id);
244
289
  return { content: chunk.content, file_path: chunk.file_path, file_type: chunk.file_type, kb_name: kbObj?.name ?? "unknown", score: r.score, snippet: chunk.content.slice(0, 200), start_line: chunk.start_line, end_line: chunk.end_line };
245
290
  });
246
- return { results, total_count: results.length, has_more: false };
291
+ return { results, total_count: results.length, has_more: false, warnings: warnings.length > 0 ? warnings : undefined };
247
292
  }
248
293
 
249
294
  const total = filtered.length;
@@ -265,13 +310,14 @@ export class KnowledgeEngine {
265
310
  };
266
311
  }).filter(Boolean) as SearchResult[];
267
312
 
268
- return { results, total_count: total, has_more: offset + limit < total };
313
+ return { results, total_count: total, has_more: offset + limit < total, warnings: warnings.length > 0 ? warnings : undefined };
269
314
  }
270
315
 
271
316
  remove(nameOrId: string): boolean {
272
317
  if (!this.db) return false;
273
318
  const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
274
319
  if (!kb) return false;
320
+ this.invalidateVectorCache(kb.id);
275
321
  deleteKB(this.db, kb.id);
276
322
  return true;
277
323
  }
@@ -284,6 +330,7 @@ export class KnowledgeEngine {
284
330
  clear(): void {
285
331
  if (!this.db) return;
286
332
  for (const kb of listKBs(this.db)) deleteKB(this.db, kb.id);
333
+ this.vectorCache.clear();
287
334
  }
288
335
 
289
336
  diagnose(): DiagnosticResult[] {
@@ -291,6 +338,40 @@ export class KnowledgeEngine {
291
338
  return listKBs(this.db).map((kb) => diagnoseKB(this.db!, kb));
292
339
  }
293
340
 
341
+ async exportKB(nameOrId: string, outputPath: string): Promise<number> {
342
+ if (!this.db) throw new Error("Engine not initialized");
343
+ const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
344
+ if (!kb) throw new Error(`Knowledge base not found: ${nameOrId}`);
345
+ const chunks = getChunksByKB(this.db, kb.id);
346
+ const { writeFileSync } = await import("node:fs");
347
+ const header = JSON.stringify({ name: kb.name, description: kb.description, source_path: kb.source_path, source_type: kb.source_type, chunk_count: chunks.length });
348
+ const lines = [header, ...chunks.map((c) => JSON.stringify({ content: c.content, file_path: c.file_path, file_type: c.file_type, start_line: c.start_line, end_line: c.end_line, metadata_json: c.metadata_json }))];
349
+ writeFileSync(outputPath, lines.join("\n") + "\n");
350
+ return chunks.length;
351
+ }
352
+
353
+ async importKB(inputPath: string, onProgress?: ProgressCallback): Promise<{ kb: KnowledgeBase; chunkCount: number }> {
354
+ if (!this.db) throw new Error("Engine not initialized");
355
+ const { readFileSync } = await import("node:fs");
356
+ const lines = readFileSync(inputPath, "utf-8").trim().split("\n");
357
+ if (lines.length < 1) throw new Error("Empty import file");
358
+ const header = JSON.parse(lines[0]);
359
+ const kb = createKB(this.db, { name: header.name, description: header.description, source_path: header.source_path, source_type: header.source_type || "text" });
360
+ updateKBStatus(this.db, kb.id, "indexing");
361
+ const chunkData = lines.slice(1).map((l) => JSON.parse(l));
362
+ onProgress?.(`Importing ${chunkData.length} chunks, embedding...`);
363
+ const texts = chunkData.map((c: any) => c.content);
364
+ const vectors = await embedDocuments(texts);
365
+ const chunks = chunkData.map((c: any) => ({ content_hash: contentHash(c.content), content: c.content, content_tokenized: preTokenizeForFTS(c.content), file_path: c.file_path, file_type: c.file_type, start_line: c.start_line, end_line: c.end_line, metadata_json: c.metadata_json || "{}" }));
366
+ insertChunks(this.db, kb.id, chunks);
367
+ const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
368
+ saveVectors(vectorPath, vectors);
369
+ this.vectorCache.set(kb.id, vectors);
370
+ updateKBCounts(this.db, kb.id, chunks.length, new Set(chunks.map((c: any) => c.file_path)).size);
371
+ updateKBStatus(this.db, kb.id, "ready");
372
+ return { kb: getKB(this.db, kb.id)!, chunkCount: chunks.length };
373
+ }
374
+
294
375
  async dispose(): Promise<void> {
295
376
  await disposeEmbedding();
296
377
  await disposeReranker();
@@ -70,7 +70,9 @@ export function walkDir(dirPath: string): ScannedFile[] {
70
70
  const results: ScannedFile[] = [];
71
71
 
72
72
  function walk(dir: string): void {
73
- const entries = readdirSync(dir, { withFileTypes: true });
73
+ let entries;
74
+ try { entries = readdirSync(dir, { withFileTypes: true }); }
75
+ catch { return; }
74
76
  for (const entry of entries) {
75
77
  const fullPath = join(dir, entry.name);
76
78
  const relPath = relative(dirPath, fullPath).split(sep).join("/");
@@ -251,13 +253,32 @@ export function chunkText(content: string, filePath: string): Omit<ChunkInsert,
251
253
 
252
254
  export async function chunkFile(content: string, filePath: string): Promise<Omit<ChunkInsert, "kb_id">[]> {
253
255
  const fileType = detectFileType(filePath);
254
- if (fileType === "markdown") return chunkMarkdown(content, filePath);
255
- if (["typescript", "javascript", "python", "go", "rust", "java"].includes(fileType)) {
256
+ let chunks: Omit<ChunkInsert, "kb_id">[] = [];
257
+
258
+ if (fileType === "markdown") {
259
+ chunks = chunkMarkdown(content, filePath);
260
+ } else if (["typescript", "javascript", "python", "go", "rust", "java"].includes(fileType)) {
256
261
  try {
257
262
  const { chunkWithAST } = await import("./chunkers/code-ast.ts");
258
- const astChunks = await chunkWithAST(content, filePath, fileType);
259
- if (astChunks.length > 0) return astChunks;
260
- } catch { /* fallback to text chunker */ }
263
+ chunks = await chunkWithAST(content, filePath, fileType);
264
+ } catch { /* fallback below */ }
261
265
  }
262
- return chunkText(content, filePath);
266
+
267
+ if (chunks.length === 0) chunks = chunkText(content, filePath);
268
+
269
+ // Fallback: if file has content but no chunks (too short for splitting), keep as single chunk
270
+ if (chunks.length === 0 && content.trim().length > 10) {
271
+ chunks = [{
272
+ content_hash: contentHash(content),
273
+ content: content.trim(),
274
+ content_tokenized: preTokenizeForFTS(content),
275
+ file_path: filePath,
276
+ file_type: fileType,
277
+ start_line: 1,
278
+ end_line: content.split("\n").length,
279
+ metadata_json: "{}",
280
+ }];
281
+ }
282
+
283
+ return chunks;
263
284
  }
@@ -113,11 +113,28 @@ export function openDatabase(knowledgeDir?: string): Database.Database {
113
113
  if (!hasVersion) {
114
114
  db.exec(SCHEMA_SQL);
115
115
  db.prepare("INSERT INTO schema_version (version) VALUES (?)").run(SCHEMA_VERSION);
116
+ } else {
117
+ const row = db.prepare("SELECT version FROM schema_version").get() as { version: number } | undefined;
118
+ const currentVersion = row?.version ?? 0;
119
+ if (currentVersion < SCHEMA_VERSION) {
120
+ runMigrations(db, currentVersion, SCHEMA_VERSION);
121
+ db.prepare("UPDATE schema_version SET version = ?").run(SCHEMA_VERSION);
122
+ }
116
123
  }
117
124
 
118
125
  return db;
119
126
  }
120
127
 
128
+ function runMigrations(db: Database.Database, from: number, to: number): void {
129
+ const migrations: Record<number, string> = {
130
+ // Future migrations go here, keyed by target version number:
131
+ // 2: "ALTER TABLE ...",
132
+ };
133
+ for (let v = from + 1; v <= to; v++) {
134
+ if (migrations[v]) db.exec(migrations[v]);
135
+ }
136
+ }
137
+
121
138
  // --- CRUD: Knowledge Bases ---
122
139
 
123
140
  export function createKB(