pi-knowledge 0.3.6 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.pi/skills/search-docs.md +24 -4
- package/CHANGELOG.md +9 -2
- package/README.md +40 -2
- package/index.ts +200 -14
- package/package.json +1 -1
- package/src/diagnostics/health.ts +84 -14
- package/src/embedding/vectors.ts +95 -12
- package/src/engine.ts +880 -140
- package/src/indexer/chunker.ts +232 -24
- package/src/search/bm25.ts +15 -4
- package/src/search/ranking.ts +4 -0
- package/src/search/vector.ts +60 -0
- package/src/storage/sqlite.ts +224 -9
package/src/storage/sqlite.ts
CHANGED
|
@@ -10,6 +10,7 @@ export interface KnowledgeBase {
|
|
|
10
10
|
description: string | null;
|
|
11
11
|
source_path: string | null;
|
|
12
12
|
source_type: "file" | "directory" | "text" | "url";
|
|
13
|
+
source_options: string | null;
|
|
13
14
|
created_at: number;
|
|
14
15
|
updated_at: number;
|
|
15
16
|
chunk_count: number;
|
|
@@ -18,6 +19,25 @@ export interface KnowledgeBase {
|
|
|
18
19
|
status: "ready" | "indexing" | "error" | "stale";
|
|
19
20
|
}
|
|
20
21
|
|
|
22
|
+
export interface IndexingJob {
|
|
23
|
+
kb_id: string;
|
|
24
|
+
operation: "add" | "update" | "import";
|
|
25
|
+
status: "running" | "succeeded" | "failed" | "cancelled";
|
|
26
|
+
phase: string;
|
|
27
|
+
message: string;
|
|
28
|
+
started_at: number;
|
|
29
|
+
updated_at: number;
|
|
30
|
+
last_progress_at: number;
|
|
31
|
+
processed_files: number;
|
|
32
|
+
processed_chunks: number;
|
|
33
|
+
total_files: number | null;
|
|
34
|
+
skipped_total: number;
|
|
35
|
+
added_chunks: number;
|
|
36
|
+
removed_chunks: number;
|
|
37
|
+
unchanged_chunks: number;
|
|
38
|
+
error_message: string | null;
|
|
39
|
+
}
|
|
40
|
+
|
|
21
41
|
export interface Chunk {
|
|
22
42
|
id: string;
|
|
23
43
|
kb_id: string;
|
|
@@ -34,7 +54,7 @@ export interface Chunk {
|
|
|
34
54
|
|
|
35
55
|
export type ChunkInsert = Omit<Chunk, "id" | "kb_id" | "indexed_at">;
|
|
36
56
|
|
|
37
|
-
const SCHEMA_VERSION =
|
|
57
|
+
const SCHEMA_VERSION = 3;
|
|
38
58
|
|
|
39
59
|
const SCHEMA_SQL = `
|
|
40
60
|
CREATE TABLE IF NOT EXISTS schema_version (
|
|
@@ -47,6 +67,7 @@ CREATE TABLE IF NOT EXISTS knowledge_bases (
|
|
|
47
67
|
description TEXT,
|
|
48
68
|
source_path TEXT,
|
|
49
69
|
source_type TEXT NOT NULL DEFAULT 'directory',
|
|
70
|
+
source_options TEXT,
|
|
50
71
|
created_at INTEGER NOT NULL,
|
|
51
72
|
updated_at INTEGER NOT NULL,
|
|
52
73
|
chunk_count INTEGER NOT NULL DEFAULT 0,
|
|
@@ -70,6 +91,26 @@ CREATE TABLE IF NOT EXISTS chunks (
|
|
|
70
91
|
FOREIGN KEY (kb_id) REFERENCES knowledge_bases(id) ON DELETE CASCADE
|
|
71
92
|
);
|
|
72
93
|
|
|
94
|
+
CREATE TABLE IF NOT EXISTS indexing_jobs (
|
|
95
|
+
kb_id TEXT PRIMARY KEY,
|
|
96
|
+
operation TEXT NOT NULL,
|
|
97
|
+
status TEXT NOT NULL,
|
|
98
|
+
phase TEXT NOT NULL,
|
|
99
|
+
message TEXT NOT NULL,
|
|
100
|
+
started_at INTEGER NOT NULL,
|
|
101
|
+
updated_at INTEGER NOT NULL,
|
|
102
|
+
last_progress_at INTEGER NOT NULL,
|
|
103
|
+
processed_files INTEGER NOT NULL DEFAULT 0,
|
|
104
|
+
processed_chunks INTEGER NOT NULL DEFAULT 0,
|
|
105
|
+
total_files INTEGER,
|
|
106
|
+
skipped_total INTEGER NOT NULL DEFAULT 0,
|
|
107
|
+
added_chunks INTEGER NOT NULL DEFAULT 0,
|
|
108
|
+
removed_chunks INTEGER NOT NULL DEFAULT 0,
|
|
109
|
+
unchanged_chunks INTEGER NOT NULL DEFAULT 0,
|
|
110
|
+
error_message TEXT,
|
|
111
|
+
FOREIGN KEY (kb_id) REFERENCES knowledge_bases(id) ON DELETE CASCADE
|
|
112
|
+
);
|
|
113
|
+
|
|
73
114
|
CREATE INDEX IF NOT EXISTS idx_chunks_kb_id ON chunks(kb_id);
|
|
74
115
|
CREATE INDEX IF NOT EXISTS idx_chunks_content_hash ON chunks(content_hash);
|
|
75
116
|
|
|
@@ -127,10 +168,36 @@ export function openDatabase(knowledgeDir?: string): Database.Database {
|
|
|
127
168
|
|
|
128
169
|
function runMigrations(db: Database.Database, from: number, to: number): void {
|
|
129
170
|
const migrations: Record<number, string> = {
|
|
130
|
-
|
|
131
|
-
|
|
171
|
+
2: `
|
|
172
|
+
CREATE TABLE IF NOT EXISTS indexing_jobs (
|
|
173
|
+
kb_id TEXT PRIMARY KEY,
|
|
174
|
+
operation TEXT NOT NULL,
|
|
175
|
+
status TEXT NOT NULL,
|
|
176
|
+
phase TEXT NOT NULL,
|
|
177
|
+
message TEXT NOT NULL,
|
|
178
|
+
started_at INTEGER NOT NULL,
|
|
179
|
+
updated_at INTEGER NOT NULL,
|
|
180
|
+
last_progress_at INTEGER NOT NULL,
|
|
181
|
+
processed_files INTEGER NOT NULL DEFAULT 0,
|
|
182
|
+
processed_chunks INTEGER NOT NULL DEFAULT 0,
|
|
183
|
+
total_files INTEGER,
|
|
184
|
+
skipped_total INTEGER NOT NULL DEFAULT 0,
|
|
185
|
+
added_chunks INTEGER NOT NULL DEFAULT 0,
|
|
186
|
+
removed_chunks INTEGER NOT NULL DEFAULT 0,
|
|
187
|
+
unchanged_chunks INTEGER NOT NULL DEFAULT 0,
|
|
188
|
+
error_message TEXT,
|
|
189
|
+
FOREIGN KEY (kb_id) REFERENCES knowledge_bases(id) ON DELETE CASCADE
|
|
190
|
+
);
|
|
191
|
+
`,
|
|
132
192
|
};
|
|
133
193
|
for (let v = from + 1; v <= to; v++) {
|
|
194
|
+
if (v === 3) {
|
|
195
|
+
const columns = db.prepare("PRAGMA table_info(knowledge_bases)").all() as Array<{ name: string }>;
|
|
196
|
+
if (!columns.some((column) => column.name === "source_options")) {
|
|
197
|
+
db.exec("ALTER TABLE knowledge_bases ADD COLUMN source_options TEXT");
|
|
198
|
+
}
|
|
199
|
+
continue;
|
|
200
|
+
}
|
|
134
201
|
if (migrations[v]) db.exec(migrations[v]);
|
|
135
202
|
}
|
|
136
203
|
}
|
|
@@ -139,14 +206,29 @@ function runMigrations(db: Database.Database, from: number, to: number): void {
|
|
|
139
206
|
|
|
140
207
|
export function createKB(
|
|
141
208
|
db: Database.Database,
|
|
142
|
-
opts: {
|
|
209
|
+
opts: {
|
|
210
|
+
name: string;
|
|
211
|
+
description?: string;
|
|
212
|
+
source_path?: string;
|
|
213
|
+
source_type: KnowledgeBase["source_type"];
|
|
214
|
+
source_options?: string;
|
|
215
|
+
},
|
|
143
216
|
): KnowledgeBase {
|
|
144
217
|
const id = randomUUID();
|
|
145
218
|
const now = Date.now();
|
|
146
219
|
db.prepare(
|
|
147
|
-
`INSERT INTO knowledge_bases (id, name, description, source_path, source_type, created_at, updated_at)
|
|
148
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
|
149
|
-
).run(
|
|
220
|
+
`INSERT INTO knowledge_bases (id, name, description, source_path, source_type, source_options, created_at, updated_at)
|
|
221
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
222
|
+
).run(
|
|
223
|
+
id,
|
|
224
|
+
opts.name,
|
|
225
|
+
opts.description ?? null,
|
|
226
|
+
opts.source_path ?? null,
|
|
227
|
+
opts.source_type,
|
|
228
|
+
opts.source_options ?? null,
|
|
229
|
+
now,
|
|
230
|
+
now,
|
|
231
|
+
);
|
|
150
232
|
const kb = getKB(db, id);
|
|
151
233
|
if (!kb) throw new Error(`Failed to create knowledge base: ${id}`);
|
|
152
234
|
return kb;
|
|
@@ -165,6 +247,7 @@ export function listKBs(db: Database.Database): KnowledgeBase[] {
|
|
|
165
247
|
}
|
|
166
248
|
|
|
167
249
|
export function deleteKB(db: Database.Database, id: string): void {
|
|
250
|
+
db.prepare("DELETE FROM indexing_jobs WHERE kb_id = ?").run(id);
|
|
168
251
|
db.prepare("DELETE FROM chunks WHERE kb_id = ?").run(id);
|
|
169
252
|
db.prepare("DELETE FROM knowledge_bases WHERE id = ?").run(id);
|
|
170
253
|
}
|
|
@@ -182,6 +265,110 @@ export function updateKBCounts(db: Database.Database, id: string, chunkCount: nu
|
|
|
182
265
|
);
|
|
183
266
|
}
|
|
184
267
|
|
|
268
|
+
// --- Indexing Jobs ---
|
|
269
|
+
|
|
270
|
+
export function startIndexingJob(
|
|
271
|
+
db: Database.Database,
|
|
272
|
+
kbId: string,
|
|
273
|
+
operation: IndexingJob["operation"],
|
|
274
|
+
message: string,
|
|
275
|
+
): void {
|
|
276
|
+
const now = Date.now();
|
|
277
|
+
db.prepare(
|
|
278
|
+
`INSERT INTO indexing_jobs (
|
|
279
|
+
kb_id, operation, status, phase, message, started_at, updated_at, last_progress_at
|
|
280
|
+
) VALUES (?, ?, 'running', 'starting', ?, ?, ?, ?)
|
|
281
|
+
ON CONFLICT(kb_id) DO UPDATE SET
|
|
282
|
+
operation = excluded.operation,
|
|
283
|
+
status = 'running',
|
|
284
|
+
phase = 'starting',
|
|
285
|
+
message = excluded.message,
|
|
286
|
+
started_at = excluded.started_at,
|
|
287
|
+
updated_at = excluded.updated_at,
|
|
288
|
+
last_progress_at = excluded.last_progress_at,
|
|
289
|
+
processed_files = 0,
|
|
290
|
+
processed_chunks = 0,
|
|
291
|
+
total_files = NULL,
|
|
292
|
+
skipped_total = 0,
|
|
293
|
+
added_chunks = 0,
|
|
294
|
+
removed_chunks = 0,
|
|
295
|
+
unchanged_chunks = 0,
|
|
296
|
+
error_message = NULL`,
|
|
297
|
+
).run(kbId, operation, message, now, now, now);
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
export function updateIndexingJob(
|
|
301
|
+
db: Database.Database,
|
|
302
|
+
kbId: string,
|
|
303
|
+
progress: {
|
|
304
|
+
phase?: string;
|
|
305
|
+
message?: string;
|
|
306
|
+
processed_files?: number;
|
|
307
|
+
processed_chunks?: number;
|
|
308
|
+
total_files?: number | null;
|
|
309
|
+
skipped_total?: number;
|
|
310
|
+
added_chunks?: number;
|
|
311
|
+
removed_chunks?: number;
|
|
312
|
+
unchanged_chunks?: number;
|
|
313
|
+
},
|
|
314
|
+
): void {
|
|
315
|
+
const current = getIndexingJob(db, kbId);
|
|
316
|
+
if (!current) return;
|
|
317
|
+
const now = Date.now();
|
|
318
|
+
db.prepare(
|
|
319
|
+
`UPDATE indexing_jobs SET
|
|
320
|
+
phase = ?,
|
|
321
|
+
message = ?,
|
|
322
|
+
updated_at = ?,
|
|
323
|
+
last_progress_at = ?,
|
|
324
|
+
processed_files = ?,
|
|
325
|
+
processed_chunks = ?,
|
|
326
|
+
total_files = ?,
|
|
327
|
+
skipped_total = ?,
|
|
328
|
+
added_chunks = ?,
|
|
329
|
+
removed_chunks = ?,
|
|
330
|
+
unchanged_chunks = ?
|
|
331
|
+
WHERE kb_id = ?`,
|
|
332
|
+
).run(
|
|
333
|
+
progress.phase ?? current.phase,
|
|
334
|
+
progress.message ?? current.message,
|
|
335
|
+
now,
|
|
336
|
+
now,
|
|
337
|
+
progress.processed_files ?? current.processed_files,
|
|
338
|
+
progress.processed_chunks ?? current.processed_chunks,
|
|
339
|
+
progress.total_files === undefined ? current.total_files : progress.total_files,
|
|
340
|
+
progress.skipped_total ?? current.skipped_total,
|
|
341
|
+
progress.added_chunks ?? current.added_chunks,
|
|
342
|
+
progress.removed_chunks ?? current.removed_chunks,
|
|
343
|
+
progress.unchanged_chunks ?? current.unchanged_chunks,
|
|
344
|
+
kbId,
|
|
345
|
+
);
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
export function finishIndexingJob(
|
|
349
|
+
db: Database.Database,
|
|
350
|
+
kbId: string,
|
|
351
|
+
status: Extract<IndexingJob["status"], "succeeded" | "failed" | "cancelled">,
|
|
352
|
+
message: string,
|
|
353
|
+
errorMessage?: string,
|
|
354
|
+
): void {
|
|
355
|
+
const now = Date.now();
|
|
356
|
+
db.prepare(
|
|
357
|
+
`UPDATE indexing_jobs SET
|
|
358
|
+
status = ?,
|
|
359
|
+
phase = ?,
|
|
360
|
+
message = ?,
|
|
361
|
+
updated_at = ?,
|
|
362
|
+
last_progress_at = ?,
|
|
363
|
+
error_message = ?
|
|
364
|
+
WHERE kb_id = ?`,
|
|
365
|
+
).run(status, status, message, now, now, errorMessage ?? null, kbId);
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
export function getIndexingJob(db: Database.Database, kbId: string): IndexingJob | undefined {
|
|
369
|
+
return db.prepare("SELECT * FROM indexing_jobs WHERE kb_id = ?").get(kbId) as IndexingJob | undefined;
|
|
370
|
+
}
|
|
371
|
+
|
|
185
372
|
// --- CRUD: Chunks ---
|
|
186
373
|
|
|
187
374
|
export function insertChunks(db: Database.Database, kbId: string, chunks: ChunkInsert[]): void {
|
|
@@ -214,11 +401,21 @@ export function getChunksByKB(db: Database.Database, kbId: string): Chunk[] {
|
|
|
214
401
|
return db.prepare("SELECT * FROM chunks WHERE kb_id = ? ORDER BY rowid").all(kbId) as Chunk[];
|
|
215
402
|
}
|
|
216
403
|
|
|
404
|
+
export function iterateChunksByKB(db: Database.Database, kbId: string): IterableIterator<Chunk> {
|
|
405
|
+
return db.prepare("SELECT * FROM chunks WHERE kb_id = ? ORDER BY rowid").iterate(kbId) as IterableIterator<Chunk>;
|
|
406
|
+
}
|
|
407
|
+
|
|
217
408
|
export function getChunkIdsByKB(db: Database.Database, kbId: string): string[] {
|
|
218
409
|
const rows = db.prepare("SELECT id FROM chunks WHERE kb_id = ? ORDER BY rowid").all(kbId) as { id: string }[];
|
|
219
410
|
return rows.map((r) => r.id);
|
|
220
411
|
}
|
|
221
412
|
|
|
413
|
+
export function iterateChunkIdsByKB(db: Database.Database, kbId: string): IterableIterator<{ id: string }> {
|
|
414
|
+
return db.prepare("SELECT id FROM chunks WHERE kb_id = ? ORDER BY rowid").iterate(kbId) as IterableIterator<{
|
|
415
|
+
id: string;
|
|
416
|
+
}>;
|
|
417
|
+
}
|
|
418
|
+
|
|
222
419
|
export function getChunkById(db: Database.Database, id: string): Chunk | undefined {
|
|
223
420
|
return db.prepare("SELECT * FROM chunks WHERE id = ?").get(id) as Chunk | undefined;
|
|
224
421
|
}
|
|
@@ -249,8 +446,14 @@ export function deleteChunksByKB(db: Database.Database, kbId: string): void {
|
|
|
249
446
|
|
|
250
447
|
export function deleteChunksByIds(db: Database.Database, ids: string[]): void {
|
|
251
448
|
if (ids.length === 0) return;
|
|
252
|
-
const
|
|
253
|
-
db.
|
|
449
|
+
const batchSize = 500;
|
|
450
|
+
const deleteBatch = db.transaction((batch: string[]) => {
|
|
451
|
+
const placeholders = batch.map(() => "?").join(",");
|
|
452
|
+
db.prepare(`DELETE FROM chunks WHERE id IN (${placeholders})`).run(...batch);
|
|
453
|
+
});
|
|
454
|
+
for (let offset = 0; offset < ids.length; offset += batchSize) {
|
|
455
|
+
deleteBatch(ids.slice(offset, offset + batchSize));
|
|
456
|
+
}
|
|
254
457
|
}
|
|
255
458
|
|
|
256
459
|
export function getChunkHashesByKB(db: Database.Database, kbId: string): Map<string, string> {
|
|
@@ -261,6 +464,18 @@ export function getChunkHashesByKB(db: Database.Database, kbId: string): Map<str
|
|
|
261
464
|
return new Map(rows.map((r) => [r.content_hash, r.id]));
|
|
262
465
|
}
|
|
263
466
|
|
|
467
|
+
export function iterateChunkHashesByKB(
|
|
468
|
+
db: Database.Database,
|
|
469
|
+
kbId: string,
|
|
470
|
+
): IterableIterator<{ id: string; content_hash: string }> {
|
|
471
|
+
return db
|
|
472
|
+
.prepare("SELECT id, content_hash FROM chunks WHERE kb_id = ? ORDER BY rowid")
|
|
473
|
+
.iterate(kbId) as IterableIterator<{
|
|
474
|
+
id: string;
|
|
475
|
+
content_hash: string;
|
|
476
|
+
}>;
|
|
477
|
+
}
|
|
478
|
+
|
|
264
479
|
export function getChunkCount(db: Database.Database, kbId: string): number {
|
|
265
480
|
const row = db.prepare("SELECT COUNT(*) as count FROM chunks WHERE kb_id = ?").get(kbId) as { count: number };
|
|
266
481
|
return row.count;
|