pi-knowledge 0.3.6 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,27 +1,12 @@
1
- import { join } from "node:path";
2
- import { getDefaultKnowledgeDir } from "../storage/sqlite.ts";
1
+ import { rerankInModelWorker } from "../model-worker-client.ts";
3
2
 
4
- type RerankerPipeline = {
5
- (input: { text: string; text_pair: string }): Promise<{ score?: number } | Array<{ score?: number }>>;
6
- dispose(): Promise<void> | void;
7
- };
8
-
9
- let rerankerPipeline: RerankerPipeline | null = null;
10
3
  let disposeTimer: ReturnType<typeof setTimeout> | null = null;
11
4
  let disposePromise: Promise<void> | null = null;
12
5
  let activeRuns = 0;
13
6
  let disposeRequested = false;
14
7
  const idleWaiters: Array<() => void> = [];
15
8
  const IDLE_MS = 30_000;
16
-
17
- async function load(): Promise<RerankerPipeline> {
18
- if (rerankerPipeline) return rerankerPipeline;
19
- if (disposePromise) await disposePromise;
20
- const { pipeline, env } = await import("@huggingface/transformers");
21
- env.cacheDir = join(getDefaultKnowledgeDir(), "models");
22
- rerankerPipeline = await pipeline("text-classification", "Xenova/ms-marco-MiniLM-L-4-v2");
23
- return rerankerPipeline;
24
- }
9
+ const ENABLE_NATIVE_IDLE_DISPOSE = process.env.PI_KNOWLEDGE_ENABLE_NATIVE_IDLE_DISPOSE === "true";
25
10
 
26
11
  function clearTimer(): void {
27
12
  if (disposeTimer) clearTimeout(disposeTimer);
@@ -31,6 +16,7 @@ function clearTimer(): void {
31
16
  function scheduleDispose(): void {
32
17
  if (activeRuns > 0 || disposeRequested) return;
33
18
  clearTimer();
19
+ if (!ENABLE_NATIVE_IDLE_DISPOSE) return;
34
20
  disposeTimer = setTimeout(() => disposeReranker(), IDLE_MS);
35
21
  }
36
22
 
@@ -56,13 +42,7 @@ export async function disposeReranker(): Promise<void> {
56
42
  if (disposePromise) return disposePromise;
57
43
  disposeRequested = true;
58
44
  await waitForNoActiveRuns();
59
- const instance = rerankerPipeline;
60
- rerankerPipeline = null;
61
- if (!instance) {
62
- disposeRequested = false;
63
- return;
64
- }
65
- disposePromise = Promise.resolve(instance.dispose()).finally(() => {
45
+ disposePromise = Promise.resolve().finally(() => {
66
46
  disposePromise = null;
67
47
  disposeRequested = false;
68
48
  });
@@ -85,20 +65,10 @@ export async function rerank(
85
65
  topK: number,
86
66
  ): Promise<Array<{ chunkId: string; score: number }>> {
87
67
  if (candidates.length === 0) return [];
88
- const pipe = await load();
89
68
  beginRun();
90
-
91
- const results: Array<{ chunkId: string; score: number }> = [];
92
69
  try {
93
- for (const c of candidates) {
94
- const output = await pipe({ text: query, text_pair: c.content });
95
- const score = Array.isArray(output) ? (output[0]?.score ?? 0) : (output?.score ?? 0);
96
- results.push({ chunkId: c.chunkId, score });
97
- }
70
+ return await rerankInModelWorker(query, candidates, topK);
98
71
  } finally {
99
72
  endRun();
100
73
  }
101
-
102
- results.sort((a, b) => b.score - a.score);
103
- return results.slice(0, topK);
104
74
  }
@@ -1,14 +1,42 @@
1
+ import { openVectorReader } from "../embedding/vectors.ts";
2
+
1
3
  export interface VectorResult {
2
4
  chunkId: string;
3
5
  score: number;
4
6
  }
5
7
 
8
+ export interface VectorFileSearchResult {
9
+ results: VectorResult[];
10
+ vectorsByChunkId: Map<string, Float32Array>;
11
+ }
12
+
13
+ type ChunkIdSource = Iterable<string | { id: string }>;
14
+
6
15
  function cosine(a: Float32Array, b: Float32Array): number {
7
16
  let dot = 0;
8
17
  for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
9
18
  return dot;
10
19
  }
11
20
 
21
+ function keepTopK(
22
+ top: Array<VectorResult & { vector: Float32Array }>,
23
+ item: VectorResult,
24
+ vector: Float32Array,
25
+ limit: number,
26
+ ): void {
27
+ if (limit <= 0) return;
28
+ if (top.length < limit) {
29
+ const stored = { ...item, vector: new Float32Array(vector) };
30
+ top.push(stored);
31
+ top.sort((a, b) => a.score - b.score);
32
+ return;
33
+ }
34
+ if (item.score <= top[0].score) return;
35
+ const stored = { ...item, vector: new Float32Array(vector) };
36
+ top[0] = stored;
37
+ top.sort((a, b) => a.score - b.score);
38
+ }
39
+
12
40
  export function searchVector(
13
41
  queryVec: Float32Array,
14
42
  vectors: Float32Array[],
@@ -19,3 +47,35 @@ export function searchVector(
19
47
  scores.sort((a, b) => b.score - a.score);
20
48
  return scores.slice(0, limit);
21
49
  }
50
+
51
+ export function searchVectorFile(
52
+ queryVec: Float32Array,
53
+ vectorPath: string,
54
+ chunkIds: ChunkIdSource,
55
+ limit = 50,
56
+ ): VectorFileSearchResult {
57
+ const reader = openVectorReader(vectorPath);
58
+ if (!reader) return { results: [], vectorsByChunkId: new Map() };
59
+ try {
60
+ if (reader.dim !== queryVec.length) return { results: [], vectorsByChunkId: new Map() };
61
+ const scratch = new Float32Array(reader.dim);
62
+ const top: Array<VectorResult & { vector: Float32Array }> = [];
63
+ let i = 0;
64
+ for (const chunkIdRef of chunkIds) {
65
+ if (i >= reader.count) break;
66
+ const vectorIndex = i;
67
+ i++;
68
+ if (!reader.readInto(vectorIndex, scratch)) continue;
69
+ const chunkId = typeof chunkIdRef === "string" ? chunkIdRef : chunkIdRef.id;
70
+ keepTopK(top, { chunkId, score: cosine(queryVec, scratch) }, scratch, limit);
71
+ }
72
+ const sorted = top.sort((a, b) => b.score - a.score);
73
+ const vectorsByChunkId = new Map(sorted.map((item) => [item.chunkId, item.vector]));
74
+ return {
75
+ results: sorted.map(({ chunkId, score }) => ({ chunkId, score })),
76
+ vectorsByChunkId,
77
+ };
78
+ } finally {
79
+ reader.close();
80
+ }
81
+ }
@@ -10,6 +10,7 @@ export interface KnowledgeBase {
10
10
  description: string | null;
11
11
  source_path: string | null;
12
12
  source_type: "file" | "directory" | "text" | "url";
13
+ source_options: string | null;
13
14
  created_at: number;
14
15
  updated_at: number;
15
16
  chunk_count: number;
@@ -18,6 +19,25 @@ export interface KnowledgeBase {
18
19
  status: "ready" | "indexing" | "error" | "stale";
19
20
  }
20
21
 
22
+ export interface IndexingJob {
23
+ kb_id: string;
24
+ operation: "add" | "update" | "import";
25
+ status: "running" | "succeeded" | "failed" | "cancelled";
26
+ phase: string;
27
+ message: string;
28
+ started_at: number;
29
+ updated_at: number;
30
+ last_progress_at: number;
31
+ processed_files: number;
32
+ processed_chunks: number;
33
+ total_files: number | null;
34
+ skipped_total: number;
35
+ added_chunks: number;
36
+ removed_chunks: number;
37
+ unchanged_chunks: number;
38
+ error_message: string | null;
39
+ }
40
+
21
41
  export interface Chunk {
22
42
  id: string;
23
43
  kb_id: string;
@@ -34,7 +54,7 @@ export interface Chunk {
34
54
 
35
55
  export type ChunkInsert = Omit<Chunk, "id" | "kb_id" | "indexed_at">;
36
56
 
37
- const SCHEMA_VERSION = 1;
57
+ const SCHEMA_VERSION = 3;
38
58
 
39
59
  const SCHEMA_SQL = `
40
60
  CREATE TABLE IF NOT EXISTS schema_version (
@@ -47,6 +67,7 @@ CREATE TABLE IF NOT EXISTS knowledge_bases (
47
67
  description TEXT,
48
68
  source_path TEXT,
49
69
  source_type TEXT NOT NULL DEFAULT 'directory',
70
+ source_options TEXT,
50
71
  created_at INTEGER NOT NULL,
51
72
  updated_at INTEGER NOT NULL,
52
73
  chunk_count INTEGER NOT NULL DEFAULT 0,
@@ -70,6 +91,26 @@ CREATE TABLE IF NOT EXISTS chunks (
70
91
  FOREIGN KEY (kb_id) REFERENCES knowledge_bases(id) ON DELETE CASCADE
71
92
  );
72
93
 
94
+ CREATE TABLE IF NOT EXISTS indexing_jobs (
95
+ kb_id TEXT PRIMARY KEY,
96
+ operation TEXT NOT NULL,
97
+ status TEXT NOT NULL,
98
+ phase TEXT NOT NULL,
99
+ message TEXT NOT NULL,
100
+ started_at INTEGER NOT NULL,
101
+ updated_at INTEGER NOT NULL,
102
+ last_progress_at INTEGER NOT NULL,
103
+ processed_files INTEGER NOT NULL DEFAULT 0,
104
+ processed_chunks INTEGER NOT NULL DEFAULT 0,
105
+ total_files INTEGER,
106
+ skipped_total INTEGER NOT NULL DEFAULT 0,
107
+ added_chunks INTEGER NOT NULL DEFAULT 0,
108
+ removed_chunks INTEGER NOT NULL DEFAULT 0,
109
+ unchanged_chunks INTEGER NOT NULL DEFAULT 0,
110
+ error_message TEXT,
111
+ FOREIGN KEY (kb_id) REFERENCES knowledge_bases(id) ON DELETE CASCADE
112
+ );
113
+
73
114
  CREATE INDEX IF NOT EXISTS idx_chunks_kb_id ON chunks(kb_id);
74
115
  CREATE INDEX IF NOT EXISTS idx_chunks_content_hash ON chunks(content_hash);
75
116
 
@@ -127,10 +168,36 @@ export function openDatabase(knowledgeDir?: string): Database.Database {
127
168
 
128
169
  function runMigrations(db: Database.Database, from: number, to: number): void {
129
170
  const migrations: Record<number, string> = {
130
- // Future migrations go here, keyed by target version number:
131
- // 2: "ALTER TABLE ...",
171
+ 2: `
172
+ CREATE TABLE IF NOT EXISTS indexing_jobs (
173
+ kb_id TEXT PRIMARY KEY,
174
+ operation TEXT NOT NULL,
175
+ status TEXT NOT NULL,
176
+ phase TEXT NOT NULL,
177
+ message TEXT NOT NULL,
178
+ started_at INTEGER NOT NULL,
179
+ updated_at INTEGER NOT NULL,
180
+ last_progress_at INTEGER NOT NULL,
181
+ processed_files INTEGER NOT NULL DEFAULT 0,
182
+ processed_chunks INTEGER NOT NULL DEFAULT 0,
183
+ total_files INTEGER,
184
+ skipped_total INTEGER NOT NULL DEFAULT 0,
185
+ added_chunks INTEGER NOT NULL DEFAULT 0,
186
+ removed_chunks INTEGER NOT NULL DEFAULT 0,
187
+ unchanged_chunks INTEGER NOT NULL DEFAULT 0,
188
+ error_message TEXT,
189
+ FOREIGN KEY (kb_id) REFERENCES knowledge_bases(id) ON DELETE CASCADE
190
+ );
191
+ `,
132
192
  };
133
193
  for (let v = from + 1; v <= to; v++) {
194
+ if (v === 3) {
195
+ const columns = db.prepare("PRAGMA table_info(knowledge_bases)").all() as Array<{ name: string }>;
196
+ if (!columns.some((column) => column.name === "source_options")) {
197
+ db.exec("ALTER TABLE knowledge_bases ADD COLUMN source_options TEXT");
198
+ }
199
+ continue;
200
+ }
134
201
  if (migrations[v]) db.exec(migrations[v]);
135
202
  }
136
203
  }
@@ -139,14 +206,29 @@ function runMigrations(db: Database.Database, from: number, to: number): void {
139
206
 
140
207
  export function createKB(
141
208
  db: Database.Database,
142
- opts: { name: string; description?: string; source_path?: string; source_type: KnowledgeBase["source_type"] },
209
+ opts: {
210
+ name: string;
211
+ description?: string;
212
+ source_path?: string;
213
+ source_type: KnowledgeBase["source_type"];
214
+ source_options?: string;
215
+ },
143
216
  ): KnowledgeBase {
144
217
  const id = randomUUID();
145
218
  const now = Date.now();
146
219
  db.prepare(
147
- `INSERT INTO knowledge_bases (id, name, description, source_path, source_type, created_at, updated_at)
148
- VALUES (?, ?, ?, ?, ?, ?, ?)`,
149
- ).run(id, opts.name, opts.description ?? null, opts.source_path ?? null, opts.source_type, now, now);
220
+ `INSERT INTO knowledge_bases (id, name, description, source_path, source_type, source_options, created_at, updated_at)
221
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
222
+ ).run(
223
+ id,
224
+ opts.name,
225
+ opts.description ?? null,
226
+ opts.source_path ?? null,
227
+ opts.source_type,
228
+ opts.source_options ?? null,
229
+ now,
230
+ now,
231
+ );
150
232
  const kb = getKB(db, id);
151
233
  if (!kb) throw new Error(`Failed to create knowledge base: ${id}`);
152
234
  return kb;
@@ -165,6 +247,7 @@ export function listKBs(db: Database.Database): KnowledgeBase[] {
165
247
  }
166
248
 
167
249
  export function deleteKB(db: Database.Database, id: string): void {
250
+ db.prepare("DELETE FROM indexing_jobs WHERE kb_id = ?").run(id);
168
251
  db.prepare("DELETE FROM chunks WHERE kb_id = ?").run(id);
169
252
  db.prepare("DELETE FROM knowledge_bases WHERE id = ?").run(id);
170
253
  }
@@ -182,6 +265,110 @@ export function updateKBCounts(db: Database.Database, id: string, chunkCount: nu
182
265
  );
183
266
  }
184
267
 
268
+ // --- Indexing Jobs ---
269
+
270
+ export function startIndexingJob(
271
+ db: Database.Database,
272
+ kbId: string,
273
+ operation: IndexingJob["operation"],
274
+ message: string,
275
+ ): void {
276
+ const now = Date.now();
277
+ db.prepare(
278
+ `INSERT INTO indexing_jobs (
279
+ kb_id, operation, status, phase, message, started_at, updated_at, last_progress_at
280
+ ) VALUES (?, ?, 'running', 'starting', ?, ?, ?, ?)
281
+ ON CONFLICT(kb_id) DO UPDATE SET
282
+ operation = excluded.operation,
283
+ status = 'running',
284
+ phase = 'starting',
285
+ message = excluded.message,
286
+ started_at = excluded.started_at,
287
+ updated_at = excluded.updated_at,
288
+ last_progress_at = excluded.last_progress_at,
289
+ processed_files = 0,
290
+ processed_chunks = 0,
291
+ total_files = NULL,
292
+ skipped_total = 0,
293
+ added_chunks = 0,
294
+ removed_chunks = 0,
295
+ unchanged_chunks = 0,
296
+ error_message = NULL`,
297
+ ).run(kbId, operation, message, now, now, now);
298
+ }
299
+
300
+ export function updateIndexingJob(
301
+ db: Database.Database,
302
+ kbId: string,
303
+ progress: {
304
+ phase?: string;
305
+ message?: string;
306
+ processed_files?: number;
307
+ processed_chunks?: number;
308
+ total_files?: number | null;
309
+ skipped_total?: number;
310
+ added_chunks?: number;
311
+ removed_chunks?: number;
312
+ unchanged_chunks?: number;
313
+ },
314
+ ): void {
315
+ const current = getIndexingJob(db, kbId);
316
+ if (!current) return;
317
+ const now = Date.now();
318
+ db.prepare(
319
+ `UPDATE indexing_jobs SET
320
+ phase = ?,
321
+ message = ?,
322
+ updated_at = ?,
323
+ last_progress_at = ?,
324
+ processed_files = ?,
325
+ processed_chunks = ?,
326
+ total_files = ?,
327
+ skipped_total = ?,
328
+ added_chunks = ?,
329
+ removed_chunks = ?,
330
+ unchanged_chunks = ?
331
+ WHERE kb_id = ?`,
332
+ ).run(
333
+ progress.phase ?? current.phase,
334
+ progress.message ?? current.message,
335
+ now,
336
+ now,
337
+ progress.processed_files ?? current.processed_files,
338
+ progress.processed_chunks ?? current.processed_chunks,
339
+ progress.total_files === undefined ? current.total_files : progress.total_files,
340
+ progress.skipped_total ?? current.skipped_total,
341
+ progress.added_chunks ?? current.added_chunks,
342
+ progress.removed_chunks ?? current.removed_chunks,
343
+ progress.unchanged_chunks ?? current.unchanged_chunks,
344
+ kbId,
345
+ );
346
+ }
347
+
348
+ export function finishIndexingJob(
349
+ db: Database.Database,
350
+ kbId: string,
351
+ status: Extract<IndexingJob["status"], "succeeded" | "failed" | "cancelled">,
352
+ message: string,
353
+ errorMessage?: string,
354
+ ): void {
355
+ const now = Date.now();
356
+ db.prepare(
357
+ `UPDATE indexing_jobs SET
358
+ status = ?,
359
+ phase = ?,
360
+ message = ?,
361
+ updated_at = ?,
362
+ last_progress_at = ?,
363
+ error_message = ?
364
+ WHERE kb_id = ?`,
365
+ ).run(status, status, message, now, now, errorMessage ?? null, kbId);
366
+ }
367
+
368
+ export function getIndexingJob(db: Database.Database, kbId: string): IndexingJob | undefined {
369
+ return db.prepare("SELECT * FROM indexing_jobs WHERE kb_id = ?").get(kbId) as IndexingJob | undefined;
370
+ }
371
+
185
372
  // --- CRUD: Chunks ---
186
373
 
187
374
  export function insertChunks(db: Database.Database, kbId: string, chunks: ChunkInsert[]): void {
@@ -214,11 +401,21 @@ export function getChunksByKB(db: Database.Database, kbId: string): Chunk[] {
214
401
  return db.prepare("SELECT * FROM chunks WHERE kb_id = ? ORDER BY rowid").all(kbId) as Chunk[];
215
402
  }
216
403
 
404
+ export function iterateChunksByKB(db: Database.Database, kbId: string): IterableIterator<Chunk> {
405
+ return db.prepare("SELECT * FROM chunks WHERE kb_id = ? ORDER BY rowid").iterate(kbId) as IterableIterator<Chunk>;
406
+ }
407
+
217
408
  export function getChunkIdsByKB(db: Database.Database, kbId: string): string[] {
218
409
  const rows = db.prepare("SELECT id FROM chunks WHERE kb_id = ? ORDER BY rowid").all(kbId) as { id: string }[];
219
410
  return rows.map((r) => r.id);
220
411
  }
221
412
 
413
+ export function iterateChunkIdsByKB(db: Database.Database, kbId: string): IterableIterator<{ id: string }> {
414
+ return db.prepare("SELECT id FROM chunks WHERE kb_id = ? ORDER BY rowid").iterate(kbId) as IterableIterator<{
415
+ id: string;
416
+ }>;
417
+ }
418
+
222
419
  export function getChunkById(db: Database.Database, id: string): Chunk | undefined {
223
420
  return db.prepare("SELECT * FROM chunks WHERE id = ?").get(id) as Chunk | undefined;
224
421
  }
@@ -249,8 +446,14 @@ export function deleteChunksByKB(db: Database.Database, kbId: string): void {
249
446
 
250
447
  export function deleteChunksByIds(db: Database.Database, ids: string[]): void {
251
448
  if (ids.length === 0) return;
252
- const placeholders = ids.map(() => "?").join(",");
253
- db.prepare(`DELETE FROM chunks WHERE id IN (${placeholders})`).run(...ids);
449
+ const batchSize = 500;
450
+ const deleteBatch = db.transaction((batch: string[]) => {
451
+ const placeholders = batch.map(() => "?").join(",");
452
+ db.prepare(`DELETE FROM chunks WHERE id IN (${placeholders})`).run(...batch);
453
+ });
454
+ for (let offset = 0; offset < ids.length; offset += batchSize) {
455
+ deleteBatch(ids.slice(offset, offset + batchSize));
456
+ }
254
457
  }
255
458
 
256
459
  export function getChunkHashesByKB(db: Database.Database, kbId: string): Map<string, string> {
@@ -261,6 +464,18 @@ export function getChunkHashesByKB(db: Database.Database, kbId: string): Map<str
261
464
  return new Map(rows.map((r) => [r.content_hash, r.id]));
262
465
  }
263
466
 
467
+ export function iterateChunkHashesByKB(
468
+ db: Database.Database,
469
+ kbId: string,
470
+ ): IterableIterator<{ id: string; content_hash: string }> {
471
+ return db
472
+ .prepare("SELECT id, content_hash FROM chunks WHERE kb_id = ? ORDER BY rowid")
473
+ .iterate(kbId) as IterableIterator<{
474
+ id: string;
475
+ content_hash: string;
476
+ }>;
477
+ }
478
+
264
479
  export function getChunkCount(db: Database.Database, kbId: string): number {
265
480
  const row = db.prepare("SELECT COUNT(*) as count FROM chunks WHERE kb_id = ?").get(kbId) as { count: number };
266
481
  return row.count;