rag-memory-epf-mcp 6.4.1 → 6.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -14,6 +14,7 @@
14
14
  // kick() merely triggers a debounced scan. All reconciliation writes are
15
15
  // per-row transactions with a CAS re-check so a foreground embed that lands
16
16
  // first is never downgraded (6R note 3).
17
+ import { searchableVectorSql, VecSlotError, deleteVector } from './vecSlots.js';
17
18
  // A handle closed by the engine's reopen (wal-sidecar-guard), or an I/O error from a sidecar the
18
19
  // sync client swapped out: the run was cut, not broken.
19
20
  const CONN_LOSS_RERUN_MAX = 3;
@@ -36,6 +37,15 @@ export class BackfillCoordinator {
36
37
  shuttingDown = false;
37
38
  connLossReruns = 0;
38
39
  scanning = false;
40
+ // vec-slot-reuse: rows whose vec0 slot has validity 0 cannot be rewritten in place (sqlite-vec refuses
41
+ // to DELETE many of them); the backfill skips them and asks for the verified rebuild instead.
42
+ repairWanted = false;
43
+ slotErrors = new Map(); // table -> unresolvable slot map message
44
+ repairRequestedAt = 0;
45
+ requestRepair() { if (!this.repairWanted)
46
+ this.repairRequestedAt = Date.now(); this.repairWanted = true; }
47
+ takeRepairRequest() { const w = this.repairWanted; this.repairWanted = false; return w; }
48
+ get repairRequested() { return this.repairWanted; }
39
49
  scanPromise = null;
40
50
  rerunRequested = false;
41
51
  snapshot = null;
@@ -43,6 +53,8 @@ export class BackfillCoordinator {
43
53
  this.deps = deps;
44
54
  }
45
55
  get reconState() { return this.recon; }
56
+ // vec-slot-reuse D1b: the idle vector rebuild must not overlap a backfill scan or a reconciliation pass.
57
+ get busy() { return this.scanning || !!this.scanPromise || this.reconActive || this.recon === 'running'; }
46
58
  get reconLastError() { return this.reconError; }
47
59
  // Shared eligibility barrier (spec §3): vector search AND automatic backfill.
48
60
  // 'n/a' (fresh DB — nothing to reconcile) counts as satisfied (6R note 1).
@@ -54,17 +66,19 @@ export class BackfillCoordinator {
54
66
  // non-atomic writes can leave metadata-without-vector and vector-without-
55
67
  // metadata split states, and neither must be miscounted as reconciled work.
56
68
  countNullWithVector(db) {
57
- const ent = db.prepare(`SELECT COUNT(*) c FROM entity_embedding_metadata m JOIN entity_embeddings v ON v.rowid = m.rowid
58
- WHERE m.provenance_state IS NULL`).get();
59
- const chk = db.prepare(`SELECT COUNT(*) c FROM chunk_metadata m JOIN chunks v ON v.rowid = m.rowid
60
- WHERE m.provenance_state IS NULL`).get();
69
+ const ent = db.prepare(`SELECT COUNT(*) c FROM entity_embedding_metadata m
70
+ WHERE m.provenance_state IS NULL AND ${searchableVectorSql('entity_embeddings', 'm.rowid')}`).get();
71
+ const chk = db.prepare(`SELECT COUNT(*) c FROM chunk_metadata m
72
+ WHERE m.provenance_state IS NULL AND ${searchableVectorSql('chunks', 'm.rowid')}`).get();
61
73
  return ent.c + chk.c;
62
74
  }
63
75
  // Sanitation (beta B3·B4) — runs on EVERY reconciliation entry, even when no
64
76
  // NULL rows exist, so profile changes and legacy split states are repaired
65
77
  // before vector eligibility can open:
66
78
  // 1. entity vectors without metadata (orphans) -> delete vector
67
- // 2. entity metadata without vector (split state) -> delete metadata (row becomes a backfill target)
79
+ // 2. stamped entity metadata whose vector row is gone -> normalized to missing, NOT deleted: the row keeps
80
+ // its rowid and becomes a backfill target (vec-slot-reuse r3 M2; validity-0 vector rows are repaired
81
+ // inline by deleteVector/writeVector or by the idle rebuild, never DELETEd here)
68
82
  // 3. provenance-stamped rows whose compatibility profile != current
69
83
  // (old engine/model profile) -> delete-to-missing
70
84
  // 4. chunk metadata stamped but vectorless -> normalize to missing (NULL provenance)
@@ -75,12 +89,14 @@ export class BackfillCoordinator {
75
89
  const profileId = this.deps.currentProfileId();
76
90
  const q = (sql, ...args) => db.prepare(sql).get(...args).c;
77
91
  return this.countNullWithVector(db)
78
- + q(`SELECT COUNT(*) c FROM entity_embeddings WHERE rowid NOT IN (SELECT rowid FROM entity_embedding_metadata)`)
79
- + q(`SELECT COUNT(*) c FROM entity_embedding_metadata WHERE rowid NOT IN (SELECT rowid FROM entity_embeddings)`)
92
+ // vec-slot-reuse r3 M2: searchable orphans only (validity-0 rows belong to the rebuild); vectorless
93
+ // metadata counts only while still stamped (it is normalized to missing, not deleted, to keep its rowid).
94
+ + q(`SELECT COUNT(*) c FROM entity_embeddings_rowids m WHERE m.rowid NOT IN (SELECT rowid FROM entity_embedding_metadata) AND ${searchableVectorSql('entity_embeddings', 'm.rowid')}`)
95
+ + q(`SELECT COUNT(*) c FROM entity_embedding_metadata WHERE provenance_state IS NOT NULL AND rowid NOT IN (SELECT rowid FROM entity_embeddings_rowids)`)
80
96
  + q(`SELECT COUNT(*) c FROM entity_embedding_metadata WHERE provenance_state IS NOT NULL AND profile_id IS NOT ?`, profileId)
81
- + q(`SELECT COUNT(*) c FROM chunk_metadata m JOIN chunks v ON v.rowid = m.rowid WHERE m.provenance_state IS NOT NULL AND m.profile_id IS NOT ?`, profileId)
97
+ + q(`SELECT COUNT(*) c FROM chunk_metadata m WHERE m.provenance_state IS NOT NULL AND m.profile_id IS NOT ? AND m.rowid IN (SELECT rowid FROM chunks_rowids)`, profileId)
82
98
  // stamped chunk metadata whose vector is gone (beta 3R M3)
83
- + q(`SELECT COUNT(*) c FROM chunk_metadata WHERE provenance_state IS NOT NULL AND rowid NOT IN (SELECT rowid FROM chunks)`);
99
+ + q(`SELECT COUNT(*) c FROM chunk_metadata WHERE provenance_state IS NOT NULL AND rowid NOT IN (SELECT rowid FROM chunks_rowids)`);
84
100
  }
85
101
  // Batched (beta 2R residual): each batch commits its own transaction and
86
102
  // yields the event loop — a large DB or a profile switch must not stall the
@@ -103,13 +119,32 @@ export class BackfillCoordinator {
103
119
  await new Promise(r => setImmediate(r));
104
120
  }
105
121
  };
106
- await batchedRun(`DELETE FROM entity_embeddings WHERE rowid IN (
107
- SELECT rowid FROM entity_embeddings WHERE rowid NOT IN (SELECT rowid FROM entity_embedding_metadata) LIMIT ${RECON_BATCH})`);
108
- await batchedRun(`DELETE FROM entity_embedding_metadata WHERE rowid IN (
109
- SELECT rowid FROM entity_embedding_metadata WHERE rowid NOT IN (SELECT rowid FROM entity_embeddings) LIMIT ${RECON_BATCH})`);
122
+ // 1. searchable entity vectors without metadata -> delete (validity-0 orphans are left to the rebuild:
123
+ // vec0 refuses to DELETE most of them, vec-slot-reuse r3 M2)
124
+ for (;;) {
125
+ if (this.shuttingDown)
126
+ throw new Error('shutdown during sanitation');
127
+ const orphans = db.prepare(`SELECT m.rowid FROM entity_embeddings_rowids m
128
+ WHERE m.rowid NOT IN (SELECT rowid FROM entity_embedding_metadata) AND ${searchableVectorSql('entity_embeddings', 'm.rowid')} LIMIT ${RECON_BATCH}`).all();
129
+ if (orphans.length === 0)
130
+ break;
131
+ batchTx(() => { for (const r of orphans) {
132
+ deleteVector(db, 'entity_embeddings', r.rowid);
133
+ touched++;
134
+ } });
135
+ if (orphans.length < RECON_BATCH)
136
+ break;
137
+ await new Promise(r => setImmediate(r));
138
+ }
139
+ // 2. stamped entity metadata whose vector row is gone -> normalize to missing (was: delete). Keeping the
140
+ // row keeps its rowid, so the backfill re-inserts at the same rowid (r3 M2 case C: restart after repair).
141
+ await batchedRun(`UPDATE entity_embedding_metadata SET input_hash = NULL, profile_id = NULL, provenance_state = NULL
142
+ WHERE rowid IN (
143
+ SELECT rowid FROM entity_embedding_metadata
144
+ WHERE provenance_state IS NOT NULL AND rowid NOT IN (SELECT rowid FROM entity_embeddings_rowids) LIMIT ${RECON_BATCH})`);
110
145
  await batchedRun(`UPDATE chunk_metadata SET input_hash = NULL, profile_id = NULL, provenance_state = NULL
111
146
  WHERE rowid IN (
112
- SELECT rowid FROM chunk_metadata WHERE provenance_state IS NOT NULL AND rowid NOT IN (SELECT rowid FROM chunks) LIMIT ${RECON_BATCH})`);
147
+ SELECT rowid FROM chunk_metadata WHERE provenance_state IS NOT NULL AND rowid NOT IN (SELECT rowid FROM chunks_rowids) LIMIT ${RECON_BATCH})`);
113
148
  // Old-profile rows: row deletes in bounded batches with yields.
114
149
  for (;;) {
115
150
  if (this.shuttingDown)
@@ -120,8 +155,8 @@ export class BackfillCoordinator {
120
155
  break;
121
156
  batchTx(() => {
122
157
  for (const r of staleEnt) {
123
- db.exec(`DELETE FROM entity_embeddings WHERE rowid = ${Number(r.rowid)}`);
124
- db.prepare(`DELETE FROM entity_embedding_metadata WHERE rowid = ?`).run(r.rowid);
158
+ deleteVector(db, 'entity_embeddings', r.rowid); // validity-0: left to the rebuild
159
+ db.prepare(`UPDATE entity_embedding_metadata SET input_hash = NULL, profile_id = NULL, provenance_state = NULL WHERE rowid = ?`).run(r.rowid);
125
160
  touched++;
126
161
  }
127
162
  });
@@ -130,13 +165,13 @@ export class BackfillCoordinator {
130
165
  for (;;) {
131
166
  if (this.shuttingDown)
132
167
  throw new Error('shutdown during sanitation');
133
- const staleChunk = db.prepare(`SELECT m.rowid FROM chunk_metadata m JOIN chunks v ON v.rowid = m.rowid
134
- WHERE m.provenance_state IS NOT NULL AND m.profile_id IS NOT ? LIMIT ${RECON_BATCH}`).all(profileId);
168
+ const staleChunk = db.prepare(`SELECT m.rowid FROM chunk_metadata m
169
+ WHERE m.provenance_state IS NOT NULL AND m.profile_id IS NOT ? AND m.rowid IN (SELECT rowid FROM chunks_rowids) LIMIT ${RECON_BATCH}`).all(profileId);
135
170
  if (staleChunk.length === 0)
136
171
  break;
137
172
  batchTx(() => {
138
173
  for (const r of staleChunk) {
139
- db.exec(`DELETE FROM chunks WHERE rowid = ${Number(r.rowid)}`);
174
+ deleteVector(db, 'chunks', r.rowid); // validity-0: left to the rebuild
140
175
  db.prepare(`UPDATE chunk_metadata SET input_hash = NULL, profile_id = NULL, provenance_state = NULL WHERE rowid = ?`).run(r.rowid);
141
176
  touched++;
142
177
  }
@@ -151,10 +186,14 @@ export class BackfillCoordinator {
151
186
  // to 'n/a'. Row-level errors are fail-closed (vector deleted -> missing);
152
187
  // systemic errors transition to 'failed' and keep eligibility shut.
153
188
  runReconciliation() {
154
- if (!this.reconPromise)
155
- this.reconPromise = this.reconcileOnce();
189
+ if (!this.reconPromise) {
190
+ // r3 M4: busy for the whole pass (sanitize included), not only while recon === 'running'.
191
+ this.reconActive = true;
192
+ this.reconPromise = this.reconcileOnce().finally(() => { this.reconActive = false; });
193
+ }
156
194
  return this.reconPromise;
157
195
  }
196
+ reconActive = false;
158
197
  async reconcileOnce() {
159
198
  const db = this.deps.db();
160
199
  if (!db) {
@@ -233,8 +272,7 @@ export class BackfillCoordinator {
233
272
  // grandfather candidates — metadata-without-vector was already removed by
234
273
  // sanitize().
235
274
  const rows = db.prepare(`SELECT m.rowid, m.entity_id, m.embedding_text FROM entity_embedding_metadata m
236
- JOIN entity_embeddings v ON v.rowid = m.rowid
237
- WHERE m.provenance_state IS NULL LIMIT ${RECON_BATCH}`).all();
275
+ WHERE m.provenance_state IS NULL AND ${searchableVectorSql('entity_embeddings', 'm.rowid')} LIMIT ${RECON_BATCH}`).all();
238
276
  if (rows.length === 0)
239
277
  return;
240
278
  for (const row of rows) {
@@ -244,7 +282,7 @@ export class BackfillCoordinator {
244
282
  const cur = db.prepare(`SELECT provenance_state FROM entity_embedding_metadata WHERE rowid = ?`).get(row.rowid);
245
283
  if (!cur || cur.provenance_state !== null)
246
284
  return;
247
- db.exec(`DELETE FROM entity_embeddings WHERE rowid = ${row.rowid}`);
285
+ deleteVector(db, 'entity_embeddings', row.rowid);
248
286
  db.prepare(`DELETE FROM entity_embedding_metadata WHERE rowid = ?`).run(row.rowid);
249
287
  });
250
288
  const stampTx = db.transaction((hash) => {
@@ -300,8 +338,8 @@ export class BackfillCoordinator {
300
338
  for (;;) {
301
339
  if (this.shuttingDown)
302
340
  throw new Error('shutdown during reconciliation');
303
- const rows = db.prepare(`SELECT m.rowid, m.text FROM chunk_metadata m JOIN chunks v ON v.rowid = m.rowid
304
- WHERE m.provenance_state IS NULL LIMIT ${RECON_BATCH}`).all();
341
+ const rows = db.prepare(`SELECT m.rowid, m.text FROM chunk_metadata m
342
+ WHERE m.provenance_state IS NULL AND ${searchableVectorSql('chunks', 'm.rowid')} LIMIT ${RECON_BATCH}`).all();
305
343
  if (rows.length === 0)
306
344
  return;
307
345
  for (const row of rows) {
@@ -313,7 +351,7 @@ export class BackfillCoordinator {
313
351
  // Vector removed; provenance stays NULL. The barrier only protects
314
352
  // VECTOR-BEARING NULL rows — vectorless rows are always legitimate
315
353
  // backfill targets (spec §6c "벡터 없음").
316
- db.exec(`DELETE FROM chunks WHERE rowid = ${row.rowid}`);
354
+ deleteVector(db, 'chunks', row.rowid);
317
355
  db.prepare(`UPDATE chunk_metadata SET input_hash = NULL, profile_id = NULL WHERE rowid = ?`)
318
356
  .run(row.rowid);
319
357
  });
@@ -452,13 +490,32 @@ export class BackfillCoordinator {
452
490
  return consecutive >= 3 ? 'abort' : 'fail';
453
491
  };
454
492
  // Phase 1: chunks.
455
- const chunkRows = db.prepare(`SELECT m.rowid, m.text, m.input_hash, m.profile_id, m.provenance_state,
456
- EXISTS(SELECT 1 FROM chunks v WHERE v.rowid = m.rowid) AS has_vec
457
- FROM chunk_metadata m WHERE m.text IS NOT NULL`).all();
493
+ let sawInvalid = false;
494
+ const errorsBefore = JSON.stringify([...this.slotErrors]);
495
+ const slotErrorsChanged = () => JSON.stringify([...this.slotErrors]) !== errorsBefore;
496
+ let chunkRows = [];
497
+ try {
498
+ chunkRows = db.prepare(`SELECT m.rowid, m.text, m.input_hash, m.profile_id, m.provenance_state,
499
+ ${searchableVectorSql('chunks', 'm.rowid')} AS has_vec,
500
+ EXISTS(SELECT 1 FROM chunks_rowids r WHERE r.rowid = m.rowid) AS has_row
501
+ FROM chunk_metadata m WHERE m.text IS NOT NULL`).all();
502
+ this.slotErrors.delete('chunks');
503
+ }
504
+ catch (e) {
505
+ if (!(e instanceof VecSlotError) && !String(e.message).includes('vec0 slot'))
506
+ throw e;
507
+ this.slotErrors.set('chunks', e.message);
508
+ console.error(`⚠️ backfill: chunk slot map unresolvable, chunk phase held: ${e.message}`);
509
+ }
458
510
  let processed = 0;
459
511
  for (const row of chunkRows) {
460
512
  if (this.shuttingDown || !this.eligible)
461
513
  return;
514
+ if (row.has_row && !row.has_vec) {
515
+ this.requestRepair();
516
+ sawInvalid = true;
517
+ continue;
518
+ } // validity 0 -> rebuild
462
519
  const currentHash = this.deps.chunkInputHash(row.text);
463
520
  const isTarget = !row.has_vec
464
521
  || (row.provenance_state !== null && (row.profile_id !== profileId || row.input_hash !== currentHash));
@@ -475,13 +532,28 @@ export class BackfillCoordinator {
475
532
  if (++processed % 8 === 0)
476
533
  await new Promise(r => setImmediate(r));
477
534
  }
478
- // Phase 2: entities.
479
- const entityRows = db.prepare(`SELECT e.id, m.input_hash, m.profile_id, m.provenance_state, m.rowid AS meta_rowid,
480
- CASE WHEN m.rowid IS NOT NULL AND EXISTS(SELECT 1 FROM entity_embeddings v WHERE v.rowid = m.rowid) THEN 1 ELSE 0 END AS has_vec
481
- FROM entities e LEFT JOIN entity_embedding_metadata m ON m.entity_id = e.id`).all();
535
+ let entityRows = [];
536
+ try {
537
+ entityRows = db.prepare(`SELECT e.id, m.input_hash, m.profile_id, m.provenance_state, m.rowid AS meta_rowid,
538
+ CASE WHEN m.rowid IS NOT NULL AND ${searchableVectorSql('entity_embeddings', 'm.rowid')} THEN 1 ELSE 0 END AS has_vec,
539
+ CASE WHEN m.rowid IS NOT NULL AND EXISTS(SELECT 1 FROM entity_embeddings_rowids r WHERE r.rowid = m.rowid) THEN 1 ELSE 0 END AS has_row
540
+ FROM entities e LEFT JOIN entity_embedding_metadata m ON m.entity_id = e.id`).all();
541
+ this.slotErrors.delete('entity_embeddings');
542
+ }
543
+ catch (e) {
544
+ if (!(e instanceof VecSlotError) && !String(e.message).includes('vec0 slot'))
545
+ throw e;
546
+ this.slotErrors.set('entity_embeddings', e.message);
547
+ console.error(`⚠️ backfill: entity slot map unresolvable, entity phase held: ${e.message}`);
548
+ }
482
549
  for (const row of entityRows) {
483
550
  if (this.shuttingDown || !this.eligible)
484
551
  return;
552
+ if (row.has_row && !row.has_vec) {
553
+ this.requestRepair();
554
+ sawInvalid = true;
555
+ continue;
556
+ } // validity 0 -> rebuild
485
557
  const currentHash = this.deps.buildEntityInputHash(row.id);
486
558
  if (currentHash === null)
487
559
  continue; // malformed entity: leave to explicit repair
@@ -501,6 +573,8 @@ export class BackfillCoordinator {
501
573
  if (++processed % 8 === 0)
502
574
  await new Promise(r => setImmediate(r));
503
575
  }
576
+ if (sawInvalid || slotErrorsChanged())
577
+ this.snapshot = null; // r3 M3: coverage reflects what the scan saw
504
578
  if (processed > 0) {
505
579
  this.snapshot = null;
506
580
  console.error(`✅ backfill pass complete (${processed} rows examined for re-embedding)`);
@@ -519,12 +593,25 @@ export class BackfillCoordinator {
519
593
  if (this.snapshot)
520
594
  return this.snapshot;
521
595
  const chunkTotal = db.prepare(`SELECT COUNT(*) c FROM chunk_metadata`).get().c;
522
- const chunkBy = db.prepare(`SELECT m.provenance_state s, COUNT(*) c FROM chunk_metadata m JOIN chunks v ON v.rowid = m.rowid
523
- GROUP BY m.provenance_state`).all();
596
+ // Searchable only (D1c). A slot map error is reported per table instead of a count.
597
+ const byState = (sql) => {
598
+ try {
599
+ return { rows: db.prepare(sql).all() };
600
+ }
601
+ catch (e) {
602
+ if (!(e instanceof VecSlotError) && !String(e.message).includes('vec0 slot'))
603
+ throw e;
604
+ return { rows: [], error: e.message };
605
+ }
606
+ };
607
+ const chunkQ = byState(`SELECT m.provenance_state s, COUNT(*) c FROM chunk_metadata m
608
+ WHERE ${searchableVectorSql('chunks', 'm.rowid')} GROUP BY m.provenance_state`);
609
+ const chunkBy = chunkQ.rows;
524
610
  const entTotal = db.prepare(`SELECT COUNT(*) c FROM entities`).get().c;
525
611
  // Vector-join (beta B3): metadata without an actual vector is missing, not embedded.
526
- const entBy = db.prepare(`SELECT m.provenance_state s, COUNT(*) c FROM entity_embedding_metadata m
527
- JOIN entity_embeddings v ON v.rowid = m.rowid GROUP BY m.provenance_state`).all();
612
+ const entQ = byState(`SELECT m.provenance_state s, COUNT(*) c FROM entity_embedding_metadata m
613
+ WHERE ${searchableVectorSql('entity_embeddings', 'm.rowid')} GROUP BY m.provenance_state`);
614
+ const entBy = entQ.rows;
528
615
  const pick = (rows, s) => rows.find(r => r.s === s)?.c ?? 0;
529
616
  // Searchable = verified + legacy_assumed. NULL rows (pre-reconciliation or
530
617
  // deferred in off mode) are NOT counted as embedded (6R note 4).
@@ -533,8 +620,10 @@ export class BackfillCoordinator {
533
620
  const entVerified = pick(entBy, 'verified');
534
621
  const entLegacy = pick(entBy, 'legacy_assumed');
535
622
  this.snapshot = {
536
- chunk: { total: chunkTotal, embedded: chunkVerified + chunkLegacy, verified: chunkVerified, legacy_assumed: chunkLegacy },
537
- entity: { total: entTotal, embedded: entVerified + entLegacy, verified: entVerified, legacy_assumed: entLegacy },
623
+ chunk: { total: chunkTotal, embedded: chunkVerified + chunkLegacy, verified: chunkVerified, legacy_assumed: chunkLegacy,
624
+ ...(chunkQ.error ? { error: chunkQ.error } : {}) },
625
+ entity: { total: entTotal, embedded: entVerified + entLegacy, verified: entVerified, legacy_assumed: entLegacy,
626
+ ...(entQ.error ? { error: entQ.error } : {}) },
538
627
  };
539
628
  return this.snapshot;
540
629
  }
@@ -1,3 +1,4 @@
1
+ import { deleteVector } from '../vecSlots.js';
1
2
  // active 행을 projection_order 순으로 모아 entities.observations 를 재작성한다.
2
3
  // 단계 1 에서는 이 배열이 여전히 FTS/벡터의 입력이다 — 그래서 projection 갱신만으로
3
4
  // 기존 entity FTS 트리거와 벡터 무효화가 따라온다(spec §4.5).
@@ -23,7 +24,7 @@ export function deleteStaleKgChunks(db, entityId) {
23
24
  const chunks = db.prepare(`SELECT rowid FROM chunk_metadata WHERE chunk_type = 'entity' AND entity_id = ?`).all(entityId);
24
25
  let n = 0;
25
26
  for (const c of chunks) {
26
- db.exec(`DELETE FROM chunks WHERE rowid = ${Number(c.rowid)}`);
27
+ deleteVector(db, 'chunks', Number(c.rowid));
27
28
  db.prepare(`DELETE FROM chunk_metadata WHERE rowid = ?`).run(c.rowid);
28
29
  n++;
29
30
  }
@@ -39,7 +40,7 @@ export function deleteKgRelationshipChunks(db, relationshipIds) {
39
40
  for (const rid of relationshipIds) {
40
41
  const chunks = db.prepare(`SELECT rowid FROM chunk_metadata WHERE chunk_type = 'relationship' AND relationship_id = ?`).all(rid);
41
42
  for (const c of chunks) {
42
- db.exec(`DELETE FROM chunks WHERE rowid = ${Number(c.rowid)}`);
43
+ deleteVector(db, 'chunks', Number(c.rowid));
43
44
  db.prepare(`DELETE FROM chunk_metadata WHERE rowid = ?`).run(c.rowid);
44
45
  n++;
45
46
  }
@@ -60,7 +61,7 @@ export function deleteKgChunkRows(db, rowids) {
60
61
  const row = db.prepare(`SELECT chunk_type FROM chunk_metadata WHERE rowid = ?`).get(rowid);
61
62
  if (!row || (row.chunk_type !== 'entity' && row.chunk_type !== 'relationship'))
62
63
  continue;
63
- db.exec(`DELETE FROM chunks WHERE rowid = ${rowid}`);
64
+ deleteVector(db, 'chunks', rowid);
64
65
  db.prepare(`DELETE FROM chunk_entities WHERE chunk_rowid = ?`).run(rowid);
65
66
  n += db.prepare(`DELETE FROM chunk_metadata WHERE rowid = ?`).run(rowid).changes;
66
67
  }
@@ -0,0 +1,51 @@
1
+ import type Database from 'better-sqlite3';
2
+ export type VecTable = 'chunks' | 'entity_embeddings';
3
+ export type SlotState = 'valid' | 'invalid' | 'shared' | 'absent';
4
+ export declare class VecSlotError extends Error {
5
+ constructor(message: string);
6
+ }
7
+ export declare function bitIsSet(validity: unknown, offset: unknown, size: unknown): boolean;
8
+ export declare function slotState(db: Database.Database, table: VecTable, rowid: number | bigint): SlotState;
9
+ export declare class InvalidSlotError extends VecSlotError {
10
+ constructor(table: string, rowid: number | bigint);
11
+ }
12
+ export declare function writeVector(db: Database.Database, table: VecTable, rowid: number | bigint, embedding: Buffer): SlotState;
13
+ export declare function deleteVector(db: Database.Database, table: VecTable, rowid: number | bigint): SlotState;
14
+ export declare function registerVecSlotFunctions(db: Database.Database): void;
15
+ export declare function searchableVectorSql(table: VecTable, rowidExpr: string): string;
16
+ export interface SlotSummary {
17
+ table: VecTable;
18
+ chunks: number;
19
+ chunkSize: number;
20
+ slots: number;
21
+ valid: number;
22
+ invalid: number;
23
+ aliased: number;
24
+ empty: number;
25
+ reclaimable: number;
26
+ reclaimableBytes: number;
27
+ rewriteBytes: number;
28
+ }
29
+ export declare function slotSummary(db: Database.Database, table: VecTable, dims?: number): SlotSummary;
30
+ export interface RebuildPolicy {
31
+ minChunks: number;
32
+ maxLive: number;
33
+ }
34
+ export declare const DEFAULT_REBUILD_POLICY: RebuildPolicy;
35
+ export declare function rebuildReason(s: SlotSummary, policy?: RebuildPolicy): string | null;
36
+ export type RebuildFault = 'after-drop' | 'after-create' | 'after-partial' | 'corrupt-one';
37
+ export interface RebuildResult {
38
+ table: VecTable;
39
+ status: 'rebuilt' | 'not-needed' | 'busy' | 'position-error' | 'over-cap' | 'failed';
40
+ reason?: string;
41
+ before?: SlotSummary;
42
+ after?: SlotSummary;
43
+ ms: number;
44
+ error?: string;
45
+ }
46
+ export declare function rebuildVecTable(db: Database.Database, table: VecTable, opts?: {
47
+ policy?: RebuildPolicy;
48
+ force?: boolean;
49
+ fault?: RebuildFault;
50
+ dims?: number;
51
+ }): RebuildResult;