rag-memory-epf-mcp 6.4.1 → 6.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -0
- package/dist/index.d.ts +19 -0
- package/dist/index.js +234 -35
- package/dist/src/backfillCoordinator.d.ts +10 -0
- package/dist/src/backfillCoordinator.js +129 -40
- package/dist/src/observations/projection.js +4 -3
- package/dist/src/vecSlots.d.ts +51 -0
- package/dist/src/vecSlots.js +323 -0
- package/docs/UPDATING.md +17 -0
- package/package.json +3 -3
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
// kick() merely triggers a debounced scan. All reconciliation writes are
|
|
15
15
|
// per-row transactions with a CAS re-check so a foreground embed that lands
|
|
16
16
|
// first is never downgraded (6R note 3).
|
|
17
|
+
import { searchableVectorSql, VecSlotError, deleteVector } from './vecSlots.js';
|
|
17
18
|
// A handle closed by the engine's reopen (wal-sidecar-guard), or an I/O error from a sidecar the
|
|
18
19
|
// sync client swapped out: the run was cut, not broken.
|
|
19
20
|
const CONN_LOSS_RERUN_MAX = 3;
|
|
@@ -36,6 +37,15 @@ export class BackfillCoordinator {
|
|
|
36
37
|
shuttingDown = false;
|
|
37
38
|
connLossReruns = 0;
|
|
38
39
|
scanning = false;
|
|
40
|
+
// vec-slot-reuse: rows whose vec0 slot has validity 0 cannot be rewritten in place (sqlite-vec refuses
|
|
41
|
+
// to DELETE many of them); the backfill skips them and asks for the verified rebuild instead.
|
|
42
|
+
repairWanted = false;
|
|
43
|
+
slotErrors = new Map(); // table -> unresolvable slot map message
|
|
44
|
+
repairRequestedAt = 0;
|
|
45
|
+
requestRepair() { if (!this.repairWanted)
|
|
46
|
+
this.repairRequestedAt = Date.now(); this.repairWanted = true; }
|
|
47
|
+
takeRepairRequest() { const w = this.repairWanted; this.repairWanted = false; return w; }
|
|
48
|
+
get repairRequested() { return this.repairWanted; }
|
|
39
49
|
scanPromise = null;
|
|
40
50
|
rerunRequested = false;
|
|
41
51
|
snapshot = null;
|
|
@@ -43,6 +53,8 @@ export class BackfillCoordinator {
|
|
|
43
53
|
this.deps = deps;
|
|
44
54
|
}
|
|
45
55
|
get reconState() { return this.recon; }
|
|
56
|
+
// vec-slot-reuse D1b: the idle vector rebuild must not overlap a backfill scan or a reconciliation pass.
|
|
57
|
+
get busy() { return this.scanning || !!this.scanPromise || this.reconActive || this.recon === 'running'; }
|
|
46
58
|
get reconLastError() { return this.reconError; }
|
|
47
59
|
// Shared eligibility barrier (spec §3): vector search AND automatic backfill.
|
|
48
60
|
// 'n/a' (fresh DB — nothing to reconcile) counts as satisfied (6R note 1).
|
|
@@ -54,17 +66,19 @@ export class BackfillCoordinator {
|
|
|
54
66
|
// non-atomic writes can leave metadata-without-vector and vector-without-
|
|
55
67
|
// metadata split states, and neither must be miscounted as reconciled work.
|
|
56
68
|
countNullWithVector(db) {
|
|
57
|
-
const ent = db.prepare(`SELECT COUNT(*) c FROM entity_embedding_metadata m
|
|
58
|
-
WHERE m.provenance_state IS NULL`).get();
|
|
59
|
-
const chk = db.prepare(`SELECT COUNT(*) c FROM chunk_metadata m
|
|
60
|
-
WHERE m.provenance_state IS NULL`).get();
|
|
69
|
+
const ent = db.prepare(`SELECT COUNT(*) c FROM entity_embedding_metadata m
|
|
70
|
+
WHERE m.provenance_state IS NULL AND ${searchableVectorSql('entity_embeddings', 'm.rowid')}`).get();
|
|
71
|
+
const chk = db.prepare(`SELECT COUNT(*) c FROM chunk_metadata m
|
|
72
|
+
WHERE m.provenance_state IS NULL AND ${searchableVectorSql('chunks', 'm.rowid')}`).get();
|
|
61
73
|
return ent.c + chk.c;
|
|
62
74
|
}
|
|
63
75
|
// Sanitation (beta B3·B4) — runs on EVERY reconciliation entry, even when no
|
|
64
76
|
// NULL rows exist, so profile changes and legacy split states are repaired
|
|
65
77
|
// before vector eligibility can open:
|
|
66
78
|
// 1. entity vectors without metadata (orphans) -> delete vector
|
|
67
|
-
// 2. entity metadata
|
|
79
|
+
// 2. stamped entity metadata whose vector row is gone -> normalized to missing, NOT deleted: the row keeps
|
|
80
|
+
// its rowid and becomes a backfill target (vec-slot-reuse r3 M2; validity-0 vector rows are repaired
|
|
81
|
+
// inline by deleteVector/writeVector or by the idle rebuild, never DELETEd here)
|
|
68
82
|
// 3. provenance-stamped rows whose compatibility profile != current
|
|
69
83
|
// (old engine/model profile) -> delete-to-missing
|
|
70
84
|
// 4. chunk metadata stamped but vectorless -> normalize to missing (NULL provenance)
|
|
@@ -75,12 +89,14 @@ export class BackfillCoordinator {
|
|
|
75
89
|
const profileId = this.deps.currentProfileId();
|
|
76
90
|
const q = (sql, ...args) => db.prepare(sql).get(...args).c;
|
|
77
91
|
return this.countNullWithVector(db)
|
|
78
|
-
|
|
79
|
-
|
|
92
|
+
// vec-slot-reuse r3 M2: searchable orphans only (validity-0 rows belong to the rebuild); vectorless
|
|
93
|
+
// metadata counts only while still stamped (it is normalized to missing, not deleted, to keep its rowid).
|
|
94
|
+
+ q(`SELECT COUNT(*) c FROM entity_embeddings_rowids m WHERE m.rowid NOT IN (SELECT rowid FROM entity_embedding_metadata) AND ${searchableVectorSql('entity_embeddings', 'm.rowid')}`)
|
|
95
|
+
+ q(`SELECT COUNT(*) c FROM entity_embedding_metadata WHERE provenance_state IS NOT NULL AND rowid NOT IN (SELECT rowid FROM entity_embeddings_rowids)`)
|
|
80
96
|
+ q(`SELECT COUNT(*) c FROM entity_embedding_metadata WHERE provenance_state IS NOT NULL AND profile_id IS NOT ?`, profileId)
|
|
81
|
-
+ q(`SELECT COUNT(*) c FROM chunk_metadata m
|
|
97
|
+
+ q(`SELECT COUNT(*) c FROM chunk_metadata m WHERE m.provenance_state IS NOT NULL AND m.profile_id IS NOT ? AND m.rowid IN (SELECT rowid FROM chunks_rowids)`, profileId)
|
|
82
98
|
// stamped chunk metadata whose vector is gone (beta 3R M3)
|
|
83
|
-
+ q(`SELECT COUNT(*) c FROM chunk_metadata WHERE provenance_state IS NOT NULL AND rowid NOT IN (SELECT rowid FROM
|
|
99
|
+
+ q(`SELECT COUNT(*) c FROM chunk_metadata WHERE provenance_state IS NOT NULL AND rowid NOT IN (SELECT rowid FROM chunks_rowids)`);
|
|
84
100
|
}
|
|
85
101
|
// Batched (beta 2R residual): each batch commits its own transaction and
|
|
86
102
|
// yields the event loop — a large DB or a profile switch must not stall the
|
|
@@ -103,13 +119,32 @@ export class BackfillCoordinator {
|
|
|
103
119
|
await new Promise(r => setImmediate(r));
|
|
104
120
|
}
|
|
105
121
|
};
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
122
|
+
// 1. searchable entity vectors without metadata -> delete (validity-0 orphans are left to the rebuild:
|
|
123
|
+
// vec0 refuses to DELETE most of them, vec-slot-reuse r3 M2)
|
|
124
|
+
for (;;) {
|
|
125
|
+
if (this.shuttingDown)
|
|
126
|
+
throw new Error('shutdown during sanitation');
|
|
127
|
+
const orphans = db.prepare(`SELECT m.rowid FROM entity_embeddings_rowids m
|
|
128
|
+
WHERE m.rowid NOT IN (SELECT rowid FROM entity_embedding_metadata) AND ${searchableVectorSql('entity_embeddings', 'm.rowid')} LIMIT ${RECON_BATCH}`).all();
|
|
129
|
+
if (orphans.length === 0)
|
|
130
|
+
break;
|
|
131
|
+
batchTx(() => { for (const r of orphans) {
|
|
132
|
+
deleteVector(db, 'entity_embeddings', r.rowid);
|
|
133
|
+
touched++;
|
|
134
|
+
} });
|
|
135
|
+
if (orphans.length < RECON_BATCH)
|
|
136
|
+
break;
|
|
137
|
+
await new Promise(r => setImmediate(r));
|
|
138
|
+
}
|
|
139
|
+
// 2. stamped entity metadata whose vector row is gone -> normalize to missing (was: delete). Keeping the
|
|
140
|
+
// row keeps its rowid, so the backfill re-inserts at the same rowid (r3 M2 case C: restart after repair).
|
|
141
|
+
await batchedRun(`UPDATE entity_embedding_metadata SET input_hash = NULL, profile_id = NULL, provenance_state = NULL
|
|
142
|
+
WHERE rowid IN (
|
|
143
|
+
SELECT rowid FROM entity_embedding_metadata
|
|
144
|
+
WHERE provenance_state IS NOT NULL AND rowid NOT IN (SELECT rowid FROM entity_embeddings_rowids) LIMIT ${RECON_BATCH})`);
|
|
110
145
|
await batchedRun(`UPDATE chunk_metadata SET input_hash = NULL, profile_id = NULL, provenance_state = NULL
|
|
111
146
|
WHERE rowid IN (
|
|
112
|
-
SELECT rowid FROM chunk_metadata WHERE provenance_state IS NOT NULL AND rowid NOT IN (SELECT rowid FROM
|
|
147
|
+
SELECT rowid FROM chunk_metadata WHERE provenance_state IS NOT NULL AND rowid NOT IN (SELECT rowid FROM chunks_rowids) LIMIT ${RECON_BATCH})`);
|
|
113
148
|
// Old-profile rows: row deletes in bounded batches with yields.
|
|
114
149
|
for (;;) {
|
|
115
150
|
if (this.shuttingDown)
|
|
@@ -120,8 +155,8 @@ export class BackfillCoordinator {
|
|
|
120
155
|
break;
|
|
121
156
|
batchTx(() => {
|
|
122
157
|
for (const r of staleEnt) {
|
|
123
|
-
db
|
|
124
|
-
db.prepare(`
|
|
158
|
+
deleteVector(db, 'entity_embeddings', r.rowid); // validity-0: left to the rebuild
|
|
159
|
+
db.prepare(`UPDATE entity_embedding_metadata SET input_hash = NULL, profile_id = NULL, provenance_state = NULL WHERE rowid = ?`).run(r.rowid);
|
|
125
160
|
touched++;
|
|
126
161
|
}
|
|
127
162
|
});
|
|
@@ -130,13 +165,13 @@ export class BackfillCoordinator {
|
|
|
130
165
|
for (;;) {
|
|
131
166
|
if (this.shuttingDown)
|
|
132
167
|
throw new Error('shutdown during sanitation');
|
|
133
|
-
const staleChunk = db.prepare(`SELECT m.rowid FROM chunk_metadata m
|
|
134
|
-
WHERE m.provenance_state IS NOT NULL AND m.profile_id IS NOT ? LIMIT ${RECON_BATCH}`).all(profileId);
|
|
168
|
+
const staleChunk = db.prepare(`SELECT m.rowid FROM chunk_metadata m
|
|
169
|
+
WHERE m.provenance_state IS NOT NULL AND m.profile_id IS NOT ? AND m.rowid IN (SELECT rowid FROM chunks_rowids) LIMIT ${RECON_BATCH}`).all(profileId);
|
|
135
170
|
if (staleChunk.length === 0)
|
|
136
171
|
break;
|
|
137
172
|
batchTx(() => {
|
|
138
173
|
for (const r of staleChunk) {
|
|
139
|
-
db
|
|
174
|
+
deleteVector(db, 'chunks', r.rowid); // validity-0: left to the rebuild
|
|
140
175
|
db.prepare(`UPDATE chunk_metadata SET input_hash = NULL, profile_id = NULL, provenance_state = NULL WHERE rowid = ?`).run(r.rowid);
|
|
141
176
|
touched++;
|
|
142
177
|
}
|
|
@@ -151,10 +186,14 @@ export class BackfillCoordinator {
|
|
|
151
186
|
// to 'n/a'. Row-level errors are fail-closed (vector deleted -> missing);
|
|
152
187
|
// systemic errors transition to 'failed' and keep eligibility shut.
|
|
153
188
|
runReconciliation() {
|
|
154
|
-
if (!this.reconPromise)
|
|
155
|
-
|
|
189
|
+
if (!this.reconPromise) {
|
|
190
|
+
// r3 M4: busy for the whole pass (sanitize included), not only while recon === 'running'.
|
|
191
|
+
this.reconActive = true;
|
|
192
|
+
this.reconPromise = this.reconcileOnce().finally(() => { this.reconActive = false; });
|
|
193
|
+
}
|
|
156
194
|
return this.reconPromise;
|
|
157
195
|
}
|
|
196
|
+
reconActive = false;
|
|
158
197
|
async reconcileOnce() {
|
|
159
198
|
const db = this.deps.db();
|
|
160
199
|
if (!db) {
|
|
@@ -233,8 +272,7 @@ export class BackfillCoordinator {
|
|
|
233
272
|
// grandfather candidates — metadata-without-vector was already removed by
|
|
234
273
|
// sanitize().
|
|
235
274
|
const rows = db.prepare(`SELECT m.rowid, m.entity_id, m.embedding_text FROM entity_embedding_metadata m
|
|
236
|
-
|
|
237
|
-
WHERE m.provenance_state IS NULL LIMIT ${RECON_BATCH}`).all();
|
|
275
|
+
WHERE m.provenance_state IS NULL AND ${searchableVectorSql('entity_embeddings', 'm.rowid')} LIMIT ${RECON_BATCH}`).all();
|
|
238
276
|
if (rows.length === 0)
|
|
239
277
|
return;
|
|
240
278
|
for (const row of rows) {
|
|
@@ -244,7 +282,7 @@ export class BackfillCoordinator {
|
|
|
244
282
|
const cur = db.prepare(`SELECT provenance_state FROM entity_embedding_metadata WHERE rowid = ?`).get(row.rowid);
|
|
245
283
|
if (!cur || cur.provenance_state !== null)
|
|
246
284
|
return;
|
|
247
|
-
db
|
|
285
|
+
deleteVector(db, 'entity_embeddings', row.rowid);
|
|
248
286
|
db.prepare(`DELETE FROM entity_embedding_metadata WHERE rowid = ?`).run(row.rowid);
|
|
249
287
|
});
|
|
250
288
|
const stampTx = db.transaction((hash) => {
|
|
@@ -300,8 +338,8 @@ export class BackfillCoordinator {
|
|
|
300
338
|
for (;;) {
|
|
301
339
|
if (this.shuttingDown)
|
|
302
340
|
throw new Error('shutdown during reconciliation');
|
|
303
|
-
const rows = db.prepare(`SELECT m.rowid, m.text FROM chunk_metadata m
|
|
304
|
-
WHERE m.provenance_state IS NULL LIMIT ${RECON_BATCH}`).all();
|
|
341
|
+
const rows = db.prepare(`SELECT m.rowid, m.text FROM chunk_metadata m
|
|
342
|
+
WHERE m.provenance_state IS NULL AND ${searchableVectorSql('chunks', 'm.rowid')} LIMIT ${RECON_BATCH}`).all();
|
|
305
343
|
if (rows.length === 0)
|
|
306
344
|
return;
|
|
307
345
|
for (const row of rows) {
|
|
@@ -313,7 +351,7 @@ export class BackfillCoordinator {
|
|
|
313
351
|
// Vector removed; provenance stays NULL. The barrier only protects
|
|
314
352
|
// VECTOR-BEARING NULL rows — vectorless rows are always legitimate
|
|
315
353
|
// backfill targets (spec §6c "벡터 없음").
|
|
316
|
-
db
|
|
354
|
+
deleteVector(db, 'chunks', row.rowid);
|
|
317
355
|
db.prepare(`UPDATE chunk_metadata SET input_hash = NULL, profile_id = NULL WHERE rowid = ?`)
|
|
318
356
|
.run(row.rowid);
|
|
319
357
|
});
|
|
@@ -452,13 +490,32 @@ export class BackfillCoordinator {
|
|
|
452
490
|
return consecutive >= 3 ? 'abort' : 'fail';
|
|
453
491
|
};
|
|
454
492
|
// Phase 1: chunks.
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
493
|
+
let sawInvalid = false;
|
|
494
|
+
const errorsBefore = JSON.stringify([...this.slotErrors]);
|
|
495
|
+
const slotErrorsChanged = () => JSON.stringify([...this.slotErrors]) !== errorsBefore;
|
|
496
|
+
let chunkRows = [];
|
|
497
|
+
try {
|
|
498
|
+
chunkRows = db.prepare(`SELECT m.rowid, m.text, m.input_hash, m.profile_id, m.provenance_state,
|
|
499
|
+
${searchableVectorSql('chunks', 'm.rowid')} AS has_vec,
|
|
500
|
+
EXISTS(SELECT 1 FROM chunks_rowids r WHERE r.rowid = m.rowid) AS has_row
|
|
501
|
+
FROM chunk_metadata m WHERE m.text IS NOT NULL`).all();
|
|
502
|
+
this.slotErrors.delete('chunks');
|
|
503
|
+
}
|
|
504
|
+
catch (e) {
|
|
505
|
+
if (!(e instanceof VecSlotError) && !String(e.message).includes('vec0 slot'))
|
|
506
|
+
throw e;
|
|
507
|
+
this.slotErrors.set('chunks', e.message);
|
|
508
|
+
console.error(`⚠️ backfill: chunk slot map unresolvable, chunk phase held: ${e.message}`);
|
|
509
|
+
}
|
|
458
510
|
let processed = 0;
|
|
459
511
|
for (const row of chunkRows) {
|
|
460
512
|
if (this.shuttingDown || !this.eligible)
|
|
461
513
|
return;
|
|
514
|
+
if (row.has_row && !row.has_vec) {
|
|
515
|
+
this.requestRepair();
|
|
516
|
+
sawInvalid = true;
|
|
517
|
+
continue;
|
|
518
|
+
} // validity 0 -> rebuild
|
|
462
519
|
const currentHash = this.deps.chunkInputHash(row.text);
|
|
463
520
|
const isTarget = !row.has_vec
|
|
464
521
|
|| (row.provenance_state !== null && (row.profile_id !== profileId || row.input_hash !== currentHash));
|
|
@@ -475,13 +532,28 @@ export class BackfillCoordinator {
|
|
|
475
532
|
if (++processed % 8 === 0)
|
|
476
533
|
await new Promise(r => setImmediate(r));
|
|
477
534
|
}
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
535
|
+
let entityRows = [];
|
|
536
|
+
try {
|
|
537
|
+
entityRows = db.prepare(`SELECT e.id, m.input_hash, m.profile_id, m.provenance_state, m.rowid AS meta_rowid,
|
|
538
|
+
CASE WHEN m.rowid IS NOT NULL AND ${searchableVectorSql('entity_embeddings', 'm.rowid')} THEN 1 ELSE 0 END AS has_vec,
|
|
539
|
+
CASE WHEN m.rowid IS NOT NULL AND EXISTS(SELECT 1 FROM entity_embeddings_rowids r WHERE r.rowid = m.rowid) THEN 1 ELSE 0 END AS has_row
|
|
540
|
+
FROM entities e LEFT JOIN entity_embedding_metadata m ON m.entity_id = e.id`).all();
|
|
541
|
+
this.slotErrors.delete('entity_embeddings');
|
|
542
|
+
}
|
|
543
|
+
catch (e) {
|
|
544
|
+
if (!(e instanceof VecSlotError) && !String(e.message).includes('vec0 slot'))
|
|
545
|
+
throw e;
|
|
546
|
+
this.slotErrors.set('entity_embeddings', e.message);
|
|
547
|
+
console.error(`⚠️ backfill: entity slot map unresolvable, entity phase held: ${e.message}`);
|
|
548
|
+
}
|
|
482
549
|
for (const row of entityRows) {
|
|
483
550
|
if (this.shuttingDown || !this.eligible)
|
|
484
551
|
return;
|
|
552
|
+
if (row.has_row && !row.has_vec) {
|
|
553
|
+
this.requestRepair();
|
|
554
|
+
sawInvalid = true;
|
|
555
|
+
continue;
|
|
556
|
+
} // validity 0 -> rebuild
|
|
485
557
|
const currentHash = this.deps.buildEntityInputHash(row.id);
|
|
486
558
|
if (currentHash === null)
|
|
487
559
|
continue; // malformed entity: leave to explicit repair
|
|
@@ -501,6 +573,8 @@ export class BackfillCoordinator {
|
|
|
501
573
|
if (++processed % 8 === 0)
|
|
502
574
|
await new Promise(r => setImmediate(r));
|
|
503
575
|
}
|
|
576
|
+
if (sawInvalid || slotErrorsChanged())
|
|
577
|
+
this.snapshot = null; // r3 M3: coverage reflects what the scan saw
|
|
504
578
|
if (processed > 0) {
|
|
505
579
|
this.snapshot = null;
|
|
506
580
|
console.error(`✅ backfill pass complete (${processed} rows examined for re-embedding)`);
|
|
@@ -519,12 +593,25 @@ export class BackfillCoordinator {
|
|
|
519
593
|
if (this.snapshot)
|
|
520
594
|
return this.snapshot;
|
|
521
595
|
const chunkTotal = db.prepare(`SELECT COUNT(*) c FROM chunk_metadata`).get().c;
|
|
522
|
-
|
|
523
|
-
|
|
596
|
+
// Searchable only (D1c). A slot map error is reported per table instead of a count.
|
|
597
|
+
const byState = (sql) => {
|
|
598
|
+
try {
|
|
599
|
+
return { rows: db.prepare(sql).all() };
|
|
600
|
+
}
|
|
601
|
+
catch (e) {
|
|
602
|
+
if (!(e instanceof VecSlotError) && !String(e.message).includes('vec0 slot'))
|
|
603
|
+
throw e;
|
|
604
|
+
return { rows: [], error: e.message };
|
|
605
|
+
}
|
|
606
|
+
};
|
|
607
|
+
const chunkQ = byState(`SELECT m.provenance_state s, COUNT(*) c FROM chunk_metadata m
|
|
608
|
+
WHERE ${searchableVectorSql('chunks', 'm.rowid')} GROUP BY m.provenance_state`);
|
|
609
|
+
const chunkBy = chunkQ.rows;
|
|
524
610
|
const entTotal = db.prepare(`SELECT COUNT(*) c FROM entities`).get().c;
|
|
525
611
|
// Vector-join (beta B3): metadata without an actual vector is missing, not embedded.
|
|
526
|
-
const
|
|
527
|
-
|
|
612
|
+
const entQ = byState(`SELECT m.provenance_state s, COUNT(*) c FROM entity_embedding_metadata m
|
|
613
|
+
WHERE ${searchableVectorSql('entity_embeddings', 'm.rowid')} GROUP BY m.provenance_state`);
|
|
614
|
+
const entBy = entQ.rows;
|
|
528
615
|
const pick = (rows, s) => rows.find(r => r.s === s)?.c ?? 0;
|
|
529
616
|
// Searchable = verified + legacy_assumed. NULL rows (pre-reconciliation or
|
|
530
617
|
// deferred in off mode) are NOT counted as embedded (6R note 4).
|
|
@@ -533,8 +620,10 @@ export class BackfillCoordinator {
|
|
|
533
620
|
const entVerified = pick(entBy, 'verified');
|
|
534
621
|
const entLegacy = pick(entBy, 'legacy_assumed');
|
|
535
622
|
this.snapshot = {
|
|
536
|
-
chunk: { total: chunkTotal, embedded: chunkVerified + chunkLegacy, verified: chunkVerified, legacy_assumed: chunkLegacy
|
|
537
|
-
|
|
623
|
+
chunk: { total: chunkTotal, embedded: chunkVerified + chunkLegacy, verified: chunkVerified, legacy_assumed: chunkLegacy,
|
|
624
|
+
...(chunkQ.error ? { error: chunkQ.error } : {}) },
|
|
625
|
+
entity: { total: entTotal, embedded: entVerified + entLegacy, verified: entVerified, legacy_assumed: entLegacy,
|
|
626
|
+
...(entQ.error ? { error: entQ.error } : {}) },
|
|
538
627
|
};
|
|
539
628
|
return this.snapshot;
|
|
540
629
|
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { deleteVector } from '../vecSlots.js';
|
|
1
2
|
// active 행을 projection_order 순으로 모아 entities.observations 를 재작성한다.
|
|
2
3
|
// 단계 1 에서는 이 배열이 여전히 FTS/벡터의 입력이다 — 그래서 projection 갱신만으로
|
|
3
4
|
// 기존 entity FTS 트리거와 벡터 무효화가 따라온다(spec §4.5).
|
|
@@ -23,7 +24,7 @@ export function deleteStaleKgChunks(db, entityId) {
|
|
|
23
24
|
const chunks = db.prepare(`SELECT rowid FROM chunk_metadata WHERE chunk_type = 'entity' AND entity_id = ?`).all(entityId);
|
|
24
25
|
let n = 0;
|
|
25
26
|
for (const c of chunks) {
|
|
26
|
-
db
|
|
27
|
+
deleteVector(db, 'chunks', Number(c.rowid));
|
|
27
28
|
db.prepare(`DELETE FROM chunk_metadata WHERE rowid = ?`).run(c.rowid);
|
|
28
29
|
n++;
|
|
29
30
|
}
|
|
@@ -39,7 +40,7 @@ export function deleteKgRelationshipChunks(db, relationshipIds) {
|
|
|
39
40
|
for (const rid of relationshipIds) {
|
|
40
41
|
const chunks = db.prepare(`SELECT rowid FROM chunk_metadata WHERE chunk_type = 'relationship' AND relationship_id = ?`).all(rid);
|
|
41
42
|
for (const c of chunks) {
|
|
42
|
-
db
|
|
43
|
+
deleteVector(db, 'chunks', Number(c.rowid));
|
|
43
44
|
db.prepare(`DELETE FROM chunk_metadata WHERE rowid = ?`).run(c.rowid);
|
|
44
45
|
n++;
|
|
45
46
|
}
|
|
@@ -60,7 +61,7 @@ export function deleteKgChunkRows(db, rowids) {
|
|
|
60
61
|
const row = db.prepare(`SELECT chunk_type FROM chunk_metadata WHERE rowid = ?`).get(rowid);
|
|
61
62
|
if (!row || (row.chunk_type !== 'entity' && row.chunk_type !== 'relationship'))
|
|
62
63
|
continue;
|
|
63
|
-
db
|
|
64
|
+
deleteVector(db, 'chunks', rowid);
|
|
64
65
|
db.prepare(`DELETE FROM chunk_entities WHERE chunk_rowid = ?`).run(rowid);
|
|
65
66
|
n += db.prepare(`DELETE FROM chunk_metadata WHERE rowid = ?`).run(rowid).changes;
|
|
66
67
|
}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import type Database from 'better-sqlite3';
|
|
2
|
+
export type VecTable = 'chunks' | 'entity_embeddings';
|
|
3
|
+
export type SlotState = 'valid' | 'invalid' | 'shared' | 'absent';
|
|
4
|
+
export declare class VecSlotError extends Error {
|
|
5
|
+
constructor(message: string);
|
|
6
|
+
}
|
|
7
|
+
export declare function bitIsSet(validity: unknown, offset: unknown, size: unknown): boolean;
|
|
8
|
+
export declare function slotState(db: Database.Database, table: VecTable, rowid: number | bigint): SlotState;
|
|
9
|
+
export declare class InvalidSlotError extends VecSlotError {
|
|
10
|
+
constructor(table: string, rowid: number | bigint);
|
|
11
|
+
}
|
|
12
|
+
export declare function writeVector(db: Database.Database, table: VecTable, rowid: number | bigint, embedding: Buffer): SlotState;
|
|
13
|
+
export declare function deleteVector(db: Database.Database, table: VecTable, rowid: number | bigint): SlotState;
|
|
14
|
+
export declare function registerVecSlotFunctions(db: Database.Database): void;
|
|
15
|
+
export declare function searchableVectorSql(table: VecTable, rowidExpr: string): string;
|
|
16
|
+
export interface SlotSummary {
|
|
17
|
+
table: VecTable;
|
|
18
|
+
chunks: number;
|
|
19
|
+
chunkSize: number;
|
|
20
|
+
slots: number;
|
|
21
|
+
valid: number;
|
|
22
|
+
invalid: number;
|
|
23
|
+
aliased: number;
|
|
24
|
+
empty: number;
|
|
25
|
+
reclaimable: number;
|
|
26
|
+
reclaimableBytes: number;
|
|
27
|
+
rewriteBytes: number;
|
|
28
|
+
}
|
|
29
|
+
export declare function slotSummary(db: Database.Database, table: VecTable, dims?: number): SlotSummary;
|
|
30
|
+
export interface RebuildPolicy {
|
|
31
|
+
minChunks: number;
|
|
32
|
+
maxLive: number;
|
|
33
|
+
}
|
|
34
|
+
export declare const DEFAULT_REBUILD_POLICY: RebuildPolicy;
|
|
35
|
+
export declare function rebuildReason(s: SlotSummary, policy?: RebuildPolicy): string | null;
|
|
36
|
+
export type RebuildFault = 'after-drop' | 'after-create' | 'after-partial' | 'corrupt-one';
|
|
37
|
+
export interface RebuildResult {
|
|
38
|
+
table: VecTable;
|
|
39
|
+
status: 'rebuilt' | 'not-needed' | 'busy' | 'position-error' | 'over-cap' | 'failed';
|
|
40
|
+
reason?: string;
|
|
41
|
+
before?: SlotSummary;
|
|
42
|
+
after?: SlotSummary;
|
|
43
|
+
ms: number;
|
|
44
|
+
error?: string;
|
|
45
|
+
}
|
|
46
|
+
export declare function rebuildVecTable(db: Database.Database, table: VecTable, opts?: {
|
|
47
|
+
policy?: RebuildPolicy;
|
|
48
|
+
force?: boolean;
|
|
49
|
+
fault?: RebuildFault;
|
|
50
|
+
dims?: number;
|
|
51
|
+
}): RebuildResult;
|