rag-memory-epf-mcp 6.4.2 → 6.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -0
- package/dist/index.d.ts +15 -0
- package/dist/index.js +220 -34
- package/dist/src/backfillCoordinator.d.ts +10 -0
- package/dist/src/backfillCoordinator.js +129 -40
- package/dist/src/observations/projection.js +4 -3
- package/dist/src/vecSlots.d.ts +51 -0
- package/dist/src/vecSlots.js +323 -0
- package/docs/UPDATING.md +10 -0
- package/package.json +2 -2
|
@@ -0,0 +1,323 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
const TABLES = new Set(['chunks', 'entity_embeddings']);
|
|
3
|
+
export class VecSlotError extends Error {
|
|
4
|
+
constructor(message) { super(message); this.name = 'VecSlotError'; }
|
|
5
|
+
}
|
|
6
|
+
function assertTable(table) {
|
|
7
|
+
if (!TABLES.has(table))
|
|
8
|
+
throw new VecSlotError(`not a vec0 table: ${table}`);
|
|
9
|
+
}
|
|
10
|
+
// One bit test, shared by the JS path and the SQL function below.
|
|
11
|
+
export function bitIsSet(validity, offset, size) {
|
|
12
|
+
if (!(validity instanceof Uint8Array) || typeof offset !== 'number' && typeof offset !== 'bigint'
|
|
13
|
+
|| typeof size !== 'number' && typeof size !== 'bigint') {
|
|
14
|
+
throw new VecSlotError(`unresolvable vec0 slot (validity ${validity === null ? 'NULL' : typeof validity}, offset ${offset}, size ${size})`);
|
|
15
|
+
}
|
|
16
|
+
const off = Number(offset), sz = Number(size);
|
|
17
|
+
// Every value must be an exact non-negative integer and agree with the bitmap; no 32-bit `>>` (it wraps).
|
|
18
|
+
if (!Number.isSafeInteger(sz) || sz <= 0 || sz > validity.length * 8) {
|
|
19
|
+
throw new VecSlotError(`vec0 chunk size ${String(size)} does not fit its ${validity.length}-byte bitmap`);
|
|
20
|
+
}
|
|
21
|
+
if (!Number.isSafeInteger(off) || off < 0 || off >= sz || BigInt(off) !== BigInt(offset)) {
|
|
22
|
+
throw new VecSlotError(`vec0 slot offset ${String(offset)} outside chunk (size ${sz}, bitmap ${validity.length} bytes)`);
|
|
23
|
+
}
|
|
24
|
+
const byte = Math.floor(off / 8);
|
|
25
|
+
return ((validity[byte] >> (off % 8)) & 1) === 1;
|
|
26
|
+
}
|
|
27
|
+
export function slotState(db, table, rowid) {
|
|
28
|
+
assertTable(table);
|
|
29
|
+
const r = db.prepare(`SELECT chunk_id, chunk_offset FROM ${table}_rowids WHERE rowid = ?`).safeIntegers(true).get(BigInt(rowid));
|
|
30
|
+
if (!r)
|
|
31
|
+
return 'absent';
|
|
32
|
+
if (r.chunk_id === null || r.chunk_offset === null)
|
|
33
|
+
throw new VecSlotError(`${table}: rowid ${rowid} has no position`);
|
|
34
|
+
const c = db.prepare(`SELECT size, validity FROM ${table}_chunks WHERE chunk_id = ?`).safeIntegers(true).get(r.chunk_id);
|
|
35
|
+
if (!c)
|
|
36
|
+
throw new VecSlotError(`${table}: rowid ${rowid} points at missing chunk ${r.chunk_id}`);
|
|
37
|
+
const bit = bitIsSet(c.validity, r.chunk_offset, c.size);
|
|
38
|
+
// A validity-0 slot looks free to vec0, so a later insert can take it: two rowids then point at one slot
|
|
39
|
+
// and neither owns it (r4 M1). That is not a searchable vector of this rowid.
|
|
40
|
+
const owners = Number(db.prepare(`SELECT COUNT(*) c FROM ${table}_rowids WHERE chunk_id = ? AND chunk_offset = ?`)
|
|
41
|
+
.safeIntegers(true).get(r.chunk_id, r.chunk_offset).c);
|
|
42
|
+
if (owners > 1)
|
|
43
|
+
return 'shared';
|
|
44
|
+
return bit ? 'valid' : 'invalid';
|
|
45
|
+
}
|
|
46
|
+
// Does chunk <chunkId> hold a validity-0 or shared row? Deleting the last valid row of such a chunk makes
|
|
47
|
+
// vec0 drop the chunk while those rows still point at it (r4 M3), so the delete repairs first.
|
|
48
|
+
function chunkHasAnomaly(db, table, chunkId) {
|
|
49
|
+
const c = db.prepare(`SELECT size, validity FROM ${table}_chunks WHERE chunk_id = ?`).safeIntegers(true).get(chunkId);
|
|
50
|
+
if (!c)
|
|
51
|
+
return true;
|
|
52
|
+
const seen = new Set();
|
|
53
|
+
for (const r of db.prepare(`SELECT chunk_offset FROM ${table}_rowids WHERE chunk_id = ?`).safeIntegers(true).iterate(chunkId)) {
|
|
54
|
+
const k = String(r.chunk_offset);
|
|
55
|
+
if (seen.has(k) || !bitIsSet(c.validity, r.chunk_offset, c.size))
|
|
56
|
+
return true;
|
|
57
|
+
seen.add(k);
|
|
58
|
+
}
|
|
59
|
+
return false;
|
|
60
|
+
}
|
|
61
|
+
// Rebuild <table> now, inside the caller's transaction (a savepoint when one is open), keeping exactly the
|
|
62
|
+
// rows that own a valid slot. Used when a write or delete meets an anomalous slot, so no ordinary operation
|
|
63
|
+
// ever runs against a validity-0 or shared slot (r4 M1·M3). Costs one rebuild (measured 0.44 s for 5,600).
|
|
64
|
+
// Not forced (r5 M2): the same reason/cap as the idle rebuild apply. Above the live-vector cap the caller's
|
|
65
|
+
// transaction rolls back with a "maintenance needed" error instead of an unbounded rebuild inside a tool call.
|
|
66
|
+
function repairInline(db, table) {
|
|
67
|
+
const r = rebuildVecTable(db, table);
|
|
68
|
+
if (r.status === 'not-needed')
|
|
69
|
+
return;
|
|
70
|
+
if (r.status !== 'rebuilt') {
|
|
71
|
+
throw new VecSlotError(`${table}: inline repair did not complete (${r.status}${r.error ? `: ${r.error}` : ''})` +
|
|
72
|
+
(r.status === 'over-cap' ? ' — run the maintenance rebuild (scripts/rag-maintenance/compact-vectors.mjs)' : ''));
|
|
73
|
+
}
|
|
74
|
+
console.error(`🧹 vec0 ${table} repaired inline: dropped validity-0 ${r.before?.invalid ?? 0}, shared ${r.before?.aliased ?? 0}, ${r.ms} ms`);
|
|
75
|
+
}
|
|
76
|
+
// Raised only if a slot is still not writable after an inline repair (should not happen; kept so callers
|
|
77
|
+
// that catch it stay correct).
|
|
78
|
+
export class InvalidSlotError extends VecSlotError {
|
|
79
|
+
constructor(table, rowid) {
|
|
80
|
+
super(`${table}: rowid ${rowid} has a validity-0 slot; it is repaired by the vector table rebuild`);
|
|
81
|
+
this.name = 'InvalidSlotError';
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
// The one same-rowid vector write. Caller owns the transaction (CAS + provenance stay in it).
|
|
85
|
+
// valid -> UPDATE (same slot, no leak)
|
|
86
|
+
// absent -> INSERT
|
|
87
|
+
// invalid / shared -> inline repair first (UPDATE would stay invisible or overwrite another rowid's vector), then INSERT
|
|
88
|
+
export function writeVector(db, table, rowid, embedding) {
|
|
89
|
+
let state = slotState(db, table, rowid);
|
|
90
|
+
const id = BigInt(rowid);
|
|
91
|
+
if (state === 'invalid' || state === 'shared') {
|
|
92
|
+
repairInline(db, table);
|
|
93
|
+
state = slotState(db, table, rowid);
|
|
94
|
+
}
|
|
95
|
+
if (state !== 'valid' && state !== 'absent')
|
|
96
|
+
throw new InvalidSlotError(table, rowid);
|
|
97
|
+
if (state === 'valid')
|
|
98
|
+
db.prepare(`UPDATE ${table} SET embedding = ? WHERE rowid = ?`).run(embedding, id);
|
|
99
|
+
else
|
|
100
|
+
db.prepare(`INSERT INTO ${table}(rowid, embedding) VALUES (?, ?)`).run(id, embedding);
|
|
101
|
+
return state;
|
|
102
|
+
}
|
|
103
|
+
// The one single-row vector delete (vec-slot-reuse r3 M2 · r4 M3). Every engine delete goes through here:
|
|
104
|
+
// valid -> DELETE (after an inline repair if its chunk holds a validity-0 or shared row, so the
|
|
105
|
+
// delete can never leave rows pointing at a chunk vec0 dropped)
|
|
106
|
+
// invalid / shared -> inline repair (it drops the row; vec0 refuses to DELETE most validity-0 rows)
|
|
107
|
+
// absent -> nothing
|
|
108
|
+
// A position error raises VecSlotError (caller's transaction rolls back), never a silent skip.
|
|
109
|
+
export function deleteVector(db, table, rowid) {
|
|
110
|
+
let state = slotState(db, table, rowid);
|
|
111
|
+
if (state === 'invalid' || state === 'shared') {
|
|
112
|
+
repairInline(db, table);
|
|
113
|
+
return state;
|
|
114
|
+
} // repair drops it
|
|
115
|
+
if (state === 'valid') {
|
|
116
|
+
const p = db.prepare(`SELECT chunk_id FROM ${table}_rowids WHERE rowid = ?`).safeIntegers(true).get(BigInt(rowid));
|
|
117
|
+
if (chunkHasAnomaly(db, table, p.chunk_id)) {
|
|
118
|
+
repairInline(db, table);
|
|
119
|
+
state = slotState(db, table, rowid);
|
|
120
|
+
if (state !== 'valid')
|
|
121
|
+
return state;
|
|
122
|
+
}
|
|
123
|
+
db.exec(`DELETE FROM ${table} WHERE rowid = ${BigInt(rowid).toString()}`);
|
|
124
|
+
}
|
|
125
|
+
return state;
|
|
126
|
+
}
|
|
127
|
+
// ---- D1c: one searchability predicate, usable from SQL --------------------------------------------
|
|
128
|
+
// Registered on every engine connection (openConnection). Given the chunk row the slot points at, it
|
|
129
|
+
// answers 1/0; a NULL chunk (LEFT JOIN miss), NULL offset or an out-of-range offset raises an error so
|
|
130
|
+
// a broken position is never counted as "no vector" or "has a vector".
|
|
131
|
+
export function registerVecSlotFunctions(db) {
|
|
132
|
+
db.function('vec_slot_valid', { deterministic: true, safeIntegers: true }, (validity, offset, size) => (bitIsSet(validity, offset, size) ? 1 : 0));
|
|
133
|
+
}
|
|
134
|
+
// SQL fragment: "a searchable vector exists in <table> for rowid <expr>". LEFT JOIN keeps a _rowids row
|
|
135
|
+
// whose chunk is gone, so vec_slot_valid sees NULL and raises instead of the row silently vanishing.
|
|
136
|
+
export function searchableVectorSql(table, rowidExpr) {
|
|
137
|
+
assertTable(table);
|
|
138
|
+
// A slot shared by two rowids is owned by neither (r4 M1): not searchable for either.
|
|
139
|
+
return `EXISTS (SELECT 1 FROM ${table}_rowids vr LEFT JOIN ${table}_chunks vc ON vc.chunk_id = vr.chunk_id
|
|
140
|
+
WHERE vr.rowid = ${rowidExpr} AND vec_slot_valid(vc.validity, vr.chunk_offset, vc.size) = 1
|
|
141
|
+
AND NOT EXISTS (SELECT 1 FROM ${table}_rowids vs WHERE vs.chunk_id = vr.chunk_id
|
|
142
|
+
AND vs.chunk_offset = vr.chunk_offset AND vs.rowid <> vr.rowid))`;
|
|
143
|
+
}
|
|
144
|
+
export function slotSummary(db, table, dims = 1024) {
|
|
145
|
+
assertTable(table);
|
|
146
|
+
const chunks = db.prepare(`SELECT chunk_id, size, validity FROM ${table}_chunks`).safeIntegers(true).all();
|
|
147
|
+
const byId = new Map(chunks.map(c => [c.chunk_id, c]));
|
|
148
|
+
let valid = 0, invalid = 0, slots = 0, chunkSize = 0;
|
|
149
|
+
for (const c of chunks) {
|
|
150
|
+
slots += Number(c.size);
|
|
151
|
+
chunkSize = Math.max(chunkSize, Number(c.size));
|
|
152
|
+
}
|
|
153
|
+
// valid = rows that alone own a slot whose bit is set · invalid = bit 0 · aliased = rows sharing a slot
|
|
154
|
+
// (neither owns it; counted only here) · empty = slots no row owns validly.
|
|
155
|
+
const rows = db.prepare(`SELECT rowid, chunk_id, chunk_offset FROM ${table}_rowids`).safeIntegers(true).all();
|
|
156
|
+
const perSlot = new Map();
|
|
157
|
+
for (const r of rows) {
|
|
158
|
+
const c = r.chunk_id === null ? undefined : byId.get(r.chunk_id);
|
|
159
|
+
if (!c)
|
|
160
|
+
throw new VecSlotError(`${table}: rowid ${r.rowid} points at missing chunk ${r.chunk_id}`);
|
|
161
|
+
const key = `${r.chunk_id}:${r.chunk_offset}`;
|
|
162
|
+
perSlot.set(key, (perSlot.get(key) ?? 0) + 1);
|
|
163
|
+
}
|
|
164
|
+
// Every position is validated first (r5 M1): sharing a slot is no evidence that the slot is a real one.
|
|
165
|
+
const bits = rows.map(r => bitIsSet(byId.get(r.chunk_id).validity, r.chunk_offset, byId.get(r.chunk_id).size));
|
|
166
|
+
let aliased = 0;
|
|
167
|
+
rows.forEach((r, i) => {
|
|
168
|
+
if (perSlot.get(`${r.chunk_id}:${r.chunk_offset}`) > 1) {
|
|
169
|
+
aliased++;
|
|
170
|
+
return;
|
|
171
|
+
}
|
|
172
|
+
if (bits[i])
|
|
173
|
+
valid++;
|
|
174
|
+
else
|
|
175
|
+
invalid++;
|
|
176
|
+
});
|
|
177
|
+
const size = chunkSize || 1024;
|
|
178
|
+
const reclaimable = Math.max(0, chunks.length - Math.ceil(valid / size));
|
|
179
|
+
return { table, chunks: chunks.length, chunkSize: size, slots, valid, invalid, aliased, empty: Math.max(0, slots - valid),
|
|
180
|
+
reclaimable, reclaimableBytes: reclaimable * size * dims * 4, rewriteBytes: valid * dims * 4 };
|
|
181
|
+
}
|
|
182
|
+
// maxLive: the automatic path holds the write lock for the whole rebuild. Measured 2026-11-02 (Mac, load
|
|
183
|
+
// ~4, complete function incl. full read-back): see specs/changes/vec-slot-reuse/proposal.md. Above the cap
|
|
184
|
+
// the rebuild is left to the owner-run maintenance script and stats shows due = 'over-cap'.
|
|
185
|
+
export const DEFAULT_REBUILD_POLICY = { minChunks: 8, maxLive: 20_000 };
|
|
186
|
+
// Why a rebuild is (not) due. Repair (validity-0 rows present) does not need the saving threshold:
|
|
187
|
+
// sqlite-vec 0.1.7 refuses to DELETE such a row whenever no higher bit in its byte is set, so the
|
|
188
|
+
// rebuild is the only path that removes it (measured 2026-11-02, probe/delete_matrix.cjs).
|
|
189
|
+
export function rebuildReason(s, policy = DEFAULT_REBUILD_POLICY) {
|
|
190
|
+
if (s.valid > policy.maxLive)
|
|
191
|
+
return null;
|
|
192
|
+
if (s.invalid > 0 || s.aliased > 0)
|
|
193
|
+
return 'repair';
|
|
194
|
+
if (s.reclaimable >= policy.minChunks && s.reclaimable * s.chunkSize >= s.valid)
|
|
195
|
+
return 'reclaim';
|
|
196
|
+
return null;
|
|
197
|
+
}
|
|
198
|
+
// Order-independent digest of (rowid, vector bytes) pairs: per-row sha256, sorted, hashed. Lets the copy
|
|
199
|
+
// and the read-back both use one full scan instead of a per-rowid lookup (measured 2026-11-02: the
|
|
200
|
+
// per-rowid version took 15.1 s / +625 MB RSS for 60,000 vectors).
|
|
201
|
+
class RowDigest {
|
|
202
|
+
parts = [];
|
|
203
|
+
add(id, buf) {
|
|
204
|
+
this.parts.push(createHash('sha256').update(String(id)).update(':').update(buf).digest('hex'));
|
|
205
|
+
}
|
|
206
|
+
get count() { return this.parts.length; }
|
|
207
|
+
hex() { return createHash('sha256').update(this.parts.sort().join('\n')).digest('hex'); }
|
|
208
|
+
}
|
|
209
|
+
// Rebuild <table> keeping exactly its searchable (rowid, vector) pairs. Called on its own: one BEGIN IMMEDIATE
|
|
210
|
+
// transaction. Called inside an open transaction (inline repair): a savepoint of the caller's transaction;
|
|
211
|
+
// busy_timeout is 0 for the attempt (another writer means "skip now", not "wait 5 s") and restored after.
|
|
212
|
+
// Everything is re-checked inside the transaction because another engine may have rebuilt already.
|
|
213
|
+
// The table is re-created from its own CREATE statement in sqlite_master. Before commit: valid count,
|
|
214
|
+
// invalid = 0 and the full rowid -> vector-bytes digest must match, else the transaction rolls back.
|
|
215
|
+
export function rebuildVecTable(db, table, opts = {}) {
|
|
216
|
+
assertTable(table);
|
|
217
|
+
const t0 = Date.now();
|
|
218
|
+
const policy = opts.policy ?? DEFAULT_REBUILD_POLICY;
|
|
219
|
+
const prevBusy = Number(db.pragma('busy_timeout', { simple: true }));
|
|
220
|
+
const prevCache = Number(db.pragma('cache_size', { simple: true }));
|
|
221
|
+
db.pragma('busy_timeout = 0');
|
|
222
|
+
try {
|
|
223
|
+
return db.transaction(() => {
|
|
224
|
+
let before;
|
|
225
|
+
try {
|
|
226
|
+
before = slotSummary(db, table, opts.dims);
|
|
227
|
+
}
|
|
228
|
+
catch (e) {
|
|
229
|
+
if (e instanceof VecSlotError)
|
|
230
|
+
return { table, status: 'position-error', error: e.message, ms: Date.now() - t0 };
|
|
231
|
+
throw e;
|
|
232
|
+
}
|
|
233
|
+
if (before.valid > policy.maxLive && !opts.force)
|
|
234
|
+
return { table, status: 'over-cap', before, ms: Date.now() - t0 };
|
|
235
|
+
const reason = opts.force ? 'forced' : rebuildReason(before, policy);
|
|
236
|
+
if (!reason)
|
|
237
|
+
return { table, status: 'not-needed', before, ms: Date.now() - t0 };
|
|
238
|
+
// The read-back below scans pages this transaction just wrote. With the page cache smaller than the
|
|
239
|
+
// rewrite they spill and are re-read from disk: 20,000 vectors took 1.57 s to read back at the engine's
|
|
240
|
+
// 32 MB cache vs 0.28 s at 200 MB (measured 2026-11-02). Size the cache to the rewrite for this call.
|
|
241
|
+
const wantKiB = Math.ceil(before.rewriteBytes * 1.25 / 1024) + 16_384;
|
|
242
|
+
if (prevCache < 0 ? wantKiB > -prevCache : true)
|
|
243
|
+
db.pragma(`cache_size = -${wantKiB}`);
|
|
244
|
+
const ddlRow = db.prepare(`SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?`).get(table);
|
|
245
|
+
if (!ddlRow?.sql)
|
|
246
|
+
throw new VecSlotError(`${table}: no CREATE statement in sqlite_master`);
|
|
247
|
+
// Keep only searchable rows: read positions, test bits, copy those vectors.
|
|
248
|
+
// rowids are read as BigInt end to end (r3 M1: a Number round-trip rewrote 2^53+1 as 2^53).
|
|
249
|
+
const chunkRows = db.prepare(`SELECT chunk_id, size, validity FROM ${table}_chunks`).safeIntegers(true).all();
|
|
250
|
+
const byId = new Map(chunkRows.map(c => [c.chunk_id, c]));
|
|
251
|
+
// Keep valid rows that own their slot alone. A validity-0 row's slot looks free to vec0, so a later
|
|
252
|
+
// insert can take it and two rowids then point at one slot (seen 2026-11-02): which vector belongs to
|
|
253
|
+
// the older rowid is unknowable, so every rowid of a shared slot is dropped and re-embedded later.
|
|
254
|
+
const rowsAll = db.prepare(`SELECT rowid, chunk_id, chunk_offset FROM ${table}_rowids`).safeIntegers(true).all();
|
|
255
|
+
const owners = new Map();
|
|
256
|
+
for (const r of rowsAll) {
|
|
257
|
+
const k = `${r.chunk_id}:${r.chunk_offset}`;
|
|
258
|
+
owners.set(k, (owners.get(k) ?? 0) + 1);
|
|
259
|
+
}
|
|
260
|
+
const validIds = new Set();
|
|
261
|
+
for (const r of rowsAll) {
|
|
262
|
+
const c = byId.get(r.chunk_id);
|
|
263
|
+
if (!c)
|
|
264
|
+
throw new VecSlotError(`${table}: rowid ${r.rowid} points at missing chunk ${r.chunk_id}`);
|
|
265
|
+
const bit = bitIsSet(c.validity, r.chunk_offset, c.size); // position validated before sharing (r5 M1)
|
|
266
|
+
if (owners.get(`${r.chunk_id}:${r.chunk_offset}`) > 1)
|
|
267
|
+
continue;
|
|
268
|
+
if (bit)
|
|
269
|
+
validIds.add(r.rowid);
|
|
270
|
+
}
|
|
271
|
+
const keep = [];
|
|
272
|
+
const wantD = new RowDigest();
|
|
273
|
+
for (const v of db.prepare(`SELECT rowid, embedding FROM ${table}`).safeIntegers(true).iterate()) {
|
|
274
|
+
if (!validIds.has(v.rowid))
|
|
275
|
+
continue;
|
|
276
|
+
const buf = Buffer.from(v.embedding);
|
|
277
|
+
keep.push([v.rowid, buf]);
|
|
278
|
+
wantD.add(v.rowid, buf);
|
|
279
|
+
}
|
|
280
|
+
if (keep.length !== validIds.size) {
|
|
281
|
+
throw new VecSlotError(`${table}: full scan returned ${keep.length} of ${validIds.size} valid vectors`);
|
|
282
|
+
}
|
|
283
|
+
const want = wantD.hex();
|
|
284
|
+
db.exec(`DROP TABLE ${table}`);
|
|
285
|
+
if (opts.fault === 'after-drop')
|
|
286
|
+
throw new Error('fault injection: after-drop');
|
|
287
|
+
db.exec(ddlRow.sql);
|
|
288
|
+
if (opts.fault === 'after-create')
|
|
289
|
+
throw new Error('fault injection: after-create');
|
|
290
|
+
const ins = db.prepare(`INSERT INTO ${table}(rowid, embedding) VALUES (?, ?)`);
|
|
291
|
+
keep.forEach(([id, buf], i) => {
|
|
292
|
+
if (opts.fault === 'after-partial' && i === Math.floor(keep.length / 2))
|
|
293
|
+
throw new Error('fault injection: after-partial');
|
|
294
|
+
if (opts.fault === 'corrupt-one' && i === 0) {
|
|
295
|
+
const bad = Buffer.from(buf);
|
|
296
|
+
bad.writeFloatLE(bad.readFloatLE(0) + 1, 0);
|
|
297
|
+
ins.run(id, bad);
|
|
298
|
+
return;
|
|
299
|
+
}
|
|
300
|
+
ins.run(id, buf);
|
|
301
|
+
});
|
|
302
|
+
const after = slotSummary(db, table, opts.dims);
|
|
303
|
+
const gotD = new RowDigest();
|
|
304
|
+
for (const v of db.prepare(`SELECT rowid, embedding FROM ${table}`).safeIntegers(true).iterate()) {
|
|
305
|
+
gotD.add(v.rowid, Buffer.from(v.embedding));
|
|
306
|
+
}
|
|
307
|
+
if (after.valid !== validIds.size || after.invalid !== 0 || after.aliased !== 0 || gotD.count !== wantD.count || gotD.hex() !== want) {
|
|
308
|
+
throw new VecSlotError(`${table}: rebuild verification failed (kept ${validIds.size}, valid after ${after.valid}, invalid ${after.invalid}, aliased ${after.aliased})`);
|
|
309
|
+
}
|
|
310
|
+
return { table, status: 'rebuilt', reason, before, after, ms: Date.now() - t0 };
|
|
311
|
+
}).immediate();
|
|
312
|
+
}
|
|
313
|
+
catch (e) {
|
|
314
|
+
const code = e.code;
|
|
315
|
+
if (code === 'SQLITE_BUSY' || code === 'SQLITE_LOCKED')
|
|
316
|
+
return { table, status: 'busy', error: String(e.message), ms: Date.now() - t0 };
|
|
317
|
+
return { table, status: 'failed', error: String(e.message), ms: Date.now() - t0 };
|
|
318
|
+
}
|
|
319
|
+
finally {
|
|
320
|
+
db.pragma(`busy_timeout = ${prevBusy}`);
|
|
321
|
+
db.pragma(`cache_size = ${prevCache}`);
|
|
322
|
+
}
|
|
323
|
+
}
|
package/docs/UPDATING.md
CHANGED
|
@@ -108,6 +108,16 @@ path and holder pid (e.g. `.download-<key>.lock`). Verify the holder process
|
|
|
108
108
|
is genuinely gone or hung (`ps -p <pid>`), then remove the lock file manually;
|
|
109
109
|
the next start becomes a clean download owner.
|
|
110
110
|
|
|
111
|
+
## v6.4.3 (schema v14, unchanged): vector slots rewritten in place, idle rebuild, searchable-only coverage
|
|
112
|
+
|
|
113
|
+
**What changes for an existing database.** Nothing is migrated and nothing is re-embedded. Re-embeds stop leaving empty
|
|
114
|
+
vector slots. On the first idle check after start (about 60 s quiet, then at most every 10 min) a table with many empty
|
|
115
|
+
slots or with damaged slots is rebuilt once in place (rowids and vector bytes kept, verified before commit); the DB file
|
|
116
|
+
does not shrink until a VACUUM (`scripts/rag-maintenance/compact-vectors.mjs` in the framework does build + VACUUM).
|
|
117
|
+
Coverage may now report a few `missing` vectors that 6.4.2 counted as embedded (invisible validity-0 rows); the backfill
|
|
118
|
+
fills them. `server.vector_slots` shows the state. An engine that is already running keeps the old behaviour until it
|
|
119
|
+
restarts on 6.4.3.
|
|
120
|
+
|
|
111
121
|
## v6.4.2 (schema v14, unchanged): CPU memory arena off, transformers 4.3.2
|
|
112
122
|
|
|
113
123
|
**What changes for an existing database.** Nothing is migrated and nothing is re-embedded (the embedding profile —
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "rag-memory-epf-mcp",
|
|
3
|
-
"version": "6.4.
|
|
3
|
+
"version": "6.4.3",
|
|
4
4
|
"engines": {
|
|
5
5
|
"node": ">=24"
|
|
6
6
|
},
|
|
@@ -45,7 +45,7 @@
|
|
|
45
45
|
"prepare": "npm run build",
|
|
46
46
|
"watch": "tsc --watch",
|
|
47
47
|
"verify:invariants": "node test/chunk-invariants.test.mjs",
|
|
48
|
-
"verify:engine": "node test/engine-smoke.test.mjs && node test/launch-smoke.test.mjs && node test/sync-atomicity.test.mjs && node test/dedup.test.mjs && node test/search-degradation.test.mjs && node test/entity-embed-cap.test.mjs && node test/migration12.test.mjs && node test/model-cache.test.mjs && node test/embedding-gate.test.mjs && node test/lazy-boot.test.mjs && node test/reconciliation.test.mjs && node test/backfill.test.mjs && node test/fts-query.test.mjs && node test/search-contracts.test.mjs && node test/tool-contracts.test.mjs && node test/bounded-exit.test.mjs && node test/observation-schema.test.mjs && node test/search-graph-default.test.mjs && node test/observation-migration.test.mjs && node test/observation-lifecycle.test.mjs && node test/observation-contracts.test.mjs && node test/observation-search.test.mjs && node test/observation-cascade.test.mjs && node test/observation-realdata.test.mjs && node --no-concurrent-sparkplug test/chunker-c.test.mjs && node test/migration14.test.mjs && node test/chunk-params-validation.test.mjs && node test/vector-reuse.test.mjs && node test/entity-range-linking.test.mjs && node test/alias-link-gate.test.mjs && node test/entity-name-boundary.test.mjs && node test/stats-chunking.test.mjs && node test/migration14-realdata.test.mjs && node test/migration14-realdata-sync.test.mjs && node test/search-summaries-off.test.mjs && node test/search-fusion-rrf.test.mjs && node test/query-input.test.mjs && node test/document-return-contracts.test.mjs && node test/observation-date-prefix.test.mjs && node test/delete-entities-cascade.test.mjs && node test/delete-entities-kg-hygiene.test.mjs && node test/backup-publish-portable.test.mjs && node test/graph-context-explain.test.mjs && node test/eval-graph-role-libs.test.mjs && node test/eval-graph-role-t5b.test.mjs && node --test test/eval-graph-role-t8-fix.test.mjs && node --test test/eval-graph-role-t7-upstream.test.mjs && node --test test/eval-graph-role-t11-decision.test.mjs && node --test test/eval-graph-role-prereq-fix.test.mjs && node test/db-path.test.mjs && node test/mmap-option.test.mjs && node test/wal-exit.test.mjs && node test/wal-sidecar-guard.test.mjs && node test/kg-evidence-core.test.mjs && node test/kg-delete-relation.test.mjs && node test/kg-confidence-copy.test.mjs && node test/kg-embed-cas.test.mjs && node test/kg-relation-evidence.test.mjs && node test/kg-detail-context.test.mjs && node test/kg-evidence-invalidation.test.mjs && node test/kg-evidence-cas.test.mjs && node test/kg-defaults.test.mjs && node test/chunker-oversize-cost.test.mjs && node test/chunker-structure-parity.test.mjs && node test/search-limit-depth.test.mjs && node test/kg-import-id-contract.test.mjs && node test/kg-evidence-guards.test.mjs && node test/model-session-options.test.mjs",
|
|
48
|
+
"verify:engine": "node test/engine-smoke.test.mjs && node test/launch-smoke.test.mjs && node test/sync-atomicity.test.mjs && node test/dedup.test.mjs && node test/search-degradation.test.mjs && node test/entity-embed-cap.test.mjs && node test/migration12.test.mjs && node test/model-cache.test.mjs && node test/embedding-gate.test.mjs && node test/lazy-boot.test.mjs && node test/reconciliation.test.mjs && node test/backfill.test.mjs && node test/fts-query.test.mjs && node test/search-contracts.test.mjs && node test/tool-contracts.test.mjs && node test/bounded-exit.test.mjs && node test/observation-schema.test.mjs && node test/search-graph-default.test.mjs && node test/observation-migration.test.mjs && node test/observation-lifecycle.test.mjs && node test/observation-contracts.test.mjs && node test/observation-search.test.mjs && node test/observation-cascade.test.mjs && node test/observation-realdata.test.mjs && node --no-concurrent-sparkplug test/chunker-c.test.mjs && node test/migration14.test.mjs && node test/chunk-params-validation.test.mjs && node test/vector-reuse.test.mjs && node test/entity-range-linking.test.mjs && node test/alias-link-gate.test.mjs && node test/entity-name-boundary.test.mjs && node test/stats-chunking.test.mjs && node test/migration14-realdata.test.mjs && node test/migration14-realdata-sync.test.mjs && node test/search-summaries-off.test.mjs && node test/search-fusion-rrf.test.mjs && node test/query-input.test.mjs && node test/document-return-contracts.test.mjs && node test/observation-date-prefix.test.mjs && node test/delete-entities-cascade.test.mjs && node test/delete-entities-kg-hygiene.test.mjs && node test/backup-publish-portable.test.mjs && node test/graph-context-explain.test.mjs && node test/eval-graph-role-libs.test.mjs && node test/eval-graph-role-t5b.test.mjs && node --test test/eval-graph-role-t8-fix.test.mjs && node --test test/eval-graph-role-t7-upstream.test.mjs && node --test test/eval-graph-role-t11-decision.test.mjs && node --test test/eval-graph-role-prereq-fix.test.mjs && node test/db-path.test.mjs && node test/mmap-option.test.mjs && node test/wal-exit.test.mjs && node test/wal-sidecar-guard.test.mjs && node test/kg-evidence-core.test.mjs && node test/kg-delete-relation.test.mjs && node test/kg-confidence-copy.test.mjs && node test/kg-embed-cas.test.mjs && node test/kg-relation-evidence.test.mjs && node test/kg-detail-context.test.mjs && node test/kg-evidence-invalidation.test.mjs && node test/kg-evidence-cas.test.mjs && node test/kg-defaults.test.mjs && node test/chunker-oversize-cost.test.mjs && node test/chunker-structure-parity.test.mjs && node test/search-limit-depth.test.mjs && node test/kg-import-id-contract.test.mjs && node test/kg-evidence-guards.test.mjs && node test/model-session-options.test.mjs && node test/write-seq.test.mjs && node test/vec-slot-reuse.test.mjs && node test/vec-rebuild.test.mjs && node test/vec-slot-invalid.test.mjs",
|
|
49
49
|
"test": "npm run build && npm run verify:invariants && npm run verify:engine",
|
|
50
50
|
"prepublishOnly": "npm run build && npm run verify:invariants && npm run verify:engine"
|
|
51
51
|
},
|