rag-memory-epf-mcp 6.4.1 → 6.4.3

package/README.md

@@ -154,6 +154,43 @@ storeDocument(id, content, metadata)
 
 ## Changelog
 
+### v6.4.3 (2026-11-02)
+
+- **Fixed — re-embedding no longer leaks vector slots.** sqlite-vec puts a new row only into a free slot of the latest
+  1,024-slot chunk, so the old DELETE + INSERT of the same rowid left a hole whenever the row sat in an older chunk (one
+  project DB reached 84 chunks for 5,578 vectors, 93.5% empty, 337 of 439 MB). Same-rowid re-embeds (document chunks,
+  KG chunks, bulk `embedChunks`, entities) now rewrite the vector in place; entity re-embeds keep their rowid.
+- **Added — idle verified rebuild of the vector tables.** When the engine is idle (no tool call, backfill or
+  reconciliation; 60 s quiet; checked at most every 10 min) and a table has at least 8 reclaimable chunks that outweigh
+  its live vectors, or holds damaged slots, it is rebuilt in one transaction keeping every searchable rowid and its
+  exact vector bytes (verified before commit; rolled back otherwise). Skipped at once if another connection holds the
+  write lock. Automatic only up to 20,000 live vectors per table (measured on one Mac: 5,600 = 0.44 s, 20,000 =
+  1.24 s and +267 MB transient). No VACUUM: the file stops growing; it shrinks only through a VACUUM.
+- **Fixed — coverage counts only searchable vectors.** A row whose slot validity bit is 0 (invisible to search) was
+  counted as embedded; it is now missing and is repaired: writes and deletes that meet such a slot, or a slot shared by
+  two rowids, rebuild the table first in the same transaction, then the backfill re-embeds at the original rowid.
+  An unresolvable slot position is reported as an error (`missing: null`), never as a count.
+- **Added — `getKnowledgeGraphStats().server.vector_slots`**: per table slots, valid, validity-0, shared, empty,
+  reclaimable chunks, whether a rebuild is due and why, and this process's last attempt.
+- **Added — write-seq signal** (`[rag-memory] write-seq N` on stderr when the engine's own connection changed rows),
+  used by the framework launcher to record a DB handoff only after the engine wrote.
+- **Changed — an entity metadata row whose vector row is gone is kept unstamped (missing) instead of deleted**, so the
+  backfill re-inserts at the same rowid; new entity rowids are allocated past both tables.
+- Not measured: Windows, Drive-synced DBs under two machines, tables above 20,000 vectors.
+
+### v6.4.2 (2026-10-31)
+
+- **Fixed — the engine no longer keeps the peak memory of its largest embedding.** The ONNX session is opened with
+  the CPU memory arena off. With it on, one long input (12,000 characters, 5,774 tokens) left the process at about
+  8 GB until it exited (1.4 GB after model load), and every MCP client runs its own engine. With it off the same
+  input left 1.5-2.4 GB in a probe and 2.7-3.2 GB in the full engine (one Mac, one run each). Vectors are
+  byte-identical with the arena on and off. The transient peak while a long input is being embedded is not lowered.
+- **Changed — `@huggingface/transformers` 4.3.2** (was `^3.5.1`; onnxruntime-node 1.30.0). The model cache layout is
+  unchanged and an existing cache loads without a re-download. Vectors differ from 3.x only in the last bits (cosine
+  0.9999995 or higher on 5 inputs); the embedding profile is unchanged, so nothing is re-embedded. Measured on one
+  Mac: a 30-character query 128 ms -> 33 ms, a 484-token chunk 739 ms -> 639 ms (3.8.1 with the arena on vs 4.3.2
+  with it off). Not measured: fixed-query ranking of existing document vectors against new query vectors, Windows.
+
 ### v6.4.1 (2026-10-16)
 
 - **Changed — new opt-in manager method `generateRelationEvidenceChunks(items, { formatVersion: 'evidence-v1' })`.**

package/dist/index.d.ts

@@ -4,14 +4,19 @@ import { EVIDENCE_FORMAT, type EvidenceItemInput, type EvidenceStatus, type Evid
 import { EmbeddingGate } from './src/embeddingGate.js';
 import type { EmbedPriority } from './src/embeddingGate.js';
 import { BackfillCoordinator } from './src/backfillCoordinator.js';
+import type { RebuildResult, VecTable } from './src/vecSlots.js';
 export declare const LEGACY_QUERY_INSTRUCTION = "Represent this sentence for searching relevant passages: ";
 export declare function embeddingInput(text: string, isQuery: boolean, defaultModel?: boolean): string;
 export declare function makeEmbedFn(model: (input: string, opts: any) => Promise<{
     data: any;
 }>, defaultModel: boolean): (text: string, dims: number, isQuery: boolean) => Promise<Float32Array<ArrayBuffer>>;
 export declare function compileFtsLiteralQuery(q: string): string | null;
+export declare function setPipelineImpl(fn: ((...args: any[]) => Promise<any>) | null): void;
 export declare function resolveDbFilePath(envValue: string | undefined, cwd: string, serverDir: string): string;
 export declare function parseMmapSize(raw: string | undefined): number;
+export declare const MODEL_SESSION_OPTIONS: Readonly<{
+    enableCpuMemArena: false;
+}>;
 interface Entity {
     name: string;
     entityType: string;
@@ -128,6 +133,20 @@ export declare class RAGKnowledgeGraphManager {
         __testForceFkOff?: boolean;
     }): Promise<void>;
     private openConnection;
+    writeSeq: number;
+    private tcBase;
+    private lastActivityAt;
+    private lastWriteAt;
+    private lastSlotCheckAt;
+    readonly vectorSlotLast: Partial<Record<VecTable, RebuildResult & {
+        at: string;
+    }>>;
+    private static slotCheckMs;
+    private static slotQuietMs;
+    maybeRebuildVectors(now?: number): RebuildResult[] | null;
+    vectorSlotStats(): Record<string, unknown>;
+    private totalChanges;
+    noteWrites(): void;
     private ensureCurrentProfile;
     private buildRealLoader;
     startReconciliation(): Promise<void>;

package/dist/index.js

@@ -34,6 +34,7 @@ import { migrations } from './src/migrations/migrations.js';
 import { EmbeddingGate, GateNotReadyError, GateDisabledError, TerminalConfigError } from './src/embeddingGate.js';
 import { resolveModelCacheDir, preflightCacheDir, artifactKey, ModelDownloadLock, handleLoaderFailure } from './src/modelCache.js';
 import { BackfillCoordinator } from './src/backfillCoordinator.js';
+import { writeVector, deleteVector, registerVecSlotFunctions, InvalidSlotError, slotSummary, rebuildVecTable, rebuildReason, DEFAULT_REBUILD_POLICY } from './src/vecSlots.js';
 import { calendarDate, resolveCalendarTimeZone, stampDatePrefix, stripDatePrefix } from './src/observations/date-prefix.js';
 import os from 'node:os';
 import { createHash } from 'crypto';
@@ -95,6 +96,12 @@ export function compileFtsLiteralQuery(q) {
         return null;
     return terms.map(t => `"${t}"`).join(' OR ');
 }
+// Test-only seam for the real model loader: replaces `pipeline` so a test can see the exact
+// arguments buildRealLoader passes (same rule as setSyncFaultPoint: an argument, never an env var).
+let __pipelineImpl = pipeline;
+export function setPipelineImpl(fn) {
+    __pipelineImpl = (fn ?? pipeline);
+}
 // Configure Hugging Face transformers for better compatibility
 if (env.backends?.onnx?.wasm) {
     env.backends.onnx.wasm.wasmPaths = './node_modules/@huggingface/transformers/dist/';
@@ -142,6 +149,12 @@ const IS_DEFAULT_MODEL_CONFIG = !process.env.EMBEDDING_MODEL || process.env.EMBE
 // on 'main' (their vectors are never auto-grandfathered anyway).
 const MODEL_REVISION = IS_DEFAULT_MODEL_CONFIG ? '4de13258303883538bd53b696b452bf8099f0858' : 'main';
 const MODEL_DTYPE = 'fp16';
+// ONNX session options for the embedding model. The CPU memory arena is off: with it on, one long
+// input (12,000 chars = 5,774 tokens) left the process holding its peak (footprint 1.4 → 8.0 GB,
+// measured 2026-10-31 on transformers 3.8.1 and 4.3.2) and every MCP client runs its own engine.
+// Arena off returned to 1.5-2.4 GB with byte-identical vectors; the cost is per-call allocation
+// (query 29 → 33 ms, 484-token chunk 574 → 639 ms on 4.3.2). Not part of the cache artifact key.
+export const MODEL_SESSION_OPTIONS = Object.freeze({ enableCpuMemArena: false });
 // Entity embedding text builder version — mixed into entity input hashes so a
 // builder change re-backfills entities without touching chunk vectors (spec §6c).
 const TEXT_BUILDER_VERSION = 'tb1';
@@ -314,6 +327,7 @@ export class RAGKnowledgeGraphManager {
     openConnection(forceFkOff = false) {
         this.db = new Database(DB_FILE_PATH);
         sqliteVec.load(this.db);
+        registerVecSlotFunctions(this.db); // vec_slot_valid() for the D1c searchability predicate
         this.db.pragma('journal_mode = WAL');
         this.db.pragma('synchronous = NORMAL');
         this.db.pragma('busy_timeout = 5000');
@@ -344,6 +358,119 @@ export class RAGKnowledgeGraphManager {
                     `for history integrity; refusing to run migrations without it.`);
             }
         }
+        // Write-sequence baseline for THIS connection (see noteWrites). A fresh connection starts
+        // its own total_changes() count, so the baseline is reset on every open; the announced
+        // sequence itself never goes back.
+        this.tcBase = this.totalChanges();
+    }
+    // --- Write-sequence signal (test/write-seq.test.mjs) ------------------------------------
+    // One stderr line per observation that THIS connection changed rows:
+    //   [rag-memory] write-seq <N>   (N = 1, 2, 3, … for the life of the process)
+    // The framework launcher (mcp-proxy.cjs) records "the DB state this machine left" only
+    // after such a line. Before this signal it recorded whenever the file changed and went
+    // quiet — which also fired when a cloud sync client swapped the other machine's file in
+    // under an idle engine, and that record then refused a correct DB (measured 2026-11-02).
+    // total_changes() counts rows changed by this connection (vec0 shadow tables included);
+    // reads and other connections do not move it. It is read outside a transaction only, so a
+    // statement still inside one is announced after it ends. It also counts rows of a
+    // transaction that was rolled back: a line means "this connection's change count went up",
+    // not "a commit happened". The launcher consumes a pending sequence once the file it last
+    // recorded is still there, the WAL is empty and the file is stable; until then a sequence that
+    // came from a rollback can still publish a file another writer or a sync client changed, and
+    // that period has no fixed upper bound (hub session 251 review r1 M1 · r3. r2: a WAL-size
+    // test was tried and removed — a header-only WAL looks non-empty and another connection's
+    // checkpoint hides a real commit).
+    // Written with fs.writeSync: the last call runs inside process 'exit', where an async
+    // stderr pipe write (macOS) can be lost.
+    writeSeq = 0;
+    tcBase = 0;
+    // ---- vec-slot-reuse D1b: idle verified rebuild of the vec0 tables -------------------------------
+    // Initial operating values (2_ s253 r2 Q3: starting values, not optima): look at most every 10 min,
+    // only after 60 s without a tool call or an own-connection write, never while a tool call, a backfill
+    // scan or a reconciliation is running. A repair request (validity-0 rows seen) skips the 10-min wait.
+    // The saving threshold and the live-vector cap live in DEFAULT_REBUILD_POLICY (src/vecSlots.ts).
+    lastActivityAt = Date.now();
+    lastWriteAt = 0;
+    lastSlotCheckAt = 0;
+    // Last attempt per table. Process memory only: unknown after a restart (stats says so).
+    vectorSlotLast = {};
+    static slotCheckMs = Number(process.env.RAG_MEMORY_SLOT_CHECK_MS ?? 600_000);
+    static slotQuietMs = Number(process.env.RAG_MEMORY_SLOT_QUIET_MS ?? 60_000);
+    maybeRebuildVectors(now = Date.now()) {
+        if (!this.db || this.shuttingDown || this.inFlight > 0 || this.db.inTransaction)
+            return null;
+        if (this.coordinator?.busy)
+            return null;
+        if (now - Math.max(this.lastActivityAt, this.lastWriteAt) < RAGKnowledgeGraphManager.slotQuietMs)
+            return null;
+        // A repair request skips the 10-min wait once: only if it arrived after the last check (r3 M5 — otherwise
+        // a busy / failed / over-cap attempt would be retried on every 1 s tick).
+        const repair = this.coordinator?.repairRequested ?? false;
+        const freshRepair = repair && (this.coordinator?.repairRequestedAt ?? 0) > this.lastSlotCheckAt;
+        if (!freshRepair && now - this.lastSlotCheckAt < RAGKnowledgeGraphManager.slotCheckMs)
+            return null;
+        this.lastSlotCheckAt = now;
+        const results = [];
+        for (const table of ['chunks', 'entity_embeddings']) {
+            const r = rebuildVecTable(this.db, table);
+            this.vectorSlotLast[table] = { ...r, at: new Date(now).toISOString() };
+            results.push(r);
+            if (r.status === 'rebuilt') {
+                console.error(`🧹 vec0 ${table} rebuilt (${r.reason}): chunks ${r.before?.chunks} -> ${r.after?.chunks}, ` +
+                    `valid ${r.after?.valid}, dropped validity-0 ${r.before?.invalid}, shared-slot ${r.before?.aliased}, ${r.ms} ms`);
+            }
+            else if (r.status !== 'not-needed') {
+                console.error(`⚠️ vec0 ${table} rebuild ${r.status}${r.error ? `: ${r.error}` : ''}`);
+            }
+        }
+        if (repair && results.every(r => r.status === 'rebuilt' || r.status === 'not-needed'))
+            this.coordinator?.takeRepairRequest();
+        if (results.some(r => r.status === 'rebuilt')) {
+            this.coordinator?.invalidateCoverage();
+            this.noteWrites();
+            this.coordinator?.kick(); // rows dropped as validity-0 are now missing -> re-embedded at their rowid
+        }
+        return results;
+    }
+    vectorSlotStats() {
+        const out = {};
+        for (const table of ['chunks', 'entity_embeddings']) {
+            let summary;
+            try {
+                summary = slotSummary(this.db, table);
+            }
+            catch (e) {
+                summary = { error: e.message };
+            }
+            const pending = 'error' in summary ? 'position-error'
+                : (summary.valid > DEFAULT_REBUILD_POLICY.maxLive ? 'over-cap' : (rebuildReason(summary) ?? null));
+            const last = this.vectorSlotLast[table];
+            out[table] = { ...summary, due: pending,
+                last_attempt: last ? { at: last.at, status: last.status, reason: last.reason ?? null, ms: last.ms, error: last.error ?? null }
+                    : 'unknown (process memory; none since start)' };
+        }
+        out.policy = { ...DEFAULT_REBUILD_POLICY, check_ms: RAGKnowledgeGraphManager.slotCheckMs, quiet_ms: RAGKnowledgeGraphManager.slotQuietMs,
+            repair_requested: this.coordinator?.repairRequested ?? false, vacuum: 'never (maintenance script only)' };
+        return out;
+    }
+    totalChanges() {
+        if (!this.db)
+            return 0;
+        return Number(this.db.prepare('select total_changes()').pluck().get());
+    }
+    noteWrites() {
+        try {
+            if (!this.db || this.db.inTransaction)
+                return;
+            const tc = this.totalChanges();
+            if (tc <= this.tcBase)
+                return;
+            this.tcBase = tc;
+            this.lastWriteAt = Date.now();
+            this.writeSeq += 1;
+            fsSync.writeSync(2, `[rag-memory] write-seq ${this.writeSeq}\n`);
+        }
+        catch { /* a dead handle is the reopen path's business; the next tick tries again */ }
     }
     // Upsert the stored-vector compatibility profile (spec §6c layer 2) and record
     // retrieval config in server_meta (layer 3 — never a backfill trigger).
@@ -387,7 +514,8 @@ export class RAGKnowledgeGraphManager {
             env.allowRemoteModels = true;
             env.allowLocalModels = true;
             console.error(`🤖 Loading embedding model: ${EMBEDDING_MODEL} (1024-dim, cache=${cacheDir})...`);
-            const model = await pipeline('feature-extraction', EMBEDDING_MODEL, { revision: MODEL_REVISION, dtype: MODEL_DTYPE, cache_dir: cacheDir });
+            const model = await __pipelineImpl('feature-extraction', EMBEDDING_MODEL, { revision: MODEL_REVISION, dtype: MODEL_DTYPE, cache_dir: cacheDir,
+                session_options: { ...MODEL_SESSION_OPTIONS } });
             // dims fail-fast (spec §2 / beta B8): probe the ACTUAL output length — a
             // 384/768-dim custom model must fail here with a clear message, not at
             // every subsequent vector write.
@@ -539,7 +667,7 @@ export class RAGKnowledgeGraphManager {
         const meta = this.db.prepare(`SELECT rowid FROM entity_embedding_metadata WHERE entity_id = ?`)
             .get(entityId);
         if (meta) {
-            this.db.exec(`DELETE FROM entity_embeddings WHERE rowid = ${Number(meta.rowid)}`);
+            deleteVector(this.db, 'entity_embeddings', Number(meta.rowid));
             this.db.prepare(`DELETE FROM entity_embedding_metadata WHERE entity_id = ?`).run(entityId);
         }
         deleteStaleKgChunks(this.db, entityId);
@@ -554,7 +682,7 @@ export class RAGKnowledgeGraphManager {
         if (!meta)
             return;
         const tx = this.db.transaction(() => {
-            this.db.exec(`DELETE FROM entity_embeddings WHERE rowid = ${Number(meta.rowid)}`);
+            deleteVector(this.db, 'entity_embeddings', Number(meta.rowid));
             this.db.prepare(`DELETE FROM entity_embedding_metadata WHERE entity_id = ?`).run(entityId);
         });
         tx();
@@ -584,8 +712,7 @@ export class RAGKnowledgeGraphManager {
                 return false; // inputs changed
             if (this.currentProfileId !== captured.profile_id)
                 return false; // profile changed during inference
-            this.db.exec(`DELETE FROM chunks WHERE rowid = ${rowid}`);
-            this.db.prepare(`INSERT INTO chunks (rowid, embedding) VALUES (${rowid}, ?)`).run(Buffer.from(embedding.buffer));
+            writeVector(this.db, 'chunks', rowid, Buffer.from(embedding.buffer)); // same slot when valid (vec-slot-reuse)
             this.db.prepare(`UPDATE chunk_metadata SET input_hash = ?, profile_id = ?, provenance_state = 'verified' WHERE rowid = ?`)
                 .run(captured.text_sha256, captured.profile_id, rowid);
             const meta = parseEvidenceMetadata(cur.metadata);
@@ -597,7 +724,17 @@ export class RAGKnowledgeGraphManager {
             this.db.prepare(`DELETE FROM embedding_backfill_failures WHERE kind = 'chunk' AND target_id = ?`).run(String(rowid));
             return true;
         });
-        const written = tx();
+        let written;
+        try {
+            written = tx();
+        }
+        catch (e) {
+            if (e instanceof InvalidSlotError) {
+                this.coordinator?.requestRepair();
+                return false;
+            }
+            throw e;
+        }
         this.coordinator?.invalidateCoverage();
         return written;
     }
@@ -635,14 +772,23 @@ export class RAGKnowledgeGraphManager {
                 const cur = this.db.prepare(`SELECT text FROM chunk_metadata WHERE rowid = ?`).get(rowid);
                 if (!cur || cur.text !== row.text)
                     return false; // superseded — discard
-                this.db.exec(`DELETE FROM chunks WHERE rowid = ${safe}`);
-                this.db.prepare(`INSERT INTO chunks (rowid, embedding) VALUES (${safe}, ?)`).run(Buffer.from(embedding.buffer));
+                writeVector(this.db, 'chunks', safe, Buffer.from(embedding.buffer));
                 this.db.prepare(`UPDATE chunk_metadata SET input_hash = ?, profile_id = ?, provenance_state = 'verified' WHERE rowid = ?`)
                     .run(hash, this.currentProfileId, rowid);
                 this.db.prepare(`DELETE FROM embedding_backfill_failures WHERE kind = 'chunk' AND target_id = ?`).run(String(rowid));
                 return true;
             });
-            const written = tx();
+            let written;
+            try {
+                written = tx();
+            }
+            catch (e) {
+                if (e instanceof InvalidSlotError) {
+                    this.coordinator?.requestRepair();
+                    return false;
+                }
+                throw e;
+            }
             this.coordinator?.invalidateCoverage();
             return written;
         }
@@ -859,6 +1005,7 @@ export class RAGKnowledgeGraphManager {
             console.error(`⚠️ reopen storm (${REOPEN_STORM_MAX} in ${REOPEN_STORM_WINDOW_MS / 1000}s) — watch paused, reason=${reason}`);
             return false;
         }
+        this.noteWrites(); // rows changed on the old connection are announced before its count is gone
         const old = this.db;
         // Salvage: frames committed to the old WAL after the last checkpoint (≤1 s) go to the main
         // file if the old descriptor is still readable. An I/O error here just means they are gone.
@@ -964,14 +1111,22 @@ export class RAGKnowledgeGraphManager {
             catch (e) {
                 console.error(`⚠️ wal tick: ${e.message}`);
             }
+            try {
+                this.maybeRebuildVectors();
+            }
+            catch (e) {
+                console.error(`⚠️ vec0 rebuild check: ${e.message}`);
+            }
             this.checkpointWal();
+            this.noteWrites();
         }, tickMs);
         this.walTimer.unref();
     }
     noteActivity(delayMs = 200) {
+        this.lastActivityAt = Date.now();
         if (this.walSoon)
             return;
-        this.walSoon = setTimeout(() => { this.walSoon = null; this.checkpointWal(); }, delayMs);
+        this.walSoon = setTimeout(() => { this.walSoon = null; this.checkpointWal(); this.noteWrites(); }, delayMs);
         this.walSoon.unref();
     }
     cleanup() {
@@ -993,6 +1148,7 @@ export class RAGKnowledgeGraphManager {
             // close() folds the WAL only for the last connection; with other engines attached it
             // would leave every frame behind.
             this.checkpointWal();
+            this.noteWrites();
             this.db.close();
             this.db = null;
         }
@@ -1306,13 +1462,11 @@ export class RAGKnowledgeGraphManager {
             SELECT rowid FROM entity_embedding_metadata WHERE entity_id = ?
           `).get(eid);
                     if (embeddingMetadata) {
-                        const embeddings = this.db.prepare(`
-              DELETE FROM entity_embeddings WHERE rowid = ?
-            `).run(embeddingMetadata.rowid);
+                        const vecState = deleteVector(this.db, 'entity_embeddings', Number(embeddingMetadata.rowid));
                         const metadata = this.db.prepare(`
               DELETE FROM entity_embedding_metadata WHERE entity_id = ?
             `).run(eid);
-                        if (embeddings.changes > 0 || metadata.changes > 0) {
+                        if (vecState === 'valid' || metadata.changes > 0) {
                             console.error(`  ├─ Removed entity embeddings for '${name}'`);
                         }
                     }
@@ -1433,7 +1587,7 @@ export class RAGKnowledgeGraphManager {
                     const meta = this.db.prepare(`SELECT rowid FROM entity_embedding_metadata WHERE entity_id = ?`)
                         .get(p.entityId);
                     if (meta) {
-                        this.db.exec(`DELETE FROM entity_embeddings WHERE rowid = ${Number(meta.rowid)}`);
+                        deleteVector(this.db, 'entity_embeddings', Number(meta.rowid));
                         this.db.prepare(`DELETE FROM entity_embedding_metadata WHERE entity_id = ?`)
                             .run(p.entityId);
                     }
@@ -2046,19 +2200,40 @@ export class RAGKnowledgeGraphManager {
           SELECT rowid FROM entity_embedding_metadata WHERE entity_id = ?
         `).get(entityId);
                 if (existingMetadata) {
-                    this.db.exec(`DELETE FROM entity_embeddings WHERE rowid = ${Number(existingMetadata.rowid)}`);
-                    this.db.prepare(`DELETE FROM entity_embedding_metadata WHERE entity_id = ?`).run(entityId);
+                    // vec-slot-reuse: keep the rowid and rewrite the vector in place (DELETE + INSERT with a new
+                    // rowid leaked one vec0 slot per re-embed). The metadata row is updated, not replaced.
+                    writeVector(this.db, 'entity_embeddings', Number(existingMetadata.rowid), Buffer.from(embedding.buffer));
+                    this.db.prepare(`
+            UPDATE entity_embedding_metadata
+               SET embedding_text = ?, input_hash = ?, profile_id = ?, provenance_state = 'verified'
+             WHERE rowid = ?
+          `).run(embeddingText, inputHash, this.currentProfileId, existingMetadata.rowid);
+                    return true;
                 }
-                const result = this.db.prepare(`
-          INSERT INTO entity_embeddings (embedding) VALUES (?)
-        `).run(Buffer.from(embedding.buffer));
+                // r4 M2: metadata rows can now outlive their vector (kept unstamped to keep the rowid), so the vec0
+                // auto rowid could collide with one. Allocate past both tables, in this transaction.
+                const nextRowid = BigInt(this.db.prepare(`SELECT MAX(r) m FROM (
+            SELECT MAX(rowid) r FROM entity_embedding_metadata UNION ALL SELECT MAX(rowid) r FROM entity_embeddings_rowids)`)
+                    .safeIntegers(true).get().m ?? 0n) + 1n;
+                writeVector(this.db, 'entity_embeddings', nextRowid, Buffer.from(embedding.buffer));
+                const result = { lastInsertRowid: nextRowid };
                 this.db.prepare(`
           INSERT INTO entity_embedding_metadata (rowid, entity_id, embedding_text, input_hash, profile_id, provenance_state)
           VALUES (?, ?, ?, ?, ?, 'verified')
         `).run(result.lastInsertRowid, entityId, embeddingText, inputHash, this.currentProfileId);
                 return true;
             });
-            const written = writeTx();
+            let written;
+            try {
+                written = writeTx();
+            }
+            catch (e) {
+                if (e instanceof InvalidSlotError) {
+                    this.coordinator?.requestRepair();
+                    return false;
+                }
+                throw e;
+            }
             if (!written) {
                 console.error(`⏭️ discarded superseded embedding for ${entityId} (entity changed during inference)`);
             }
@@ -2280,7 +2455,7 @@ export class RAGKnowledgeGraphManager {
         let deletedAssociations = 0;
         for (const chunk of existingChunks) {
             // Delete vector embeddings (vec0 needs literal integer, not parameterized)
-            this.db.exec(`DELETE FROM chunks WHERE rowid = ${safeRowid(chunk.rowid)}`);
+            deleteVector(this.db, 'chunks', safeRowid(chunk.rowid));
             deletedVectors++;
             const associations = this.db.prepare(`
         DELETE FROM chunk_entities WHERE chunk_rowid = ?
@@ -2625,7 +2800,7 @@ export class RAGKnowledgeGraphManager {
                 const existing = db.prepare(`SELECT rowid FROM chunk_metadata WHERE document_id = ?`).all(documentId);
                 for (const ch of existing) {
                     db.prepare(`DELETE FROM chunk_entities WHERE chunk_rowid = ?`).run(ch.rowid);
-                    db.exec(`DELETE FROM chunks WHERE rowid = ${safeRowid(ch.rowid)}`);
+                    deleteVector(db, 'chunks', safeRowid(ch.rowid));
                 }
                 db.prepare(`DELETE FROM chunk_metadata WHERE document_id = ?`).run(documentId);
                 deleteKgChunkRows(db, citedEvidence);
@@ -2640,9 +2815,23 @@ export class RAGKnowledgeGraphManager {
           `).run(chunkId, documentId, seg.chunk_index, seg.text, seg.start_pos, seg.end_pos, seg.start_token, seg.end_token);
                     const rowid = Number(info.lastInsertRowid);
                     if (vec) {
-                        db.prepare(`INSERT INTO chunks (rowid, embedding) VALUES (${rowid}, ?)`).run(vec);
-                        db.prepare(`UPDATE chunk_metadata SET input_hash = ?, profile_id = ?, provenance_state = ? WHERE rowid = ?`)
-                            .run(shaHex(seg.text), this.currentProfileId, provenance, rowid);
+                        // A validity-0 row left at a reused rowid cannot be written over (vec-slot-reuse): the chunk then stays
+                        // vectorless (missing) until the idle rebuild drops that row and the backfill embeds it.
+                        let wrote = true;
+                        try {
+                            writeVector(db, 'chunks', rowid, vec);
+                        }
+                        catch (e) {
+                            if (e instanceof InvalidSlotError) {
+                                wrote = false;
+                                this.coordinator?.requestRepair();
+                            }
+                            else
+                                throw e;
+                        }
+                        if (wrote)
+                            db.prepare(`UPDATE chunk_metadata SET input_hash = ?, profile_id = ?, provenance_state = ? WHERE rowid = ?`)
+                                .run(shaHex(seg.text), this.currentProfileId, provenance, rowid);
                     }
                 }
                 if (oldRowids.length > 0) {
@@ -2789,10 +2978,7 @@ export class RAGKnowledgeGraphManager {
             // Store in vector table (+ verified provenance, §6a-2 atomic)
             try {
                 const tx = this.db.transaction(() => {
-                    this.db.exec(`DELETE FROM chunks WHERE rowid = ${safeRowid(rowid)}`);
-                    this.db.prepare(`
-            INSERT INTO chunks (rowid, embedding) VALUES (${rowid}, ?)
-          `).run(Buffer.from(embedding.buffer));
+                    writeVector(this.db, 'chunks', safeRowid(rowid), Buffer.from(embedding.buffer));
                     this.db.prepare(`UPDATE chunk_metadata SET input_hash = ?, profile_id = ?, provenance_state = 'verified' WHERE rowid = ?`)
                         .run(createHash('sha256').update(chunk.text).digest('hex'), this.currentProfileId, rowid);
                 });
@@ -2800,6 +2986,8 @@ export class RAGKnowledgeGraphManager {
                 embeddedCount++;
             }
             catch (error) {
+                if (error instanceof InvalidSlotError)
+                    this.coordinator?.requestRepair();
                 const errMsg = `chunk ${chunk.chunk_id} (rowid=${rowid}, type=${typeof chunk.rowid}): ${error instanceof Error ? error.message : String(error)}`;
                 console.error(`Failed to embed ${errMsg}`);
                 errors.push(errMsg);
@@ -3238,7 +3426,7 @@ export class RAGKnowledgeGraphManager {
         `).run(chunk.rowid);
                 deletedAssociations += associations.changes;
                 // Delete vector embeddings (vec0 needs literal integer, not parameterized)
-                db.exec(`DELETE FROM chunks WHERE rowid = ${safeRowid(chunk.rowid)}`);
+                deleteVector(db, 'chunks', safeRowid(chunk.rowid));
                 deletedVectors++;
             }
             const metadata = db.prepare(`
@@ -3486,10 +3674,12 @@ export class RAGKnowledgeGraphManager {
                 const orphanChunks = this.db.prepare(`SELECT rowid FROM chunk_metadata WHERE chunk_type IN ('entity','relationship')`)
                     .all();
                 for (const c of orphanChunks) {
-                    this.db.exec(`DELETE FROM chunks WHERE rowid = ${Number(c.rowid)}`);
+                    deleteVector(this.db, 'chunks', Number(c.rowid));
                     this.db.prepare(`DELETE FROM chunk_metadata WHERE rowid = ?`).run(c.rowid);
                 }
-                this.db.exec(`DELETE FROM entity_embeddings WHERE rowid IN (SELECT rowid FROM entity_embedding_metadata)`);
+                for (const m of this.db.prepare(`SELECT rowid FROM entity_embedding_metadata`).all()) {
+                    deleteVector(this.db, 'entity_embeddings', Number(m.rowid)); // validity-0 rows are left to the rebuild
+                }
                 this.db.exec(`DELETE FROM entity_embedding_metadata`);
                 console.error('🗑️ Cleared existing data for full import');
             }
@@ -4374,9 +4564,17 @@ export class RAGKnowledgeGraphManager {
                 reconciliation_state: this.coordinator?.reconState ?? 'n/a',
                 reconciliation_last_error: this.coordinator?.reconLastError ?? null,
                 coverage: cov ? {
-                    chunk: { total: cov.chunk.total, embedded: cov.chunk.embedded, verified: cov.chunk.verified, legacy_assumed: cov.chunk.legacy_assumed, missing: cov.chunk.total - cov.chunk.embedded },
-                    entity: { total: cov.entity.total, embedded: cov.entity.embedded, verified: cov.entity.verified, legacy_assumed: cov.entity.legacy_assumed, missing: cov.entity.total - cov.entity.embedded },
+                    // vec-slot-reuse D1c: an unresolvable slot map is reported as error with missing = null, not as a count.
+                    chunk: cov.chunk.error
+                        ? { total: cov.chunk.total, embedded: null, verified: null, legacy_assumed: null, missing: null, error: cov.chunk.error }
+                        : { total: cov.chunk.total, embedded: cov.chunk.embedded, verified: cov.chunk.verified, legacy_assumed: cov.chunk.legacy_assumed, missing: cov.chunk.total - cov.chunk.embedded },
+                    entity: cov.entity.error
+                        ? { total: cov.entity.total, embedded: null, verified: null, legacy_assumed: null, missing: null, error: cov.entity.error }
+                        : { total: cov.entity.total, embedded: cov.entity.embedded, verified: cov.entity.verified, legacy_assumed: cov.entity.legacy_assumed, missing: cov.entity.total - cov.entity.embedded },
                 } : null,
+                // vec-slot-reuse: per vec0 table = slots, searchable, validity-0 rows, reclaimable chunks, why a rebuild
+                // is (not) due, and the last attempt of this process. Read-only: computing it writes nothing.
+                vector_slots: this.db ? this.vectorSlotStats() : null,
             }
         };
     }
@@ -4888,6 +5086,7 @@ async function main() {
             // Synchronously, before anything is awaited: a host may SIGKILL before the settle
             // sequence below completes (codex: ~185 ms after SIGTERM).
             ragKgManager.checkpointWal();
+            ragKgManager.noteWrites();
             void (async () => {
                 try {
                     await server.close();

package/dist/src/backfillCoordinator.d.ts

@@ -6,12 +6,14 @@ export interface CoverageSnapshot {
         embedded: number;
         verified: number;
         legacy_assumed: number;
+        error?: string;
     };
     entity: {
         total: number;
         embedded: number;
         verified: number;
         legacy_assumed: number;
+        error?: string;
     };
 }
 export interface CoordinatorDeps {
@@ -37,17 +39,25 @@ export declare class BackfillCoordinator {
     private shuttingDown;
     private connLossReruns;
     private scanning;
+    private repairWanted;
+    readonly slotErrors: Map<string, string>;
+    repairRequestedAt: number;
+    requestRepair(): void;
+    takeRepairRequest(): boolean;
+    get repairRequested(): boolean;
     private scanPromise;
     private rerunRequested;
     private snapshot;
     constructor(deps: CoordinatorDeps);
     get reconState(): ReconState;
+    get busy(): boolean;
     get reconLastError(): string | undefined;
     get eligible(): boolean;
     private countNullWithVector;
     private countRepairables;
     private sanitize;
     runReconciliation(): Promise<void>;
+    private reconActive;
     private reconcileOnce;
     private reconcileEntities;
     private reconcileChunks;