rag-memory-epf-mcp 6.4.2 → 6.4.4

package/README.md

@@ -154,6 +154,48 @@ storeDocument(id, content, metadata)
 
 ## Changelog
 
+### v6.4.4 (2026-11-07)
+
+- **Fixed — the similarity numbers are real cosine similarities.** The vec0 tables use the default L2 metric and the
+  embeddings are unit-normalized, so cosine similarity = 1 − d²/2. The engine used 1 − d/2 (L2 read as a cosine
+  distance): `searchNodes` `similarity` and the default `hybridSearch` `vector_similarity` were too low above cos 0.5
+  (cos 0.8 showed ≈0.684) and too high below it (orthogonal vectors showed ≈0.293 instead of 0). Ranking of the default
+  search is unchanged (rank fusion uses ranks only).
+- **Unchanged on purpose — the legacy paths keep the 6.3.2 score.** `hybridSearch` with `useGraph: true` or with
+  summaries on, and the graph-seed threshold (`> 0.4`, about cosine 0.28), stay byte-identical to 6.3.2 for evaluation
+  comparability; that score is named `legacyL2AsCosineScore` and is not a cosine.
+- **Fixed — chunk vectors without metadata are cleaned up.** Reconciliation now deletes searchable chunk vectors that
+  have no `chunk_metadata` row (same rule and batching as entity orphans; off mode only classifies it as deferred).
+  `importGraph` no longer creates them: a full replace (`merge: false`) and a same-id document replace delete that
+  document's chunk vectors first. Rowids past 2^53 are kept exact.
+- **Added — `getKnowledgeGraphStats().server.coverage.{chunk,entity}.orphan_vectors`**: searchable vectors with no
+  metadata row (`null` when the slot map cannot be resolved). `missing` keeps its meaning.
+- Not measured: Windows, Drive-synced DBs under two machines.
+
+### v6.4.3 (2026-11-02)
+
+- **Fixed — re-embedding no longer leaks vector slots.** sqlite-vec puts a new row only into a free slot of the latest
+  1,024-slot chunk, so the old DELETE + INSERT of the same rowid left a hole whenever the row sat in an older chunk (one
+  project DB reached 84 chunks for 5,578 vectors, 93.5% empty, 337 of 439 MB). Same-rowid re-embeds (document chunks,
+  KG chunks, bulk `embedChunks`, entities) now rewrite the vector in place; entity re-embeds keep their rowid.
+- **Added — idle verified rebuild of the vector tables.** When the engine is idle (no tool call, backfill or
+  reconciliation; 60 s quiet; checked at most every 10 min) and a table has at least 8 reclaimable chunks that outweigh
+  its live vectors, or holds damaged slots, it is rebuilt in one transaction keeping every searchable rowid and its
+  exact vector bytes (verified before commit; rolled back otherwise). Skipped at once if another connection holds the
+  write lock. Automatic only up to 20,000 live vectors per table (measured on one Mac: 5,600 = 0.44 s, 20,000 =
+  1.24 s and +267 MB transient). No VACUUM: the file stops growing; it shrinks only through a VACUUM.
+- **Fixed — coverage counts only searchable vectors.** A row whose slot validity bit is 0 (invisible to search) was
+  counted as embedded; it is now missing and is repaired: writes and deletes that meet such a slot, or a slot shared by
+  two rowids, rebuild the table first in the same transaction, then the backfill re-embeds at the original rowid.
+  An unresolvable slot position is reported as an error (`missing: null`), never as a count.
+- **Added — `getKnowledgeGraphStats().server.vector_slots`**: per table slots, valid, validity-0, shared, empty,
+  reclaimable chunks, whether a rebuild is due and why, and this process's last attempt.
+- **Added — write-seq signal** (`[rag-memory] write-seq N` on stderr when the engine's own connection changed rows),
+  used by the framework launcher to record a DB handoff only after the engine wrote.
+- **Changed — an entity metadata row whose vector row is gone is kept unstamped (missing) instead of deleted**, so the
+  backfill re-inserts at the same rowid; new entity rowids are allocated past both tables.
+- Not measured: Windows, Drive-synced DBs under two machines, tables above 20,000 vectors.
+
 ### v6.4.2 (2026-10-31)
 
 - **Fixed — the engine no longer keeps the peak memory of its largest embedding.** The ONNX session is opened with

package/dist/index.d.ts

@@ -4,6 +4,7 @@ import { EVIDENCE_FORMAT, type EvidenceItemInput, type EvidenceStatus, type Evid
 import { EmbeddingGate } from './src/embeddingGate.js';
 import type { EmbedPriority } from './src/embeddingGate.js';
 import { BackfillCoordinator } from './src/backfillCoordinator.js';
+import type { RebuildResult, VecTable } from './src/vecSlots.js';
 export declare const LEGACY_QUERY_INSTRUCTION = "Represent this sentence for searching relevant passages: ";
 export declare function embeddingInput(text: string, isQuery: boolean, defaultModel?: boolean): string;
 export declare function makeEmbedFn(model: (input: string, opts: any) => Promise<{
@@ -132,6 +133,20 @@ export declare class RAGKnowledgeGraphManager {
         __testForceFkOff?: boolean;
     }): Promise<void>;
     private openConnection;
+    writeSeq: number;
+    private tcBase;
+    private lastActivityAt;
+    private lastWriteAt;
+    private lastSlotCheckAt;
+    readonly vectorSlotLast: Partial<Record<VecTable, RebuildResult & {
+        at: string;
+    }>>;
+    private static slotCheckMs;
+    private static slotQuietMs;
+    maybeRebuildVectors(now?: number): RebuildResult[] | null;
+    vectorSlotStats(): Record<string, unknown>;
+    private totalChanges;
+    noteWrites(): void;
     private ensureCurrentProfile;
     private buildRealLoader;
     startReconciliation(): Promise<void>;
@@ -474,7 +489,7 @@ export declare class RAGKnowledgeGraphManager {
     }>;
     /**
      * Diagnostic seam for the graph re-ranker (evaluation change graph-role-evaluation, R2).
-     * Returns the seed entities (query-vector matched, similarity > 0.4, top-10 per variant) and the
+     * Returns the seed entities (query-vector matched, legacy score > 0.4 = cosine ≈0.28, top-10 per variant) and the
      * 1-hop connected entities exactly as hybridSearch(useGraph:true) computes them, plus edge detail
      * that hybridSearch itself does not use (edge id, type, direction, confidence). It never generates
      * candidates and never changes ranking; hybridSearch consumes only the name sets.

package/dist/index.js

@@ -34,6 +34,8 @@ import { migrations } from './src/migrations/migrations.js';
 import { EmbeddingGate, GateNotReadyError, GateDisabledError, TerminalConfigError } from './src/embeddingGate.js';
 import { resolveModelCacheDir, preflightCacheDir, artifactKey, ModelDownloadLock, handleLoaderFailure } from './src/modelCache.js';
 import { BackfillCoordinator } from './src/backfillCoordinator.js';
+import { l2ToCosineSimilarity, legacyL2AsCosineScore } from './src/vecSimilarity.js';
+import { writeVector, deleteVector, registerVecSlotFunctions, InvalidSlotError, slotSummary, rebuildVecTable, rebuildReason, DEFAULT_REBUILD_POLICY } from './src/vecSlots.js';
 import { calendarDate, resolveCalendarTimeZone, stampDatePrefix, stripDatePrefix } from './src/observations/date-prefix.js';
 import os from 'node:os';
 import { createHash } from 'crypto';
@@ -326,6 +328,7 @@ export class RAGKnowledgeGraphManager {
     openConnection(forceFkOff = false) {
         this.db = new Database(DB_FILE_PATH);
         sqliteVec.load(this.db);
+        registerVecSlotFunctions(this.db); // vec_slot_valid() for the D1c searchability predicate
         this.db.pragma('journal_mode = WAL');
         this.db.pragma('synchronous = NORMAL');
         this.db.pragma('busy_timeout = 5000');
@@ -356,6 +359,119 @@ export class RAGKnowledgeGraphManager {
                     `for history integrity; refusing to run migrations without it.`);
             }
         }
+        // Write-sequence baseline for THIS connection (see noteWrites). A fresh connection starts
+        // its own total_changes() count, so the baseline is reset on every open; the announced
+        // sequence itself never goes back.
+        this.tcBase = this.totalChanges();
+    }
+    // --- Write-sequence signal (test/write-seq.test.mjs) ------------------------------------
+    // One stderr line per observation that THIS connection changed rows:
+    //   [rag-memory] write-seq <N>   (N = 1, 2, 3, … for the life of the process)
+    // The framework launcher (mcp-proxy.cjs) records "the DB state this machine left" only
+    // after such a line. Before this signal it recorded whenever the file changed and went
+    // quiet — which also fired when a cloud sync client swapped the other machine's file in
+    // under an idle engine, and that record then refused a correct DB (measured 2026-11-02).
+    // total_changes() counts rows changed by this connection (vec0 shadow tables included);
+    // reads and other connections do not move it. It is read outside a transaction only, so a
+    // statement still inside one is announced after it ends. It also counts rows of a
+    // transaction that was rolled back: a line means "this connection's change count went up",
+    // not "a commit happened". The launcher consumes a pending sequence once the file it last
+    // recorded is still there, the WAL is empty and the file is stable; until then a sequence that
+    // came from a rollback can still publish a file another writer or a sync client changed, and
+    // that period has no fixed upper bound (hub session 251 review r1 M1 · r3. r2: a WAL-size
+    // test was tried and removed — a header-only WAL looks non-empty and another connection's
+    // checkpoint hides a real commit).
+    // Written with fs.writeSync: the last call runs inside process 'exit', where an async
+    // stderr pipe write (macOS) can be lost.
+    writeSeq = 0;
+    tcBase = 0;
+    // ---- vec-slot-reuse D1b: idle verified rebuild of the vec0 tables -------------------------------
+    // Initial operating values (2_ s253 r2 Q3: starting values, not optima): look at most every 10 min,
+    // only after 60 s without a tool call or an own-connection write, never while a tool call, a backfill
+    // scan or a reconciliation is running. A repair request (validity-0 rows seen) skips the 10-min wait.
+    // The saving threshold and the live-vector cap live in DEFAULT_REBUILD_POLICY (src/vecSlots.ts).
+    lastActivityAt = Date.now();
+    lastWriteAt = 0;
+    lastSlotCheckAt = 0;
+    // Last attempt per table. Process memory only: unknown after a restart (stats says so).
+    vectorSlotLast = {};
+    static slotCheckMs = Number(process.env.RAG_MEMORY_SLOT_CHECK_MS ?? 600_000);
+    static slotQuietMs = Number(process.env.RAG_MEMORY_SLOT_QUIET_MS ?? 60_000);
+    maybeRebuildVectors(now = Date.now()) {
+        if (!this.db || this.shuttingDown || this.inFlight > 0 || this.db.inTransaction)
+            return null;
+        if (this.coordinator?.busy)
+            return null;
+        if (now - Math.max(this.lastActivityAt, this.lastWriteAt) < RAGKnowledgeGraphManager.slotQuietMs)
+            return null;
+        // A repair request skips the 10-min wait once: only if it arrived after the last check (r3 M5 — otherwise
+        // a busy / failed / over-cap attempt would be retried on every 1 s tick).
+        const repair = this.coordinator?.repairRequested ?? false;
+        const freshRepair = repair && (this.coordinator?.repairRequestedAt ?? 0) > this.lastSlotCheckAt;
+        if (!freshRepair && now - this.lastSlotCheckAt < RAGKnowledgeGraphManager.slotCheckMs)
+            return null;
+        this.lastSlotCheckAt = now;
+        const results = [];
+        for (const table of ['chunks', 'entity_embeddings']) {
+            const r = rebuildVecTable(this.db, table);
+            this.vectorSlotLast[table] = { ...r, at: new Date(now).toISOString() };
+            results.push(r);
+            if (r.status === 'rebuilt') {
+                console.error(`🧹 vec0 ${table} rebuilt (${r.reason}): chunks ${r.before?.chunks} -> ${r.after?.chunks}, ` +
+                    `valid ${r.after?.valid}, dropped validity-0 ${r.before?.invalid}, shared-slot ${r.before?.aliased}, ${r.ms} ms`);
+            }
+            else if (r.status !== 'not-needed') {
+                console.error(`⚠️ vec0 ${table} rebuild ${r.status}${r.error ? `: ${r.error}` : ''}`);
+            }
+        }
+        if (repair && results.every(r => r.status === 'rebuilt' || r.status === 'not-needed'))
+            this.coordinator?.takeRepairRequest();
+        if (results.some(r => r.status === 'rebuilt')) {
+            this.coordinator?.invalidateCoverage();
+            this.noteWrites();
+            this.coordinator?.kick(); // rows dropped as validity-0 are now missing -> re-embedded at their rowid
+        }
+        return results;
+    }
+    vectorSlotStats() {
+        const out = {};
+        for (const table of ['chunks', 'entity_embeddings']) {
+            let summary;
+            try {
+                summary = slotSummary(this.db, table);
+            }
+            catch (e) {
+                summary = { error: e.message };
+            }
+            const pending = 'error' in summary ? 'position-error'
+                : (summary.valid > DEFAULT_REBUILD_POLICY.maxLive ? 'over-cap' : (rebuildReason(summary) ?? null));
+            const last = this.vectorSlotLast[table];
+            out[table] = { ...summary, due: pending,
+                last_attempt: last ? { at: last.at, status: last.status, reason: last.reason ?? null, ms: last.ms, error: last.error ?? null }
+                    : 'unknown (process memory; none since start)' };
+        }
+        out.policy = { ...DEFAULT_REBUILD_POLICY, check_ms: RAGKnowledgeGraphManager.slotCheckMs, quiet_ms: RAGKnowledgeGraphManager.slotQuietMs,
+            repair_requested: this.coordinator?.repairRequested ?? false, vacuum: 'never (maintenance script only)' };
+        return out;
+    }
+    totalChanges() {
+        if (!this.db)
+            return 0;
+        return Number(this.db.prepare('select total_changes()').pluck().get());
+    }
+    noteWrites() {
+        try {
+            if (!this.db || this.db.inTransaction)
+                return;
+            const tc = this.totalChanges();
+            if (tc <= this.tcBase)
+                return;
+            this.tcBase = tc;
+            this.lastWriteAt = Date.now();
+            this.writeSeq += 1;
+            fsSync.writeSync(2, `[rag-memory] write-seq ${this.writeSeq}\n`);
+        }
+        catch { /* a dead handle is the reopen path's business; the next tick tries again */ }
     }
     // Upsert the stored-vector compatibility profile (spec §6c layer 2) and record
     // retrieval config in server_meta (layer 3 — never a backfill trigger).
@@ -552,7 +668,7 @@ export class RAGKnowledgeGraphManager {
         const meta = this.db.prepare(`SELECT rowid FROM entity_embedding_metadata WHERE entity_id = ?`)
             .get(entityId);
         if (meta) {
-            this.db.exec(`DELETE FROM entity_embeddings WHERE rowid = ${Number(meta.rowid)}`);
+            deleteVector(this.db, 'entity_embeddings', Number(meta.rowid));
             this.db.prepare(`DELETE FROM entity_embedding_metadata WHERE entity_id = ?`).run(entityId);
         }
         deleteStaleKgChunks(this.db, entityId);
@@ -567,7 +683,7 @@ export class RAGKnowledgeGraphManager {
         if (!meta)
             return;
         const tx = this.db.transaction(() => {
-            this.db.exec(`DELETE FROM entity_embeddings WHERE rowid = ${Number(meta.rowid)}`);
+            deleteVector(this.db, 'entity_embeddings', Number(meta.rowid));
             this.db.prepare(`DELETE FROM entity_embedding_metadata WHERE entity_id = ?`).run(entityId);
         });
         tx();
@@ -597,8 +713,7 @@ export class RAGKnowledgeGraphManager {
                 return false; // inputs changed
             if (this.currentProfileId !== captured.profile_id)
                 return false; // profile changed during inference
-            this.db.exec(`DELETE FROM chunks WHERE rowid = ${rowid}`);
-            this.db.prepare(`INSERT INTO chunks (rowid, embedding) VALUES (${rowid}, ?)`).run(Buffer.from(embedding.buffer));
+            writeVector(this.db, 'chunks', rowid, Buffer.from(embedding.buffer)); // same slot when valid (vec-slot-reuse)
             this.db.prepare(`UPDATE chunk_metadata SET input_hash = ?, profile_id = ?, provenance_state = 'verified' WHERE rowid = ?`)
                 .run(captured.text_sha256, captured.profile_id, rowid);
             const meta = parseEvidenceMetadata(cur.metadata);
@@ -610,7 +725,17 @@ export class RAGKnowledgeGraphManager {
             this.db.prepare(`DELETE FROM embedding_backfill_failures WHERE kind = 'chunk' AND target_id = ?`).run(String(rowid));
             return true;
         });
-        const written = tx();
+        let written;
+        try {
+            written = tx();
+        }
+        catch (e) {
+            if (e instanceof InvalidSlotError) {
+                this.coordinator?.requestRepair();
+                return false;
+            }
+            throw e;
+        }
         this.coordinator?.invalidateCoverage();
         return written;
     }
@@ -648,14 +773,23 @@ export class RAGKnowledgeGraphManager {
                 const cur = this.db.prepare(`SELECT text FROM chunk_metadata WHERE rowid = ?`).get(rowid);
                 if (!cur || cur.text !== row.text)
                     return false; // superseded — discard
-                this.db.exec(`DELETE FROM chunks WHERE rowid = ${safe}`);
-                this.db.prepare(`INSERT INTO chunks (rowid, embedding) VALUES (${safe}, ?)`).run(Buffer.from(embedding.buffer));
+                writeVector(this.db, 'chunks', safe, Buffer.from(embedding.buffer));
                 this.db.prepare(`UPDATE chunk_metadata SET input_hash = ?, profile_id = ?, provenance_state = 'verified' WHERE rowid = ?`)
                     .run(hash, this.currentProfileId, rowid);
                 this.db.prepare(`DELETE FROM embedding_backfill_failures WHERE kind = 'chunk' AND target_id = ?`).run(String(rowid));
                 return true;
             });
-            const written = tx();
+            let written;
+            try {
+                written = tx();
+            }
+            catch (e) {
+                if (e instanceof InvalidSlotError) {
+                    this.coordinator?.requestRepair();
+                    return false;
+                }
+                throw e;
+            }
             this.coordinator?.invalidateCoverage();
             return written;
         }
@@ -872,6 +1006,7 @@ export class RAGKnowledgeGraphManager {
             console.error(`⚠️ reopen storm (${REOPEN_STORM_MAX} in ${REOPEN_STORM_WINDOW_MS / 1000}s) — watch paused, reason=${reason}`);
             return false;
         }
+        this.noteWrites(); // rows changed on the old connection are announced before its count is gone
         const old = this.db;
         // Salvage: frames committed to the old WAL after the last checkpoint (≤1 s) go to the main
         // file if the old descriptor is still readable. An I/O error here just means they are gone.
@@ -977,14 +1112,22 @@ export class RAGKnowledgeGraphManager {
             catch (e) {
                 console.error(`⚠️ wal tick: ${e.message}`);
             }
+            try {
+                this.maybeRebuildVectors();
+            }
+            catch (e) {
+                console.error(`⚠️ vec0 rebuild check: ${e.message}`);
+            }
             this.checkpointWal();
+            this.noteWrites();
         }, tickMs);
         this.walTimer.unref();
     }
     noteActivity(delayMs = 200) {
+        this.lastActivityAt = Date.now();
         if (this.walSoon)
             return;
-        this.walSoon = setTimeout(() => { this.walSoon = null; this.checkpointWal(); }, delayMs);
+        this.walSoon = setTimeout(() => { this.walSoon = null; this.checkpointWal(); this.noteWrites(); }, delayMs);
         this.walSoon.unref();
     }
     cleanup() {
@@ -1006,6 +1149,7 @@ export class RAGKnowledgeGraphManager {
             // close() folds the WAL only for the last connection; with other engines attached it
             // would leave every frame behind.
             this.checkpointWal();
+            this.noteWrites();
             this.db.close();
             this.db = null;
         }
@@ -1319,13 +1463,11 @@ export class RAGKnowledgeGraphManager {
             SELECT rowid FROM entity_embedding_metadata WHERE entity_id = ?
           `).get(eid);
                     if (embeddingMetadata) {
-                        const embeddings = this.db.prepare(`
-              DELETE FROM entity_embeddings WHERE rowid = ?
-            `).run(embeddingMetadata.rowid);
+                        const vecState = deleteVector(this.db, 'entity_embeddings', Number(embeddingMetadata.rowid));
                         const metadata = this.db.prepare(`
               DELETE FROM entity_embedding_metadata WHERE entity_id = ?
             `).run(eid);
-                        if (embeddings.changes > 0 || metadata.changes > 0) {
+                        if (vecState === 'valid' || metadata.changes > 0) {
                             console.error(`  ├─ Removed entity embeddings for '${name}'`);
                         }
                     }
@@ -1446,7 +1588,7 @@ export class RAGKnowledgeGraphManager {
                     const meta = this.db.prepare(`SELECT rowid FROM entity_embedding_metadata WHERE entity_id = ?`)
                         .get(p.entityId);
                     if (meta) {
-                        this.db.exec(`DELETE FROM entity_embeddings WHERE rowid = ${Number(meta.rowid)}`);
+                        deleteVector(this.db, 'entity_embeddings', Number(meta.rowid));
                         this.db.prepare(`DELETE FROM entity_embedding_metadata WHERE entity_id = ?`)
                             .run(p.entityId);
                     }
@@ -1753,7 +1895,7 @@ export class RAGKnowledgeGraphManager {
             name: result.name,
             entityType: result.entityType,
             observations: JSON.parse(result.observations),
-            similarity: Math.max(0, 1 - result.distance / 2) // Convert cosine distance (0-2) to similarity (1-0)
+            similarity: l2ToCosineSimilarity(result.distance) // vec0 L2 distance of unit vectors -> cosine similarity
         }));
         // hybrid-partial (spec §4): entities without vectors must not vanish from
         // search while backfill catches up — merge lexical FTS hits for the gap.
@@ -2059,19 +2201,40 @@ export class RAGKnowledgeGraphManager {
           SELECT rowid FROM entity_embedding_metadata WHERE entity_id = ?
         `).get(entityId);
                 if (existingMetadata) {
-                    this.db.exec(`DELETE FROM entity_embeddings WHERE rowid = ${Number(existingMetadata.rowid)}`);
-                    this.db.prepare(`DELETE FROM entity_embedding_metadata WHERE entity_id = ?`).run(entityId);
+                    // vec-slot-reuse: keep the rowid and rewrite the vector in place (DELETE + INSERT with a new
+                    // rowid leaked one vec0 slot per re-embed). The metadata row is updated, not replaced.
+                    writeVector(this.db, 'entity_embeddings', Number(existingMetadata.rowid), Buffer.from(embedding.buffer));
+                    this.db.prepare(`
+            UPDATE entity_embedding_metadata
+               SET embedding_text = ?, input_hash = ?, profile_id = ?, provenance_state = 'verified'
+             WHERE rowid = ?
+          `).run(embeddingText, inputHash, this.currentProfileId, existingMetadata.rowid);
+                    return true;
                 }
-                const result = this.db.prepare(`
-          INSERT INTO entity_embeddings (embedding) VALUES (?)
-        `).run(Buffer.from(embedding.buffer));
+                // r4 M2: metadata rows can now outlive their vector (kept unstamped to keep the rowid), so the vec0
+                // auto rowid could collide with one. Allocate past both tables, in this transaction.
+                const nextRowid = BigInt(this.db.prepare(`SELECT MAX(r) m FROM (
+            SELECT MAX(rowid) r FROM entity_embedding_metadata UNION ALL SELECT MAX(rowid) r FROM entity_embeddings_rowids)`)
+                    .safeIntegers(true).get().m ?? 0n) + 1n;
+                writeVector(this.db, 'entity_embeddings', nextRowid, Buffer.from(embedding.buffer));
+                const result = { lastInsertRowid: nextRowid };
                 this.db.prepare(`
           INSERT INTO entity_embedding_metadata (rowid, entity_id, embedding_text, input_hash, profile_id, provenance_state)
           VALUES (?, ?, ?, ?, ?, 'verified')
         `).run(result.lastInsertRowid, entityId, embeddingText, inputHash, this.currentProfileId);
                 return true;
             });
-            const written = writeTx();
+            let written;
+            try {
+                written = writeTx();
+            }
+            catch (e) {
+                if (e instanceof InvalidSlotError) {
+                    this.coordinator?.requestRepair();
+                    return false;
+                }
+                throw e;
+            }
             if (!written) {
                 console.error(`⏭️ discarded superseded embedding for ${entityId} (entity changed during inference)`);
             }
@@ -2293,7 +2456,7 @@ export class RAGKnowledgeGraphManager {
         let deletedAssociations = 0;
         for (const chunk of existingChunks) {
             // Delete vector embeddings (vec0 needs literal integer, not parameterized)
-            this.db.exec(`DELETE FROM chunks WHERE rowid = ${safeRowid(chunk.rowid)}`);
+            deleteVector(this.db, 'chunks', safeRowid(chunk.rowid));
             deletedVectors++;
             const associations = this.db.prepare(`
         DELETE FROM chunk_entities WHERE chunk_rowid = ?
@@ -2638,7 +2801,7 @@ export class RAGKnowledgeGraphManager {
                 const existing = db.prepare(`SELECT rowid FROM chunk_metadata WHERE document_id = ?`).all(documentId);
                 for (const ch of existing) {
                     db.prepare(`DELETE FROM chunk_entities WHERE chunk_rowid = ?`).run(ch.rowid);
-                    db.exec(`DELETE FROM chunks WHERE rowid = ${safeRowid(ch.rowid)}`);
+                    deleteVector(db, 'chunks', safeRowid(ch.rowid));
                 }
                 db.prepare(`DELETE FROM chunk_metadata WHERE document_id = ?`).run(documentId);
                 deleteKgChunkRows(db, citedEvidence);
@@ -2653,9 +2816,23 @@ export class RAGKnowledgeGraphManager {
           `).run(chunkId, documentId, seg.chunk_index, seg.text, seg.start_pos, seg.end_pos, seg.start_token, seg.end_token);
                     const rowid = Number(info.lastInsertRowid);
                     if (vec) {
-                        db.prepare(`INSERT INTO chunks (rowid, embedding) VALUES (${rowid}, ?)`).run(vec);
-                        db.prepare(`UPDATE chunk_metadata SET input_hash = ?, profile_id = ?, provenance_state = ? WHERE rowid = ?`)
-                            .run(shaHex(seg.text), this.currentProfileId, provenance, rowid);
+                        // A validity-0 row left at a reused rowid cannot be written over (vec-slot-reuse): the chunk then stays
+                        // vectorless (missing) until the idle rebuild drops that row and the backfill embeds it.
+                        let wrote = true;
+                        try {
+                            writeVector(db, 'chunks', rowid, vec);
+                        }
+                        catch (e) {
+                            if (e instanceof InvalidSlotError) {
+                                wrote = false;
+                                this.coordinator?.requestRepair();
+                            }
+                            else
+                                throw e;
+                        }
+                        if (wrote)
+                            db.prepare(`UPDATE chunk_metadata SET input_hash = ?, profile_id = ?, provenance_state = ? WHERE rowid = ?`)
+                                .run(shaHex(seg.text), this.currentProfileId, provenance, rowid);
                     }
                 }
                 if (oldRowids.length > 0) {
@@ -2802,10 +2979,7 @@ export class RAGKnowledgeGraphManager {
             // Store in vector table (+ verified provenance, §6a-2 atomic)
             try {
                 const tx = this.db.transaction(() => {
-                    this.db.exec(`DELETE FROM chunks WHERE rowid = ${safeRowid(rowid)}`);
-                    this.db.prepare(`
-            INSERT INTO chunks (rowid, embedding) VALUES (${rowid}, ?)
-          `).run(Buffer.from(embedding.buffer));
+                    writeVector(this.db, 'chunks', safeRowid(rowid), Buffer.from(embedding.buffer));
                     this.db.prepare(`UPDATE chunk_metadata SET input_hash = ?, profile_id = ?, provenance_state = 'verified' WHERE rowid = ?`)
                         .run(createHash('sha256').update(chunk.text).digest('hex'), this.currentProfileId, rowid);
                 });
@@ -2813,6 +2987,8 @@ export class RAGKnowledgeGraphManager {
                 embeddedCount++;
             }
             catch (error) {
+                if (error instanceof InvalidSlotError)
+                    this.coordinator?.requestRepair();
                 const errMsg = `chunk ${chunk.chunk_id} (rowid=${rowid}, type=${typeof chunk.rowid}): ${error instanceof Error ? error.message : String(error)}`;
                 console.error(`Failed to embed ${errMsg}`);
                 errors.push(errMsg);
@@ -3251,7 +3427,7 @@ export class RAGKnowledgeGraphManager {
         `).run(chunk.rowid);
                 deletedAssociations += associations.changes;
                 // Delete vector embeddings (vec0 needs literal integer, not parameterized)
-                db.exec(`DELETE FROM chunks WHERE rowid = ${safeRowid(chunk.rowid)}`);
+                deleteVector(db, 'chunks', safeRowid(chunk.rowid));
                 deletedVectors++;
             }
             const metadata = db.prepare(`
@@ -3490,19 +3666,27 @@ export class RAGKnowledgeGraphManager {
                 this.db.exec(`DELETE FROM entity_observations`);
                 this.db.exec(`DELETE FROM observation_roots`);
                 this.db.exec(`DELETE FROM entities`);
+                // documents → chunk_metadata is a CASCADE, but vec0 has no FK: delete the document chunk vectors first,
+                // or they stay searchable-less orphans (IN-42 · same order as storeDocument's replace path).
+                // rowids stay BigInt end to end (2_ r1 R1: a Number cast past 2^53 deletes the wrong vector)
+                for (const c of this.db.prepare(`SELECT rowid FROM chunk_metadata WHERE document_id IS NOT NULL`).safeIntegers(true).all()) {
+                    deleteVector(this.db, 'chunks', c.rowid);
+                }
                 this.db.exec(`DELETE FROM documents`);
                 // entities 를 지워도 파생 데이터는 따라오지 않는다: chunk_metadata 에는
                 // entities 로 가는 FK 가 없고 entity_embedding_metadata.entity_id 는 UNIQUE 일
                 // 뿐이다. 그래서 replace-import 뒤에 **사라진 entity 의 벡터와 KG chunk 가
-                // 검색에 남았다**(advisor beta 발견 2). document chunk 는 documents 의
-                // CASCADE 로 이미 정리되므로 여기서는 entity·relationship chunk 만 지운다.
+                // 검색에 남았다**(advisor beta 발견 2). document chunk 메타는 documents 의
+                // CASCADE 로 정리되고 그 벡터는 위에서 먼저 지웠다(IN-42) — 여기서는 entity·relationship chunk 만 지운다.
                 const orphanChunks = this.db.prepare(`SELECT rowid FROM chunk_metadata WHERE chunk_type IN ('entity','relationship')`)
                     .all();
                 for (const c of orphanChunks) {
-                    this.db.exec(`DELETE FROM chunks WHERE rowid = ${Number(c.rowid)}`);
+                    deleteVector(this.db, 'chunks', Number(c.rowid));
                     this.db.prepare(`DELETE FROM chunk_metadata WHERE rowid = ?`).run(c.rowid);
                 }
-                this.db.exec(`DELETE FROM entity_embeddings WHERE rowid IN (SELECT rowid FROM entity_embedding_metadata)`);
+                for (const m of this.db.prepare(`SELECT rowid FROM entity_embedding_metadata`).all()) {
+                    deleteVector(this.db, 'entity_embeddings', Number(m.rowid)); // validity-0 rows are left to the rebuild
+                }
                 this.db.exec(`DELETE FROM entity_embedding_metadata`);
                 console.error('🗑️ Cleared existing data for full import');
             }
@@ -3555,6 +3739,10 @@ export class RAGKnowledgeGraphManager {
                     // doc.id is a string here (checked above, before any write).
                     for (const r of collectEvidenceRowidsCitingDocument(this.db, doc.id))
                         citedEvidence.add(r);
+                    // REPLACE deletes an existing row → its chunk_metadata goes by CASCADE, the vec0 vectors would not (IN-42).
+                    for (const c of this.db.prepare(`SELECT rowid FROM chunk_metadata WHERE document_id = ?`).safeIntegers(true).all(doc.id)) {
+                        deleteVector(this.db, 'chunks', c.rowid);
+                    }
                     const result = stmt.run(doc.id, doc.content, JSON.stringify(doc.metadata || {}), doc.created_at || new Date().toISOString());
                     if (result.changes > 0) {
                         imported.documents++;
@@ -3724,7 +3912,7 @@ export class RAGKnowledgeGraphManager {
     }
     /**
      * Diagnostic seam for the graph re-ranker (evaluation change graph-role-evaluation, R2).
-     * Returns the seed entities (query-vector matched, similarity > 0.4, top-10 per variant) and the
+     * Returns the seed entities (query-vector matched, legacy score > 0.4 = cosine ≈0.28, top-10 per variant) and the
      * 1-hop connected entities exactly as hybridSearch(useGraph:true) computes them, plus edge detail
      * that hybridSearch itself does not use (edge id, type, direction, confidence). It never generates
      * candidates and never changes ranking; hybridSearch consumes only the name sets.
@@ -3776,7 +3964,7 @@ export class RAGKnowledgeGraphManager {
             WHERE r.source_entity = ? OR r.target_entity = ?
             ORDER BY r.id`);
             for (const entity of similar) {
-                const similarity = Math.max(0, 1 - entity.distance / 2);
+                const similarity = legacyL2AsCosineScore(entity.distance); // legacy seed score, pinned (4ac3155) — not a cosine
                 if (similarity > 0.4) {
                     seeds.push({ entity_id: entity.entity_id, name: entity.name, similarity });
                     for (const row of edgeStmt.all(entity.entity_id, entity.entity_id, entity.entity_id, entity.entity_id, entity.entity_id)) {
@@ -4123,7 +4311,8 @@ export class RAGKnowledgeGraphManager {
                 ({ summary, keyHighlight, relevanceScore } = await this.generateContentSummary(result.text, primaryQueryEmbedding, chunkEntities, result.chunk_type === 'relationship' ? 1 : 2 // Shorter summary for relationships
                 ));
             }
-            const vectorSimilarity = Math.max(0, 1 - result.distance / 2);
+            // default (rank fusion) = real cosine · legacy paths (useGraph:true or summaries on) keep the pinned 6.3.2 score
+            const vectorSimilarity = rankFusion ? l2ToCosineSimilarity(result.distance) : legacyL2AsCosineScore(result.distance);
             const ftsBoost = ftsBoostMap.get(result.chunk_id) || 0; // = 1/(60 + FTS rank), 0 if not an FTS hit
             const vectorRank = vectorRankOf.get(result.chunk_id);
             const finalScore = rankFusion
@@ -4387,9 +4576,17 @@ export class RAGKnowledgeGraphManager {
                 reconciliation_state: this.coordinator?.reconState ?? 'n/a',
                 reconciliation_last_error: this.coordinator?.reconLastError ?? null,
                 coverage: cov ? {
-                    chunk: { total: cov.chunk.total, embedded: cov.chunk.embedded, verified: cov.chunk.verified, legacy_assumed: cov.chunk.legacy_assumed, missing: cov.chunk.total - cov.chunk.embedded },
-                    entity: { total: cov.entity.total, embedded: cov.entity.embedded, verified: cov.entity.verified, legacy_assumed: cov.entity.legacy_assumed, missing: cov.entity.total - cov.entity.embedded },
+                    // vec-slot-reuse D1c: an unresolvable slot map is reported as error with missing = null, not as a count.
+                    chunk: cov.chunk.error
+                        ? { total: cov.chunk.total, embedded: null, verified: null, legacy_assumed: null, missing: null, orphan_vectors: null, error: cov.chunk.error }
+                        : { total: cov.chunk.total, embedded: cov.chunk.embedded, verified: cov.chunk.verified, legacy_assumed: cov.chunk.legacy_assumed, missing: cov.chunk.total - cov.chunk.embedded, orphan_vectors: cov.chunk.orphan_vectors ?? null },
+                    entity: cov.entity.error
+                        ? { total: cov.entity.total, embedded: null, verified: null, legacy_assumed: null, missing: null, orphan_vectors: null, error: cov.entity.error }
+                        : { total: cov.entity.total, embedded: cov.entity.embedded, verified: cov.entity.verified, legacy_assumed: cov.entity.legacy_assumed, missing: cov.entity.total - cov.entity.embedded, orphan_vectors: cov.entity.orphan_vectors ?? null },
                 } : null,
+                // vec-slot-reuse: per vec0 table = slots, searchable, validity-0 rows, reclaimable chunks, why a rebuild
+                // is (not) due, and the last attempt of this process. Read-only: computing it writes nothing.
+                vector_slots: this.db ? this.vectorSlotStats() : null,
             }
         };
     }
@@ -4901,6 +5098,7 @@ async function main() {
             // Synchronously, before anything is awaited: a host may SIGKILL before the settle
             // sequence below completes (codex: ~185 ms after SIGTERM).
             ragKgManager.checkpointWal();
+            ragKgManager.noteWrites();
             void (async () => {
                 try {
                     await server.close();

package/dist/src/backfillCoordinator.d.ts

@@ -6,12 +6,16 @@ export interface CoverageSnapshot {
         embedded: number;
         verified: number;
         legacy_assumed: number;
+        orphan_vectors?: number;
+        error?: string;
     };
     entity: {
         total: number;
         embedded: number;
         verified: number;
         legacy_assumed: number;
+        orphan_vectors?: number;
+        error?: string;
     };
 }
 export interface CoordinatorDeps {
@@ -37,17 +41,25 @@ export declare class BackfillCoordinator {
     private shuttingDown;
     private connLossReruns;
     private scanning;
+    private repairWanted;
+    readonly slotErrors: Map<string, string>;
+    repairRequestedAt: number;
+    requestRepair(): void;
+    takeRepairRequest(): boolean;
+    get repairRequested(): boolean;
     private scanPromise;
     private rerunRequested;
     private snapshot;
     constructor(deps: CoordinatorDeps);
     get reconState(): ReconState;
+    get busy(): boolean;
     get reconLastError(): string | undefined;
     get eligible(): boolean;
     private countNullWithVector;
     private countRepairables;
     private sanitize;
     runReconciliation(): Promise<void>;
+    private reconActive;
     private reconcileOnce;
     private reconcileEntities;
     private reconcileChunks;