xtctx 0.11.1 → 0.11.4

package/dist/src/handoff/sqlite-index.js

@@ -1,24 +1,37 @@
 import { createHash } from "node:crypto";
-import { mkdir } from "node:fs/promises";
-import { dirname } from "node:path";
+import { mkdir, readdir, rm } from "node:fs/promises";
+import { dirname, join } from "node:path";
 import Database from "better-sqlite3";
-import { DEFAULT_EMBEDDING_MODEL, TransformersEmbeddingProvider, } from "./embeddings.js";
+import { DEFAULT_EMBEDDING_MODEL, TransformersEmbeddingProvider, poolVectors, splitTextForEmbedding, } from "./embeddings.js";
 import { cosineSimilarity, deserializeVector, serializeVector } from "./vector.js";
+/**
+ * Bumped whenever the schema shape changes. The index is derived data, so a
+ * version mismatch (older or newer) triggers a full rebuild rather than a
+ * migration — the transcript stores remain authoritative.
+ */
+const SCHEMA_VERSION = 1;
 const DEFAULT_LIMIT = 5;
 const MAX_LIMIT = 100;
 const DEFAULT_WINDOW_SIZE = 8;
 const DEFAULT_WINDOW_STRIDE = 4;
 const MAX_MATCHES_PER_SESSION = 3;
+/**
+ * Minimum raw cosine similarity for a retrieval window to count as a semantic
+ * match. Unrelated sentence-transformer pairs sit near 0; related ones are
+ * comfortably above this.
+ */
+const MIN_SEMANTIC_COSINE = 0.15;
 const SOURCE_CURSOR_OVERLAP_MS = 1_000;
 export class SqliteHandoffIndex {
     dbPath;
     projectRoot;
     tools;
     db = null;
+    stmts = null;
     initialized;
     refreshPromise = null;
     lastRefreshMs = 0;
-    refreshTtlMs = 1_000;
+    refreshTtlMs = 5_000;
     embeddingProvider;
     windowSize;
     windowStride;
@@ -31,6 +44,10 @@ export class SqliteHandoffIndex {
         this.windowSize = Math.max(2, Math.floor(options.windowSize ?? DEFAULT_WINDOW_SIZE));
         this.windowStride = Math.max(1, Math.floor(options.windowStride ?? DEFAULT_WINDOW_STRIDE));
         this.initialized = this.initialize();
+        // Attach a no-op handler so a failed open cannot become an unhandled
+        // rejection (which would kill the process) before the first caller
+        // awaits; each awaiter of `initialized` still observes the rejection.
+        this.initialized.catch(() => { });
     }
     async listRecentSessions(limit, toolFilter) {
         await this.refresh({ toolFilter });
@@ -42,11 +59,21 @@ export class SqliteHandoffIndex {
             .prepare(`SELECT session_ref, tool, started_at, last_activity_at, message_count, preview, source_path
          FROM sessions
          ${where}
-         ORDER BY datetime(last_activity_at) DESC
+         ORDER BY last_activity_at DESC
          LIMIT ?`)
             .all(...filters, normalizedLimit);
         return rows.map(formatSessionRow);
     }
+    async getSessionByRef(sessionRef) {
+        await this.refresh({ sessionRef });
+        const db = this.getDb();
+        const row = db
+            .prepare(`SELECT session_ref, tool, started_at, last_activity_at, message_count, preview, source_path
+         FROM sessions
+         WHERE session_ref = ?`)
+            .get(sessionRef);
+        return row ? formatSessionRow(row) : null;
+    }
     async getSessionDetail(sessionRef, offset, limit) {
         await this.refresh({ sessionRef });
         const db = this.getDb();
@@ -56,7 +83,7 @@ export class SqliteHandoffIndex {
             .prepare(`SELECT id, timestamp, role, content, message_index, source_pointer
          FROM messages
          WHERE session_ref = ?
-         ORDER BY datetime(timestamp) ASC, message_index ASC, id ASC
+         ORDER BY timestamp ASC, message_index ASC, id ASC
          LIMIT ? OFFSET ?`)
             .all(sessionRef, normalizedLimit, normalizedOffset);
         return rows.map((row) => ({
@@ -77,10 +104,18 @@ export class SqliteHandoffIndex {
             return this.keywordSearch(trimmed, limit, toolFilter);
         }
         try {
-            return await this.semanticSearch(trimmed, limit, toolFilter, normalizedMode);
+            const results = await this.semanticSearch(trimmed, limit, toolFilter, normalizedMode);
+            clearSetting(this.getDb(), "last_error:embeddings");
+            return results;
         }
         catch (error) {
             if (normalizedMode === "hybrid") {
+                // Degrading to keyword keeps search useful, but a broken embedding
+                // path must not be invisible: it silently returned keyword-only
+                // results for every user while status still read healthy.
+                const message = error instanceof Error ? error.message : String(error);
+                setSetting(this.getDb(), "last_error:embeddings", message);
+                process.stderr.write(`xtctx: semantic search unavailable, using keyword only (${message})\n`);
                 return this.keywordSearch(trimmed, limit, toolFilter);
             }
             throw error;
@@ -113,6 +148,7 @@ export class SqliteHandoffIndex {
                 indexed_sessions: indexed?.sessions ?? 0,
                 indexed_messages: indexed?.messages ?? 0,
                 last_indexed_at: indexed?.last_indexed_at ?? null,
+                last_error: getSetting(db, `last_error:${tool}`),
             };
         }));
         return {
@@ -124,11 +160,12 @@ export class SqliteHandoffIndex {
             retrieval_units: retrievalUnitCount,
             vectorized_units: vectorizedUnitCount,
             vector_model: this.embeddingProvider.model,
+            embedding_error: getSetting(db, "last_error:embeddings"),
             tools,
         };
     }
     async close() {
-        await this.initialized;
+        await this.initialized.catch(() => { });
         this.db?.close();
         this.db = null;
     }
@@ -142,6 +179,9 @@ export class SqliteHandoffIndex {
         }
         if (!this.refreshPromise) {
             this.refreshPromise = this.refreshNow().finally(() => {
+                // Stamp the TTL on failure as well as success so a persistently
+                // broken refresh backs off instead of re-running on every call.
+                this.lastRefreshMs = Date.now();
                 this.refreshPromise = null;
             });
         }
@@ -171,32 +211,29 @@ export class SqliteHandoffIndex {
                         lastTimestamp: overlapTimestamp(latestTimestamp),
                     });
                 }
+                clearSetting(db, `last_error:${scraper.tool}`);
             }
             catch (error) {
                 setSetting(db, `last_error:${scraper.tool}`, error instanceof Error ? error.message : String(error));
-                if (latestTimestamp) {
-                    try {
-                        await scraper.saveScrapedPosition({
-                            lastTimestamp: overlapTimestamp(latestTimestamp),
-                        });
-                    }
-                    catch {
-                        // Ignore position save failures inside catch
-                    }
-                }
+                // Deliberately do NOT advance the cursor here: chunks yielded before
+                // the failure may sort after content in files never reached, and
+                // advancing would skip that content permanently. Re-scraping the
+                // same window is safe (message ids are deterministic hashes).
             }
         }
         for (const sessionRef of touchedSessions) {
+            // Roll up message_count/preview once per touched session rather than
+            // once per inserted message (which made indexing O(N²) per session).
+            this.prepared().sessionRollup.run(sessionRef);
             this.rebuildRetrievalUnitsForSession(sessionRef);
         }
         setSetting(db, "last_scan_at", startedAt);
-        this.lastRefreshMs = Date.now();
     }
     upsertChunk(chunk) {
         if (!chunk.content.trim()) {
             return null;
         }
-        const db = this.getDb();
+        const stmts = this.prepared();
         const timestamp = chunk.timestamp.toISOString();
         const sessionRef = `${chunk.tool}:${chunk.sessionId}`;
         const messageIndex = chunk.metadata.messageIndex ?? 0;
@@ -212,68 +249,52 @@ export class SqliteHandoffIndex {
         const now = new Date().toISOString();
         const metadataJson = JSON.stringify(chunk.metadata ?? {});
         const sourcePointer = sourcePathFromMetadata(chunk.metadata);
-        db.prepare(`INSERT INTO sessions
-       (session_ref, tool, source_session_id, project_root, started_at, last_activity_at,
-        message_count, preview, source_path, updated_at)
-       VALUES (?, ?, ?, ?, ?, ?, 0, NULL, ?, ?)
-       ON CONFLICT(session_ref) DO UPDATE SET
-         started_at = CASE
-           WHEN excluded.started_at < started_at THEN excluded.started_at
-           ELSE started_at
-         END,
-         last_activity_at = CASE
-           WHEN excluded.last_activity_at > last_activity_at THEN excluded.last_activity_at
-           ELSE last_activity_at
-         END,
-         source_path = COALESCE(source_path, excluded.source_path),
-         updated_at = excluded.updated_at`).run(sessionRef, chunk.tool, chunk.sessionId, this.projectRoot, timestamp, timestamp, sourcePointer, now);
-        db.prepare(`INSERT OR IGNORE INTO messages
-       (id, session_ref, tool, source_session_id, timestamp, role, content,
-        message_index, content_hash, metadata_json, source_pointer, indexed_at)
-       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(id, sessionRef, chunk.tool, chunk.sessionId, timestamp, chunk.role, chunk.content, messageIndex, contentHash, metadataJson, sourcePointer, now);
-        db.prepare(`INSERT OR IGNORE INTO messages_fts(rowid, session_ref, tool, role, timestamp, content)
-       SELECT rowid, session_ref, tool, role, timestamp, content
-       FROM messages
-       WHERE id = ?`).run(id);
-        db.prepare(`UPDATE sessions
-       SET message_count = (
-             SELECT COUNT(*) FROM messages WHERE messages.session_ref = sessions.session_ref
-           ),
-           preview = COALESCE(
-             (
-               SELECT substr(content, 1, 240)
-               FROM messages
-               WHERE messages.session_ref = sessions.session_ref
-               ORDER BY datetime(timestamp) ASC, message_index ASC, id ASC
-               LIMIT 1
-             ),
-             preview
-           )
-       WHERE session_ref = ?`).run(sessionRef);
+        stmts.upsertChunkTxn([
+            sessionRef,
+            chunk.tool,
+            chunk.sessionId,
+            this.projectRoot,
+            timestamp,
+            timestamp,
+            sourcePointer,
+            now,
+        ], [
+            id,
+            sessionRef,
+            chunk.tool,
+            chunk.sessionId,
+            timestamp,
+            chunk.role,
+            chunk.content,
+            messageIndex,
+            contentHash,
+            metadataJson,
+            sourcePointer,
+            now,
+        ]);
         return sessionRef;
     }
     rebuildRetrievalUnitsForSession(sessionRef) {
         const db = this.getDb();
-        const messages = db
-            .prepare(`SELECT id, timestamp, role, content, message_index, source_pointer
-         FROM messages
-         WHERE session_ref = ?
-         ORDER BY datetime(timestamp) ASC, message_index ASC, id ASC`)
-            .all(sessionRef);
-        db.prepare("DELETE FROM retrieval_units_fts WHERE session_ref = ?").run(sessionRef);
-        db.prepare("DELETE FROM retrieval_units WHERE session_ref = ?").run(sessionRef);
+        const stmts = this.prepared();
+        const messages = stmts.selectSessionMessages.all(sessionRef);
         if (messages.length === 0) {
+            db.prepare("DELETE FROM retrieval_units_fts WHERE session_ref = ?").run(sessionRef);
+            db.prepare("DELETE FROM retrieval_units WHERE session_ref = ?").run(sessionRef);
             return;
         }
-        const session = db
-            .prepare("SELECT tool FROM sessions WHERE session_ref = ?")
-            .get(sessionRef);
+        const session = stmts.selectSessionTool.get(sessionRef);
         if (!session) {
             return;
         }
+        // Diff the desired windows against what is stored instead of deleting
+        // everything: unit ids are deterministic content hashes, so unchanged
+        // windows (and, via the FK, their vectors) survive a re-index untouched.
         const now = new Date().toISOString();
+        const desired = new Map();
         for (const window of buildMessageWindows(messages, this.windowSize, this.windowStride)) {
             const content = formatRetrievalUnitContent(sessionRef, window.messages);
+            const searchableText = window.messages.map((message) => message.content).join("\n");
             const contentHash = hashParts([content]);
             const unitId = hashParts([
                 "retrieval-unit",
@@ -282,17 +303,104 @@ export class SqliteHandoffIndex {
                 String(window.end.message_index),
                 contentHash,
             ]);
-            db.prepare(`INSERT INTO retrieval_units
+            desired.set(unitId, {
+                start: window.start,
+                end: window.end,
+                content,
+                searchableText,
+                contentHash,
+            });
+        }
+        const existing = new Set(stmts.selectUnitIds.all(sessionRef).map((row) => row.id));
+        const applyDiff = db.transaction(() => {
+            for (const unitId of existing) {
+                if (!desired.has(unitId)) {
+                    stmts.deleteUnitFts.run(unitId);
+                    stmts.deleteUnit.run(unitId);
+                }
+            }
+            for (const [unitId, unit] of desired) {
+                if (existing.has(unitId)) {
+                    continue;
+                }
+                stmts.insertUnit.run(unitId, sessionRef, session.tool, unit.start.message_index, unit.end.message_index, unit.start.timestamp, unit.end.timestamp, unit.content, unit.contentHash, now);
+                // Only the transcript text is keyword-indexed. `unit.content` also
+                // carries the window scaffolding ("Session: …", "Turn 1/8 |
+                // message_index=0 | user @ …") that gives the embedding model
+                // ordering context; indexing it made `message_index` or a tool name
+                // match every session.
+                stmts.insertUnitFts.run(unitId, sessionRef, session.tool, unit.searchableText);
+            }
+        });
+        applyDiff();
+    }
+    prepared() {
+        if (this.stmts) {
+            return this.stmts;
+        }
+        const db = this.getDb();
+        const upsertSession = db.prepare(`INSERT INTO sessions
+       (session_ref, tool, source_session_id, project_root, started_at, last_activity_at,
+        message_count, preview, source_path, updated_at)
+       VALUES (?, ?, ?, ?, ?, ?, 0, NULL, ?, ?)
+       ON CONFLICT(session_ref) DO UPDATE SET
+         started_at = CASE
+           WHEN excluded.started_at < started_at THEN excluded.started_at
+           ELSE started_at
+         END,
+         last_activity_at = CASE
+           WHEN excluded.last_activity_at > last_activity_at THEN excluded.last_activity_at
+           ELSE last_activity_at
+         END,
+         source_path = COALESCE(source_path, excluded.source_path),
+         updated_at = excluded.updated_at`);
+        const insertMessage = db.prepare(`INSERT OR IGNORE INTO messages
+       (id, session_ref, tool, source_session_id, timestamp, role, content,
+        message_index, content_hash, metadata_json, source_pointer, indexed_at)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
+        this.stmts = {
+            upsertSession,
+            insertMessage,
+            upsertChunkTxn: db.transaction((sessionArgs, messageArgs) => {
+                upsertSession.run(...sessionArgs);
+                insertMessage.run(...messageArgs);
+            }),
+            sessionRollup: db.prepare(`UPDATE sessions
+         SET message_count = (
+               SELECT COUNT(*) FROM messages WHERE messages.session_ref = sessions.session_ref
+             ),
+             preview = COALESCE(
+               (
+                 SELECT substr(content, 1, 240)
+                 FROM messages
+                 WHERE messages.session_ref = sessions.session_ref
+                 ORDER BY timestamp ASC, message_index ASC, id ASC
+                 LIMIT 1
+               ),
+               preview
+             )
+         WHERE session_ref = ?`),
+            selectSessionMessages: db.prepare(`SELECT id, timestamp, role, content, message_index, source_pointer
+         FROM messages
+         WHERE session_ref = ?
+         ORDER BY timestamp ASC, message_index ASC, id ASC`),
+            selectSessionTool: db.prepare("SELECT tool FROM sessions WHERE session_ref = ?"),
+            selectUnitIds: db.prepare("SELECT id FROM retrieval_units WHERE session_ref = ?"),
+            insertUnit: db.prepare(`INSERT INTO retrieval_units
          (id, session_ref, tool, message_start_index, message_end_index,
           started_at, ended_at, content, content_hash, updated_at)
-         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(unitId, sessionRef, session.tool, window.start.message_index, window.end.message_index, window.start.timestamp, window.end.timestamp, content, contentHash, now);
-            db.prepare(`INSERT INTO retrieval_units_fts(unit_id, session_ref, tool, content)
-         VALUES (?, ?, ?, ?)`).run(unitId, sessionRef, session.tool, content);
-        }
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`),
+            insertUnitFts: db.prepare(`INSERT INTO retrieval_units_fts(unit_id, session_ref, tool, content)
+         VALUES (?, ?, ?, ?)`),
+            deleteUnit: db.prepare("DELETE FROM retrieval_units WHERE id = ?"),
+            deleteUnitFts: db.prepare("DELETE FROM retrieval_units_fts WHERE unit_id = ?"),
+        };
+        return this.stmts;
     }
     async keywordSearch(query, limit, toolFilter) {
         const rows = this.queryKeywordUnits(query, limit, toolFilter);
-        return groupUnits(rows, new Map(), "keyword", normalizeLimit(limit, DEFAULT_LIMIT));
+        // Rank by BM25 position so relevance, not recency, dominates ordering.
+        return groupUnits(rows, rankKeywordRows(rows), "keyword", normalizeLimit(limit, DEFAULT_LIMIT));
     }
     async semanticSearch(query, limit, toolFilter, mode) {
         const normalizedLimit = normalizeLimit(limit, DEFAULT_LIMIT);
@@ -318,12 +426,14 @@ export class SqliteHandoffIndex {
         const timeRange = getTimeRange(rows.map((row) => row.ended_at));
         const scored = rows
             .map((row) => {
-            const semanticScore = normalizeCosine(cosineSimilarity(queryVector, deserializeVector(row.vector, row.dimensions)));
+            const rawCosine = cosineSimilarity(queryVector, deserializeVector(row.vector, row.dimensions));
+            const semanticScore = normalizeCosine(rawCosine);
             const keywordScore = keywordScores.get(row.unit_id) ?? 0;
             const recencyScore = scoreRecency(row.ended_at, timeRange);
             const continuityScore = scoreContinuity(row.message_end_index, row.session_message_count);
             return {
                 row,
+                rawCosine,
                 score: blendScores(mode, semanticScore, keywordScore, recencyScore, continuityScore),
                 semanticScore,
                 keywordScore,
@@ -331,6 +441,12 @@ export class SqliteHandoffIndex {
                 continuityScore,
             };
         })
+            // Require actual evidence. Cosine normalises to ~0.5 for unrelated
+            // content, so recency and continuity alone were enough to return the
+            // entire corpus for a query matching nothing — formatted exactly like a
+            // real hit. A unit qualifies on semantic similarity or a keyword match;
+            // "no matching sessions" is a more useful answer than a nearest vector.
+            .filter((item) => item.rawCosine >= MIN_SEMANTIC_COSINE || item.keywordScore > 0)
             .sort((left, right) => right.score - left.score);
         return groupScoredUnits(scored, mode, normalizedLimit);
     }
@@ -349,7 +465,7 @@ export class SqliteHandoffIndex {
          JOIN retrieval_units u ON u.id = f.unit_id
          JOIN sessions s ON s.session_ref = u.session_ref
          WHERE retrieval_units_fts MATCH ? ${toolWhere}
-         ORDER BY bm25(retrieval_units_fts), datetime(u.ended_at) DESC
+         ORDER BY bm25(retrieval_units_fts), u.ended_at DESC
          LIMIT ?`)
             .all(ftsQuery, ...filters, normalizedLimit * MAX_MATCHES_PER_SESSION);
     }
@@ -365,36 +481,87 @@ export class SqliteHandoffIndex {
           AND v.model = ?
           AND v.content_hash = u.content_hash
          WHERE v.unit_id IS NULL ${toolWhere}
-         ORDER BY datetime(u.ended_at) DESC`)
+         ORDER BY u.ended_at DESC`)
             .all(this.embeddingProvider.model, ...filters);
         if (rows.length === 0) {
             return;
         }
-        const vectors = await this.embeddingProvider.embedBatch(rows.map((row) => row.content));
-        const now = new Date().toISOString();
         const upsert = db.prepare(`INSERT INTO retrieval_unit_vectors
        (unit_id, model, dimensions, content_hash, vector, created_at)
        VALUES (?, ?, ?, ?, ?, ?)
-       ON CONFLICT(unit_id) DO UPDATE SET
-         model = excluded.model,
+       ON CONFLICT(unit_id, model) DO UPDATE SET
          dimensions = excluded.dimensions,
          content_hash = excluded.content_hash,
          vector = excluded.vector,
          created_at = excluded.created_at`);
-        const transaction = db.transaction(() => {
-            rows.forEach((row, index) => {
-                const vector = vectors[index];
-                upsert.run(row.unit_id, this.embeddingProvider.model, vector.length, row.content_hash, serializeVector(vector), now);
+        // Bounded batches keep memory flat on a first-time index of a large
+        // history, and each batch commits before the next one embeds.
+        const unitBatchSize = 64;
+        for (let start = 0; start < rows.length; start += unitBatchSize) {
+            const batch = rows.slice(start, start + unitBatchSize);
+            // Long windows are segmented to the model's sequence budget and
+            // mean-pooled, so content beyond the window's opening still shapes
+            // the unit's vector.
+            const segmented = batch.map((row) => splitTextForEmbedding(row.content));
+            const segmentVectors = await this.embeddingProvider.embedBatch(segmented.flat());
+            let cursor = 0;
+            const pooled = segmented.map((segments) => {
+                const slice = segmentVectors.slice(cursor, cursor + segments.length);
+                cursor += segments.length;
+                return poolVectors(slice);
             });
-        });
-        transaction();
+            const now = new Date().toISOString();
+            const transaction = db.transaction(() => {
+                batch.forEach((row, index) => {
+                    const vector = pooled[index];
+                    upsert.run(row.unit_id, this.embeddingProvider.model, vector.length, row.content_hash, serializeVector(vector), now);
+                });
+            });
+            transaction();
+        }
     }
     async initialize() {
         await mkdir(dirname(this.dbPath), { recursive: true });
-        this.db = new Database(this.dbPath);
-        this.db.pragma("journal_mode = WAL");
-        this.db.pragma("foreign_keys = ON");
-        createSchema(this.db);
+        try {
+            this.db = openDatabase(this.dbPath);
+        }
+        catch {
+            // The index is derived data: a corrupt or schema-incompatible database
+            // is discarded and rebuilt from the transcript stores on next refresh.
+            await this.deleteDatabaseFiles();
+            this.db = openDatabase(this.dbPath);
+        }
+        // One rule covers every way the index can end up empty — deleted by a
+        // user (the recovery the docs invite), rebuilt after corruption, or
+        // wiped by hand: an empty index cannot honour a scraper cursor, because
+        // the cursor would skip straight past history still sitting on disk.
+        // On a genuine first run there are no cursor files, so this is a no-op.
+        const sessionCount = this.db.prepare("SELECT COUNT(*) AS count FROM sessions").get().count;
+        if (sessionCount === 0) {
+            await this.clearScraperCursors();
+        }
+    }
+    async deleteDatabaseFiles() {
+        for (const suffix of ["", "-wal", "-shm"]) {
+            await rm(`${this.dbPath}${suffix}`, { force: true });
+        }
+    }
+    /**
+     * Drop every scraper's saved position so the next refresh re-scans from the
+     * beginning. Called whenever the index is empty; see initialize().
+     */
+    async clearScraperCursors() {
+        const stateDir = dirname(this.dbPath);
+        try {
+            for (const entry of await readdir(stateDir)) {
+                if (entry.endsWith("-state.json")) {
+                    await rm(join(stateDir, entry), { force: true });
+                }
+            }
+        }
+        catch {
+            // No state directory yet: nothing to reset.
+        }
     }
     getDb() {
         if (!this.db) {
@@ -403,6 +570,25 @@ export class SqliteHandoffIndex {
         return this.db;
     }
 }
+function openDatabase(dbPath) {
+    const db = new Database(dbPath);
+    try {
+        db.pragma("journal_mode = WAL");
+        db.pragma("foreign_keys = ON");
+        const objectCount = db.prepare("SELECT COUNT(*) AS count FROM sqlite_master").get().count;
+        const version = db.pragma("user_version", { simple: true });
+        if (objectCount > 0 && version !== SCHEMA_VERSION) {
+            throw new Error(`xtctx index schema version ${version} does not match supported version ${SCHEMA_VERSION}`);
+        }
+        createSchema(db);
+        db.pragma(`user_version = ${SCHEMA_VERSION}`);
+        return db;
+    }
+    catch (error) {
+        db.close();
+        throw error;
+    }
+}
 function createSchema(db) {
     db.exec(`
     CREATE TABLE IF NOT EXISTS sessions (
@@ -437,9 +623,6 @@ function createSchema(db) {
     CREATE INDEX IF NOT EXISTS idx_messages_session_order
       ON messages(session_ref, timestamp, message_index, id);
 
-    CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts
-      USING fts5(session_ref UNINDEXED, tool UNINDEXED, role UNINDEXED, timestamp UNINDEXED, content);
-
     CREATE TABLE IF NOT EXISTS retrieval_units (
       id TEXT PRIMARY KEY,
       session_ref TEXT NOT NULL REFERENCES sessions(session_ref) ON DELETE CASCADE,
@@ -462,12 +645,13 @@ function createSchema(db) {
       USING fts5(unit_id UNINDEXED, session_ref UNINDEXED, tool UNINDEXED, content);
 
     CREATE TABLE IF NOT EXISTS retrieval_unit_vectors (
-      unit_id TEXT PRIMARY KEY REFERENCES retrieval_units(id) ON DELETE CASCADE,
+      unit_id TEXT NOT NULL REFERENCES retrieval_units(id) ON DELETE CASCADE,
       model TEXT NOT NULL,
       dimensions INTEGER NOT NULL,
       content_hash TEXT NOT NULL,
       vector BLOB NOT NULL,
-      created_at TEXT NOT NULL
+      created_at TEXT NOT NULL,
+      PRIMARY KEY (unit_id, model)
     );
 
     CREATE INDEX IF NOT EXISTS idx_retrieval_unit_vectors_model
@@ -542,7 +726,7 @@ function formatRetrievalUnitContent(sessionRef, messages) {
 function groupUnits(rows, keywordScores, retrieval, limit) {
     const timeRange = getTimeRange(rows.map((row) => row.ended_at));
     const scored = rows.map((row) => {
-        const keywordScore = keywordScores.get(row.unit_id) ?? 1;
+        const keywordScore = keywordScores.get(row.unit_id) ?? 0;
         const recencyScore = scoreRecency(row.ended_at, timeRange);
         const continuityScore = scoreContinuity(row.message_end_index, row.session_message_count);
         return {
@@ -692,6 +876,9 @@ function setSetting(db, key, value) {
      VALUES (?, ?)
      ON CONFLICT(key) DO UPDATE SET value = excluded.value`).run(key, value);
 }
+function clearSetting(db, key) {
+    db.prepare("DELETE FROM settings WHERE key = ?").run(key);
+}
 function sourcePathFromMetadata(metadata) {
     const value = metadata.sourcePath;
     return typeof value === "string" && value.trim().length > 0 ? value : null;