renor-engine 0.0.0-stage → 1.0.0

package/dist/db/client.js

@@ -0,0 +1,147 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.openDb = openDb;
+exports.openIndexDb = openIndexDb;
+exports.openReadOnlyDb = openReadOnlyDb;
+const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const migrations_1 = require("./migrations");
+const indexSalvage_1 = require("./indexSalvage");
+/**
+ * Map up to this much of the database file into memory for reads (`PRAGMA mmap_size`). SQLite maps only
+ * what exists, so a small database is untouched; the ceiling is address space, not memory.
+ *
+ * Why: a sqlite-vec nearest-neighbour query reads every vector chunk, each a multi-megabyte BLOB, and
+ * SQLite reads a BLOB's overflow pages straight from the file, one pread() per 4 KB page, bypassing its
+ * page cache (so a bigger cache_size does not help). On a 10,000-note workspace that was ~60% kernel time
+ * per query. Reading through the mapping turns those syscalls into copies and cut the scan's CPU by about
+ * a third (scripts/bench/mcp-latency.ts). Only reads change: writes still go through write(), and answers
+ * are identical.
+ *
+ * The one thing a mapped read cannot survive is the file shrinking under it. The engine never VACUUMs or
+ * truncates its database, and the data-deletion runbook stops the service before touching the files.
+ * Not applied to `openReadOnlyDb`, which inspects or imports from files this process does not own.
+ */
+const MMAP_SIZE_BYTES = 1024 * 1024 * 1024;
+function openDb(dbPath) {
+    const onDisk = dbPath !== ':memory:' && dbPath !== '';
+    if (onDisk) {
+        // The index holds every remembered statement in plain text. A new one is created owner-only whatever the
+        // umask; SQLite gives its -wal and -shm files the main file's mode. An existing file keeps its mode.
+        fs_1.default.mkdirSync(path_1.default.dirname(dbPath), { recursive: true, mode: 0o700 });
+        fs_1.default.closeSync(fs_1.default.openSync(dbPath, fs_1.default.constants.O_CREAT | fs_1.default.constants.O_WRONLY, 0o600));
+    }
+    const db = new better_sqlite3_1.default(dbPath);
+    try {
+        db.pragma('journal_mode = WAL');
+        db.pragma('foreign_keys = ON');
+        db.pragma(`mmap_size = ${MMAP_SIZE_BYTES}`);
+    }
+    catch (err) {
+        // A file that is not a database fails on its first pragma; do not leave its handle open for the caller that never gets it.
+        db.close();
+        throw err;
+    }
+    return db;
+}
+/**
+ * Opens the derived index, and rebuilds it when it is damaged. The index is rebuilt from the ledger and the Markdown
+ * (the CLI banner says so), so a damaged `index.db` is not a reason to refuse to start: it is moved aside, with its
+ * `-wal` and `-shm`, to `index.db.corrupt-<timestamp>` (kept for forensics) and the caller gets a fresh index, exactly
+ * as for a missing one. Only corruption is moved aside: an open that fails for another reason (permissions, I/O) throws.
+ * The move needs the ledger to rebuild from, so with no ledger a damaged index throws rather than being set aside.
+ * The ledger itself is never touched here.
+ */
+function openIndexDb(dbPath, ledgerPath) {
+    let damage;
+    try {
+        const db = openDb(dbPath);
+        let verdict;
+        try {
+            verdict = db.pragma('quick_check', { simple: true });
+        }
+        catch (err) {
+            db.close();
+            throw err;
+        }
+        if (verdict === 'ok')
+            return db;
+        db.close();
+        damage = String(verdict);
+    }
+    catch (err) {
+        if (!isCorruptionError(err))
+            throw err;
+        damage = err.message;
+    }
+    if (!fs_1.default.existsSync(ledgerPath)) {
+        throw new Error(`index ${dbPath} is damaged (${damage}) and there is no ledger to rebuild it from; it was not moved aside. Restore the ledger, or move the index aside by hand.`);
+    }
+    // Read what the damaged file still gives before it is moved: the pending requests and cursors are not in the ledger.
+    const salvage = readDamagedIndex(dbPath);
+    const aside = setIndexAside(dbPath);
+    const db = openDb(dbPath);
+    (0, migrations_1.runMigrations)(db);
+    const { restored, rejected } = (0, indexSalvage_1.restoreIndexTables)(db, salvage);
+    console.error(`[index] ${dbPath} is damaged (${damage}); moved it and its -wal and -shm files to ${aside} and rebuilt it from the ledger. ${(0, indexSalvage_1.describeSalvage)(salvage, restored, rejected)}`);
+    return db;
+}
+/** The rows the ledger does not hold, read from the damaged file, read-only (openReadOnlyDb: the file is not written). */
+function readDamagedIndex(dbPath) {
+    let db;
+    try {
+        db = openReadOnlyDb(dbPath);
+    }
+    catch (err) {
+        return (0, indexSalvage_1.unreadableSalvage)(err instanceof Error ? err.message : String(err));
+    }
+    try {
+        return (0, indexSalvage_1.salvageIndexTables)(db);
+    }
+    finally {
+        db.close();
+    }
+}
+/** A SQLite error that says the file is damaged, as opposed to unreadable for some other reason. */
+function isCorruptionError(err) {
+    const code = err.code;
+    return typeof code === 'string' && (code === 'SQLITE_NOTADB' || code === 'SQLITE_CORRUPT' || code.startsWith('SQLITE_CORRUPT_'));
+}
+/**
+ * Renames the index's side files first and the main file last: if this is interrupted, the damaged main file is still
+ * there to be found again on the next start, and no WAL is ever left beside a fresh database it does not belong to.
+ */
+function setIndexAside(dbPath) {
+    const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+    let aside = `${dbPath}.corrupt-${stamp}`;
+    for (let n = 2; ['', '-wal', '-shm'].some(side => fs_1.default.existsSync(aside + side)); n++)
+        aside = `${dbPath}.corrupt-${stamp}-${n}`;
+    for (const side of ['-wal', '-shm', '']) {
+        if (fs_1.default.existsSync(dbPath + side))
+            fs_1.default.renameSync(dbPath + side, aside + side);
+    }
+    return aside;
+}
+/**
+ * Read-only open of a SQLite database by path — a product index being inspected, or a
+ * source index an import reads from.
+ *
+ * Read-only is enforced by SQLite itself rather than by convention, so a read can never
+ * write to the store it is reading. That matters for the WAL side files too: a
+ * write-capable open of a WAL database can create `-wal`/`-shm` and checkpoint back into
+ * the main file. `journal_mode` is deliberately NOT set here, because setting it is a
+ * write. Product code must reach product SQLite through this module
+ * (`tests/eval/product-path-contract.test.ts` guards the `openDb` boundary).
+ */
+function openReadOnlyDb(dbPath) {
+    if (!fs_1.default.existsSync(dbPath)) {
+        throw new Error(`database not found: ${dbPath}`);
+    }
+    const db = new better_sqlite3_1.default(dbPath, { readonly: true, fileMustExist: true });
+    db.pragma('foreign_keys = ON');
+    return db;
+}

package/dist/db/indexSalvage.js

@@ -0,0 +1,81 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.INDEX_ONLY_TABLES = void 0;
+exports.unreadableSalvage = unreadableSalvage;
+exports.salvageIndexTables = salvageIndexTables;
+exports.restoreIndexTables = restoreIndexTables;
+exports.describeSalvage = describeSalvage;
+/**
+ * The index is rebuilt from the ledger, but a few of its tables hold state that no ledger event rebuilds: the pending
+ * agent requests, the what's-new cursors, and the connection names the workspace has seen (known_agents). When the
+ * index is damaged, those rows are read out of the damaged file, read-only, and written into the rebuilt one.
+ */
+exports.INDEX_ONLY_TABLES = ['agent_requests', 'agent_cursors', 'known_agents'];
+/** What a damaged index gives when it cannot be opened at all: every table is unreadable, for the same reason. */
+function unreadableSalvage(reason) {
+    const salvage = { rows: {}, unreadable: {} };
+    for (const table of exports.INDEX_ONLY_TABLES) {
+        salvage.rows[table] = [];
+        salvage.unreadable[table] = reason;
+    }
+    return salvage;
+}
+/** Reads what an open (read-only) damaged index still gives. Never throws: a table that does not read says why. */
+function salvageIndexTables(db) {
+    const salvage = { rows: {}, unreadable: {} };
+    for (const table of exports.INDEX_ONLY_TABLES) {
+        const rows = [];
+        salvage.rows[table] = rows;
+        try {
+            for (const row of db.prepare(`SELECT rowid AS __rowid, * FROM "${table}"`).iterate()) {
+                rows.push(row);
+            }
+        }
+        catch (err) {
+            // An index from before this table existed has nothing to salvage here; that is not a loss.
+            if (!/no such table/.test(errorText(err)))
+                salvage.unreadable[table] = errorText(err);
+        }
+    }
+    return salvage;
+}
+/** Writes the salvaged rows into a freshly migrated index. A row that will not go in is counted, never skipped silently. */
+function restoreIndexTables(db, salvage) {
+    const restored = {};
+    const rejected = {};
+    for (const table of exports.INDEX_ONLY_TABLES) {
+        restored[table] = 0;
+        rejected[table] = 0;
+        for (const row of salvage.rows[table] ?? []) {
+            const { __rowid, ...values } = row;
+            const cols = ['rowid', ...Object.keys(values)];
+            const sql = `INSERT OR IGNORE INTO "${table}" (${cols.map(c => `"${c}"`).join(', ')}) VALUES (${cols.map(() => '?').join(', ')})`;
+            try {
+                const info = db.prepare(sql).run(__rowid, ...Object.keys(values).map(c => values[c]));
+                restored[table] += info.changes;
+            }
+            catch {
+                rejected[table] += 1;
+            }
+        }
+    }
+    return { restored, rejected };
+}
+/** One sentence for the log: what came back from the damaged index, and what did not. */
+function describeSalvage(salvage, restored, rejected) {
+    const kept = exports.INDEX_ONLY_TABLES.map(t => `${t} ${restored[t] ?? 0}`).join(', ');
+    const lost = exports.INDEX_ONLY_TABLES
+        .map(t => {
+        const parts = [];
+        if (rejected[t])
+            parts.push(`${rejected[t]} row(s) could not be written`);
+        if (salvage.unreadable[t])
+            parts.push(`the damaged file could not be read (${salvage.unreadable[t]}), so rows past the damage are gone and not counted`);
+        return parts.length ? `${t}: ${parts.join('; ')}` : '';
+    })
+        .filter(Boolean);
+    return `Kept from the damaged index: ${kept}.${lost.length ? ` NOT RECOVERED: ${lost.join('. ')}.` : ''}`;
+}
+function errorText(err) {
+    return err instanceof Error ? err.message : String(err);
+}