renor-engine 0.0.0-stage → 1.0.0

package/dist/db/readCache.js

@@ -0,0 +1,109 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.tableHasColumns = tableHasColumns;
+exports.inReadSnapshot = inReadSnapshot;
+exports.databaseStamp = databaseStamp;
+exports.readUntilWrite = readUntilWrite;
+const statements_1 = require("./statements");
+/**
+ * Two things a read asks of the database again and again and that change rarely: the shape of a table, and a value
+ * derived from every row of one (the ledger's closed events). A `get_context` asked for each of them six times.
+ */
+const withColumns = new WeakMap();
+/**
+ * Whether `table` has every one of `columns` (the reads that predate a column ask, so an old index still answers).
+ * Only a yes is kept for the connection: a table that lacks a column may get it from a migration, and a column that
+ * is there is never dropped, so a no is asked again each time and a yes never is. The names are the code's own, never
+ * a caller's.
+ */
+function tableHasColumns(db, table, columns) {
+    const key = `${table}(${columns.join(',')})`;
+    let known = withColumns.get(db);
+    if (known?.has(key))
+        return true;
+    const present = new Set(db.prepare(`PRAGMA table_info(${table})`).all().map(c => c.name));
+    if (!columns.every(c => present.has(c)))
+        return false;
+    if (!known)
+        withColumns.set(db, known = new Map());
+    known.set(key, true);
+    return true;
+}
+const kept = new WeakMap();
+/** Values kept per connection. A key may carry what a caller sent (a scope), so the oldest goes when the room is used. */
+const MAX_KEPT = 64;
+/** The connections inside a snapshot (`inReadSnapshot`). */
+const snapshots = new WeakSet();
+/**
+ * What the database has been through: changes this connection committed (`total_changes()`, which counts every row an
+ * INSERT, UPDATE or DELETE touched, triggers and virtual tables included) and changes any other connection committed
+ * (`PRAGMA data_version`, which a connection's own writes do not move). Both only ever go up, so two equal stamps are
+ * the same data. Null inside a transaction: it can still be rolled back, and its changes would be counted for good. A
+ * snapshot is the exception: it is ended with a COMMIT and never rolled back, so what it has counted stays counted.
+ */
+function stampOf(db) {
+    if (db.inTransaction && !snapshots.has(db))
+        return null;
+    const version = (0, statements_1.prepareOnce)(db, 'PRAGMA data_version').pluck().get();
+    const changes = (0, statements_1.prepareOnce)(db, 'SELECT total_changes()').pluck().get();
+    return `${version}:${changes}`;
+}
+/**
+ * Runs `read` in ONE transaction. A statement that runs on its own begins a read transaction and ends it again, and in WAL
+ * mode that is two locks on the shared-memory index and a file-size call: about 5 microseconds for a lookup that takes 1.5,
+ * and a `get_context` runs a hundred of them. Inside a transaction the lock is taken once, and every statement after the first
+ * pays for what it reads. The reads see the database as it was when the first of them began; a commit by another process
+ * meanwhile is in the next read, not in a mix of the two.
+ *
+ * `read` is for a stretch of reads that holds no `await` and writes nothing: nothing else of this process can run in a
+ * stretch that does not yield, and a write inside it would not be durable until the transaction ended. When a transaction is
+ * open already (a write transaction, or a snapshot outside this one) `read` just runs in it. The stamps (`databaseStamp`,
+ * `readUntilWrite`) hold for a snapshot, which is committed and never rolled back.
+ */
+function inReadSnapshot(db, read) {
+    if (db.inTransaction)
+        return read();
+    (0, statements_1.prepareOnce)(db, 'BEGIN').run();
+    snapshots.add(db);
+    let done = false;
+    try {
+        const result = read();
+        done = true;
+        return result;
+    }
+    finally {
+        snapshots.delete(db);
+        try {
+            (0, statements_1.prepareOnce)(db, 'COMMIT').run();
+        }
+        catch (err) {
+            if (done)
+                throw err; // the read failed first: that is the error to report
+        }
+    }
+}
+/** The stamp `readUntilWrite` keeps a value under. Null inside a transaction, which can still roll back. */
+function databaseStamp(db) {
+    return stampOf(db);
+}
+/**
+ * `compute()` for the database as it is now, read once and kept until anything is written to it, by this connection or
+ * another. The same answer a fresh `compute()` gives: nothing is kept across a write, and nothing at all inside a
+ * transaction. `compute` must read only the database, and what it returns is shared: callers must not change it.
+ */
+function readUntilWrite(db, key, compute) {
+    const stamp = stampOf(db);
+    if (stamp === null)
+        return compute();
+    let byKey = kept.get(db);
+    if (!byKey)
+        kept.set(db, byKey = new Map());
+    const held = byKey.get(key);
+    if (held && held.stamp === stamp)
+        return held.value;
+    const value = compute();
+    if (!byKey.has(key) && byKey.size >= MAX_KEPT)
+        byKey.delete(byKey.keys().next().value);
+    byKey.set(key, { stamp, value });
+    return value;
+}

package/dist/db/schema.js

@@ -0,0 +1,5 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runMigrations = void 0;
+var migrations_1 = require("./migrations");
+Object.defineProperty(exports, "runMigrations", { enumerable: true, get: function () { return migrations_1.runMigrations; } });

package/dist/db/statements.js

@@ -0,0 +1,89 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.prepareOnce = prepareOnce;
+exports.listStatement = listStatement;
+exports.paddedList = paddedList;
+exports.transactionOnce = transactionOnce;
+/**
+ * One prepared statement per connection and SQL text, for the statements a read or a save runs again on every request.
+ *
+ * Preparing a statement costs a parse and a plan, and a `get_context` runs a few dozen small ones: preparing them again
+ * on every call was about 2 ms of its 20 (profiled with scripts/bench/mcp-latency.ts --cpu-profile). A prepared
+ * statement is reused as it is, and SQLite prepares it again by itself when the schema changes, so a migration that runs
+ * after the first use is seen. The result of a statement does not depend on how long ago it was prepared.
+ *
+ * What is shared is the statement, so what a caller leaves on it is shared too:
+ *  - Do not leave a mode on it (`.pluck()`, `.raw()`, `.expand()`, `.safeIntegers()`, `.bind()`) unless every caller of
+ *    that text asks for the same mode. Give a statement that is read another way its own text.
+ *  - Do not cache a statement that is iterated (`.iterate()`): an open iteration holds it, and a second use fails.
+ *  - Do not cache SQL whose text varies with its input (an `IN (?, ?, ...)` list of any length): each text is kept.
+ * The cache stops growing at MAX_PREPARED texts per connection; past it a statement is prepared and not kept.
+ */
+const MAX_PREPARED = 256;
+const prepared = new WeakMap();
+function prepareOnce(db, sql) {
+    let byText = prepared.get(db);
+    if (!byText)
+        prepared.set(db, byText = new Map());
+    let statement = byText.get(sql);
+    if (statement)
+        return statement;
+    statement = db.prepare(sql);
+    if (byText.size < MAX_PREPARED)
+        byText.set(sql, statement);
+    return statement;
+}
+/**
+ * The most values a list is padded to a power of two for. A longer list is prepared as it is and not kept: the padding would
+ * double it, and a list that long is not the usual one (SQLite takes 32,766 values in a statement).
+ */
+const MAX_PADDED_LIST = 256;
+/** The number of values a list of `count` is padded to: the next power of two, and at least four. */
+function paddedSize(count) {
+    return count <= 4 ? 4 : 2 ** (32 - Math.clz32(count - 1));
+}
+/**
+ * A statement over an `IN (?, ?, ...)` list whose length follows the question (the notes a read hit, the entities of those
+ * notes). The text of such a statement changes with the length, so `prepareOnce` would keep one for every length and fill its
+ * room (see the third rule above), and preparing it on every call was about a tenth of a `get_context`'s statement time (ten
+ * statements prepared again on each call). Here the list is made a power of two long, at least four, and the values are padded
+ * with `paddedList`: a NULL in an IN list matches no row, so the rows are the rows of the list itself. A connection prepares
+ * one statement for each size, a few in all, and the next call finds them ready. `build` makes the SQL from the placeholders.
+ * A list over MAX_PADDED_LIST is prepared as it is, for this call only. Use it where the list is the whole of an IN: not
+ * under NOT IN, where a NULL in the list would make the whole test unknown.
+ */
+function listStatement(db, count, build) {
+    if (count > MAX_PADDED_LIST)
+        return db.prepare(build('?,'.repeat(count - 1) + '?'));
+    return prepareOnce(db, build('?,'.repeat(paddedSize(count) - 1) + '?'));
+}
+/** The values of a list as `listStatement` binds them: padded with NULL to the size it was prepared for. */
+function paddedList(values) {
+    const padded = values.slice();
+    if (values.length > MAX_PADDED_LIST)
+        return padded;
+    for (let size = paddedSize(values.length); padded.length < size;)
+        padded.push(null);
+    return padded;
+}
+/**
+ * One transaction wrapper per connection and body, for a transaction a replay runs once per ledger event.
+ *
+ * `db.transaction(fn)` builds the wrapper's four variants (default, deferred, immediate, exclusive) every time it is called, which
+ * is about 0.05 ms: the ledger replay at startup called it twice per event, about 35 ms of the 190 ms replay of a 1,000-fact
+ * workspace. The wrapper does not depend on what the body closes over, so the body takes the connection and its inputs as
+ * arguments, is a module-level function, and is wrapped once. What runs in the transaction, and when it commits or rolls back, is
+ * unchanged; a call inside another transaction is a savepoint, as before.
+ */
+const transactions = new WeakMap();
+function transactionOnce(db, body) {
+    let byBody = transactions.get(db);
+    if (!byBody)
+        transactions.set(db, byBody = new Map());
+    let tx = byBody.get(body);
+    if (!tx) {
+        tx = db.transaction((...args) => body(db, ...args));
+        byBody.set(body, tx);
+    }
+    return tx;
+}

package/dist/digests/digestCache.js

@@ -0,0 +1,142 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DigestCache = exports.DIGEST_CACHE_SCHEMA_VERSION = void 0;
+const node_fs_1 = __importDefault(require("node:fs"));
+const node_crypto_1 = __importDefault(require("node:crypto"));
+const node_path_1 = __importDefault(require("node:path"));
+const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
+/**
+ * Content-addressed store for per-note digests (RENOR_DIGESTS, src/digests/digests.ts).
+ *
+ *   digest key = sha256(schema \0 model \0 reasoning effort \0 sha256(prompt template) \0 input version \0 note_hash)
+ *
+ * A note's digest is keyed by the hash of the note's file (the index's `note_hash`), so an unchanged note is never
+ * digested twice, an edited note gets a new digest, and a changed prompt, model or input rule re-digests every note.
+ * An empty digest (the note states nothing about the user's life) is stored too, so it is not asked for again.
+ *
+ * The item vectors live beside the digest, one set per embedding space (provider, model, dimensions): a digest
+ * outlives a change of embedding model, only its vectors are rebuilt. A vector from another space is never read.
+ *
+ * Design follows src/router/routerCache.ts: SQLite + WAL + busy_timeout=10000 + INSERT OR IGNORE.
+ */
+exports.DIGEST_CACHE_SCHEMA_VERSION = 'note-digest-v1';
+const sha256 = (text) => node_crypto_1.default.createHash('sha256').update(text).digest('hex');
+class DigestCache {
+    db;
+    getDigestStmt;
+    putDigestStmt;
+    getVectorsStmt;
+    putVectorsStmt;
+    constructor(dbPath, opts = {}) {
+        if (!opts.readonly)
+            node_fs_1.default.mkdirSync(node_path_1.default.dirname(dbPath), { recursive: true });
+        this.db = new better_sqlite3_1.default(dbPath, opts.readonly ? { readonly: true, fileMustExist: true } : undefined);
+        if (!opts.readonly) {
+            this.db.pragma('journal_mode = WAL');
+            this.db.exec(`
+        CREATE TABLE IF NOT EXISTS note_digests (
+          cache_key TEXT PRIMARY KEY,
+          schema_version TEXT NOT NULL,
+          model TEXT NOT NULL,
+          note_hash TEXT NOT NULL,
+          items_json TEXT NOT NULL,
+          created_at TEXT NOT NULL
+        );
+        CREATE TABLE IF NOT EXISTS digest_vectors (
+          cache_key TEXT NOT NULL,
+          space TEXT NOT NULL,
+          dims INTEGER NOT NULL,
+          vectors BLOB NOT NULL,
+          PRIMARY KEY (cache_key, space)
+        );
+        -- What the background worker (src/digests/worker.ts) spent per UTC day: the per-workspace cost guard reads it.
+        CREATE TABLE IF NOT EXISTS digest_spend (
+          day TEXT PRIMARY KEY,
+          calls INTEGER NOT NULL DEFAULT 0,
+          prompt_tokens INTEGER NOT NULL DEFAULT 0,
+          completion_tokens INTEGER NOT NULL DEFAULT 0,
+          micro_usd INTEGER NOT NULL DEFAULT 0
+        );
+      `);
+        }
+        this.db.pragma('busy_timeout = 10000');
+        this.getDigestStmt = this.db.prepare('SELECT items_json, model FROM note_digests WHERE cache_key = ?');
+        this.putDigestStmt = this.db.prepare(`INSERT OR IGNORE INTO note_digests (cache_key, schema_version, model, note_hash, items_json, created_at)
+       VALUES (?, ?, ?, ?, ?, ?)`);
+        this.getVectorsStmt = this.db.prepare('SELECT dims, vectors FROM digest_vectors WHERE cache_key = ? AND space = ?');
+        this.putVectorsStmt = this.db.prepare('INSERT OR IGNORE INTO digest_vectors (cache_key, space, dims, vectors) VALUES (?, ?, ?, ?)');
+    }
+    static digestKey(model, effort, promptTemplate, inputVersion, noteHash) {
+        return sha256([exports.DIGEST_CACHE_SCHEMA_VERSION, model, effort, sha256(promptTemplate), inputVersion, noteHash].join('\0'));
+    }
+    getDigest(key) {
+        const row = this.getDigestStmt.get(key);
+        if (!row)
+            return null;
+        return { items: JSON.parse(row.items_json), model: row.model };
+    }
+    putDigest(key, noteHash, entry) {
+        this.putDigestStmt.run(key, exports.DIGEST_CACHE_SCHEMA_VERSION, entry.model, noteHash, JSON.stringify(entry.items), new Date().toISOString());
+    }
+    /** The item vectors of one digest in one embedding space, or null when they are absent or do not match the items. */
+    getVectors(key, space, count) {
+        const row = this.getVectorsStmt.get(key, space);
+        if (!row || row.dims <= 0 || row.vectors.length !== count * row.dims * 4)
+            return null;
+        const floats = new Float32Array(row.vectors.buffer.slice(row.vectors.byteOffset, row.vectors.byteOffset + row.vectors.length));
+        const out = [];
+        for (let i = 0; i < count; i++)
+            out.push(Array.from(floats.subarray(i * row.dims, (i + 1) * row.dims)));
+        return out;
+    }
+    putVectors(key, space, vectors) {
+        if (vectors.length === 0)
+            return;
+        const dims = vectors[0].length;
+        const floats = new Float32Array(vectors.length * dims);
+        vectors.forEach((vector, i) => floats.set(vector, i * dims));
+        this.putVectorsStmt.run(key, space, dims, Buffer.from(floats.buffer));
+    }
+    /**
+     * Keep only the digests of the notes whose file hash is in `noteHashes`, and the vectors that go with them: a digest
+     * of text a note no longer holds (it was edited, forgotten, or its file is gone) is derived from content the user
+     * removed, so it is not kept. Only the workspace's own cache calls this: an eval cache is shared by many vaults.
+     * Returns how many digests it removed.
+     */
+    retainOnly(noteHashes) {
+        const rows = this.db.prepare('SELECT cache_key, note_hash FROM note_digests').all();
+        const stale = rows.filter(row => !noteHashes.has(row.note_hash));
+        if (stale.length === 0)
+            return 0;
+        const dropDigest = this.db.prepare('DELETE FROM note_digests WHERE cache_key = ?');
+        const dropVectors = this.db.prepare('DELETE FROM digest_vectors WHERE cache_key = ?');
+        this.db.transaction(() => {
+            for (const row of stale) {
+                dropVectors.run(row.cache_key);
+                dropDigest.run(row.cache_key);
+            }
+        })();
+        return stale.length;
+    }
+    /** What the worker has spent on `day` (a UTC `YYYY-MM-DD`), in USD. */
+    spentUsd(day) {
+        const row = this.db.prepare('SELECT micro_usd FROM digest_spend WHERE day = ?').get(day);
+        return (row?.micro_usd ?? 0) / 1e6;
+    }
+    addSpend(day, promptTokens, completionTokens, usd) {
+        this.db.prepare(`INSERT INTO digest_spend (day, calls, prompt_tokens, completion_tokens, micro_usd) VALUES (?, 1, ?, ?, ?)
+       ON CONFLICT(day) DO UPDATE SET calls = calls + 1, prompt_tokens = prompt_tokens + excluded.prompt_tokens,
+         completion_tokens = completion_tokens + excluded.completion_tokens, micro_usd = micro_usd + excluded.micro_usd`).run(day, Math.max(0, Math.round(promptTokens)), Math.max(0, Math.round(completionTokens)), Math.max(0, Math.round(usd * 1e6)));
+    }
+    /** Spend rows older than the newest `keepDays` days are dropped, so the table does not grow without bound. */
+    pruneSpend(keepDays) {
+        this.db.prepare('DELETE FROM digest_spend WHERE day NOT IN (SELECT day FROM digest_spend ORDER BY day DESC LIMIT ?)').run(keepDays);
+    }
+    close() {
+        this.db.close();
+    }
+}
+exports.DigestCache = DigestCache;