renor-engine 0.0.0-stage → 1.0.0

package/dist/retrieval/semantic.js

@@ -0,0 +1,67 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.byPlace = byPlace;
+exports.byScoreThenPlace = byScoreThenPlace;
+exports.semanticSearch = semanticSearch;
+const readCache_1 = require("../db/readCache");
+const statements_1 = require("../db/statements");
+/**
+ * Note path, then chunk id: the order of chunks whose scores are equal. A note's chunks are written top to bottom (by
+ * the indexer, and by the importer, which copies them in that order), so within one note the id is the chunk's place,
+ * the same in every index of the same notes. Across notes ids are not: they follow the order indexing wrote the notes
+ * in, which concurrent indexing does not fix.
+ */
+function byPlace(a, b) {
+    if (a.notePath !== b.notePath)
+        return a.notePath < b.notePath ? -1 : 1;
+    return a.chunkId - b.chunkId;
+}
+/** Highest score first; equal scores by place (`byPlace`). */
+function byScoreThenPlace(a, b) {
+    return b.score - a.score || byPlace(a, b);
+}
+async function semanticSearch(db, vectorIndex, provider, query, topK, probe, 
+/** Only the chunks it shows are candidates: the nearest `topK` of them, however many others are nearer. */
+scope) {
+    // A fresh/empty memory has nothing to compare against. Avoid paying the
+    // provider/network cost for a query embedding that cannot produce a hit.
+    // Ask vec0's rowid shadow table, not the virtual table: `SELECT ... FROM chunk_embeddings` without a
+    // MATCH walks every vector (milliseconds at 10,000 notes, growing with the index) to learn "not empty".
+    // `chunk_embeddings_rowids` holds one row per stored vector.
+    const hasVectors = (0, statements_1.prepareOnce)(db, 'SELECT 1 FROM chunk_embeddings_rowids LIMIT 1').get();
+    if (!hasVectors || topK <= 0)
+        return [];
+    const queryEmbedding = provider.embedQuery
+        ? await provider.embedQuery(query)
+        : (await provider.embed([query]))[0];
+    if (probe)
+        probe.queryVector = queryEmbedding;
+    // The chunks of the `k` nearest vectors (of those `among` names, when it is given), best first, and how many vectors
+    // were read: more than `k` when the farthest of them is tied, and the cut is made by place, not by where vectors are
+    // stored; fewer only when there are no more.
+    const nearest = (k, among) => {
+        const hits = vectorIndex.search(queryEmbedding, k, among);
+        if (hits.length === 0)
+            return { vectors: 0, chunks: [] };
+        const scoreMap = new Map(hits.map(h => [h.chunkId, h.score]));
+        const rows = (0, statements_1.prepareOnce)(db, `SELECT id, note_path, heading_path, text, start_line, end_line FROM chunks WHERE id IN (SELECT value FROM json_each(?))`).all(JSON.stringify(hits.map(h => h.chunkId)));
+        return {
+            vectors: hits.length,
+            chunks: rows
+                .map(r => ({ notePath: r.note_path, chunkId: r.id, headingPath: r.heading_path, text: r.text, score: scoreMap.get(r.id) ?? 0, startLine: r.start_line, endLine: r.end_line }))
+                .sort(byScoreThenPlace),
+        };
+    };
+    // The query is embedded: nothing below waits for anything, so the nearest chunks are read in one transaction (inReadSnapshot).
+    return (0, readCache_1.inReadSnapshot)(db, () => {
+        if (!scope)
+            return nearest(topK).chunks.slice(0, topK);
+        // Judged before the cut: when fewer than `topK` of the nearest chunks are shown (other chunks are nearer), the nearest
+        // of the shown chunks themselves are read instead, so however many others are nearer, none takes their places.
+        const first = nearest(topK);
+        const shown = first.chunks.filter(scope.shows);
+        if (shown.length >= topK || first.vectors < topK)
+            return shown.slice(0, topK);
+        return nearest(topK, scope.chunkIds()).chunks.slice(0, topK);
+    });
+}

package/dist/router/routerCache.js

@@ -0,0 +1,106 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RouterCache = exports.ROUTER_CACHE_SCHEMA_VERSION = void 0;
+const node_fs_1 = __importDefault(require("node:fs"));
+const node_crypto_1 = __importDefault(require("node:crypto"));
+const node_path_1 = __importDefault(require("node:path"));
+const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
+/**
+ * Content-addressed cache for the session router (RENOR_SESSION_ROUTER, src/router/sessionRouter.ts): the
+ * per-note summaries the router reads, and the router's own answers.
+ *
+ *   summary key = sha256(schema \0 'summary' \0 model \0 sha256(summary prompt template) \0 note_hash)
+ *   route key   = sha256(schema \0 'route'   \0 model \0 exact rendered routing prompt)
+ *
+ * A note's summary is keyed by the hash of the note's file (the index's `note_hash`), so an edited note gets a new
+ * summary and an unchanged one is never paid for twice; a changed prompt template invalidates every summary.
+ * A routing answer is keyed by the exact prompt, which already holds the question and every node it was shown.
+ * Only answers that parsed are stored (as in the evidence cache, a malformed response is never cached), so a
+ * replay is always a usable answer.
+ *
+ * Design follows src/evidence/evidenceCache.ts: SQLite + WAL + busy_timeout=10000 + INSERT OR IGNORE.
+ */
+exports.ROUTER_CACHE_SCHEMA_VERSION = 'session-router-v1';
+const sha256 = (text) => node_crypto_1.default.createHash('sha256').update(text).digest('hex');
+class RouterCache {
+    db;
+    getSummaryStmt;
+    putSummaryStmt;
+    getRouteStmt;
+    putRouteStmt;
+    touchRouteStmt;
+    constructor(dbPath) {
+        node_fs_1.default.mkdirSync(node_path_1.default.dirname(dbPath), { recursive: true });
+        this.db = new better_sqlite3_1.default(dbPath);
+        this.db.pragma('journal_mode = WAL');
+        this.db.pragma('busy_timeout = 10000');
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS note_summaries (
+        cache_key TEXT PRIMARY KEY,
+        schema_version TEXT NOT NULL,
+        model TEXT NOT NULL,
+        note_hash TEXT NOT NULL,
+        title TEXT NOT NULL,
+        summary TEXT NOT NULL,
+        created_at TEXT NOT NULL
+      );
+      CREATE TABLE IF NOT EXISTS router_answers (
+        cache_key TEXT PRIMARY KEY,
+        schema_version TEXT NOT NULL,
+        model TEXT NOT NULL,
+        node_ids_json TEXT NOT NULL,
+        thinking TEXT NOT NULL,
+        prompt_tokens INTEGER NOT NULL DEFAULT 0,
+        completion_tokens INTEGER NOT NULL DEFAULT 0,
+        latency_ms INTEGER NOT NULL DEFAULT 0,
+        created_at TEXT NOT NULL,
+        hits INTEGER NOT NULL DEFAULT 0
+      );
+    `);
+        this.getSummaryStmt = this.db.prepare('SELECT title, summary, model FROM note_summaries WHERE cache_key = ?');
+        this.putSummaryStmt = this.db.prepare(`INSERT OR IGNORE INTO note_summaries (cache_key, schema_version, model, note_hash, title, summary, created_at)
+       VALUES (?, ?, ?, ?, ?, ?, ?)`);
+        this.getRouteStmt = this.db.prepare('SELECT node_ids_json, thinking, model, prompt_tokens, completion_tokens, latency_ms FROM router_answers WHERE cache_key = ?');
+        this.putRouteStmt = this.db.prepare(`INSERT OR IGNORE INTO router_answers
+       (cache_key, schema_version, model, node_ids_json, thinking, prompt_tokens, completion_tokens, latency_ms, created_at, hits)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 0)`);
+        this.touchRouteStmt = this.db.prepare('UPDATE router_answers SET hits = hits + 1 WHERE cache_key = ?');
+    }
+    static summaryKey(model, promptTemplate, noteHash) {
+        return sha256(`${exports.ROUTER_CACHE_SCHEMA_VERSION}\0summary\0${model}\0${sha256(promptTemplate)}\0${noteHash}`);
+    }
+    static routeKey(model, prompt) {
+        return sha256(`${exports.ROUTER_CACHE_SCHEMA_VERSION}\0route\0${model}\0${prompt}`);
+    }
+    getSummary(key) {
+        const row = this.getSummaryStmt.get(key);
+        return row ?? null;
+    }
+    putSummary(key, noteHash, entry) {
+        this.putSummaryStmt.run(key, exports.ROUTER_CACHE_SCHEMA_VERSION, entry.model, noteHash, entry.title, entry.summary, new Date().toISOString());
+    }
+    getRoute(key) {
+        const row = this.getRouteStmt.get(key);
+        if (!row)
+            return null;
+        this.touchRouteStmt.run(key);
+        return {
+            nodeIds: JSON.parse(row.node_ids_json),
+            thinking: row.thinking,
+            model: row.model,
+            promptTokens: row.prompt_tokens,
+            completionTokens: row.completion_tokens,
+            latencyMs: row.latency_ms,
+        };
+    }
+    putRoute(key, entry) {
+        this.putRouteStmt.run(key, exports.ROUTER_CACHE_SCHEMA_VERSION, entry.model, JSON.stringify(entry.nodeIds), entry.thinking, entry.promptTokens, entry.completionTokens, entry.latencyMs, new Date().toISOString());
+    }
+    close() {
+        this.db.close();
+    }
+}
+exports.RouterCache = RouterCache;