renor-engine 0.0.0-stage → 1.0.0

package/dist/retrieval/keyword.js

@@ -0,0 +1,84 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.keywordSearch = keywordSearch;
+const readCache_1 = require("../db/readCache");
+const statements_1 = require("../db/statements");
+const semantic_1 = require("./semantic");
+const ROWS = 'SELECT c.id AS chunkId, c.note_path AS notePath, c.heading_path AS headingPath, c.text, f.score AS score';
+/**
+ * get_context's first keyword window is `topK * 10` (60 for the 6 it asks) and the next read, when that window is
+ * turned away, is four times that (240). A wider page is a walk toward every match; keeping its text is not worth it.
+ */
+const CACHED_PAGE_LIMIT = 256;
+/** Distinct pages kept per connection. The oldest goes when another is kept. */
+const CACHED_PAGE_COUNT = 256;
+const cachedPages = new WeakMap();
+function copyHits(hits) {
+    return hits.map(hit => ({ chunkId: hit.chunkId, notePath: hit.notePath, headingPath: hit.headingPath, text: hit.text, score: hit.score }));
+}
+/**
+ * The `limit` chunks that best match an FTS5 `match` expression, best first, and of equal scores by place (byPlace).
+ * BM25 ties often (one sentence in several notes, one word once in chunks of one length), and FTS5 gives ties in
+ * row-id order, which is the order indexing wrote the notes in: concurrent indexing does not fix it.
+ *
+ * Ranked and cut inside the full-text query, which reads no chunk row, one row past the page: joining `chunks` before
+ * the cut would read the text of every matching chunk (thousands, for a common word) to keep a few. Only when that
+ * extra row scores like the last one does the page end inside a tie; then every chunk with that score is read, so the
+ * tie is cut by place too.
+ *
+ * The same page is kept until this connection or another writes. The stamp is taken before the ranking and the page is
+ * kept only when that stamp is still current: a write during the ranking is stored under neither stamp, and nothing is
+ * kept inside a transaction. Callers may change the hits they are given, so both the kept page and a hit are copies.
+ */
+function keywordSearch(db, match, limit) {
+    if (limit < 1)
+        return [];
+    const key = `${limit}\0${match}`;
+    const before = limit <= CACHED_PAGE_LIMIT ? (0, readCache_1.databaseStamp)(db) : null;
+    if (before !== null) {
+        const held = cachedPages.get(db)?.get(key);
+        if (held && held.stamp === before)
+            return copyHits(held.hits);
+    }
+    const rows = rankKeywordHits(db, match, limit);
+    if (before !== null && (0, readCache_1.databaseStamp)(db) === before) {
+        let pages = cachedPages.get(db);
+        if (!pages)
+            cachedPages.set(db, pages = new Map());
+        if (!pages.has(key) && pages.size >= CACHED_PAGE_COUNT)
+            pages.delete(pages.keys().next().value);
+        pages.set(key, { stamp: before, hits: copyHits(rows) });
+    }
+    return rows;
+}
+function rankKeywordHits(db, match, limit) {
+    // The page is the `limit` best rows, read with one more ranked row that is not returned: its score (`next`) says whether
+    // the page ends inside a tie. The ranking is read once, and the extra row costs no chunk text and no row of its own.
+    const paged = (0, statements_1.prepareOnce)(db, `WITH ranked AS MATERIALIZED (
+       SELECT rowid AS rid, bm25(chunks_fts) AS score
+       FROM chunks_fts
+       WHERE chunks_fts MATCH ?
+       ORDER BY score
+       LIMIT ?
+     )
+     SELECT c.id AS chunkId, c.note_path AS notePath, c.heading_path AS headingPath, c.text, r.score AS score,
+       (SELECT score FROM ranked ORDER BY score LIMIT 1 OFFSET ?) AS next
+     FROM ranked r
+     JOIN chunks c ON c.id = r.rid
+     ORDER BY r.score
+     LIMIT ?`).all(match, limit + 1, limit, limit);
+    const page = paged.map(({ chunkId, notePath, headingPath, text, score }) => ({ chunkId, notePath, headingPath, text, score }));
+    const edge = page.length === limit && paged[0].next === page[limit - 1].score ? page[limit - 1].score : null;
+    const rows = edge === null ? page : [
+        ...page.filter(hit => hit.score < edge),
+        ...(0, statements_1.prepareOnce)(db, `${ROWS}
+       FROM (
+         SELECT rowid, bm25(chunks_fts) AS score
+         FROM chunks_fts
+         WHERE chunks_fts MATCH ?
+       ) f
+       JOIN chunks c ON c.id = f.rowid
+       WHERE f.score = ?`).all(match, edge),
+    ];
+    return rows.sort((a, b) => a.score - b.score || (0, semantic_1.byPlace)(a, b)).slice(0, limit);
+}

package/dist/retrieval/ledgerEventTime.js

@@ -0,0 +1,51 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createLedgerEventClock = createLedgerEventClock;
+const readCache_1 = require("../db/readCache");
+const statements_1 = require("../db/statements");
+const lifecycle_1 = require("../memory/lifecycle");
+const ledgerLiveness_1 = require("./ledgerLiveness");
+/**
+ * When the ledger events a hit carries happened.
+ *
+ * The ledger is ONE note, so the date of the note (when the file was last written) is the same for every event
+ * in it, and says nothing about when any of them was saved. Each event has its own time: the lifecycle layer
+ * projects every event into `facts` (event id, `valid_from` = the event's time), the same projection a scoped
+ * read asks which scope an event belongs to (mvp/readScope.ts). That row, not the header text of a chunk, is
+ * what makes an id an event, so a line that only looks like an event header in some other note dates nothing.
+ *
+ * `valid_from` is the canonical UTC ISO form (ms precision, `Z`) the ledger writer and replay both enforce, so
+ * comparing two of them as strings is comparing the times.
+ *
+ * Nothing is read until a hit carries an event id: text that holds none costs no query, and the one statement
+ * is prepared once however many hits ask.
+ */
+function createLedgerEventClock(db) {
+    let lookup; // undefined: not needed yet; null: no projection to ask
+    const known = new Map();
+    const timeOf = (id) => {
+        const key = id.toUpperCase(); // projected ids are upper-case, and the unique index on facts(event_id) serves the lookup
+        if (!known.has(key)) {
+            if (lookup === undefined)
+                lookup = prepareLookup(db);
+            known.set(key, lookup?.get(key, lifecycle_1.LIFECYCLE_NOTE_PATH)?.at);
+        }
+        return known.get(key);
+    };
+    // A chunk is one event (the chunker splits at every event heading); one that carries several, as a legacy or a
+    // hand-merged note can, is as new as the newest of them, the way a note is as new as its last write.
+    return hit => {
+        let newest;
+        for (const id of (0, ledgerLiveness_1.ledgerEventIds)(hit)) {
+            const at = timeOf(id);
+            if (at !== undefined && (newest === undefined || at > newest))
+                newest = at;
+        }
+        return newest;
+    };
+}
+function prepareLookup(db) {
+    if (!(0, readCache_1.tableHasColumns)(db, 'facts', ['event_id', 'source_path']))
+        return null;
+    return (0, statements_1.prepareOnce)(db, 'SELECT valid_from AS at FROM facts WHERE event_id = ? AND source_path = ?');
+}

package/dist/retrieval/ledgerLiveness.js

@@ -0,0 +1,83 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SUPERSEDED_LABEL = void 0;
+exports.closedLedgerEvents = closedLedgerEvents;
+exports.ledgerEventIds = ledgerEventIds;
+exports.resolveLedgerHits = resolveLedgerHits;
+const readCache_1 = require("../db/readCache");
+const statements_1 = require("../db/statements");
+/**
+ * The lifecycle projection is updated on write, independently of the chunk
+ * index. Resolve event identity here rather than comparing object strings:
+ * A -> B -> A still closes the first A, and unindexed corrections take effect.
+ */
+function closedLedgerEvents(db) {
+    if (!(0, readCache_1.tableHasColumns)(db, 'facts', ['event_id']))
+        return Object.assign(new Set(), { hidden: new Set() });
+    // One read serves every ask until a fact is written: a search asks several times (each lane, each page of a read).
+    return (0, readCache_1.readUntilWrite)(db, 'closedLedgerEvents', () => {
+        const rows = (0, statements_1.prepareOnce)(db, `SELECT event_id FROM facts
+      WHERE source_path = '_mcp_' AND event_id IS NOT NULL AND valid_to IS NOT NULL`).all();
+        const hidden = (0, readCache_1.tableHasColumns)(db, 'retired_facts', ['event_id']) && (0, readCache_1.tableHasColumns)(db, 'fact_retirements', ['event_id'])
+            ? (0, statements_1.prepareOnce)(db, 'SELECT event_id FROM retired_facts UNION SELECT event_id FROM fact_retirements').all()
+            : [];
+        const hiddenIds = new Set(hidden.map(r => r.event_id.toLowerCase()));
+        return Object.assign(new Set([...rows.map(r => r.event_id.toLowerCase()), ...hiddenIds]), { hidden: hiddenIds });
+    });
+}
+/**
+ * The ledger events (lower-case ids) a hit carries: the ids on its `## MEM_… · ` headers or, for a chunk cut
+ * below its header, the id in its heading path. Empty for anything that is not a ledger event.
+ */
+function ledgerEventIds(hit) {
+    // A text without `mem_` has no `## MEM_… · ` header (NAMES_AN_EVENT, below): most hits, asked about several times each.
+    const ids = NAMES_AN_EVENT.test(hit.text) ? [...hit.text.matchAll(/^## (MEM_[A-Za-z0-9]+) · /gim)].map(m => m[1].toLowerCase()) : [];
+    if (ids.length)
+        return ids;
+    const fromHeading = hit.headingPath?.match(/\b(MEM_[A-Za-z0-9]+)\b/i)?.[1];
+    return fromHeading ? [fromHeading.toLowerCase()] : [];
+}
+/** Whether a text can name a ledger event at all: every pattern that reads an event id below starts with `MEM_`, in either case. */
+const NAMES_AN_EVENT = /mem_/i;
+/** The line that opens a superseded event shown on request (include_history). */
+exports.SUPERSEDED_LABEL = '[Superseded memory event; historical evidence only]';
+/**
+ * Also handles legacy/large chunks containing several ledger event blocks. An event in `closed.hidden`
+ * (closedLedgerEvents) is dropped even with includeHistory.
+ */
+function resolveLedgerHits(hits, closed, includeHistory) {
+    if (!closed.size)
+        return hits;
+    const hidden = closed.hidden;
+    return hits.flatMap(hit => {
+        // An event is named by `## MEM_… · ` in the text or by `MEM_…` in the heading path, in either case. A hit with
+        // neither holds no block to close, so it comes back as it is, without the split and the matches below.
+        if (!NAMES_AN_EVENT.test(hit.text) && !(hit.headingPath && NAMES_AN_EVENT.test(hit.headingPath)))
+            return [hit];
+        let superseded = false;
+        let dropped = false;
+        const blocks = hit.text.replace(/\r\n/g, '\n').split(/(?=^## MEM_[A-Za-z0-9]+ · )/im);
+        const kept = blocks.flatMap(block => {
+            const id = block.match(/^## (MEM_[A-Za-z0-9]+) · /im)?.[1]
+                ?? (blocks.length === 1 ? hit.headingPath?.match(/\b(MEM_[A-Za-z0-9]+)\b/i)?.[1] : undefined);
+            const isClosed = !!id && closed.has(id.toLowerCase());
+            if (!isClosed)
+                return [block];
+            if (hidden?.has(id.toLowerCase())) {
+                dropped = true;
+                return [];
+            }
+            superseded = true;
+            return includeHistory ? [`${exports.SUPERSEDED_LABEL}\n${block}`] : [];
+        });
+        if (!superseded && !dropped)
+            return [hit];
+        const text = kept.join('').trim();
+        const history = includeHistory && superseded;
+        if (!text || (!history && !kept.some(b => /^## MEM_/im.test(b))))
+            return [];
+        return [{ ...hit, text,
+                ...(!history ? { headingPath: text.match(/^## (MEM_[A-Za-z0-9]+ · .+)$/im)?.[1] ?? hit.headingPath } : {}),
+                reason: history ? `${hit.reason}+superseded` : hit.reason }];
+    });
+}

package/dist/retrieval/ledgerPresentation.js

@@ -0,0 +1,161 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ledgerNoteMatcher = ledgerNoteMatcher;
+exports.isLedgerPreamble = isLedgerPreamble;
+exports.dropLedgerPreamble = dropLedgerPreamble;
+exports.keywordLedgerHits = keywordLedgerHits;
+exports.presentLedgerEvents = presentLedgerEvents;
+exports.withoutLedgerMetadata = withoutLedgerMetadata;
+exports.presentEvents = presentEvents;
+exports.factText = factText;
+const path_1 = __importDefault(require("path"));
+const inventory_1 = require("../memory/inventory");
+const ledgerLiveness_1 = require("./ledgerLiveness");
+function ledgerNoteMatcher(vault) {
+    const stored = (0, inventory_1.ledgerNotePath)(vault);
+    if (stored === null)
+        return () => false; // a path that cannot name a file names no ledger
+    const shown = path_1.default.relative(path_1.default.resolve(vault.memoryDir), stored);
+    return notePath => notePath === stored || notePath === shown;
+}
+/** A chunk of the ledger that no event owns: its header (nothing else comes before the first event). */
+function isLedgerPreamble(hit, isLedger) {
+    return isLedger !== undefined && isLedger(hit.notePath) && (0, ledgerLiveness_1.ledgerEventIds)(hit).length === 0;
+}
+/**
+ * The hits without the ledger's header. For a caller that cuts a page: the header has to come off before the cut, as
+ * a closed event does, so that it never takes a place that a memory should have had.
+ */
+function dropLedgerPreamble(hits, isLedger) {
+    return isLedger === undefined ? hits : hits.filter(hit => !isLedgerPreamble(hit, isLedger));
+}
+/**
+ * The hits a keyword search may show, before its page is cut: no header of the ledger, and an event of it only when a
+ * word of the question is in what the event says. The search matched the record's markup too ("kind", "store", "ledger",
+ * a year), and a hit that way is no match for the question.
+ */
+function keywordLedgerHits(hits, isLedger, query, held) {
+    if (isLedger === undefined)
+        return hits;
+    const words = wordsOf(query);
+    return dropLedgerPreamble(hits, isLedger).filter(hit => {
+        if (!isLedger(hit.notePath) || words.length === 0)
+            return true;
+        const events = parseEvents(hit.text, held);
+        return events.length === 0 || events.some(event => {
+            const said = says(event);
+            return words.some(word => said.has(word));
+        });
+    });
+}
+/** The hits as a client is shown them: no header of the ledger, and each event of it as the user's words (or the fact). */
+function presentLedgerEvents(hits, isLedger, held) {
+    if (isLedger === undefined)
+        return hits;
+    return dropLedgerPreamble(hits, isLedger).flatMap(hit => {
+        if (!isLedger(hit.notePath))
+            return [hit];
+        const text = presentEvents(hit.text, held);
+        // An event is one chunk that opens with its own header line (the chunker cuts at every `##` heading and never inside
+        // a paragraph, and a record has no blank line), so a ledger text with no record in it is one already presented.
+        if (text === null)
+            return [hit];
+        return text === '' ? [] : [{ ...hit, text }]; // a record that says nothing is not a memory
+    });
+}
+/**
+ * The hits as a client is shown their metadata: a hit of the ledger names no note and no heading (both empty). Its path
+ * is the engine's own file and its heading the record's (`Memory Events Ledger > MEM_… · time`): bookkeeping too, which
+ * every ledger hit carried, at about 46 tokens (live dogfood, 2026-10-03, D8). What it says is its text, and when, its
+ * session_date. Run last: liveness, scope and a repeat of a fact are judged by the event id in the heading.
+ */
+function withoutLedgerMetadata(hits, isLedger) {
+    if (isLedger === undefined)
+        return hits;
+    return hits.map(hit => (isLedger(hit.notePath) ? { ...hit, notePath: '', headingPath: '' } : hit));
+}
+const EVENT_HEADER = /^## MEM_[A-Za-z0-9]+ · /i;
+// A field value is the rest of its physical line (dotAll: U+2028/U+2029 and a lone \r belong to a value, as in the ledger parser).
+const FIELD = /^- ([A-Za-z][\w-]*): (.*)$/s;
+const RECORDS_HELD_MAX = 256;
+const RECORDS_HELD_MAX_CHARS = 8 * 1024;
+/** The events in a chunk's text, as the lines of their records say. Tolerant: it only reads, and never throws. */
+function parseEvents(text, held) {
+    const kept = held?.get(text);
+    if (kept !== undefined)
+        return kept;
+    const events = readEvents(text);
+    if (held && held.size < RECORDS_HELD_MAX && text.length <= RECORDS_HELD_MAX_CHARS)
+        held.set(text, events);
+    return events;
+}
+function readEvents(text) {
+    const events = [];
+    let labelled = false;
+    for (const line of text.replace(/\r\n/g, '\n').split('\n')) {
+        if (line === ledgerLiveness_1.SUPERSEDED_LABEL) {
+            labelled = true;
+        }
+        else if (EVENT_HEADER.test(line)) {
+            events.push({ fields: new Map(), superseded: labelled });
+            labelled = false;
+        }
+        else if (events.length > 0) {
+            const field = FIELD.exec(line);
+            const fields = events[events.length - 1].fields;
+            if (field && !fields.has(field[1]))
+                fields.set(field[1], field[2]);
+        }
+    }
+    return events;
+}
+/**
+ * The events in a chunk's text, one after another: the user's statement as they said it, else `subject — predicate:
+ * object`. A superseded event (shown only on request) keeps the label that says so. Null when the text holds no event,
+ * so a text that is already presented is left as it is.
+ */
+function presentEvents(text, held) {
+    const events = parseEvents(text, held);
+    return events.length ? events.map(show).filter(Boolean).join('\n\n') : null;
+}
+function show({ fields, superseded }) {
+    const said = statementOf(fields.get('statement'));
+    const body = said ?? factText(fields.get('subject'), fields.get('predicate'), fields.get('object'));
+    return body && superseded ? `${ledgerLiveness_1.SUPERSEDED_LABEL}\n${body}` : body;
+}
+/**
+ * A fact as it is shown when no words of the user's were kept: `subject — predicate: object`. The one form for a fact,
+ * whether an indexed event of the ledger shows it or a read that finds it in the facts table before the index does.
+ */
+function factText(subject, predicate, object) {
+    return [subject, [predicate, object].filter(Boolean).join(': ')].filter(Boolean).join(' — ');
+}
+/** The statement the ledger kept: one JSON string literal. Anything else, or nothing said, is not the user's words. */
+function statementOf(raw) {
+    if (raw === undefined)
+        return undefined;
+    try {
+        const value = JSON.parse(raw);
+        return typeof value === 'string' && value.trim() !== '' ? value : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+/** The words of what an event says: the statement and the fact, without the record's markup. Worked out once for a record. */
+function says(record) {
+    const { fields } = record;
+    return record.said ??= new Set(wordsOf([statementOf(fields.get('statement')), fields.get('subject'), fields.get('predicate'), fields.get('object')]
+        .filter(Boolean)
+        .join(' ')));
+}
+/**
+ * The words of a text as the keyword index reads them: runs of letters and digits, lower case, accents off. Whole
+ * words, as the index matches them, so that "mem" of an event id is not found in "remember".
+ */
+function wordsOf(text) {
+    return text.normalize('NFD').replace(/\p{M}+/gu, '').toLowerCase().match(/[\p{L}\p{N}]+/gu) ?? [];
+}

package/dist/retrieval/noteIdentity.js

@@ -0,0 +1,57 @@
+"use strict";
+/**
+ * Note-identity recall for exact terms that chunk full-text search cannot match.
+ *
+ * `chunks_fts` indexes chunk text and heading path; `note_path` is UNINDEXED and
+ * `parseNote` drops YAML frontmatter from the indexed body. So an identifier that
+ * *names* a note — a task id, a slug, a file name, a frontmatter `id` — was invisible
+ * to a keyword search even though the note is right there in the index.
+ *
+ * Migration 13 indexes those names in `note_identity_fts`, one row per note, with
+ * `-_.%` as token characters so an identifier is one whole token. This module asks
+ * that index for notes the term names. Callers keep their normal full-text page
+ * untouched and add these notes beside it, so no prose hit is ever lost or reordered.
+ *
+ * Only terms shaped like an identifier (a `-`, `_` or `.` between word characters)
+ * are asked: `md`, `notes` or `OR` are words, not names, and would match every note.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.NOTE_IDENTITY_EXTRA = void 0;
+exports.selectNoteIdentityHits = selectNoteIdentityHits;
+const statements_1 = require("../db/statements");
+/** Most named notes added beside a full prose page — the page itself is never shortened. */
+exports.NOTE_IDENTITY_EXTRA = 5;
+/** `deploy-unit-required.test.sh`, `MVP_MCP_JSON`, `literally-100%` — not `md`, `.md`, `notes`. */
+function looksLikeIdentifier(term) {
+    return /^\w[\w.%-]*[\w%]$/.test(term) && /[-_.]/.test(term);
+}
+/**
+ * The opening chunk of every note named by `term`, ordered by path so the result is
+ * deterministic. `exclude` lists note paths the caller already returns; `accept`
+ * judges a candidate by its opening chunk (e.g. by scope: a ledger opens with an event, and
+ * the ledger's path says nothing about that event's scope) *before* the limit applies.
+ */
+function selectNoteIdentityHits(db, term, limit, exclude = new Set(), accept = () => true) {
+    const q = (term ?? '').trim();
+    const max = Math.max(1, Math.floor(limit) || 1);
+    if (!looksLikeIdentifier(q))
+        return [];
+    const named = (0, statements_1.prepareOnce)(db, `SELECT n.path FROM note_identity_fts f JOIN notes n ON n.rowid = f.rowid
+     WHERE note_identity_fts MATCH ? ORDER BY n.path`).all(`"${q.replace(/"/g, '""')}"`);
+    const first = (0, statements_1.prepareOnce)(db, `SELECT id AS chunkId, note_path AS notePath, heading_path AS headingPath, text
+     FROM chunks WHERE note_path = ? ORDER BY start_line, id LIMIT 1`);
+    const rows = [];
+    for (const { path: notePath } of named) {
+        if (rows.length === max)
+            break;
+        if (exclude.has(notePath))
+            continue;
+        const opening = first.get(notePath);
+        if (opening && accept(notePath, opening))
+            rows.push(opening);
+    }
+    // bm25() does not apply to a name match. Consumers read `score` as "higher is
+    // better", so hand back a descending ordinal rather than a fake BM25 value: the
+    // ordering is real (path order), the magnitude is not.
+    return rows.map((r, i) => ({ ...r, score: rows.length - i, reason: 'note-identity' }));
+}