renor-engine 0.0.0-stage → 1.0.0

package/dist/mvp/factsLane.js

@@ -0,0 +1,203 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.factHits = factHits;
+exports.withFactHits = withFactHits;
+exports.indexBehindLedger = indexBehindLedger;
+exports.indexNotice = indexNotice;
+const path_1 = __importDefault(require("path"));
+const fs_1 = __importDefault(require("fs"));
+const orchestrator_1 = require("../retrieval/orchestrator");
+const statements_1 = require("../db/statements");
+const agentMemory_1 = require("./agentMemory");
+const lifecycle_1 = require("../memory/lifecycle");
+const writerConflicts_1 = require("../memory/writerConflicts");
+const factTerms_1 = require("../memory/factTerms");
+const ledgerLiveness_1 = require("../retrieval/ledgerLiveness");
+const ledgerPresentation_1 = require("../retrieval/ledgerPresentation");
+const scope_1 = require("./scope");
+const digests_1 = require("../digests/digests");
+const sourceName_1 = require("../memory/sourceName");
+/**
+ * The fact as the index shows it, so a hit reads the same before and after index_memory: the fact itself, never the
+ * markup of the ledger's record (its `## MEM_… · time` header and `- subject: …` lines are bookkeeping, and no read
+ * returns them). The event and its time are in the hit's heading path.
+ */
+function factBlock(row) {
+    return (0, ledgerPresentation_1.factText)(row.subject, row.predicate, row.object);
+}
+/**
+ * One current fact as a hit. `score` is left at 0 here and set by `factHits`, which knows the share of the query's
+ * words the fact holds; a fact named by a caller instead (`withFactHits` `also`) was found by meaning rather than by
+ * those words, so it keeps 0 — the convention session-expand hits use.
+ */
+function factHit(row, ledger, date, reader, conflict) {
+    const notePath = path_1.default.relative(path_1.default.resolve(ledger.memoryDir), (0, lifecycle_1.resolveLedgerPath)(ledger.memoryDir, ledger.ledgerRelPath));
+    return {
+        notePath,
+        headingPath: row.eventId ? `${row.eventId} · ${row.validFrom}` : `${row.subject} > ${row.predicate}`,
+        text: factBlock(row),
+        score: 0,
+        reason: 'fact',
+        ...(date ? { session_date: date(row.validFrom) } : {}),
+        // Who saved it, as get_context's facts show it: another agent's name, and untrusted for an import.
+        ...(0, sourceName_1.authorMarker)(row.author, reader),
+        // The values another writer replaced that stay open against it (memory/writerConflicts.ts), when there are any.
+        ...(conflict !== undefined ? { writerConflict: conflict } : {}),
+    };
+}
+/** Facts can be returned before the ledger is indexed. Bound their clock by the actual ledger write, once per read. */
+function ledgerTimeBound(ledger) {
+    let read = false;
+    let modified;
+    return at => {
+        if (!read) {
+            read = true;
+            try {
+                modified = fs_1.default.statSync((0, lifecycle_1.resolveLedgerPath)(ledger.memoryDir, ledger.ledgerRelPath)).mtime.toISOString();
+            }
+            catch { /* No trusted write time: the fact stays undated. */ }
+        }
+        return (0, orchestrator_1.clampSessionDate)(at, modified);
+    };
+}
+/**
+ * Current facts that match `query`, as search hits.
+ *
+ * `all`: every query word must be among the fact's words (search_text, a keyword tool).
+ * `any`: one is enough, and more shared words rank higher (search_memory, a natural-language
+ * tool). Stop words only count when nothing else is left. Facts whose ledger event is in
+ * `exclude` (lowercased ids the index already returned) are skipped before the `limit`, so they
+ * never use up a slot. Scope follows the facts lane of get_context: omitted means every scope,
+ * a given scope means exactly that one.
+ */
+function matchedFacts(db, args, terms) {
+    if (terms.length === 0)
+        return [];
+    const scope = args.scope === undefined ? undefined : (0, scope_1.normalizeScope)(args.scope);
+    const exclude = args.exclude ?? new Set();
+    // At most |exclude| of the best matches are dropped below, so this many always leaves `limit`.
+    return (0, factTerms_1.findCurrentFacts)(db, { terms: [...terms], scope, match: args.match, limit: args.limit + exclude.size })
+        .filter(row => !(row.eventId && exclude.has(row.eventId.toLowerCase())))
+        .slice(0, args.limit);
+}
+function factHits(db, ledger, args) {
+    const terms = (0, agentMemory_1.factQueryTerms)(args.query);
+    const date = args.dated ? ledgerTimeBound(ledger) : undefined;
+    return matchedFacts(db, args, terms)
+        .map(row => ({ ...factHit(row, ledger, date, args.reader), score: Math.round((row.shared / terms.length) * 100) / 100 }));
+}
+/**
+ * What a fact claims, in its own scope: two current facts with the same key are the same fact whatever ledger event
+ * backs them, and one derived from a note has none. So a fact already on the page is recognised without an event id
+ * to compare, which is the only way a fact saved through `remember` (not through `save_memory`) could be listed twice.
+ */
+function factKey(row) {
+    return `${(0, scope_1.normalizeScope)(row.scope)}\u0000${row.subject}\u0000${row.predicate}`;
+}
+/**
+ * The index's hits plus the current facts that match `args.query`, at most `args.limit` hits in all (the search's
+ * top_k): the facts take the places the index's own hits leave, so no index hit is lost or reordered, and a full page
+ * of index hits gets no fact (they used to come on top of it, up to top_k more, matched by any word: live dogfood,
+ * 2026-10-03, D14). Fact hits go first, after a leading evidence summary if there is one, which is outside top_k as
+ * before. A fact whose ledger event the index already returned is not repeated.
+ *
+ * `also` are facts a caller names because it is about to report one of them as its closest match, so they are listed
+ * whether or not the query's words found them — the same rule get_context applies to the fact its verdict was judged on
+ * (agentMemory.ts `missed`). They go after the facts the words found, in the order given, and only out of the room top_k
+ * leaves: a page already full of index hits gets none of them, and a full room of facts the words found gives way to
+ * them. So the page never passes top_k, and the verdict keeps its match whenever the page can carry it. A fact already on
+ * the page is not repeated, by its ledger event or by the claim it makes in its own scope — a fact saved through
+ * `remember` has no event to compare.
+ */
+function withFactHits(db, ledger, indexHits, args, also = []) {
+    const lead = (0, digests_1.leadingSummaryHits)(indexHits);
+    const room = args.limit - (indexHits.length - lead);
+    const listed = new Set(indexHits.flatMap(ledgerLiveness_1.ledgerEventIds));
+    const terms = (0, agentMemory_1.factQueryTerms)(args.query);
+    const words = room > 0 ? matchedFacts(db, { ...args, limit: room, exclude: listed }, terms) : [];
+    const claimed = new Set(words.map(factKey));
+    // The named facts the page does not hold yet, so each is listed once and none is pushed out by another.
+    const pending = [];
+    for (const row of also) {
+        const event = row.eventId?.toLowerCase();
+        if (claimed.has(factKey(row)) || (event !== undefined && listed.has(event)))
+            continue;
+        if (event !== undefined)
+            listed.add(event);
+        claimed.add(factKey(row));
+        pending.push(row);
+    }
+    // Only as many named facts as the page has room for; a page top_k has already filled takes none of them.
+    const named = pending.slice(0, Math.max(0, room));
+    const date = args.dated ? ledgerTimeBound(ledger) : undefined;
+    const disputes = (0, writerConflicts_1.writerConflictLines)(db, [...words, ...named].map(row => row.eventId));
+    const line = (row) => (row.eventId === null ? undefined : disputes.get(row.eventId));
+    const facts = [
+        ...words.slice(0, Math.max(0, room - named.length)).map(row => ({ ...factHit(row, ledger, date, args.reader, line(row)), score: Math.round((row.shared / terms.length) * 100) / 100 })),
+        ...named.map(row => factHit(row, ledger, date, args.reader, line(row))),
+    ];
+    return [...indexHits.slice(0, lead), ...facts, ...indexHits.slice(lead)];
+}
+/**
+ * Whether the newest saved fact is missing from the chunk index.
+ *
+ * The ledger is append-only and indexed whole, so the index holds a prefix of it: if the newest
+ * event is there, every older one is too. The facts table knows the newest event; the chunk
+ * full-text index finds its heading (the id, as the tokens `mem` and the rest) without reading the
+ * ledger's chunks.
+ *
+ * A read is told only about the events it can see (a scoped read returns the ledger events saved in
+ * its own scope, readScope.ts): a named scope or `global` by the newest event of that scope, a read
+ * without a scope by the newest event of all.
+ */
+function indexBehindLedger(db, ledgerPath, scope) {
+    const newest = (0, factTerms_1.newestFactEvent)(db, scope);
+    if (!newest)
+        return false;
+    const phrase = `"${newest.replace(/"/g, '""').replace(/_/g, ' ')}"`;
+    const indexed = (0, statements_1.prepareOnce)(db, `SELECT 1 FROM chunks_fts JOIN chunks c ON c.id = chunks_fts.rowid
+     WHERE chunks_fts MATCH ? AND c.note_path = ? LIMIT 1`).get(phrase, ledgerPath);
+    return !indexed;
+}
+/** Added to a response only while the index is behind. Silence means no known lag. */
+const INDEX_BEHIND = {
+    stale: true,
+    detail: 'The search index has not caught up with some saved facts, so they are matched by keyword only. Run index_memory to find them by meaning too.',
+};
+/**
+ * How many saved note files of the scope (every scope when none is given) are still waiting for their embedding. A
+ * note whose embedding failed or ran past the write budget keeps its chunks, which keyword search finds, and its file
+ * stays unfinished (note_hash ''); search by meaning cannot see it until a later pass embeds it. The partial index of
+ * migration 15 answers this at the cost of those few rows.
+ */
+function unembeddedNoteCount(db, scope) {
+    const pred = scope === undefined ? null : (0, scope_1.scopeSqlPredicate)((0, scope_1.normalizeScope)(scope), 'path');
+    const sql = `SELECT COUNT(*) FROM notes WHERE note_hash = ''${pred ? ` AND ${pred.sql}` : ''}`;
+    return (0, statements_1.prepareOnce)(db, sql).pluck().get(...(pred?.params ?? []));
+}
+/**
+ * `{ index }` for a read that uses meaning-based search while the index is behind the ledger, or while notes of its
+ * scope wait for their embedding (`unembeddedNotes`: the count, and search_text or a retry finds them).
+ * `semantic` is whether an embedding provider is configured; without one there is no meaning
+ * search for index_memory to catch up. Silence means nothing is known to be behind or waiting.
+ */
+function indexNotice(db, semantic, ledger, scope) {
+    if (!semantic)
+        return {};
+    const behind = indexBehindLedger(db, (0, lifecycle_1.resolveLedgerPath)(ledger.memoryDir, ledger.ledgerRelPath), scope === undefined ? undefined : (0, scope_1.normalizeScope)(scope));
+    const unembedded = unembeddedNoteCount(db, scope);
+    if (unembedded === 0)
+        return behind ? { index: INDEX_BEHIND } : {};
+    const it = unembedded === 1 ? 'it' : 'them';
+    const waiting = `${unembedded} saved note${unembedded === 1 ? ' is' : 's are'} still waiting for an embedding, so search_memory cannot find ${it} by meaning yet. search_text finds ${it} by exact words now; a retry finds ${it} once the embedder answers.`;
+    return {
+        index: {
+            stale: true,
+            detail: behind ? `${INDEX_BEHIND.detail} ${waiting}` : waiting,
+            unembeddedNotes: unembedded,
+        },
+    };
+}