renor-engine 0.0.0-stage → 1.0.0

package/dist/context/readHonesty.js

@@ -0,0 +1,478 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.NO_ANSWER_STATEMENT = exports.DEFAULT_EVIDENCE_BAR = exports.READ_HONESTY_FLAG = void 0;
+exports.readHonestyEnabled = readHonestyEnabled;
+exports.resolveEvidenceBar = resolveEvidenceBar;
+exports.contentTerms = contentTerms;
+exports.termStem = termStem;
+exports.evidenceCoverage = evidenceCoverage;
+exports.factProvenance = factProvenance;
+exports.noteProvenance = noteProvenance;
+exports.questionKey = questionKey;
+exports.acrossScopes = acrossScopes;
+exports.currentFactsForSubjects = currentFactsForSubjects;
+exports.detectFactConflicts = detectFactConflicts;
+exports.conflictAcrossScopes = conflictAcrossScopes;
+exports.fitConflicts = fitConflicts;
+exports.querySubjects = querySubjects;
+exports.assessAbstention = assessAbstention;
+exports.applyReadHonesty = applyReadHonesty;
+exports.readHonestyForItems = readHonestyForItems;
+const rowStore_1 = require("./rowStore");
+const factConflicts_1 = require("../memory/factConflicts");
+const scope_1 = require("../mvp/scope");
+/**
+ * Read-time honesty (LANE 3): provenance, contradiction, abstention.
+ *
+ * Three behaviours on the product read path, all gated by ONE flag
+ * (RENOR_READ_HONESTY, default OFF — nothing here runs unless an operator
+ * arms it):
+ *
+ *   1. provenance  — every returned item says where it came from and when it
+ *                    was last verified, so a consuming agent can decide
+ *                    whether to trust or discount it.
+ *   2. conflicts   — when two CURRENT facts about the same subject+predicate
+ *                    disagree, the read names both values and says which one
+ *                    is newer, instead of silently returning both as if they
+ *                    were compatible. This is the agent-dogfood D2 want: two
+ *                    contradictory statements in one source, where nothing
+ *                    surfaced the conflict. A read that names no scope
+ *                    compares every scope and names each value with every
+ *                    scope it is current in; a scoped read compares its own
+ *                    scope only, as get_context's default conflicts do
+ *                    (memory/factConflicts.ts).
+ *   3. abstention  — when the query's content terms are mostly absent from the
+ *                    returned evidence, the read says so explicitly
+ *                    ("no good answer on file for this") instead of presenting
+ *                    weak hits as if they answered the question. D6: weak hits
+ *                    train an agent to trust noise.
+ *
+ * Everything here is gold-blind and product-observable: the inputs are the
+ * query text and rows the product itself stored (notes, chunks, facts,
+ * entities). No benchmark category, gold label, `shouldAbstain` or session id
+ * is readable from this module's signatures.
+ */
+/** RENOR_READ_HONESTY (default OFF). Opt in with 1/true/yes/on. */
+exports.READ_HONESTY_FLAG = 'RENOR_READ_HONESTY';
+function readHonestyEnabled(env = process.env) {
+    const raw = (env[exports.READ_HONESTY_FLAG] ?? '').trim().toLowerCase();
+    return raw === '1' || raw === 'true' || raw === 'yes' || raw === 'on';
+}
+/**
+ * Default evidence bar: the share of the query's content terms that must be
+ * present in the returned evidence before a hit counts as an answer.
+ *
+ * MEASURED, not invented (scripts/lane3-probe.ts, synthetic 10-note vault,
+ * eval-hash embeddings): on-topic questions cover 0.67–1.00 of their content
+ * terms, off-topic questions 0.00–0.33. 0.5 sits between the two populations
+ * and is deliberately the conservative side — it abstains only when most of
+ * what the question is about is nowhere in the returned evidence.
+ *
+ * Vector similarity alone does NOT separate these populations on that fixture
+ * (an off-topic question scored 0.555 while an on-topic one scored 0.364), so
+ * a similarity threshold would be the wrong instrument.
+ */
+exports.DEFAULT_EVIDENCE_BAR = 0.5;
+function resolveEvidenceBar(env = process.env) {
+    const raw = Number(env.RENOR_READ_HONESTY_BAR);
+    if (Number.isFinite(raw) && raw >= 0 && raw <= 1)
+        return raw;
+    return exports.DEFAULT_EVIDENCE_BAR;
+}
+/**
+ * Query words that carry no answer content. Deliberately a stoplist, not a
+ * stemmer: the bar must be deterministic and explainable.
+ */
+const STOPWORDS = new Set([
+    'the', 'and', 'for', 'are', 'was', 'were', 'did', 'does', 'how', 'what',
+    'when', 'which', 'who', 'why', 'many', 'much', 'about', 'from', 'into',
+    'with', 'this', 'that', 'there', 'their', 'have', 'has', 'had', 'you',
+    'your', 'our', 'his', 'her', 'its', 'not', 'but', 'any', 'all', 'can',
+    'will', 'would', 'should', 'use', 'used', 'take', 'took', 'name', 'start',
+    'here', 'near', 'last', 'over', 'under', 'each', 'than', 'then',
+]);
+/** Content terms of a query: >=3 chars, not a stopword, de-duplicated in order.
+ *
+ *  DELIBERATELY ASCII (`[^a-z0-9%.]`), even though `rowStore.splitWords` now cuts every
+ *  other read-side tokenizer on the Unicode write-side rule. This feeds `evidenceCoverage`
+ *  and `assessAbstention`, and those terms are the DENOMINATOR of the abstention bar: every
+ *  term must be findable in the evidence or coverage falls and the read abstains. Widening
+ *  the cut therefore mints content terms the English-only stoplist cannot cover — measured
+ *  here: `Quando são as reuniões?` against the on-file `Eu prefiro reuniões às sextas-feiras`
+ *  went from terms `[quando, reuni]` (coverage 0.50, answers) to `[quando, são, reuniões]`
+ *  (coverage 0.33, abstains "no good answer on file for this"), because `são` is a Portuguese
+ *  function word that no evidence sentence contains. Widening this lane trades a false
+ *  abstention for a false answer in every language the stoplist does not cover.
+ *
+ *  A stoplist per language is the real fix and it is a separate, measured change; the
+ *  entity-resolution lane below is what this PR widens. Pinned by
+ *  `tests/context/readSideUnicodeTokens.test.ts`, which also asserts the abstention outcome
+ *  of Portuguese query/fact pairs is unchanged. */
+function contentTerms(query) {
+    return Array.from(new Set((query ?? '')
+        .toLowerCase()
+        .split(/[^a-z0-9%.]+/)
+        .filter((t) => t.length >= 3 && !STOPWORDS.has(t))));
+}
+/** The same rule as `contentTerms`, cut on the Unicode write-side class instead. Used ONLY
+ *  by `querySubjects`, whose terms SELECT entity names rather than score evidence coverage:
+ *  a name and the query that names it are cut by the same rule, so a subject in any script
+ *  resolves. Widening is safe here precisely because nothing divides by the term count.
+ *  Kept as a separate function rather than a flag so the abstention lane above cannot be
+ *  widened by accident. */
+function subjectTerms(query) {
+    return Array.from(new Set((0, rowStore_1.splitWords)(query, true).filter((t) => t.length >= 3 && !STOPWORDS.has(t))));
+}
+/**
+ * Length-2 trailing trim ("powered" -> "power", "flights" -> "flight").
+ * A prefix test, not a linguistic stemmer — measured to cover the plural and
+ * participle forms in the fixture without matching unrelated words.
+ */
+function termStem(term) {
+    return term.slice(0, Math.max(4, term.length - 2));
+}
+/** Fraction of the query's content terms that appear in the returned evidence. */
+function evidenceCoverage(query, texts) {
+    const terms = contentTerms(query);
+    if (terms.length === 0)
+        return 1; // nothing contentful to require
+    const haystack = texts.join('\n').toLowerCase();
+    const covered = terms.filter((t) => haystack.includes(termStem(t))).length;
+    return covered / terms.length;
+}
+/** Provenance of a lifecycle fact: the ledger is the source of truth, valid_from is its verification date. */
+function factProvenance(validFrom, ledgerRelPath) {
+    return { source: ledgerRelPath, lastVerified: validFrom ?? null, kind: 'ledger' };
+}
+/** Provenance of a note-backed item: the note is the source, its session/note date is the verification date. */
+function noteProvenance(notePath, lastVerified) {
+    return { source: notePath, lastVerified: lastVerified ?? null, kind: 'note' };
+}
+function instantOf(value, fallbackId) {
+    const ms = value ? Date.parse(value) : Number.NaN;
+    return Number.isFinite(ms) ? ms : Number.NEGATIVE_INFINITY + fallbackId / 1e6;
+}
+/** One question: a subject and a predicate, compared as the structured-row store compares values. */
+function questionKey(subject, predicate) {
+    return `${(0, rowStore_1.normDedupeValue)(subject)}\u0000${(0, rowStore_1.normDedupeValue)(predicate)}`;
+}
+/** Whether a conflict's values are current in different scopes: its sides then carry their scope. */
+function acrossScopes(conflict) {
+    return conflict.newer.scope !== undefined;
+}
+function newestFirst(a, b) {
+    return instantOf(b.validFrom, b.id) - instantOf(a.validFrom, a.id);
+}
+/** Where a side's value is on file: its source, its scope (when the values span scopes) and its date. */
+function where(side) {
+    return `${side.source}${side.scope ? `, scope ${side.scope}` : ''}${side.lastVerified ? `, ${side.lastVerified}` : ', date unknown'}`;
+}
+/**
+ * Current facts for a subject set, all source paths included.
+ *
+ * Reads `valid_to IS NULL` rows across the WHOLE table, not just the
+ * `_mcp_` lifecycle stream: the contradiction that matters is cross-source
+ * (a note asserting one value while another note or the ledger asserts
+ * another). Restricting to one writer would hide exactly the conflict this
+ * behaviour exists to surface.
+ *
+ * `scope` is the read's scope: a scoped read sees that scope's facts only
+ * (under any spelling of it, savedScopeSqlPredicate), so it is never told what
+ * another scope holds; null (a read that names no scope) sees every scope.
+ */
+function currentFactsForSubjects(db, subjects, scope = null) {
+    const names = Array.from(new Set(subjects.map((s) => (s ?? '').trim()).filter(Boolean)));
+    if (names.length === 0)
+        return [];
+    const placeholders = names.map(() => '?').join(', ');
+    const inScope = scope === null ? null : (0, scope_1.savedScopeSqlPredicate)((0, scope_1.normalizeScope)(scope), 'f.scope'); // any spelling of it
+    const rows = db
+        .prepare(`SELECT f.id AS id, e.canonical_name AS subject, f.predicate AS predicate,
+              f.object_text AS object, f.source_path AS sourcePath,
+              COALESCE(f.scope, 'global') AS scope, f.valid_from AS validFrom
+       FROM facts f
+       JOIN entities e ON e.id = f.subject_entity_id
+       WHERE f.valid_to IS NULL
+         AND e.canonical_name COLLATE NOCASE IN (${placeholders})${inScope ? `
+         AND ${inScope.sql}` : ''}
+       ORDER BY f.valid_from DESC, f.id DESC`)
+        .all(...names, ...(inScope?.params ?? []));
+    return rows.map((r) => ({
+        id: r.id,
+        subject: r.subject,
+        predicate: r.predicate,
+        object: r.object ?? '',
+        sourcePath: r.sourcePath,
+        scope: r.scope ?? 'global',
+        validFrom: r.validFrom ?? null,
+    }));
+}
+/**
+ * Detect read-time contradictions: >=2 CURRENT facts sharing subject+predicate
+ * but carrying different values. Values are compared after the same
+ * NFKD/ASCII/lowercase normalization the structured-row store uses, so case
+ * and accent differences are not reported as disagreements.
+ *
+ * Deterministic: within a group, sides are ordered by instant desc then id desc.
+ */
+function detectFactConflicts(rows) {
+    const groups = new Map();
+    for (const row of rows) {
+        if (!row.object.trim())
+            continue;
+        // Subject and predicate key the group, across scopes: the rows are the
+        // read's own (currentFactsForSubjects keeps a scoped read inside its
+        // scope), so a read that names no scope is told when one question has
+        // different current values in two scopes, as get_context's default
+        // conflicts tell it (memory scout finding 13).
+        const key = questionKey(row.subject, row.predicate);
+        const bucket = groups.get(key);
+        if (bucket)
+            bucket.push(row);
+        else
+            groups.set(key, [row]);
+    }
+    const conflicts = [];
+    for (const bucket of groups.values()) {
+        const byValue = new Map();
+        for (const row of bucket) {
+            const vkey = (0, rowStore_1.normDedupeValue)(row.object);
+            const list = byValue.get(vkey);
+            if (list)
+                list.push(row);
+            else
+                byValue.set(vkey, [row]);
+        }
+        if (byValue.size < 2)
+            continue; // a single value, however many sources: no disagreement
+        // One representative per distinct value: the newest row carrying it.
+        const reps = Array.from(byValue.values()).map((rowsForValue) => [...rowsForValue].sort(newestFirst)[0]);
+        reps.sort(newestFirst);
+        // Values current in different scopes: each value is named in every scope it is current in, with that scope, as
+        // get_context's default conflicts list them (memory/factConflicts.ts). Every value once, newest first, then the
+        // other scopes that hold one of them; within a scope a value's newest row stands for it, as it does across
+        // sources. A conflict inside one scope keeps its shape.
+        if (new Set(bucket.map((r) => r.scope)).size > 1) {
+            const valueInScope = (r) => `${(0, rowStore_1.normDedupeValue)(r.object)}\u0000${r.scope}`;
+            const named = new Set(reps.map(valueInScope));
+            const elsewhere = [...bucket].sort(newestFirst).filter((r) => {
+                if (named.has(valueInScope(r)))
+                    return false;
+                named.add(valueInScope(r));
+                return true;
+            });
+            const sides = [...reps, ...elsewhere].map((r) => ({ value: r.object, source: r.sourcePath, lastVerified: r.validFrom, scope: r.scope }));
+            conflicts.push(conflictAcrossScopes(reps[0].subject, reps[0].predicate, sides.slice(0, factConflicts_1.MAX_DRIFT_VALUES), sides.length));
+            continue;
+        }
+        const sides = reps.map((r) => ({
+            value: r.object,
+            source: r.sourcePath,
+            lastVerified: r.validFrom,
+        }));
+        if (sides.length < 2)
+            continue;
+        const [newer, older] = sides;
+        conflicts.push({
+            subject: reps[0].subject,
+            predicate: reps[0].predicate,
+            values: sides.map((s) => s.value),
+            newer,
+            older,
+            statement: `Contradiction on ${reps[0].subject} / ${reps[0].predicate}: ` +
+                `"${newer.value}" (${where(newer)}) is newer than ` +
+                `"${older.value}" (${where(older)}). ` +
+                `Both are current in the store — do not treat them as compatible.`,
+        });
+    }
+    // Stable order: subject, then predicate, then newest value.
+    conflicts.sort((a, b) => {
+        if (a.subject !== b.subject)
+            return a.subject < b.subject ? -1 : 1;
+        if (a.predicate !== b.predicate)
+            return a.predicate < b.predicate ? -1 : 1;
+        return a.newer.value < b.newer.value ? -1 : a.newer.value > b.newer.value ? 1 : 0;
+    });
+    return conflicts;
+}
+/**
+ * A contradiction among values current in different scopes, naming each of its `sides` with its scope, in the order
+ * `FactConflict.alsoCurrent` describes (so the first two disagree). `total` is how many sides there are when `sides`
+ * holds only the first of them (the MAX_DRIFT_VALUES cap, or `fitConflicts`); the statement then counts the rest.
+ */
+function conflictAcrossScopes(subject, predicate, sides, total = sides.length) {
+    const [newer, older, ...also] = sides;
+    const values = [];
+    const seen = new Set();
+    for (const side of sides) {
+        if (seen.has((0, rowStore_1.normDedupeValue)(side.value)))
+            continue;
+        seen.add((0, rowStore_1.normDedupeValue)(side.value));
+        values.push(side.value);
+    }
+    const unnamed = total - sides.length;
+    const named = also.map((side) => `"${side.value}" (${where(side)})`).join(', ');
+    const rest = `${named}${unnamed > 0 ? `${named ? ' and ' : ''}${unnamed} more` : ''}`;
+    return {
+        subject,
+        predicate,
+        values,
+        newer,
+        older,
+        ...(also.length > 0 ? { alsoCurrent: also } : {}),
+        ...(unnamed > 0 ? { total } : {}),
+        statement: `Contradiction on ${subject} / ${predicate}: ` +
+            `"${newer.value}" (${where(newer)}) is newer than ` +
+            `"${older.value}" (${where(older)})${rest ? `; also current: ${rest}` : ''}. ` +
+            `${total === 2 ? 'Both are' : `All ${total} are`} current in the store — do not treat them as compatible.`,
+    };
+}
+/**
+ * The conflicts cut to `budget`: a number of characters of their JSON, or whether a list of them fits (get_context: its
+ * whole reply inside max_tokens, as the engine counts tokens). `conflicts` are as detectFactConflicts returns them, and
+ * come back unchanged when they fit. Otherwise a conflict across scopes names the two values that disagree (newer,
+ * older), then more of its sides while the list fits, first conflict first, and `total` counts them all; a conflict
+ * inside one scope is kept as it is. So however many scopes hold a value, the conflicts stay within the budget, except
+ * for the two values that make each contradiction: those are always named.
+ */
+function fitConflicts(conflicts, budget) {
+    const fits = typeof budget === 'number' ? (shown) => JSON.stringify(shown).length <= budget : budget;
+    if (fits(conflicts))
+        return conflicts;
+    const sides = conflicts.map((c) => [c.newer, c.older, ...(c.alsoCurrent ?? [])]);
+    const naming = (i, n) => {
+        const c = conflicts[i];
+        return n >= sides[i].length ? c : conflictAcrossScopes(c.subject, c.predicate, sides[i].slice(0, n), c.total ?? sides[i].length);
+    };
+    const shown = conflicts.map((c, i) => (c.alsoCurrent ? naming(i, 2) : c));
+    for (let i = 0; i < shown.length; i++) {
+        if (!conflicts[i].alsoCurrent)
+            continue;
+        for (let n = 3; n <= sides[i].length; n++) {
+            const next = naming(i, n);
+            if (!fits(shown.map((c, j) => (j === i ? next : c))))
+                break;
+            shown[i] = next;
+        }
+    }
+    return shown;
+}
+/** Same rule as `subjectTerms`, applied to a canonical name or alias, so a name and the query
+ *  that names it are cut identically and a subject in any script resolves. */
+function entityTokens(value) {
+    return (0, rowStore_1.splitWords)(value).filter((t) => t.length >= 3 && !STOPWORDS.has(t));
+}
+/**
+ * Gold-blind query -> subject resolution: canonical names (or aliases) that
+ * share a content token with the query, plus any canonical name stated whole in
+ * the query. Reads only the query text and the product's own entity table.
+ */
+function querySubjects(db, query, limit = 20) {
+    const terms = subjectTerms(query);
+    if (terms.length === 0)
+        return [];
+    const lowerQuery = ` ${(0, rowStore_1.matchForm)(query)} `;
+    const found = new Set();
+    const select = db.prepare(`SELECT DISTINCT canonical_name AS name, aliases_json AS aliases
+     FROM entities
+     WHERE lower(canonical_name) LIKE '%' || ? || '%'
+     LIMIT 200`);
+    for (const term of terms.slice(0, 8)) {
+        const rows = select.all(term);
+        for (const row of rows) {
+            const name = String(row.name);
+            const tokens = entityTokens(name);
+            let aliases = [];
+            try {
+                const parsed = JSON.parse(row.aliases ?? '[]');
+                if (Array.isArray(parsed))
+                    aliases = parsed.map(String);
+            }
+            catch {
+                aliases = [];
+            }
+            const aliasTokens = aliases.flatMap(entityTokens);
+            const wholeName = name.length >= 3 && lowerQuery.includes(` ${(0, rowStore_1.matchForm)(name)} `);
+            if (wholeName || tokens.includes(term) || aliasTokens.includes(term)) {
+                found.add(name);
+            }
+            if (found.size >= limit)
+                return Array.from(found);
+        }
+    }
+    return Array.from(found);
+}
+exports.NO_ANSWER_STATEMENT = 'no good answer on file for this';
+/**
+ * Honest abstention. Returns null when the evidence clears the bar (the read
+ * may answer), or an explicit no-answer record when it does not. An empty
+ * evidence set is always a no-answer — there is nothing on file at all.
+ */
+function assessAbstention(input) {
+    const bar = input.bar ?? exports.DEFAULT_EVIDENCE_BAR;
+    const considered = input.considered ?? input.texts.length;
+    const terms = contentTerms(input.query);
+    if (considered === 0) {
+        return {
+            status: 'no-answer',
+            reason: 'no-evidence',
+            statement: exports.NO_ANSWER_STATEMENT,
+            coverage: 0,
+            bar,
+            terms,
+            considered: 0,
+        };
+    }
+    const coverage = evidenceCoverage(input.query, input.texts);
+    if (coverage < bar) {
+        return {
+            status: 'no-answer',
+            reason: 'below-evidence-bar',
+            statement: exports.NO_ANSWER_STATEMENT,
+            coverage,
+            bar,
+            terms,
+            considered,
+        };
+    }
+    return null;
+}
+function applyReadHonesty(options) {
+    const bar = resolveEvidenceBar(options.env);
+    const subjects = Array.from(new Set([...(options.subjects ?? []), ...querySubjects(options.db, options.query)]));
+    const returned = options.facts && new Set(options.facts.map((f) => questionKey(f.subject, f.predicate)));
+    const conflicts = detectFactConflicts(currentFactsForSubjects(options.db, subjects, options.scope ?? null))
+        .filter((c) => !returned || !acrossScopes(c) || returned.has(questionKey(c.subject, c.predicate)));
+    const abstention = assessAbstention({
+        query: options.query,
+        texts: options.texts,
+        considered: options.considered,
+        bar,
+    });
+    // A contradiction is itself evidence that something is on file: an agent
+    // told "no good answer" while being handed a conflict would be misled in the
+    // opposite direction. The conflict is reported, the abstention stands down.
+    if (abstention && conflicts.length > 0)
+        return { conflicts, abstention: null };
+    return { conflicts, abstention };
+}
+/**
+ * Read honesty for a retrieval-style read (the CLI `search` path and any other
+ * surface that returns note hits rather than lifecycle facts). Subjects come
+ * from the product's own entity table, joined to the query text — never from a
+ * caller-supplied list, so this cannot be fed gold labels.
+ */
+function readHonestyForItems(options) {
+    const honesty = applyReadHonesty({
+        db: options.db,
+        query: options.query,
+        ledgerRelPath: options.ledgerRelPath,
+        texts: options.items.map((i) => i.text),
+        considered: options.items.length,
+        env: options.env,
+    });
+    return {
+        ...honesty,
+        provenance: options.items.map((i) => noteProvenance(i.notePath, i.session_date ?? null)),
+    };
+}