renor-engine 0.0.0-stage → 1.0.0

package/dist/mvp/contextRelevance.js

@@ -0,0 +1,413 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FACT_CEILING = exports.SEARCH_RELEVANCE_GUIDANCE = exports.CONTEXT_BUDGET_KEEPS_NEWEST = exports.CONTEXT_RELEVANCE_KEEPS_MATCH = exports.CONTEXT_RELEVANCE_GUIDANCE = exports.UNKNOWN_NAME_MARGIN = exports.notMentioned = exports.NO_STRONG_MATCH = exports.CALIBRATED_BARS = exports.CONTEXT_RELEVANCE_DEFAULT_ON = exports.CONTEXT_RELEVANCE_MIN_ENV = exports.CONTEXT_RELEVANCE_FLAG = void 0;
+exports.requestNames = requestNames;
+exports.contextRelevanceEnabled = contextRelevanceEnabled;
+exports.embeddingModelKey = embeddingModelKey;
+exports.resolveRelevanceBar = resolveRelevanceBar;
+exports.contextRelevanceArmed = contextRelevanceArmed;
+exports.renderFactForRelevance = renderFactForRelevance;
+exports.factLane = factLane;
+exports.similaritiesTo = similaritiesTo;
+exports.warmFactVectors = warmFactVectors;
+exports.combineSimilarity = combineSimilarity;
+exports.assessContextRelevance = assessContextRelevance;
+exports.assessQueryRelevance = assessQueryRelevance;
+const readCache_1 = require("../db/readCache");
+const statements_1 = require("../db/statements");
+const factTerms_1 = require("../memory/factTerms");
+const factLanePack_1 = require("./factLanePack");
+const scope_1 = require("./scope");
+/**
+ * get_context relevance signal: "does anything on file actually match the request?"
+ *
+ * Without it an agent gets the same five confident-looking memories for a question
+ * nothing on file answers, and fills the gap by guessing. The signal is the best
+ * embedding similarity between the query and (a) the note chunks get_context returned
+ * and (b) a clean "attribute: value" rendering of every current fact, compared with a
+ * bar calibrated per embedding model on fixtures/relevance (method, confusion matrices
+ * and the chosen bar: docs/evals/results/RELEVANCE-SIGNAL-20261002.md).
+ *
+ * Why not the lexical term coverage RENOR_READ_HONESTY uses: coverage counts query
+ * words found in the best item, so a paraphrase ("Where am I based?" for
+ * home_city = Lisbon) scores 0 even when retrieval found the right memory. Why a fact
+ * rendering and not just the ledger chunk: the chunk is mostly event boilerplate (ids,
+ * timestamps, "kind: store"), which flattens every similarity toward the same value;
+ * the rendering is only the claim.
+ *
+ * The signal is cheap, and an agent reads `relevant: true` as "the answer is below", so the
+ * response holds the match it reports (`matchedOn`), the closest fact or note chunk: the
+ * budget makes room for it, cutting other items if it must (agentMemory.ts), and when it
+ * does not fit max_tokens even on its own there is no verdict rather than one about something
+ * the response lacks. The current fact closest to the request, when it clears the bar on its
+ * own (`closestFact`), is listed when the words of the request did not find it: in their
+ * place when they found none, else after the facts they found, in the room the rest leaves
+ * unless it is the match reported. It never takes the place of a fact they found: a
+ * rendering has no subject, so the closest one is often a fact that mentions the project
+ * ("role: product manager for Atlas" is near every Atlas question) or another ticket's
+ * status, and room taken from the rest for it cost answers. Nothing else is added, removed or
+ * reordered, and it adds no embedding call to get_context: the query vector comes from the
+ * semantic search that already ran, and fact vectors are embedded once, in the background.
+ * Two cases are judged without a cosine to compare: a read that found no note by meaning and
+ * holds no current fact is "not relevant" (similarity null), and a request that names
+ * something nothing in the scope read mentions needs a match closer by UNKNOWN_NAME_MARGIN.
+ * It is absent (never a guess) whenever it cannot judge: flag off, no query vector, no
+ * calibrated bar for the embedding model, or fact vectors still warming - a "not relevant"
+ * verdict needs every current fact compared.
+ */
+/** RENOR_CONTEXT_RELEVANCE: 1/true/yes/on enables, 0/false/no/off disables. */
+exports.CONTEXT_RELEVANCE_FLAG = 'RENOR_CONTEXT_RELEVANCE';
+/** RENOR_CONTEXT_RELEVANCE_MIN: explicit bar, for an embedding model with no calibrated one. */
+exports.CONTEXT_RELEVANCE_MIN_ENV = 'RENOR_CONTEXT_RELEVANCE_MIN';
+/**
+ * Shipped default: on, because the held-out calibration met the pre-registered bar (false abstentions on
+ * answerable questions < 5% AND most unanswerable flagged, at both workspace sizes). It can only be on, and
+ * can only speak, for an embedding model with a calibrated bar (CALIBRATED_BARS) or an explicit
+ * RENOR_CONTEXT_RELEVANCE_MIN. tests/mvp/context-relevance-calibration.test.ts asserts this constant and the
+ * recorded calibration cannot disagree.
+ */
+exports.CONTEXT_RELEVANCE_DEFAULT_ON = true;
+/**
+ * Calibrated bars, keyed `<MVP_EMBEDDING_PROVIDER>:<model>:<dimensions>`. Cosine scales
+ * differ by model and output size, so a bar never transfers between them.
+ */
+exports.CALIBRATED_BARS = {
+    'vertex:gemini-embedding-001:768': 0.63,
+};
+exports.NO_STRONG_MATCH = 'no stored fact or note closely matches this request; the listed items are only the nearest ones';
+/** When `names` were looked for and nothing on file mentions them. */
+const notMentioned = (names) => `nothing on file mentions ${names.join(', ')}; the listed items are only the nearest ones`;
+exports.notMentioned = notMentioned;
+/**
+ * How much closer a match must be when the request names something that nothing on file mentions ("What is the
+ * Quillmoor pricing?" on a workspace that never said Quillmoor): such a request is about something else than anything
+ * on file, and its match is on the generic part ("pricing"). Live, those sat at 0.646-0.651 over the 0.63 bar. Chosen
+ * on the dev questions, where a name a semantic model reads across languages ("Wie heißt meine Frau?", 0.708) must
+ * stay relevant, and checked on the held-out ones: no answerable question is lost
+ * (tests/mvp/context-relevance-calibration.test.ts).
+ */
+exports.UNKNOWN_NAME_MARGIN = 0.05;
+/** Capitalized in English, but a time rather than the name of something on file. */
+const CALENDAR_WORDS = new Set([
+    'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday',
+    'january', 'february', 'march', 'april', 'may', 'june', 'july', 'august', 'september', 'october', 'november', 'december',
+]);
+/**
+ * The names in a request: capitalized words that do not open a sentence ("Quillmoor" in "What is the Quillmoor
+ * pricing?"), without a possessive. Not "I", a weekday or a month. A sentence's first word is capitalized for that
+ * alone, so it is never taken for a name.
+ */
+function requestNames(query) {
+    const names = new Set();
+    let opening = true;
+    for (const [token] of query.matchAll(/[\p{L}\p{N}][\p{L}\p{N}\p{M}'’-]*|[.?!:;]+(?=\s|$)|[¿¡]/gu)) {
+        if (!/^[\p{L}\p{N}]/u.test(token)) {
+            opening = true;
+            continue;
+        }
+        const word = token.replace(/['’]s$/i, '').replace(/['’-]+$/, '');
+        if (!opening && /^\p{Lu}/u.test(word) && !/^I(['’]\p{L}+)?$/u.test(word) && !CALENDAR_WORDS.has(word.toLowerCase()))
+            names.add(word);
+        opening = false;
+    }
+    return [...names];
+}
+function contextRelevanceEnabled(env = process.env) {
+    const raw = (env[exports.CONTEXT_RELEVANCE_FLAG] ?? '').trim().toLowerCase();
+    if (raw === '1' || raw === 'true' || raw === 'yes' || raw === 'on')
+        return true;
+    if (raw === '0' || raw === 'false' || raw === 'no' || raw === 'off')
+        return false;
+    return exports.CONTEXT_RELEVANCE_DEFAULT_ON;
+}
+function embeddingModelKey(provider, model, dims) {
+    return `${provider}:${model ?? ''}:${dims ?? ''}`;
+}
+function resolveRelevanceBar(modelKey, env = process.env) {
+    const raw = (env[exports.CONTEXT_RELEVANCE_MIN_ENV] ?? '').trim();
+    if (raw !== '') {
+        const explicit = Number(raw);
+        if (Number.isFinite(explicit))
+            return explicit;
+    }
+    return modelKey && modelKey in exports.CALIBRATED_BARS ? exports.CALIBRATED_BARS[modelKey] : null;
+}
+/** Whether get_context can return the signal, i.e. whether agents should be told about it: on, embeddings configured, a bar to judge by. */
+function contextRelevanceArmed(engine, env = process.env) {
+    return Boolean(engine.provider) && contextRelevanceEnabled(env) && resolveRelevanceBar(engine.modelKey, env) !== null;
+}
+/** Appended to the get_context tool description only while the signal is armed. */
+exports.CONTEXT_RELEVANCE_GUIDANCE = ' If the result has relevance.relevant === false, nothing on file closely matches this request: '
+    + 'the listed items are only the nearest ones, so do not present them as an answer unless one clearly '
+    + 'answers it; say you have no memory of it and ask the user.';
+/**
+ * Appended to the get_context description after the guidance, while the signal is armed: the one case in which
+ * max_tokens does not drop the item that ranks last first (agentMemory.ts keeps what a relevant verdict was judged on).
+ */
+exports.CONTEXT_RELEVANCE_KEEPS_MATCH = ' When relevance.relevant is true, the closest match is in the result, or the newer relevant match the budget kept in its place: max_tokens keeps one of them even if another item has to be dropped to make room for it.';
+/**
+ * Appended to the get_context description after the two sentences above, while the signal is armed (the model has a bar to
+ * judge by): what the budget keeps first when the matches do not all fit (agentMemory.ts newestMatchAbove, sessionMatesAbove).
+ */
+exports.CONTEXT_BUDGET_KEEPS_NEWEST = ' When the matches do not all fit, the newest match above the relevance bar is kept first, with the pieces of its session that say most about the question (at most four, in at most 60% of the budget), then the best-scoring ones, so the latest relevant statement is not the one the budget drops.';
+/**
+ * Appended to the search_memory description while the signal is armed. `relevance.fact` is search's alone, and only
+ * for the one case where the page could not carry the match: the fact is named, not listed.
+ */
+exports.SEARCH_RELEVANCE_GUIDANCE = ' A relevance object says whether anything on file closely matches the query: relevant is true only when the closest match is among the hits, and when it is false the hits are only the nearest ones, so do not present them as an answer unless one clearly answers it.'
+    + ' When relevant is true and the closest match is a remembered fact that top_k left no room to list, relevance.fact names it (subject, predicate, object, scope); you can answer from it, and the hits are still at most top_k in all.';
+/**
+ * Most current facts the lane compares per call. A tenant with more is not silently truncated: the lane is
+ * reported incomplete, so it can still prove relevance but can never be the basis of "nothing matches".
+ * The fact list is read once until the next write. Each call then dots the query with the unit vectors
+ * already stored for those facts (scripts/eval/relevance-latency.ts measured the uncached list).
+ */
+exports.FACT_CEILING = 5_000;
+/** Vectors of no-longer-current facts kept before the store is pruned to the live set. */
+const STALE_VECTORS_KEPT = 256;
+/** Texts per embedding request, and requests in flight, while fact vectors are filled. */
+const FILL_BATCH = 32;
+const FILL_CONCURRENCY = 4;
+/** After a failed fill, how long reads leave the embedding backend alone before trying again. */
+const FILL_RETRY_MS = 30_000;
+/** Per embedding provider, so two models (or two test engines) never share vectors. */
+const slots = new WeakMap();
+function slotFor(provider) {
+    let slot = slots.get(provider);
+    if (!slot)
+        slots.set(provider, slot = { vectors: new Map(), filling: null, retryAfter: 0, epoch: 0, packed: new Map() });
+    return slot;
+}
+function renderFactForRelevance(predicate, object) {
+    return `${predicate.replace(/_/g, ' ')}: ${object}`;
+}
+function unit(v) {
+    let sum = 0;
+    for (let i = 0; i < v.length; i++)
+        sum += v[i] * v[i];
+    const norm = Math.sqrt(sum) || 1;
+    const out = new Float32Array(v.length);
+    for (let i = 0; i < v.length; i++)
+        out[i] = v[i] / norm;
+    return out;
+}
+function dot(a, b) {
+    let s = 0;
+    for (let i = 0; i < a.length; i++)
+        s += a[i] * b[i];
+    return s;
+}
+/** Embeds `texts` into the slot, FILL_CONCURRENCY requests at a time. A failed batch leaves its texts for the next call. */
+async function fill(provider, slot, texts) {
+    const batches = [];
+    for (let i = 0; i < texts.length; i += FILL_BATCH)
+        batches.push(texts.slice(i, i + FILL_BATCH));
+    let next = 0;
+    const worker = async () => {
+        for (let b = next++; b < batches.length; b = next++) {
+            const batch = batches[b];
+            const fresh = await provider.embed(batch);
+            if (fresh.length !== batch.length)
+                throw new Error('embedding provider returned a wrong vector count');
+            batch.forEach((text, i) => slot.vectors.set(text, unit(fresh[i])));
+            slot.epoch++;
+        }
+    };
+    await Promise.all(Array.from({ length: Math.min(FILL_CONCURRENCY, batches.length) }, worker));
+}
+/** What a lane was asked for (any scope, or one; how many facts at most): the key its snapshot and its pack are kept under. */
+function factLaneKey(scope, ceiling) {
+    return `${scope === undefined ? '*' : (0, scope_1.normalizeScope)(scope)}\0${ceiling}`;
+}
+function loadFactSnapshot(db, scope, ceiling) {
+    const inScope = scope === undefined ? undefined : (0, scope_1.savedScopeSqlPredicate)((0, scope_1.normalizeScope)(scope), 'f.scope'); // any spelling of it
+    const select = `SELECT f.id AS id, f.predicate AS predicate, f.object_text AS object
+     FROM facts f
+     WHERE f.valid_to IS NULL AND f.source_path = '_mcp_'${inScope ? ` AND ${inScope.sql}` : ''}`;
+    const scopeArgs = inScope?.params ?? [];
+    let rows = (0, statements_1.prepareOnce)(db, `${select} LIMIT ?`).all(...scopeArgs, ceiling + 1);
+    const truncated = rows.length > ceiling;
+    if (truncated) {
+        // Newest by row id: a reverse scan of the primary key, no sort. Only positives can come of this subset.
+        rows = (0, statements_1.prepareOnce)(db, `${select} ORDER BY f.id DESC LIMIT ?`).all(...scopeArgs, ceiling);
+    }
+    // Each rendering once, with the newest fact rendered that way: the one a match is shown as.
+    const newest = new Map();
+    for (const r of rows) {
+        const rendering = renderFactForRelevance(r.predicate, r.object);
+        const seen = newest.get(rendering);
+        if (seen === undefined || r.id > seen)
+            newest.set(rendering, r.id);
+    }
+    return { pairs: [...newest], truncated };
+}
+/** The best fact when there is no matrix to read (vectors of two lengths) or the query is of another length: one lookup and one dot per fact. */
+function bestFromSlot(q, snapshot, slot) {
+    let best = null;
+    let id = -1;
+    for (const [rendering, factId] of snapshot.pairs) {
+        const v = slot.vectors.get(rendering);
+        if (!v)
+            continue;
+        const s = dot(q, v);
+        if (best === null || s > best) {
+            best = s;
+            id = factId;
+        }
+    }
+    return best === null ? null : { score: best, id };
+}
+/** Drop vectors of facts that are no longer in this snapshot, then embed what is still missing. Same trigger as before. */
+function fillMissing(provider, slot, snapshot, missing) {
+    if (missing.length === 0 || slot.filling || Date.now() < slot.retryAfter)
+        return;
+    if (slot.vectors.size > snapshot.pairs.length + STALE_VECTORS_KEPT) {
+        const live = new Set(snapshot.pairs.map(([rendering]) => rendering));
+        let dropped = false;
+        for (const key of slot.vectors.keys())
+            if (!live.has(key)) {
+                slot.vectors.delete(key);
+                dropped = true;
+            }
+        // Live rows were not dropped, so a lane packed from them is still the same dots.
+        if (dropped)
+            slot.epoch++;
+    }
+    const run = slot.filling = fill(provider, slot, missing)
+        .catch(() => { slot.retryAfter = Date.now() + FILL_RETRY_MS; /* advisory: retried later for what is still missing */ })
+        .finally(() => { if (slot.filling === run)
+        slot.filling = null; });
+}
+/**
+ * Best similarity between the query vector and the current facts (scope-filtered).
+ * Never waits on the embedding backend: facts without a vector yet start a background
+ * fill and are reported through `complete: false`; what is left is a dot product per fact.
+ * `complete` is true only when EVERY in-scope current fact was compared: past `ceiling`
+ * facts only the newest are, and the lane says so.
+ */
+function factLane(db, provider, queryVector, scope, ceiling = exports.FACT_CEILING) {
+    const key = factLaneKey(scope, ceiling);
+    // The rows change only when something is written. Vectors arrive later, from the background fill.
+    const snapshot = (0, readCache_1.readUntilWrite)(db, `factLane\0${key}`, () => loadFactSnapshot(db, scope, ceiling));
+    const slot = slotFor(provider);
+    let packed = slot.packed.get(key);
+    if (!packed || packed.snapshot !== snapshot || packed.epoch !== slot.epoch) {
+        packed = (0, factLanePack_1.packLane)(snapshot, slot.vectors, slot.epoch);
+        if (slot.packed.size >= 8)
+            slot.packed.clear();
+        slot.packed.set(key, packed);
+    }
+    fillMissing(provider, slot, snapshot, packed.missing);
+    if (packed.epoch !== slot.epoch)
+        packed.epoch = slot.epoch;
+    const q = unit(queryVector);
+    const found = packed.matrix && q.length === packed.dim
+        ? (() => {
+            const best = (0, factLanePack_1.bestPacked)(q, packed.matrix, packed.ids.length, packed.dim);
+            return best ? { score: best.score, id: packed.ids[best.index] } : null;
+        })()
+        : bestFromSlot(q, snapshot, slot);
+    return {
+        similarity: found ? found.score : null,
+        complete: !snapshot.truncated && packed.missing.length === 0,
+        ...(found ? { bestFactId: found.id } : {}),
+    };
+}
+/**
+ * Cosine between `queryVector` and each of `texts` (fact renderings, `renderFactForRelevance`), read from the same
+ * per-provider store the fact lane fills, through the same embedding provider the notes are searched with. A text the
+ * store lacks is embedded now (awaited, FILL_BATCH at a time), so a caller that needs the answer for this read gets it.
+ * Null when that fails, or while the backend is in the pause after a failed fill: the caller answers by words instead.
+ */
+async function similaritiesTo(provider, texts, queryVector) {
+    const slot = slotFor(provider);
+    const missing = [...new Set(texts)].filter(text => !slot.vectors.has(text));
+    if (missing.length > 0) {
+        if (Date.now() < slot.retryAfter)
+            return null;
+        try {
+            await fill(provider, slot, missing);
+        }
+        catch {
+            slot.retryAfter = Date.now() + FILL_RETRY_MS;
+            return null;
+        }
+    }
+    const q = unit(queryVector);
+    const out = new Map();
+    for (const text of texts) {
+        const v = slot.vectors.get(text);
+        if (v && v.length === q.length)
+            out.set(text, dot(q, v));
+    }
+    return out;
+}
+/** Resolves once the fact vectors a call would need are embedded (or a fill failed): for tests, calibration, warm-up. */
+async function warmFactVectors(db, provider, scope) {
+    const probeVector = new Float32Array(provider.dimensions).fill(1);
+    slotFor(provider).retryAfter = 0; // an explicit warm-up is not held back by an earlier failure
+    for (let attempt = 0; attempt < 4; attempt++) {
+        if (factLane(db, provider, probeVector, scope).complete)
+            return true;
+        await slotFor(provider).filling;
+    }
+    return factLane(db, provider, probeVector, scope).complete;
+}
+/** The signal is the best match across both lanes; either lane alone may be unavailable. */
+function combineSimilarity(chunk, fact) {
+    if (chunk === null)
+        return fact;
+    if (fact === null)
+        return chunk;
+    return Math.max(chunk, fact);
+}
+function assessContextRelevance(topSimilarity, bar) {
+    if (topSimilarity === null || bar === null || !Number.isFinite(topSimilarity))
+        return null;
+    // Decide on the rounded value the caller sees, so a reported similarity equal to
+    // the bar can never be reported irrelevant.
+    const similarity = Math.round(topSimilarity * 1000) / 1000;
+    const relevant = similarity >= bar;
+    return { relevant, ...(relevant ? {} : { reason: exports.NO_STRONG_MATCH }), similarity, bar };
+}
+/** Advisory by design: any failure here yields no signal rather than a failed get_context. */
+function assessQueryRelevance(input) {
+    const { db, engine, scope, probe, env } = input;
+    const bar = resolveRelevanceBar(engine.modelKey, env);
+    // No query vector means retrieval embedded nothing (empty index): the signal does not pay for one.
+    if (bar === null || !engine.provider || !probe.queryVector)
+        return null;
+    try {
+        const chunk = probe.topSimilarity ?? null;
+        const fact = factLane(db, engine.provider, probe.queryVector, scope);
+        if (input.components) {
+            input.components.chunk = chunk;
+            input.components.fact = fact.similarity;
+        }
+        const best = combineSimilarity(chunk, fact.similarity);
+        // Nothing to compare: the read found no note by meaning and holds no current fact, so nothing on file is close to
+        // the request (withheld while fact vectors are warming, as any "not relevant" is).
+        if (best === null)
+            return fact.complete ? { relevance: { relevant: false, reason: exports.NO_STRONG_MATCH, similarity: null, bar } } : null;
+        // A match just over the bar for a request that names something nothing on file mentions has to be closer.
+        const raised = Math.round((bar + exports.UNKNOWN_NAME_MARGIN) * 1000) / 1000;
+        const unknown = input.onFile && assessContextRelevance(best, bar)?.relevant && !assessContextRelevance(best, raised)?.relevant
+            ? requestNames(input.query ?? '').filter(name => !input.onFile(name))
+            : [];
+        const judgedBar = unknown.length > 0 ? raised : bar;
+        const verdict = assessContextRelevance(best, judgedBar);
+        // A match found among some facts proves relevance; "nothing matches" needs all of them compared.
+        if (!verdict || (!verdict.relevant && !fact.complete))
+            return null;
+        if (unknown.length > 0)
+            verdict.reason = (0, exports.notMentioned)(unknown);
+        const closestFact = fact.bestFactId !== undefined && assessContextRelevance(fact.similarity, judgedBar)?.relevant
+            ? (0, factTerms_1.currentFactById)(db, fact.bestFactId)
+            : undefined;
+        const matchedOn = !verdict.relevant ? undefined : chunk === null || (fact.similarity !== null && fact.similarity >= chunk) ? 'fact' : 'chunk';
+        return { relevance: verdict, ...(closestFact ? { closestFact } : {}), ...(matchedOn ? { matchedOn } : {}) };
+    }
+    catch {
+        return null;
+    }
+}