renor-engine 0.0.0-stage → 1.0.0

package/dist/eval/hashEmbedding.js

@@ -0,0 +1,35 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EvalHashEmbeddingProvider = void 0;
+/**
+ * Deterministic lexical hash embeddings for eval infrastructure smoke tests.
+ *
+ * This provider intentionally has no semantic-quality claim. It exists so CI can
+ * execute indexing/vector-search/scoring paths without network/model downloads.
+ * Any manifest using it must be marked qualityEligible=false.
+ */
+class EvalHashEmbeddingProvider {
+    dimensions = 256;
+    identity = { kind: 'eval-hash', model: 'lexical-hash-v1' };
+    vector(text) {
+        const out = new Array(this.dimensions).fill(0);
+        const words = text.toLowerCase().match(/[\p{L}\p{N}]+/gu) ?? [];
+        for (const word of words) {
+            let hash = 2166136261;
+            for (let i = 0; i < word.length; i++) {
+                hash ^= word.charCodeAt(i);
+                hash = Math.imul(hash, 16777619);
+            }
+            out[(hash >>> 0) % this.dimensions] += 1;
+        }
+        const norm = Math.sqrt(out.reduce((sum, value) => sum + value * value, 0)) || 1;
+        return out.map(value => value / norm);
+    }
+    async embed(texts) {
+        return texts.map(text => this.vector(text));
+    }
+    async embedQuery(text) {
+        return this.vector(text);
+    }
+}
+exports.EvalHashEmbeddingProvider = EvalHashEmbeddingProvider;

package/dist/eval/longmemeval.js

@@ -0,0 +1,56 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseTimestamp = parseTimestamp;
+exports.validateLongMemRows = validateLongMemRows;
+/** Upstream timestamps have no zone. Interpret them as UTC, never host-local time. */
+function parseTimestamp(value) {
+    const official = /^(\d{4})\/(\d{2})\/(\d{2})(?: \([A-Za-z]{3}\))?(?: (\d{2}):(\d{2}))?$/.exec(value);
+    const iso = official ? `${official[1]}-${official[2]}-${official[3]}T${official[4] ?? '00'}:${official[5] ?? '00'}:00Z` : value;
+    if (!/(Z|[+-]\d{2}:\d{2})$/.test(iso))
+        throw new Error(`Ambiguous timestamp: ${value}`);
+    const date = new Date(iso);
+    if (Number.isNaN(date.getTime()) || (official && date.toISOString().slice(0, 16) !== iso.slice(0, 16)))
+        throw new Error(`Invalid timestamp: ${value}`);
+    return date;
+}
+function uniqueStrings(values, label) {
+    if (!Array.isArray(values) || values.some(v => typeof v !== 'string' || !v.trim()))
+        throw new Error(`Invalid ${label}`);
+    if (new Set(values).size !== values.length)
+        throw new Error(`Duplicate ${label}`);
+}
+function validateLongMemRows(rows) {
+    if (!Array.isArray(rows) || !rows.length)
+        throw new Error('Empty/invalid LongMemEval dataset');
+    uniqueStrings(rows.map(r => r.question_id), 'question id');
+    for (const row of rows) {
+        if (!row.question?.trim() || !row.question_type?.trim())
+            throw new Error('Missing question/type');
+        parseTimestamp(row.question_date);
+        uniqueStrings(row.answer_session_ids, 'answer session id');
+        // Official data may repeat a noise haystack session id; a repeated ANSWER id
+        // would make evidence mapping ambiguous and stays rejected.
+        const seenAnswer = new Set(row.answer_session_ids);
+        const haystackCounts = new Map();
+        for (const id of row.haystack_session_ids) {
+            haystackCounts.set(id, (haystackCounts.get(id) ?? 0) + 1);
+            if ((haystackCounts.get(id) ?? 0) > 1 && seenAnswer.has(id)) {
+                throw new Error(`Duplicated answer session id in haystack: ${id}`);
+            }
+        }
+        if (!Array.isArray(row.haystack_sessions) || !Array.isArray(row.haystack_dates)
+            || row.haystack_sessions.length !== row.haystack_session_ids.length
+            || row.haystack_dates.length !== row.haystack_session_ids.length)
+            throw new Error('Haystack arrays differ in length');
+        row.haystack_dates.forEach(parseTimestamp);
+        // Official data legitimately contains empty noise sessions; roles/content stay strict.
+        for (const turns of row.haystack_sessions) {
+            if (!Array.isArray(turns) || turns.some(t => !['user', 'assistant'].includes(t.role) || typeof t.content !== 'string')) {
+                throw new Error('Invalid session turns');
+            }
+        }
+        if (!row.question_id.endsWith('_abs') && (!row.answer_session_ids.length
+            || row.answer_session_ids.some(id => !row.haystack_session_ids.includes(id))))
+            throw new Error('Missing answer evidence sessions');
+    }
+}

package/dist/eval/metrics.js

@@ -0,0 +1,181 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.roleOf = roleOf;
+exports.hitMatchesTarget = hitMatchesTarget;
+exports.matchedTargetIndexes = matchedTargetIndexes;
+exports.hitAtK = hitAtK;
+exports.recallAtK = recallAtK;
+exports.recallAllAtK = recallAllAtK;
+exports.forbiddenRetrieved = forbiddenRetrieved;
+exports.reciprocalRank = reciprocalRank;
+exports.ndcgAtK = ndcgAtK;
+exports.contradictionFirst = contradictionFirst;
+/**
+ * Role resolution for a gold target.
+ *
+ * Explicit `role` always wins. Otherwise RenorBench v0 semantics apply:
+ * - relevance >= 2  -> positive (creditable for Hit/Recall/MRR and nDCG)
+ * - relevance === 1 -> forbidden (superseded/contradictory value that is NOT
+ *   the answer to the query; never creditable, and surfacing it first is an
+ *   error tracked by contradictionFirst)
+ * - relevance absent -> positive (the target is the only/primary gold)
+ *
+ * Use explicit `role: 'supporting'` when a low-grade target is genuinely
+ * acceptable evidence (nDCG-graded but never counted by binary metrics).
+ */
+function roleOf(target) {
+    if (target.role)
+        return target.role;
+    if (target.relevance === undefined)
+        return 'positive';
+    return target.relevance >= 2 ? 'positive' : 'forbidden';
+}
+function normalized(value) {
+    return (value ?? '').trim().toLowerCase();
+}
+/**
+ * Matching semantics:
+ *
+ * - Path-pinned targets (notePath set) require the hit to come from exactly
+ *   that note, and credit a needle occurring in the hit text or in the pinned
+ *   path itself (session-id style metadata matching, used by LongMemEval).
+ * - Text-scoped targets (no notePath) match against the retrieved text ONLY.
+ *   A needle appearing merely in a filename never earns textual relevance
+ *   credit, and unrelated notes can never satisfy the target.
+ */
+function hitMatchesTarget(hit, target) {
+    const needle = normalized(target.match);
+    if (!needle)
+        return false;
+    if (target.notePath !== undefined) {
+        if (normalized(hit.notePath) !== normalized(target.notePath))
+            return false;
+        return normalized(hit.text).includes(needle) || normalized(target.notePath).includes(needle);
+    }
+    return normalized(hit.text).includes(needle);
+}
+function creditableTargets(targets) {
+    return targets.filter(target => roleOf(target) === 'positive');
+}
+function matchedTargetIndexes(hits, targets, k) {
+    const matched = new Set();
+    for (const hit of hits.slice(0, k)) {
+        targets.forEach((target, index) => {
+            if (hitMatchesTarget(hit, target))
+                matched.add(index);
+        });
+    }
+    return matched;
+}
+/**
+ * Binary Hit@K: credit only positive evidence. Stale/forbidden or merely
+ * supporting gold targets can never produce a Hit.
+ */
+function hitAtK(hits, targets, k) {
+    const positive = creditableTargets(targets);
+    if (positive.length === 0)
+        return 0;
+    return matchedTargetIndexes(hits, positive, k).size > 0 ? 1 : 0;
+}
+/**
+ * Recall@K over positive evidence only.
+ */
+function recallAtK(hits, targets, k) {
+    const positive = creditableTargets(targets);
+    if (positive.length === 0)
+        return 0;
+    return matchedTargetIndexes(hits, positive, k).size / positive.length;
+}
+/**
+ * All-required-evidence retrieval (multi-hop): 1 only when every required
+ * target is matched within top-K. Single-target lists reduce to Hit@K.
+ */
+function recallAllAtK(hits, targets, k) {
+    const positive = creditableTargets(targets);
+    if (positive.length === 0)
+        return 0;
+    return matchedTargetIndexes(hits, positive, k).size === positive.length ? 1 : 0;
+}
+/**
+ * True when any forbidden-evidence target is retrieved within top-K.
+ * Used for deletion/erasure cases: retrieving deleted or must-not-surface
+ * content is a violation regardless of ordering.
+ */
+function forbiddenRetrieved(hits, targets, k) {
+    const forbidden = targets.filter(target => roleOf(target) === 'forbidden');
+    if (forbidden.length === 0)
+        return false;
+    return matchedTargetIndexes(hits, forbidden, k).size > 0;
+}
+/**
+ * Reciprocal rank of the first positive-evidence hit.
+ */
+function reciprocalRank(hits, targets) {
+    const positive = creditableTargets(targets);
+    if (positive.length === 0)
+        return 0;
+    const rank = hits.findIndex(hit => positive.some(target => hitMatchesTarget(hit, target)));
+    return rank === -1 ? 0 : 1 / (rank + 1);
+}
+function dcg(relevances) {
+    return relevances.reduce((sum, relevance, index) => {
+        if (relevance <= 0)
+            return sum;
+        return sum + (Math.pow(2, relevance) - 1) / Math.log2(index + 2);
+    }, 0);
+}
+/**
+ * Graded nDCG where each gold target can receive credit at most once.
+ * This prevents duplicated retrieval rows from inflating the score.
+ * Forbidden targets are excluded entirely; supporting targets keep their grade.
+ */
+function ndcgAtK(hits, targets, k) {
+    const graded = targets
+        .map((target, index) => ({ target, index }))
+        .filter(({ target }) => roleOf(target) !== 'forbidden');
+    if (graded.length === 0)
+        return 0;
+    const credited = new Set();
+    const observed = [];
+    for (const hit of hits.slice(0, k)) {
+        let bestIndex = -1;
+        let bestRelevance = 0;
+        graded.forEach(({ target, index }) => {
+            if (credited.has(index) || !hitMatchesTarget(hit, target))
+                return;
+            const relevance = target.relevance ?? 1;
+            if (relevance > bestRelevance) {
+                bestRelevance = relevance;
+                bestIndex = index;
+            }
+        });
+        if (bestIndex >= 0)
+            credited.add(bestIndex);
+        observed.push(bestRelevance);
+    }
+    const ideal = graded
+        .map(({ target }) => target.relevance ?? 1)
+        .sort((a, b) => b - a)
+        .slice(0, k);
+    const idealDcg = dcg(ideal);
+    return idealDcg === 0 ? 0 : dcg(observed) / idealDcg;
+}
+/**
+ * True when the highest-ranked gold-matching hit matches only forbidden
+ * targets: an obsolete/contradictory answer was surfaced ahead of the
+ * current one. Cases without both positive and forbidden targets never
+ * trigger this.
+ */
+function contradictionFirst(hits, targets) {
+    const hasPositive = targets.some(target => roleOf(target) === 'positive');
+    const hasForbidden = targets.some(target => roleOf(target) === 'forbidden');
+    if (!hasPositive || !hasForbidden)
+        return false;
+    for (const hit of hits) {
+        const matched = targets.filter(target => hitMatchesTarget(hit, target));
+        if (matched.length === 0)
+            continue;
+        return matched.every(target => roleOf(target) === 'forbidden');
+    }
+    return false;
+}