renor-engine 0.0.0-stage → 1.0.0

package/dist/eval/renorbenchBaseline.js

@@ -0,0 +1,486 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DEFAULT_BASE_REF = exports.RENORBENCH_ARMS = exports.DOCUMENTS_ARM = exports.PRODUCT_V2_ARM = exports.PRODUCT_ARM = exports.CHUNK_ARM = exports.DOCUMENTS_BASELINE_RELATIVE_PATH = exports.PRODUCT_V2_BASELINE_RELATIVE_PATH = exports.PRODUCT_BASELINE_RELATIVE_PATH = exports.BASELINE_RELATIVE_PATH = exports.BASELINE_BENCHMARK = exports.BASELINE_SCHEMA = void 0;
+exports.armFor = armFor;
+exports.loadRunDir = loadRunDir;
+exports.buildBaseline = buildBaseline;
+exports.readBaseline = readBaseline;
+exports.parseBaseline = parseBaseline;
+exports.checkAgainstBaseline = checkAgainstBaseline;
+exports.readMergeBaseState = readMergeBaseState;
+exports.compareToFloor = compareToFloor;
+exports.recordsLoweringOf = recordsLoweringOf;
+exports.compareToOwnScore = compareToOwnScore;
+exports.checkWithMergeBase = checkWithMergeBase;
+exports.planRecord = planRecord;
+/**
+ * The committed RenorBench baseline and the no-regression check the agent gate runs against it.
+ *
+ * The baseline is a compact record of one hash-embedder run (manifest invariants + scored metrics),
+ * small enough that a PR that moves it shows exactly which numbers moved. The verdict comes from the
+ * repo's own comparator (`compareRuns`, tolerance 0), never from a second opinion about what
+ * "worse" means. Latency is not recorded: it is noise and is not a gate metric.
+ *
+ * This file is part of the frozen evaluator (EVALUATOR_SOURCES): the gate cannot loosen its own exam
+ * without moving the evaluator hash, which the check refuses until the baseline is deliberately refreshed.
+ *
+ * The baseline file is an ordinary tracked file, so no number in it is trusted: `check` re-scores the run and
+ * requires the committed summary to EQUAL it, on every path, before anything else. The merge-base (origin/main)
+ * baseline is read only to authorise a LOWER baseline: it must record a deliberate lowering that names the exact
+ * merge-base baseline it lowers (`lowering.fromBaselineSha256`) and exactly the metrics that fell, written by
+ * `record --accept-lower`.
+ *
+ * There are four arms, told apart by the `benchmark` id of a run and of its baseline file: the chunk arm (the
+ * retrieval hits at top-K 10), the product arm (the reply of the shipped `search_memory` tool), the product arm on the
+ * v2 suite (a larger vault, with questions that need session expansion, the newest-value rules and the verdict) and the
+ * product arm on the documents suite (handbook-style notes: what the expansion delivers from a document). Each has
+ * its own baseline file, tracked metrics, scorer and identity fields (RENORBENCH_ARMS); every rule below applies to all of
+ * them, with the arm taken from the data, so a check can never judge a run by another arm's metrics.
+ */
+const node_child_process_1 = require("node:child_process");
+const node_fs_1 = __importDefault(require("node:fs"));
+const node_path_1 = __importDefault(require("node:path"));
+const comparison_1 = require("./comparison");
+const scoring_1 = require("./scoring");
+const renorbenchDocuments_1 = require("./renorbenchDocuments");
+const renorbenchProduct_1 = require("./renorbenchProduct");
+const artifacts_1 = require("./artifacts");
+exports.BASELINE_SCHEMA = 1;
+/** The chunk arm's benchmark id: what `benchmark` reads in its runs and in its baseline. */
+exports.BASELINE_BENCHMARK = 'renorbench-v1';
+/** The chunk arm's baseline, relative to the engine root. */
+exports.BASELINE_RELATIVE_PATH = 'docs/evals/baselines/renorbench-v1-hash.json';
+/** The product arm's baseline, relative to the engine root. */
+exports.PRODUCT_BASELINE_RELATIVE_PATH = 'docs/evals/baselines/renorbench-v1-product-hash.json';
+/** The product arm's baseline on the v2 suite, relative to the engine root. */
+exports.PRODUCT_V2_BASELINE_RELATIVE_PATH = 'docs/evals/baselines/renorbench-v2-product-hash.json';
+/** The product arm's baseline on the documents suite, relative to the engine root. */
+exports.DOCUMENTS_BASELINE_RELATIVE_PATH = 'docs/evals/baselines/renorbench-docs-product-hash.json';
+exports.CHUNK_ARM = {
+    benchmark: exports.BASELINE_BENCHMARK,
+    name: 'chunk',
+    baselineRelativePath: exports.BASELINE_RELATIVE_PATH,
+    metrics: comparison_1.QUALITY_METRICS,
+    score: (cases, results) => (0, scoring_1.scoreRetrieval)(cases, results),
+    annotationKeys: ['toolPath'],
+    identityFields: [],
+    runCommand: 'npm run eval:renor:hash',
+};
+exports.PRODUCT_ARM = {
+    benchmark: renorbenchProduct_1.PRODUCT_BENCHMARK,
+    name: 'product',
+    baselineRelativePath: exports.PRODUCT_BASELINE_RELATIVE_PATH,
+    metrics: renorbenchProduct_1.PRODUCT_QUALITY_METRICS,
+    score: (cases, results) => (0, renorbenchProduct_1.scoreProductArm)(cases, results),
+    annotationKeys: ['productPath'],
+    identityFields: ['entryPoint', 'productMode'],
+    runCommand: 'npm run eval:renor:product:hash',
+};
+exports.PRODUCT_V2_ARM = {
+    benchmark: renorbenchProduct_1.PRODUCT_V2_BENCHMARK,
+    name: 'product-v2',
+    baselineRelativePath: exports.PRODUCT_V2_BASELINE_RELATIVE_PATH,
+    metrics: renorbenchProduct_1.PRODUCT_V2_QUALITY_METRICS,
+    score: (cases, results) => (0, renorbenchProduct_1.scoreProductArm)(cases, results),
+    annotationKeys: ['productPath'],
+    identityFields: ['entryPoint', 'productMode'],
+    runCommand: 'npm run eval:renor:product:v2:hash',
+};
+/**
+ * The documents suite: handbook-style notes with unique headings, which the other suites have none of. It is scored by
+ * whether the probed section reaches the reply, in `search_memory` and in `get_context` at 800 tokens, never by quality: the
+ * run is the hash embedder's and reports as infrastructure. `documentsPath` (what a reply costs) is annotated, not scored.
+ */
+exports.DOCUMENTS_ARM = {
+    benchmark: renorbenchDocuments_1.DOCUMENTS_BENCHMARK,
+    name: 'documents',
+    baselineRelativePath: exports.DOCUMENTS_BASELINE_RELATIVE_PATH,
+    metrics: renorbenchDocuments_1.DOCUMENTS_QUALITY_METRICS,
+    score: (cases, results) => (0, renorbenchDocuments_1.scoreDocumentsArm)(cases, results),
+    annotationKeys: ['productPath', 'documentsPath'],
+    identityFields: ['entryPoint', 'productMode'],
+    runCommand: 'npm run eval:renor:product:docs:hash',
+};
+exports.RENORBENCH_ARMS = [exports.CHUNK_ARM, exports.PRODUCT_ARM, exports.PRODUCT_V2_ARM, exports.DOCUMENTS_ARM];
+/** The arm a run or baseline belongs to, by its `benchmark` id. An id that names no arm is refused, never guessed. */
+function armFor(benchmark) {
+    const arm = exports.RENORBENCH_ARMS.find(candidate => candidate.benchmark === benchmark);
+    if (!arm)
+        throw new Error(`benchmark ${String(benchmark)} is not a RenorBench arm (expected ${exports.RENORBENCH_ARMS.map(a => a.benchmark).join(' or ')})`);
+    return arm;
+}
+/** The command that re-records the baseline of `arm`, as the messages below quote it. */
+function recordCommand(arm) {
+    return `npm run eval:renorbench:baseline -- ${arm.name === 'chunk' ? '' : `--arm ${arm.name} `}record <run dir> --reason "<why>"`;
+}
+function refreshHint(arm) {
+    const changed = arm.name === 'chunk' ? 'the evaluator, chunking or fixtures' : 'the evaluator, chunking, fixtures or how the arm is run';
+    return `If you changed ${changed} on purpose, refresh the baseline with \`${recordCommand(arm)}\` and say why in the PR body.`;
+}
+function parseJsonl(bytes) {
+    return bytes.toString().trim().split(/\r?\n/).filter(Boolean).map(line => JSON.parse(line));
+}
+/**
+ * Load a run directory and re-score it from its raw results: a supplied summary is never trusted. The arm that
+ * scores it is the one its manifest names. The runner's own annotation (`toolPath`, `productPath`) next to the
+ * scored summary is not part of it.
+ */
+function loadRunDir(dir) {
+    const manifest = JSON.parse(node_fs_1.default.readFileSync(node_path_1.default.join(dir, 'manifest.json'), 'utf8'));
+    const arm = armFor(manifest.benchmark);
+    const casesBytes = node_fs_1.default.readFileSync(node_path_1.default.join(dir, 'cases.jsonl'));
+    if (!manifest.caseSnapshotSha256 || (0, artifacts_1.sha256)(casesBytes) !== manifest.caseSnapshotSha256) {
+        throw new Error(`cases.jsonl does not match the manifest hash: ${dir}`);
+    }
+    const resultsBytes = node_fs_1.default.readFileSync(node_path_1.default.join(dir, 'results.jsonl'));
+    if (manifest.resultsSha256 && (0, artifacts_1.sha256)(resultsBytes) !== manifest.resultsSha256) {
+        throw new Error(`results.jsonl does not match the manifest hash: ${dir}`);
+    }
+    const summary = arm.score(parseJsonl(casesBytes), parseJsonl(resultsBytes));
+    const supplied = JSON.parse(node_fs_1.default.readFileSync(node_path_1.default.join(dir, 'summary.json'), 'utf8'));
+    for (const key of arm.annotationKeys)
+        delete supplied[key];
+    if (JSON.stringify(summary) !== JSON.stringify(supplied)) {
+        throw new Error(`summary.json does not match the re-scored results (tampered or stale): ${dir}`);
+    }
+    return { manifest, summary };
+}
+function withoutLatency(map) {
+    return Object.fromEntries(Object.entries(map).filter(([key]) => !/LatencyMs$/.test(key)));
+}
+function buildBaseline(input) {
+    const { manifest, summary } = input.run;
+    const arm = armFor(manifest.benchmark);
+    return {
+        schema: exports.BASELINE_SCHEMA,
+        benchmark: arm.benchmark,
+        reason: input.reason,
+        recordedAt: input.recordedAt,
+        gitSha: manifest.gitSha ?? null,
+        manifest: {
+            benchmark: manifest.benchmark,
+            casesSha256: manifest.casesSha256,
+            fixtureSha256: manifest.fixtureSha256,
+            topK: manifest.topK,
+            provider: manifest.provider,
+            qualityEligible: manifest.qualityEligible,
+            evaluatorSha256: manifest.evaluatorSha256,
+            chunking: manifest.chunking,
+            gitDirty: manifest.gitDirty,
+            ...Object.fromEntries(arm.identityFields.filter(key => manifest[key] !== undefined).map(key => [key, manifest[key]])),
+        },
+        summary: {
+            overall: withoutLatency(summary.overall),
+            byCategory: Object.fromEntries(Object.entries(summary.byCategory ?? {}).map(([name, map]) => [name, withoutLatency(map)])),
+        },
+        ...(input.lowering ? { lowering: input.lowering } : {}),
+    };
+}
+/**
+ * Read the committed baseline. Throws an Error whose message names what is wrong with it. With `arm`, a file that is
+ * another arm's baseline is refused, so a path that holds the wrong one can never judge a run.
+ */
+function readBaseline(file, arm) {
+    let text;
+    try {
+        text = node_fs_1.default.readFileSync(file, 'utf8');
+    }
+    catch (err) {
+        throw new Error(`baseline ${file} is missing or unreadable (${err.code ?? 'read error'})`);
+    }
+    return parseBaseline(text, file, arm);
+}
+function parseBaseline(text, file, expected) {
+    let parsed;
+    try {
+        parsed = JSON.parse(text);
+    }
+    catch {
+        throw new Error(`baseline ${file} is not valid JSON`);
+    }
+    if (parsed.schema !== exports.BASELINE_SCHEMA)
+        throw new Error(`baseline ${file} has schema ${String(parsed.schema)}, expected ${exports.BASELINE_SCHEMA}`);
+    const arm = exports.RENORBENCH_ARMS.find(candidate => candidate.benchmark === parsed.benchmark);
+    if (!arm || (expected && arm !== expected)) {
+        throw new Error(`baseline ${file} is for benchmark ${String(parsed.benchmark)}, expected ${(expected ?? exports.CHUNK_ARM).benchmark}`);
+    }
+    if (!parsed.manifest || !parsed.summary?.overall)
+        throw new Error(`baseline ${file} has no manifest or no summary.overall`);
+    const missing = arm.metrics.map(metric => metric.key)
+        .filter(key => typeof parsed.summary.overall[key] !== 'number');
+    if (missing.length > 0)
+        throw new Error(`baseline ${file} lacks tracked metric(s): ${missing.join(', ')}`);
+    if (parsed.lowering !== undefined) {
+        const l = parsed.lowering;
+        if (!l || typeof l.reason !== 'string' || l.reason.trim().length < 12 || typeof l.fromBaselineSha256 !== 'string' || l.fromBaselineSha256.length === 0
+            || !Array.isArray(l.movedMetrics) || !l.movedMetrics.every((m) => typeof m === 'string')) {
+            throw new Error(`baseline ${file} has a malformed "lowering" (needs a reason of at least 12 characters, fromBaselineSha256 and movedMetrics, a list of metric names)`);
+        }
+    }
+    return parsed;
+}
+function movesOf(result, tracked) {
+    const metrics = [];
+    const regressed = [];
+    const improved = [];
+    for (const { key, higherBetter } of tracked) {
+        const move = { metric: key, ...result.quality[key] };
+        metrics.push(move);
+        if (move.baseline !== null && move.candidate === null) {
+            regressed.push(move); // a tracked metric that vanished is not "no change"
+        }
+        else if (move.delta !== null && move.delta !== 0) {
+            const better = higherBetter === move.delta > 0;
+            // More empty answers on abstention cases is not a quality gain: it can fail the gate, never praise it.
+            if (better && key === 'abstentionEmptyRate')
+                continue;
+            (better ? improved : regressed).push(move);
+        }
+    }
+    return { metrics, regressed, improved };
+}
+function incomparable(problem) {
+    return { ok: false, verdict: 'INCOMPARABLE', problem, regressed: [], improved: [], categoryRegressions: [], metrics: [] };
+}
+/** The no-regression verdict: any tracked quality metric that drops fails, equal and better pass. */
+function checkAgainstBaseline(baseline, candidate) {
+    const arm = armFor(baseline.benchmark);
+    let result;
+    try {
+        result = (0, comparison_1.compareRuns)(baseline, candidate, { regressionGate: true, maxQualityRegression: 0, metrics: arm.metrics });
+    }
+    catch (err) {
+        return incomparable(`${err.message} ${refreshHint(arm)}`);
+    }
+    if (result.verdict === 'DESCRIPTIVE_ONLY') {
+        return incomparable(`the run is not comparable with the baseline: ${result.reasons.join(' ')} ${refreshHint(arm)}`);
+    }
+    // The same numbers from another configuration (a different tool call, an armed product flag, another verdict bar)
+    // measure something else: the run is judged only against a baseline made the way it was made.
+    const moved = arm.identityFields.filter(key => canonicalJson(baseline.manifest[key]) !== canonicalJson(candidate.manifest[key]));
+    if (moved.length > 0) {
+        const show = (manifest, key) => canonicalJson(manifest[key]).slice(0, 200);
+        return incomparable(`the run was not made the way the baseline was (${moved.map(key => `${key}: ${show(baseline.manifest, key)} (baseline) vs ${show(candidate.manifest, key)} (this run)`).join('; ')}). ${refreshHint(arm)}`);
+    }
+    const { metrics, regressed, improved } = movesOf(result, arm.metrics);
+    const failed = regressed.length > 0 || result.categoryRegressions.length > 0;
+    return {
+        ok: !failed,
+        verdict: failed ? 'REGRESSED' : improved.length > 0 ? 'IMPROVED' : 'EQUAL',
+        problem: null,
+        regressed,
+        improved,
+        categoryRegressions: result.categoryRegressions,
+        metrics,
+    };
+}
+exports.DEFAULT_BASE_REF = 'origin/main';
+function git(cwd, args) {
+    return (0, node_child_process_1.execFileSync)('git', args, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim();
+}
+/**
+ * Read the baseline file as it was at the merge-base of HEAD and `baseRef`. Fails CLOSED: when git
+ * cannot say what the merge-base had, the floor is unknown, and an unknown floor is not a pass. The one
+ * exception is a baseline file that does not exist at the merge-base (the PR that introduces it).
+ */
+function readMergeBaseState(baselineFile, baseRef = exports.DEFAULT_BASE_REF, arm) {
+    const dir = node_fs_1.default.existsSync(node_path_1.default.dirname(node_path_1.default.resolve(baselineFile))) ? node_fs_1.default.realpathSync(node_path_1.default.dirname(node_path_1.default.resolve(baselineFile))) : node_path_1.default.dirname(node_path_1.default.resolve(baselineFile));
+    const unavailable = (why) => ({
+        kind: 'unavailable',
+        problem: `cannot read the merge-base baseline (${why}); the floor is unknown, so the check fails closed. `
+            + `Fetch ${baseRef} (git fetch origin main; a shallow clone also needs git fetch --unshallow) and rerun, from a git checkout.`,
+    });
+    let cwd;
+    let mergeBase;
+    let spec;
+    try {
+        // Run from the repository root: ls-tree and show would otherwise read a path relative to the cwd.
+        cwd = git(dir, ['rev-parse', '--show-toplevel']);
+        mergeBase = git(cwd, ['merge-base', 'HEAD', baseRef]);
+        spec = node_path_1.default.relative(node_fs_1.default.realpathSync(cwd), node_path_1.default.join(dir, node_path_1.default.basename(baselineFile))).split(node_path_1.default.sep).join('/');
+    }
+    catch (err) {
+        const stderr = String(err.stderr ?? '').trim().split('\n')[0];
+        return unavailable(`git merge-base HEAD ${baseRef} failed${stderr ? `: ${stderr}` : ''}`);
+    }
+    let text;
+    try {
+        if (git(cwd, ['ls-tree', mergeBase, '--', spec]) === '')
+            return { kind: 'new' };
+        text = (0, node_child_process_1.execFileSync)('git', ['show', `${mergeBase}:${spec}`], { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], maxBuffer: 64 * 1024 * 1024 });
+    }
+    catch (err) {
+        return unavailable(`git show ${mergeBase.slice(0, 8)}:${spec} failed: ${err.message.split('\n')[0]}`);
+    }
+    try {
+        return { kind: 'file', baseline: parseBaseline(text, `${baseRef} merge-base ${mergeBase.slice(0, 8)}:${spec}`, arm), sha256: (0, artifacts_1.sha256)(Buffer.from(text)) };
+    }
+    catch (err) {
+        return unavailable(err.message);
+    }
+}
+/**
+ * Hold a run to a baseline's NUMBERS, whatever else moved. Unlike `checkAgainstBaseline` it tolerates a
+ * changed evaluator, chunking or provider: those legitimately move the identity, but a lower score under
+ * a new identity is still a lower score. Only different benchmark data (cases / fixture) makes the two
+ * incomparable, and then the new baseline's identity is the reviewer's to read.
+ */
+function compareToFloor(floor, candidate) {
+    const arm = armFor(floor.benchmark);
+    let result;
+    try {
+        result = (0, comparison_1.compareRuns)(floor, candidate, { regressionGate: true, allowEvaluatorChange: true, maxQualityRegression: 0, metrics: arm.metrics });
+    }
+    catch (err) {
+        return { comparable: false, note: err.message, regressed: [], categoryRegressions: [] };
+    }
+    return { comparable: true, note: null, regressed: movesOf(result, arm.metrics).regressed, categoryRegressions: result.categoryRegressions };
+}
+/** Does the committed baseline record a lowering of exactly this merge-base baseline? */
+function recordsLoweringOf(baseline, mergeBase) {
+    return baseline.lowering?.fromBaselineSha256 === mergeBase.sha256;
+}
+function canonicalJson(value) {
+    if (Array.isArray(value))
+        return `[${value.map(canonicalJson).join(',')}]`;
+    if (value && typeof value === 'object') {
+        const entries = Object.entries(value).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
+        return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${canonicalJson(v)}`).join(',')}}`;
+    }
+    return JSON.stringify(value) ?? 'null';
+}
+function flattenSummary(summary) {
+    const flat = new Map();
+    for (const [key, value] of Object.entries(summary.overall))
+        flat.set(`overall.${key}`, value);
+    for (const [name, map] of Object.entries(summary.byCategory ?? {})) {
+        for (const [key, value] of Object.entries(map))
+            flat.set(`${name}.${key}`, value);
+    }
+    return flat;
+}
+/**
+ * Is the committed baseline's summary exactly what this tree scores? The check re-scores the live run
+ * from its raw results, so this is the one statement that binds the committed numbers to a measurement:
+ * a hand-edited number cannot equal it. A real lowering ships the worse code and is re-recorded, so its
+ * numbers match by construction.
+ */
+function compareToOwnScore(baseline, run) {
+    const own = flattenSummary(buildBaseline({ run, reason: 'own score', recordedAt: '' }).summary);
+    const committed = flattenSummary(baseline.summary);
+    const differing = [];
+    const detail = [];
+    for (const key of new Set([...own.keys(), ...committed.keys()])) {
+        if (canonicalJson(own.get(key)) === canonicalJson(committed.get(key)))
+            continue;
+        differing.push(key);
+        detail.push(`${key} ${canonicalJson(committed.get(key))} (baseline) vs ${canonicalJson(own.get(key))} (this tree)`);
+    }
+    return { same: differing.length === 0, differing, detail };
+}
+/** At most a handful of names in a one-line message: the full lists stay in `regressed[]`. */
+function briefList(names, max = 8) {
+    return names.length <= max ? names.join(', ') : `${names.slice(0, max).join(', ')}, ... (+${names.length - max} more)`;
+}
+function notOwnScore(check, own, arm) {
+    const shown = own.detail.slice(0, 4).join('; ');
+    const better = check.improved.length > 0
+        ? ` The tree scores better than the baseline on ${briefList(check.improved.map(m => m.metric))}: that is progress, so re-record to ratchet the floor up.`
+        : '';
+    return {
+        ...check,
+        ok: false,
+        verdict: 'INCOMPARABLE',
+        problem: `the committed baseline is not this tree's own score (${own.differing.length} number(s) differ: ${shown}${own.detail.length > 4 ? '; ...' : ''}).${better} `
+            + 'The baseline must equal what this tree measures, number for number: re-record it from a clean committed tree '
+            + `(\`${recordCommand(arm)}\`); a hand-edited number is refused.`,
+    };
+}
+/**
+ * The no-regression verdict the gate uses. Two rules, in this order:
+ *
+ * 1. The committed baseline must EQUAL this tree's own re-scored run, every stored number, on every path
+ *    (new file, floor present, incomparable data, lowered or not). It is checked first and unconditionally,
+ *    so a hand-edited number can never become the floor, whatever else the PR changed. Better code is
+ *    ratcheted by re-recording in the same PR; worse code ships its lowering the same way.
+ * 2. The merge-base floor exists only to authorise a LOWER baseline: when the tree scores below the baseline
+ *    the merge-base has, the file must record a deliberate lowering of that exact baseline, naming exactly the
+ *    metrics that fell. The verdict is then LOWERED and never promotable. No merge-base, no pass (the file
+ *    being new on the merge-base is the one case that needs no floor).
+ */
+function checkWithMergeBase(baseline, candidate, mergeBase) {
+    const arm = armFor(baseline.benchmark);
+    const own = compareToOwnScore(baseline, candidate);
+    const check = checkAgainstBaseline(baseline, candidate);
+    if (!own.same)
+        return check.ok ? notOwnScore(check, own, arm) : check;
+    if (!check.ok)
+        return check;
+    if (mergeBase.kind === 'unavailable')
+        return incomparable(mergeBase.problem);
+    if (mergeBase.kind === 'new')
+        return { ...check, mergeBase: { state: 'new', sha256: null, comparable: true, note: 'the baseline file is new on the merge-base' } };
+    const floor = compareToFloor(mergeBase.baseline, candidate);
+    const info = { state: 'file', sha256: mergeBase.sha256, comparable: floor.comparable, note: floor.note };
+    const names = [...new Set([...floor.regressed.map(m => m.metric), ...floor.categoryRegressions])];
+    if (names.length === 0)
+        return { ...check, mergeBase: info };
+    const refuse = (why) => ({
+        ...check,
+        ok: false,
+        verdict: 'REGRESSED',
+        problem: `the run is below the baseline the merge-base has (${briefList(names)}): ${why} `
+            + `Fix the regression, or record it with \`${recordCommand(arm).replace('<run dir>', '<dir>')} --accept-lower\` and defend it in the PR body.`,
+        regressed: floor.regressed,
+        categoryRegressions: floor.categoryRegressions,
+        mergeBase: info,
+    });
+    if (!baseline.lowering)
+        return refuse('this baseline does not record a deliberate lowering of it.');
+    if (!recordsLoweringOf(baseline, mergeBase))
+        return refuse('the baseline records a lowering, but of a different baseline than the merge-base has (stale).');
+    const recorded = [...new Set(baseline.lowering.movedMetrics)].sort();
+    const actual = [...names].sort();
+    if (canonicalJson(recorded) !== canonicalJson(actual)) {
+        return refuse(`the recorded lowering names [${briefList(recorded)}] but the metrics actually below the floor are [${briefList(actual)}]; a lowering excuses exactly what it names.`);
+    }
+    return { ...check, ok: true, verdict: 'LOWERED', lowering: { reason: baseline.lowering.reason, moved: floor.regressed }, categoryRegressions: floor.categoryRegressions, mergeBase: info };
+}
+/**
+ * What `record` writes, or why it refuses. Numbers are judged against the merge-base's baseline and the
+ * file as it stands, across an evaluator or chunking change: only different benchmark data is incomparable.
+ * `lowering.movedMetrics` is derived from the merge-base floor alone, because that is what `check` compares it with.
+ */
+function planRecord(input) {
+    const { run, previous, mergeBase, acceptLower } = input;
+    if (mergeBase.kind === 'unavailable')
+        throw new Error(mergeBase.problem);
+    const reason = input.reason.trim();
+    const arm = armFor(run.manifest.benchmark);
+    if (run.manifest.provider?.kind !== 'hash')
+        throw new Error(`the baseline is the hash-embedder run: re-run with \`${arm.runCommand}\``);
+    if (run.manifest.gitDirty !== false)
+        throw new Error('the run came from a dirty tree: commit first, then record, so the baseline names the code that produced it');
+    if (reason.length < 12)
+        throw new Error('--reason must say why (at least 12 characters)');
+    const againstMergeBase = mergeBase.kind === 'file' ? compareToFloor(mergeBase.baseline, run) : null;
+    const againstFile = previous ? compareToFloor(previous, run) : null;
+    const regressed = [...(againstMergeBase?.regressed ?? []), ...(againstFile?.regressed ?? [])];
+    const categoryRegressions = [...(againstMergeBase?.categoryRegressions ?? []), ...(againstFile?.categoryRegressions ?? [])];
+    const lowered = regressed.length > 0 || categoryRegressions.length > 0;
+    if (lowered && !acceptLower) {
+        const err = new Error('the new run is LOWER than the baseline (merge-base or committed): fix the regression, or pass --accept-lower and say why in the PR');
+        err.plan = { regressed, categoryRegressions };
+        throw err;
+    }
+    const floorMoved = againstMergeBase
+        ? [...new Set([...againstMergeBase.regressed.map(m => m.metric), ...againstMergeBase.categoryRegressions])]
+        : [];
+    const lowering = mergeBase.kind === 'file' && floorMoved.length > 0
+        ? { reason, fromBaselineSha256: mergeBase.sha256, movedMetrics: floorMoved }
+        : undefined;
+    return { baseline: buildBaseline({ run, reason, recordedAt: input.recordedAt, lowering }), lowered, regressed, categoryRegressions };
+}

package/dist/eval/renorbenchDocuments.js

@@ -0,0 +1,134 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DOCUMENTS_QUALITY_METRICS = exports.DOCUMENTS_DIRECT_CATEGORY = exports.DOCUMENTS_CONTEXT_MAX_TOKENS = exports.DOCUMENTS_BENCHMARK = void 0;
+exports.scoreDocumentsArm = scoreDocumentsArm;
+exports.documentsReplyTokens = documentsReplyTokens;
+const metrics_1 = require("./metrics");
+const scoring_1 = require("./scoring");
+/**
+ * The documents arm of RenorBench: a vault of handbook-style notes (scripts/eval/generate-renorbench-docs.ts), read through
+ * the shipped `search_memory` tool and through `get_context` at 800 tokens, in process, and scored on one thing: whether the
+ * probed section of a note (the one with the case's needle) is in the reply an agent receives.
+ *
+ * Why it exists. RenorBench v1 and v2 have no document in them: v2's notes are chats (a turn is a section, and the turns of a
+ * chat share their headings) and short logs, so a change to what the session expansion may add from a DOCUMENT (the budget,
+ * the number of notes, the size of a statement, the chunker's cut or its heading rule) moved no gated number. A flat 100-token
+ * cap on the expansion's statements removed 186 of the 309 answer-bearing statements it delivered on a vault of documents and
+ * passed every gate. This arm is made of documents so that such a change turns the gate red.
+ *
+ * Nothing here needs a model, a network or a quality claim: the gold is the case's own needle, the embedder is the free hash
+ * stand-in (`qualityEligible: false`), and the numbers say what the MECHANISM delivers on this fixture, never how well Renor
+ * remembers. They are reported as infrastructure.
+ *
+ * The categories are the size band of the probed section: `adjacent-le100`, `adjacent-101-200` and `adjacent-201-400` hold
+ * the cases that name a neighbour of the section (its title, or the section before it), which the section reaches only when
+ * the session expansion adds it; `direct` holds the control, the opening of the section itself, which retrieval finds on its
+ * own. The 101-200 band is the cap's cliff: a statement limit below 200 takes every section of it out of reach.
+ *
+ * Reply sizes are annotated (`documentsPath.replyTokens`), not scored: a tripwire for delivery has no business failing a
+ * change that merely costs more or less, and the other arms already hold the reply's size.
+ */
+exports.DOCUMENTS_BENCHMARK = 'renorbench-docs-product';
+/** What `get_context` is asked for: its own default budget, the one an agent gets when it does not ask for more. */
+exports.DOCUMENTS_CONTEXT_MAX_TOKENS = 800;
+/** The category of the control: a question that names the probed section's own opening. */
+exports.DOCUMENTS_DIRECT_CATEGORY = 'direct';
+/**
+ * What the gate holds the arm to, overall and per category (any one falling fails it). All three only go up. A metric a
+ * category has no case for reads null and is not compared there (`directDelivered` of an adjacent band, say). The control
+ * is recorded at 1: a direct hit is the statement's own chunk, and no limit on the expansion may take it away.
+ */
+exports.DOCUMENTS_QUALITY_METRICS = [
+    { key: 'adjacentDelivered', higherBetter: true },
+    { key: 'directDelivered', higherBetter: true },
+    { key: 'contextDelivered', higherBetter: true },
+];
+function round(value) {
+    return Math.round(value * 10000) / 10000;
+}
+function mean(values) {
+    return values.length === 0 ? null : round(values.reduce((sum, value) => sum + value, 0) / values.length);
+}
+function percentile95(values) {
+    if (values.length === 0)
+        return null;
+    const sorted = [...values].sort((a, b) => a - b);
+    return sorted[Math.max(0, Math.min(sorted.length - 1, Math.ceil(0.95 * sorted.length) - 1))];
+}
+function emptyAggregate() {
+    return { cases: 0, adjacent: 0, adjacentDelivered: 0, direct: 0, directDelivered: 0, contextDelivered: 0 };
+}
+/** The probed section is in the reply: a hit of the case's note that holds the needle. */
+function delivered(test, hits) {
+    return (0, metrics_1.hitAtK)(hits, test.relevant, hits.length) === 1;
+}
+function addCase(agg, test, result) {
+    agg.cases += 1;
+    const found = delivered(test, result.hits);
+    if (test.category === exports.DOCUMENTS_DIRECT_CATEGORY) {
+        agg.direct += 1;
+        if (found)
+            agg.directDelivered += 1;
+    }
+    else {
+        agg.adjacent += 1;
+        if (found)
+            agg.adjacentDelivered += 1;
+    }
+    if (delivered(test, result.context.hits))
+        agg.contextDelivered += 1;
+}
+function summarize(agg) {
+    return {
+        cases: agg.cases,
+        adjacentCases: agg.adjacent,
+        directCases: agg.direct,
+        adjacentDelivered: agg.adjacent ? round(agg.adjacentDelivered / agg.adjacent) : null,
+        directDelivered: agg.direct ? round(agg.directDelivered / agg.direct) : null,
+        contextDelivered: agg.cases ? round(agg.contextDelivered / agg.cases) : null,
+    };
+}
+function validateResults(cases, results) {
+    // The cases, the ids and the hits are validated by the chunk scorer's own rules, so that every arm agrees on what a
+    // well-formed run is.
+    (0, scoring_1.scoreRetrieval)(cases, results.map(result => ({ id: result.id, hits: result.hits, latencyMs: result.latencyMs, contextTokens: result.replyTokens })));
+    for (const test of cases) {
+        if (test.shouldAbstain || test.relevant.length === 0)
+            throw new Error(`documents case ${test.id} has no gold: every case probes one section`);
+    }
+    for (const result of results) {
+        const context = result.context;
+        if (!context || !Array.isArray(context.hits) || typeof context.replyTokens !== 'number')
+            throw new Error(`Missing get_context read for result: ${result.id}`);
+        if (typeof result.replyTokens !== 'number')
+            throw new Error(`Missing replyTokens for result: ${result.id}`);
+    }
+}
+function scoreDocumentsArm(cases, results) {
+    validateResults(cases, results);
+    const byId = new Map(results.map(result => [result.id, result]));
+    const overall = emptyAggregate();
+    const categories = new Map();
+    for (const test of cases) {
+        const result = byId.get(test.id); // validateResults has already refused a missing id
+        addCase(overall, test, result);
+        const category = categories.get(test.category) ?? emptyAggregate();
+        addCase(category, test, result);
+        categories.set(test.category, category);
+    }
+    return {
+        overall: summarize(overall),
+        byCategory: Object.fromEntries([...categories.entries()].map(([name, agg]) => [name, summarize(agg)])),
+    };
+}
+/** What a reply costs, per tool: recorded in summary.json for the reader, never part of the score. */
+function documentsReplyTokens(results) {
+    const search = results.map(result => result.replyTokens);
+    const context = results.map(result => result.context.replyTokens);
+    return {
+        searchMean: mean(search),
+        searchP95: percentile95(search),
+        contextMean: mean(context),
+        contextP95: percentile95(context),
+    };
+}