renor-engine 0.0.0-stage → 1.0.0

package/dist/eval/renorbenchProduct.js

@@ -0,0 +1,182 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PRODUCT_V2_QUALITY_METRICS = exports.PRODUCT_QUALITY_METRICS = exports.HASH_RELEVANCE_BAR = exports.PRODUCT_SUITES = exports.PRODUCT_V2_BENCHMARK = exports.PRODUCT_BENCHMARK = void 0;
+exports.scoreProductArm = scoreProductArm;
+const metrics_1 = require("./metrics");
+const scoring_1 = require("./scoring");
+const renorbenchDocuments_1 = require("./renorbenchDocuments");
+/**
+ * The product arm of RenorBench: the same questions as the chunk arm, answered through the shipped `search_memory`
+ * tool, scored on the reply an agent actually receives. Nothing here needs a model: the gold is the case's own notes,
+ * the verdict is the reply's own `relevance` object, and the cost is the reply's own token count.
+ *
+ * What it measures that the chunk arm cannot (the chunk arm scores `retrieveContext` hits at top-K 10 and never sees
+ * the tool): the reply as a whole (the window, the session-expansion statements and the facts the tool adds, at the
+ * tool's own top_k), the order the tool presents it in, the `relevance` verdict, and what the reply costs in tokens.
+ *
+ * Rank-based scores (hit@1, MRR, nDCG) are deliberately absent: the tool's default order is newest first, and it tells
+ * the agent to judge a hit by its score, never its position, so a rank score would measure dates. The delivery scores
+ * count what is in the reply. Order matters only where it can mislead: `contradictionFirstRate`.
+ */
+exports.PRODUCT_BENCHMARK = 'renorbench-v1-product';
+exports.PRODUCT_V2_BENCHMARK = 'renorbench-v2-product';
+/**
+ * What the product arm can run: the questions, the vault they are asked of, and the benchmark id that its runs and its
+ * baseline carry. v1 is the original (191 questions of a 30-note vault) and stays as it was, so its scores stay comparable
+ * with every run recorded before v2 existed. v2 (scripts/eval/generate-renorbench-v2.ts) is a larger vault of chats and
+ * logs with questions that need session expansion, the newest-value rules and the verdict, which v1 is too small to show.
+ */
+exports.PRODUCT_SUITES = {
+    v1: { benchmark: exports.PRODUCT_BENCHMARK, cases: 'docs/evals/renorbench-v1.dev.jsonl', fixture: 'fixtures/eval/renorbench-v1' },
+    v2: { benchmark: exports.PRODUCT_V2_BENCHMARK, cases: 'docs/evals/renorbench-v2.dev.jsonl', fixture: 'fixtures/eval/renorbench-v2' },
+    // Forty handbook-style notes (scripts/eval/generate-renorbench-docs.ts), read by search_memory and by get_context at 800
+    // tokens, and scored by delivery alone (src/eval/renorbenchDocuments.ts): what the other suites cannot see of a document.
+    documents: { benchmark: renorbenchDocuments_1.DOCUMENTS_BENCHMARK, cases: 'docs/evals/renorbench-docs.dev.jsonl', fixture: 'fixtures/eval/renorbench-docs' },
+};
+/**
+ * The verdict's bar for the hash embedder. The product judges "does anything on file match?" by the cosine of the best
+ * hit against a bar calibrated per embedding model (CALIBRATED_BARS), and the hash stand-in has no calibration, so it
+ * gets no verdict at all unless a bar is set (RENOR_CONTEXT_RELEVANCE_MIN, the product's own setting for a model with
+ * no calibrated bar). This is that setting for the stand-in. It is a constant of this harness, not a property of any
+ * model: the midpoint of the median best-hit cosine of the answerable cases (0.407) and of the abstention cases
+ * (0.3425) on this fixture, rounded, so that both kinds of case sit at a mixed operating point where a change to the
+ * verdict can move them. Lexical vectors cannot separate the two kinds of case (at best about two thirds are judged
+ * right), so under this bar the verdict scores say that the verdict still works, never how well a real model judges:
+ * a real embedder runs on its calibrated bar and never on this one.
+ */
+exports.HASH_RELEVANCE_BAR = 0.375;
+/**
+ * What the gate holds the product arm to, overall and per category (any one falling fails it). Delivery and the
+ * verdict only ever go up, erasure and a stale value read first only ever go down, and so does what a reply costs:
+ * cost is part of quality, so a reply that grows is a regression the PR must defend, not a number that drifts.
+ */
+exports.PRODUCT_QUALITY_METRICS = [
+    { key: 'replyRecall', higherBetter: true },
+    { key: 'replyRecallAll', higherBetter: true },
+    { key: 'contradictionFirstRate', higherBetter: false },
+    { key: 'forbiddenInReplyRate', higherBetter: false },
+    { key: 'verdictRelevantRate', higherBetter: true },
+    { key: 'verdictAbstainRate', higherBetter: true },
+    { key: 'meanReplyTokens', higherBetter: false },
+];
+/**
+ * What the gate holds the v2 suite to: the same, but for `forbiddenInReplyRate`. v2 has no erasure case, so that rate has
+ * nothing to count (it reads null), and a null is not a number a floor can be held to.
+ */
+exports.PRODUCT_V2_QUALITY_METRICS = exports.PRODUCT_QUALITY_METRICS.filter(metric => metric.key !== 'forbiddenInReplyRate');
+function emptyAggregate() {
+    return {
+        count: 0, answerable: 0, abstention: 0, hit: 0, recall: 0, recallAll: 0,
+        supersessionSensitive: 0, contradictionFirst: 0, forbiddenSensitive: 0, forbiddenInReply: 0,
+        verdictRelevant: 0, verdictAbstain: 0, verdictMissing: 0, latencyMs: [], replyTokens: [],
+    };
+}
+function round(value) {
+    return Math.round(value * 10000) / 10000;
+}
+function mean(values) {
+    return values.length === 0 ? null : values.reduce((sum, value) => sum + value, 0) / values.length;
+}
+function percentile(values, p) {
+    if (values.length === 0)
+        return null;
+    const sorted = [...values].sort((a, b) => a - b);
+    return sorted[Math.max(0, Math.min(sorted.length - 1, Math.ceil(p * sorted.length) - 1))];
+}
+function rounded(value) {
+    return value === null ? null : round(value);
+}
+function addCase(agg, test, result) {
+    agg.count += 1;
+    // The whole reply is read, so "in the reply" is every hit it carries.
+    const reply = result.hits.length;
+    if (test.forbiddenEvidence && test.forbiddenEvidence.length > 0) {
+        agg.forbiddenSensitive += 1;
+        if ((0, metrics_1.forbiddenRetrieved)(result.hits, test.forbiddenEvidence, reply))
+            agg.forbiddenInReply += 1;
+    }
+    // A reply with no verdict is a wrong verdict for either kind of case: the product could not say, and the agent
+    // cannot tell "the answer is on file" from "cannot judge" (see contextRelevance.ts).
+    if (result.verdict === null)
+        agg.verdictMissing += 1;
+    if (Boolean(test.shouldAbstain) || test.relevant.length === 0) {
+        agg.abstention += 1;
+        if (result.verdict?.relevant === false)
+            agg.verdictAbstain += 1;
+    }
+    else {
+        agg.answerable += 1;
+        if (result.verdict?.relevant === true)
+            agg.verdictRelevant += 1;
+        agg.hit += (0, metrics_1.hitAtK)(result.hits, test.relevant, reply);
+        agg.recall += (0, metrics_1.recallAtK)(result.hits, test.relevant, reply);
+        const required = test.requiredEvidence ?? test.relevant.filter(target => (0, metrics_1.roleOf)(target) === 'positive');
+        if (required.length > 0)
+            agg.recallAll += (0, metrics_1.recallAllAtK)(result.hits, required, reply);
+        if (test.relevant.some(target => (0, metrics_1.roleOf)(target) === 'positive') && test.relevant.some(target => (0, metrics_1.roleOf)(target) === 'forbidden')) {
+            agg.supersessionSensitive += 1;
+            if ((0, metrics_1.contradictionFirst)(result.hits, test.relevant))
+                agg.contradictionFirst += 1;
+        }
+    }
+    if (typeof result.latencyMs === 'number')
+        agg.latencyMs.push(result.latencyMs);
+    agg.replyTokens.push(result.replyTokens);
+}
+function summarize(agg) {
+    const a = Math.max(1, agg.answerable);
+    const z = Math.max(1, agg.abstention);
+    return {
+        cases: agg.count,
+        answerableCases: agg.answerable,
+        abstentionCases: agg.abstention,
+        replyHit: round(agg.hit / a),
+        replyRecall: round(agg.recall / a),
+        replyRecallAll: round(agg.recallAll / a),
+        contradictionFirstRate: agg.supersessionSensitive ? round(agg.contradictionFirst / agg.supersessionSensitive) : null,
+        supersessionSensitiveCases: agg.supersessionSensitive,
+        forbiddenInReplyRate: agg.forbiddenSensitive ? round(agg.forbiddenInReply / agg.forbiddenSensitive) : null,
+        forbiddenSensitiveCases: agg.forbiddenSensitive,
+        verdictRelevantRate: agg.answerable ? round(agg.verdictRelevant / a) : null,
+        verdictAbstainRate: agg.abstention ? round(agg.verdictAbstain / z) : null,
+        verdictMissingCases: agg.verdictMissing,
+        meanReplyTokens: rounded(mean(agg.replyTokens)),
+        p95ReplyTokens: rounded(percentile(agg.replyTokens, 0.95)),
+        meanLatencyMs: rounded(mean(agg.latencyMs)),
+        p50LatencyMs: rounded(percentile(agg.latencyMs, 0.5)),
+        p95LatencyMs: rounded(percentile(agg.latencyMs, 0.95)),
+    };
+}
+function validateResults(cases, results) {
+    // The cases, the ids, the hits and the token and time samples are validated by the chunk scorer's own rules, so the
+    // two arms can never disagree about what a well-formed run is (a run that fails there fails here).
+    (0, scoring_1.scoreRetrieval)(cases, results.map(result => ({ id: result.id, hits: result.hits, latencyMs: result.latencyMs, contextTokens: result.replyTokens })));
+    for (const result of results) {
+        if (typeof result.replyTokens !== 'number')
+            throw new Error(`Missing replyTokens for result: ${result.id}`);
+        const verdict = result.verdict;
+        if (verdict === null)
+            continue;
+        if (!verdict || typeof verdict.relevant !== 'boolean' || !Number.isFinite(verdict.bar)
+            || (verdict.similarity !== null && !Number.isFinite(verdict.similarity))) {
+            throw new Error(`Invalid verdict for result: ${result.id}`);
+        }
+    }
+}
+function scoreProductArm(cases, results) {
+    validateResults(cases, results);
+    const byId = new Map(results.map(result => [result.id, result]));
+    const overall = emptyAggregate();
+    const categories = new Map();
+    for (const test of cases) {
+        const result = byId.get(test.id); // validateResults has already refused a missing id
+        addCase(overall, test, result);
+        const category = categories.get(test.category) ?? emptyAggregate();
+        addCase(category, test, result);
+        categories.set(test.category, category);
+    }
+    return {
+        overall: summarize(overall),
+        byCategory: Object.fromEntries([...categories.entries()].map(([name, agg]) => [name, summarize(agg)])),
+    };
+}

package/dist/eval/runLedger.js

@@ -0,0 +1,279 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RunRefusedError = exports.ORDINARY_CREDENTIAL_NAMED_ENV = exports.PRODUCT_ENV_NAMESPACES = exports.RUN_RECORD_SCHEMA = exports.RUN_LEDGER_ENV = void 0;
+exports.resolveRunLedgerPath = resolveRunLedgerPath;
+exports.canonicalJson = canonicalJson;
+exports.runFingerprint = runFingerprint;
+exports.codeIdentity = codeIdentity;
+exports.questionSet = questionSet;
+exports.isPresenceOnlyEnv = isPresenceOnlyEnv;
+exports.productEnvironment = productEnvironment;
+exports.readRunLedger = readRunLedger;
+exports.admitRun = admitRun;
+exports.recordRun = recordRun;
+exports.archiveRow = archiveRow;
+exports.saveRunOutcome = saveRunOutcome;
+const node_crypto_1 = __importDefault(require("node:crypto"));
+const node_fs_1 = __importDefault(require("node:fs"));
+const node_os_1 = __importDefault(require("node:os"));
+const node_path_1 = __importDefault(require("node:path"));
+/**
+ * The eval run ledger: every LongMemEval run is on record before it spends anything.
+ *
+ * WHY THIS EXISTS
+ *   Full 500-question runs were repeated with configurations that had already been measured, because what
+ *   ran lived in disposable worktrees and private reports. The ledger is one JSON line per run, appended to
+ *   RENOR_EVAL_RUN_LEDGER (default ~/.local/share/renor/eval-run-ledger.jsonl, next to the eval and embedding
+ *   caches, so every worktree on the machine reads and writes the same file). Each line carries the run's
+ *   configuration, its fingerprint and where its per-question results are written.
+ *
+ * THE RULE
+ *   A runner calls admitRun() before its first provider call. A run whose exact fingerprint is already on
+ *   record is refused, unless the caller states why it must run again (--rerun-reason). The reason is
+ *   written into the new record. Continuing a stopped run in its own output directory is the same run, not
+ *   a repeat.
+ *
+ * WHAT SURVIVES THE WORKTREE
+ *   The record is appended before the first spend, so a run that is killed is still on record. Per-question
+ *   rows are copied into eval-runs/<record id>/ beside the ledger as they are written, and the run's JSON
+ *   artifacts plus outcome.json when it ends. A run with no outcome.json was stopped, killed or is still
+ *   running.
+ *
+ * The fingerprint is sha256 over the canonical JSON of the configuration the runner builds: models and
+ * endpoints, prompt flags, product settings, the question set, the dataset and the content of the code
+ * the run executes. It never contains a credential value.
+ */
+exports.RUN_LEDGER_ENV = 'RENOR_EVAL_RUN_LEDGER';
+exports.RUN_RECORD_SCHEMA = 'renor-eval-run/v1';
+const sha256 = (data) => node_crypto_1.default.createHash('sha256').update(data).digest('hex');
+function resolveRunLedgerPath(env = process.env) {
+    const raw = env[exports.RUN_LEDGER_ENV]?.trim();
+    if (raw === 'off' || raw === '0') {
+        throw new Error(`${exports.RUN_LEDGER_ENV} cannot turn the run ledger off: point it at a scratch file for runs that must not be on the shared record`);
+    }
+    return raw ? node_path_1.default.resolve(raw) : node_path_1.default.join(env.HOME || node_os_1.default.homedir(), '.local', 'share', 'renor', 'eval-run-ledger.jsonl');
+}
+/** JSON with object keys sorted at every depth and undefined members dropped, so equal configurations serialize equally. */
+function canonicalJson(value) {
+    const sorted = (item) => {
+        if (Array.isArray(item))
+            return item.map(sorted);
+        if (item !== null && typeof item === 'object') {
+            return Object.fromEntries(Object.keys(item).sort()
+                .filter(key => item[key] !== undefined)
+                .map(key => [key, sorted(item[key])]));
+        }
+        return item;
+    };
+    return JSON.stringify(sorted(value));
+}
+function runFingerprint(config) {
+    return sha256(canonicalJson(config));
+}
+/**
+ * Content identity of the code a run executes: sha256 over every file under `roots` (paths relative to
+ * `engineRoot`), each as `path\0sha256(content)`, sorted by path. The same bytes give the same identity on
+ * any commit and in any worktree, so a commit that touches only other files does not make a measured
+ * configuration new again; any edit to these files, committed or not, does. A missing root throws.
+ */
+function codeIdentity(engineRoot, roots) {
+    const files = new Set();
+    const walk = (relative) => {
+        const full = node_path_1.default.join(engineRoot, relative);
+        const stat = node_fs_1.default.statSync(full);
+        if (stat.isDirectory()) {
+            for (const entry of node_fs_1.default.readdirSync(full))
+                walk(node_path_1.default.posix.join(relative, entry));
+        }
+        else if (stat.isFile()) {
+            files.add(relative);
+        }
+    };
+    for (const root of roots)
+        walk(root.split(node_path_1.default.sep).join('/'));
+    return sha256([...files].sort()
+        .map(relative => `${relative}\0${sha256(node_fs_1.default.readFileSync(node_path_1.default.join(engineRoot, relative)))}`)
+        .join('\n'));
+}
+/** A question set as configuration: its size and a hash of its sorted ids (selection order does not change what is measured). */
+function questionSet(ids) {
+    return { count: ids.length, sha256: sha256([...ids].sort().join('\n')) };
+}
+/**
+ * The namespaces the product reads its settings from (`process.env` in engine/src). Every variable of a product
+ * process's environment inside them is configuration, whoever set it: the runner, the operator's shell or a parent
+ * process. A setting such as RENOR_NEWEST_FIRST or CHUNK_MAX_TOKENS changes what a run measures even when no runner
+ * names it. tests/eval/run-ledger.test.ts scans engine/src and fails on a variable read outside these namespaces that
+ * it does not list as non-configuration, so a new setting cannot escape the fingerprint.
+ */
+exports.PRODUCT_ENV_NAMESPACES = /^(RENOR_|MVP_|EMBEDDING_|OPENAI_|OLLAMA_|GOOGLE_|CHUNK_|INDEX_|LLM_|MEMORY_|CONVERSATIONS_|PRIORITY_)|^TZ$/;
+/**
+ * Inside those namespaces: the eval runners' own variables, and a spend cap, which can stop a run but not change its
+ * result. Settings that look operational but can change a result stay configuration: index concurrency and embedding
+ * batch size can change tie order, and the extractor timeout decides whether an armed extraction lands. A repeat that
+ * differs only in one of them is therefore not refused, which is the safe direction.
+ */
+const NOT_PRODUCT_CONFIGURATION = /^RENOR_QA_|^RENOR_EVAL_RUN_LEDGER$|^RENOR_EVAL_MAX_EMBED_TOKENS$/;
+/**
+ * Names whose values never enter a run record: any name with a credential word as a whole `_`-separated part
+ * (`KEY`, `KEYS`, `API_KEY`, `APIKEY`, `TOKEN`, `SECRET(S)`, `PASSWORD`, `PASSWD`, `CREDENTIAL(S)`) anywhere in it,
+ * cloud project ids, and cache locations (where a cache lives is not what is measured). Their presence is recorded.
+ *
+ * The position of the word does not matter: `OPENAI_API_KEY_BENCH`, `OLLAMA_CLOUD_KEY_4` and `OPENAI_API_KEY_BENCH_TTL_SECONDS`
+ * are all credential names, because the words around the credential word say which credential, not that the value is
+ * safe to write. A part that only contains such a word (`RENOR_KEYWORD_WEIGHT`, `RENOR_EVIDENCE_MAX_TOKENS`) is not one.
+ *
+ * Fail closed: a new ordinary setting whose name has a credential part is masked until it is listed in
+ * ORDINARY_CREDENTIAL_NAMED_ENV, which holds the full names of the settings that really are read from the environment
+ * (tests/eval/run-ledger.test.ts fails when a listed name is no longer read by engine/src).
+ */
+const CREDENTIAL_PART = /(?:^|_)(?:API_?)?(?:KEYS?|TOKEN|SECRETS?|PASSWORD|PASSWD|CREDENTIALS?)(?:_|$)/;
+/**
+ * Settings whose names carry a credential word and whose values are not credentials; their values stay in the record,
+ * so two runs that differ in one never share a fingerprint. Only full names: a suffix or prefix never makes a name
+ * ordinary.
+ * - MVP_ACCESS_TOKEN_TTL_SECONDS: how long an issued access token lives (src/mvp/config.ts).
+ * - RENOR_SECRET_GUARD: on/off switch of the secret-looking-note filter in import (src/extraction/secretGuard.ts).
+ */
+exports.ORDINARY_CREDENTIAL_NAMED_ENV = new Set([
+    'MVP_ACCESS_TOKEN_TTL_SECONDS',
+    'RENOR_SECRET_GUARD',
+]);
+/** True for a name whose value must be recorded as `set`, never as itself. */
+function isPresenceOnlyEnv(name) {
+    if (exports.ORDINARY_CREDENTIAL_NAMED_ENV.has(name))
+        return false;
+    return CREDENTIAL_PART.test(name) || /_PROJECT$|_CACHE$/.test(name);
+}
+/** The configuration a product process's environment carries: its product-namespace variables, credentials and caches by presence only. */
+function productEnvironment(env) {
+    return Object.fromEntries(Object.entries(env)
+        .filter((entry) => entry[1] !== undefined
+        && exports.PRODUCT_ENV_NAMESPACES.test(entry[0]) && !NOT_PRODUCT_CONFIGURATION.test(entry[0]))
+        .map(([name, value]) => [name, isPresenceOnlyEnv(name) ? 'set' : value]));
+}
+class RunRefusedError extends Error {
+}
+exports.RunRefusedError = RunRefusedError;
+/** Every record on the ledger. A torn last line (an append in progress) is skipped; any other unreadable line fails closed. */
+function readRunLedger(ledgerPath) {
+    if (!node_fs_1.default.existsSync(ledgerPath))
+        return [];
+    const lines = node_fs_1.default.readFileSync(ledgerPath, 'utf8').split('\n');
+    const records = [];
+    lines.forEach((line, index) => {
+        if (!line.trim())
+            return;
+        let record;
+        try {
+            record = JSON.parse(line);
+        }
+        catch {
+            if (index === lines.length - 1)
+                return;
+            throw new Error(`Run ledger ${ledgerPath} line ${index + 1} is not JSON: cannot tell whether this run repeats one on record`);
+        }
+        if (typeof record.fingerprint !== 'string' || typeof record.outputDir !== 'string') {
+            throw new Error(`Run ledger ${ledgerPath} line ${index + 1} has no fingerprint or outputDir: cannot tell whether this run repeats one on record`);
+        }
+        records.push(record);
+    });
+    return records;
+}
+/**
+ * Decide, before any provider call, whether this run may start. `resuming` is true when the output
+ * directory already holds this run's checkpoint. Throws RunRefusedError when the exact fingerprint is
+ * already on record for another output directory and no reason was given.
+ */
+function admitRun(args) {
+    const reason = args.rerunReason?.trim() ?? '';
+    if (args.rerunReason !== undefined && !reason) {
+        throw new RunRefusedError('Refused: --rerun-reason is empty. Say why this configuration must run again; the reason is recorded with the run.');
+    }
+    const outputDir = node_path_1.default.resolve(args.outputDir);
+    const fingerprint = runFingerprint(args.config);
+    const matches = readRunLedger(args.ledgerPath).filter(record => record.fingerprint === fingerprint);
+    const resumes = args.resuming ? matches.filter(record => record.outputDir === outputDir) : [];
+    const priors = resumes.length > 0 ? [] : matches;
+    if (priors.length > 0 && !reason) {
+        throw new RunRefusedError(refusalMessage(args.ledgerPath, fingerprint, priors));
+    }
+    return {
+        ledgerPath: args.ledgerPath,
+        runner: args.runner,
+        fingerprint,
+        config: args.config,
+        outputDir,
+        resumes,
+        priors,
+        rerunReason: reason || null,
+    };
+}
+function describePrior(record) {
+    const outcomePath = node_path_1.default.join(record.archiveDir ?? '', 'outcome.json');
+    if (record.archiveDir && node_fs_1.default.existsSync(outcomePath)) {
+        try {
+            const outcome = JSON.parse(node_fs_1.default.readFileSync(outcomePath, 'utf8'));
+            const score = typeof outcome.correct === 'number' ? `${outcome.correct} correct, ` : '';
+            return `${outcome.status}: ${score}${outcome.casesCompleted}/${outcome.casesTotal} questions; results ${record.archiveDir}`;
+        }
+        catch { /* an unreadable outcome reads as no outcome */ }
+    }
+    const archived = record.archiveDir ? node_path_1.default.join(record.archiveDir, node_path_1.default.basename(record.perQuestionResults ?? '')) : '';
+    const rows = archived && node_fs_1.default.existsSync(archived)
+        ? node_fs_1.default.readFileSync(archived, 'utf8').split('\n').filter(Boolean).length
+        : 0;
+    return `no outcome recorded (stopped, killed or still running); ${rows} per-question row(s) in ${archived || record.perQuestionResults}`;
+}
+function refusalMessage(ledgerPath, fingerprint, priors) {
+    const shown = priors.slice(-5).map(record => `  - ${record.recordedAt}  ${record.outputDir}\n    ${describePrior(record)}${record.rerun ? `\n    (itself a re-run: ${record.rerun.reason})` : ''}`);
+    return [
+        `Refused: this exact configuration (fingerprint ${fingerprint.slice(0, 16)}) has already run ${priors.length} time(s) on this machine:`,
+        ...shown,
+        'Running it again pays for a result that is already on record. Read the results above. To finish a stopped run, pass its',
+        '--output directory again. If a repeat is the point (a fresh answer draw, results that were lost), pass --rerun-reason "<why>":',
+        'the reason is recorded with the new run.',
+        `Ledger: ${ledgerPath}`,
+    ].join('\n');
+}
+/** Append the run's record. Call it once the run will spend (it has questions to answer), after the output directory is claimed. */
+function recordRun(admission, details) {
+    const recordedAt = new Date().toISOString();
+    const id = `${recordedAt.replace(/[-:]/g, '').slice(0, 15)}-${admission.fingerprint.slice(0, 12)}-${node_crypto_1.default.randomBytes(3).toString('hex')}`;
+    const archiveDir = node_path_1.default.join(node_path_1.default.dirname(admission.ledgerPath), 'eval-runs', id);
+    node_fs_1.default.mkdirSync(archiveDir, { recursive: true });
+    const record = {
+        schema: exports.RUN_RECORD_SCHEMA,
+        id,
+        recordedAt,
+        runner: admission.runner,
+        fingerprint: admission.fingerprint,
+        config: admission.config,
+        gitSha: details.gitSha,
+        gitDirty: details.gitDirty,
+        outputDir: admission.outputDir,
+        perQuestionResults: node_path_1.default.resolve(details.perQuestionResults),
+        archiveDir,
+        resumes: admission.resumes.map(prior => prior.id),
+        rerun: admission.rerunReason ? { reason: admission.rerunReason, priorRecordIds: admission.priors.map(prior => prior.id) } : null,
+    };
+    node_fs_1.default.appendFileSync(admission.ledgerPath, JSON.stringify(record) + '\n', 'utf8');
+    return record;
+}
+/** Copy one per-question row into the run's archive as it is written, so a run that is killed still leaves its results. */
+function archiveRow(record, line) {
+    node_fs_1.default.appendFileSync(node_path_1.default.join(record.archiveDir, node_path_1.default.basename(record.perQuestionResults)), line.endsWith('\n') ? line : `${line}\n`, 'utf8');
+}
+/** When the run ends: copy its JSON artifacts from the output directory into the archive, then write outcome.json. */
+function saveRunOutcome(record, outcome) {
+    for (const name of node_fs_1.default.readdirSync(record.outputDir)) {
+        const from = node_path_1.default.join(record.outputDir, name);
+        if (!/\.(json|jsonl|sha256)$/.test(name) || !node_fs_1.default.statSync(from).isFile())
+            continue;
+        node_fs_1.default.copyFileSync(from, node_path_1.default.join(record.archiveDir, name), node_fs_1.default.constants.COPYFILE_FICLONE);
+    }
+    node_fs_1.default.writeFileSync(node_path_1.default.join(record.archiveDir, 'outcome.json'), JSON.stringify({ recordId: record.id, ...outcome }, null, 2) + '\n', 'utf8');
+}