renor-engine 0.0.0-stage → 1.0.0

package/dist/embeddings/vertex.js

@@ -0,0 +1,224 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.VertexEmbeddingProvider = void 0;
+const node_fs_1 = __importDefault(require("node:fs"));
+const node_crypto_1 = __importDefault(require("node:crypto"));
+const types_1 = require("./types");
+const TOKEN_MARGIN_MS = 5 * 60 * 1000;
+/** Quota (429) and transient (500/503) answers are retried: 8 attempts, waiting 10 s, 20 s ... up to 120 s between them. */
+const DEFAULT_MAX_ATTEMPTS = 8;
+const defaultRetryWaitMs = (attempt) => Math.min(120_000, 5_000 * 2 ** attempt) + Math.random() * 2_000;
+/** The endpoint is built from the location; a test points it at a local stub instead. */
+function endpointBase(baseUrl, location) {
+    if (baseUrl === undefined)
+        return `https://${location}-aiplatform.googleapis.com`;
+    const url = new URL(baseUrl);
+    const loopback = url.hostname === '127.0.0.1' || url.hostname === 'localhost' || url.hostname === '[::1]';
+    if (url.protocol !== 'https:' && !(url.protocol === 'http:' && loopback))
+        throw new Error('Vertex baseUrl must be https, or http on loopback');
+    return url.origin;
+}
+/** Wait `ms`, or until `signal` aborts, which ends the call that is waiting. */
+function sleepUnlessAborted(ms, signal) {
+    return new Promise((resolve, reject) => {
+        if (signal?.aborted)
+            return reject(signal.reason);
+        const onAbort = () => {
+            clearTimeout(timer);
+            reject(signal.reason);
+        };
+        const timer = setTimeout(() => {
+            signal?.removeEventListener('abort', onAbort);
+            resolve();
+        }, ms);
+        signal?.addEventListener('abort', onAbort, { once: true });
+    });
+}
+function b64url(input) {
+    return Buffer.from(input).toString('base64url');
+}
+class VertexEmbeddingProvider {
+    dimensions;
+    kind = 'vertex';
+    model;
+    identity;
+    project;
+    location;
+    key;
+    maxInstancesPerRequest;
+    maxInputTokens;
+    base;
+    maxAttempts;
+    retryWaitMs;
+    accessToken = null;
+    tokenExpiresAt = 0;
+    usage;
+    constructor(options) {
+        const keyFile = options?.keyFile
+            ?? process.env.RENOR_EVAL_VERTEX_KEY_FILE
+            ?? process.env.GOOGLE_APPLICATION_CREDENTIALS
+            ?? '';
+        if (!keyFile || !node_fs_1.default.existsSync(keyFile)) {
+            throw new Error('Vertex provider requires a service-account key file (GOOGLE_APPLICATION_CREDENTIALS or RENOR_EVAL_VERTEX_KEY_FILE)');
+        }
+        const raw = JSON.parse(node_fs_1.default.readFileSync(keyFile, 'utf8'));
+        if (!raw.client_email || !raw.private_key || !raw.token_uri) {
+            throw new Error('Service-account key file must contain client_email, private_key, and token_uri');
+        }
+        this.key = raw;
+        this.project = options?.project ?? process.env.RENOR_EVAL_VERTEX_PROJECT ?? '';
+        if (!/^[a-z0-9-]+$/.test(this.project))
+            throw new Error('RENOR_EVAL_VERTEX_PROJECT is required');
+        this.location = options?.location ?? process.env.RENOR_EVAL_VERTEX_LOCATION ?? '';
+        if (!this.location)
+            throw new Error('RENOR_EVAL_VERTEX_LOCATION is required');
+        if (!/^[a-z0-9-]+$/.test(this.location))
+            throw new Error('Invalid Vertex location');
+        this.model = options?.model ?? process.env.RENOR_EVAL_MODEL ?? 'gemini-embedding-001';
+        this.identity = { kind: this.kind, model: this.model };
+        const dimensions = options?.dimensions ?? Number(process.env.RENOR_EVAL_EMBEDDING_DIMS ?? 768);
+        if (!Number.isInteger(dimensions) || ![768, 1536, 3072].includes(dimensions)) {
+            throw new Error('gemini-embedding-001 dimensions must be 768, 1536, or 3072');
+        }
+        this.dimensions = dimensions;
+        this.maxInstancesPerRequest = options?.maxInstancesPerRequest ?? 32;
+        const maxInputTokens = options?.maxInputTokens
+            ?? Number(process.env.RENOR_EVAL_MAX_EMBED_TOKENS ?? 80_000_000);
+        if (!Number.isInteger(maxInputTokens) || maxInputTokens < 1_000) {
+            throw new Error('RENOR_EVAL_MAX_EMBED_TOKENS must be a positive integer >= 1000');
+        }
+        this.maxInputTokens = maxInputTokens;
+        this.base = endpointBase(options?.baseUrl, this.location);
+        this.maxAttempts = options?.retry?.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
+        this.retryWaitMs = options?.retry?.waitMs ?? defaultRetryWaitMs;
+        this.usage = { requests: 0, inputTokens: 0, maxInputTokens };
+    }
+    /** OAuth2 JWT-bearer grant: SA key -> short-lived access token. */
+    async getAccessToken(signal) {
+        if (this.accessToken && Date.now() < this.tokenExpiresAt - TOKEN_MARGIN_MS)
+            return this.accessToken;
+        const now = Math.floor(Date.now() / 1000);
+        const header = b64url(JSON.stringify({ alg: 'RS256', typ: 'JWT' }));
+        const claims = b64url(JSON.stringify({
+            iss: this.key.client_email,
+            scope: 'https://www.googleapis.com/auth/cloud-platform',
+            aud: this.key.token_uri,
+            iat: now,
+            exp: now + 3600,
+        }));
+        const signature = b64url(node_crypto_1.default.createSign('RSA-SHA256')
+            .update(`${header}.${claims}`)
+            .sign(this.key.private_key));
+        const response = await fetch(this.key.token_uri, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+            body: new URLSearchParams({
+                grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer',
+                assertion: `${header}.${claims}.${signature}`,
+            }).toString(),
+            signal,
+        });
+        if (!response.ok)
+            throw new Error(`Vertex token exchange failed: ${response.status}`);
+        const data = await response.json();
+        this.accessToken = data.access_token;
+        this.tokenExpiresAt = Date.now() + data.expires_in * 1000;
+        return this.accessToken;
+    }
+    /**
+     * `call` says how long this one may take (see EmbedOptions): `deadline` is the epoch ms by which it gives up, and a
+     * retry whose wait would end past it is not waited for, so a refusing service is reported at once rather than at the
+     * deadline. Without one the retries run their whole course.
+     */
+    async predict(contents, call) {
+        const token = await this.getAccessToken(call.signal);
+        const url = `${this.base}/v1/projects/${this.project}/locations/${this.location}/publishers/google/models/${this.model}:predict`;
+        const body = JSON.stringify({
+            instances: contents.map(({ text, taskType }) => ({
+                content: text,
+                task_type: taskType,
+            })),
+            parameters: { outputDimensionality: this.dimensions, autoTruncate: true },
+        });
+        // Quota (429) and transient (500/503) errors are expected on new projects:
+        // back off with jitter instead of failing the run. Retries cost nothing —
+        // 429 responses are not billed.
+        const maxAttempts = this.maxAttempts;
+        let attempt = 0;
+        let response;
+        for (;;) {
+            response = await fetch(url, {
+                method: 'POST',
+                headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' },
+                body,
+                signal: call.signal,
+            });
+            if (response.ok || ![429, 500, 503].includes(response.status))
+                break;
+            attempt += 1;
+            if (attempt >= maxAttempts) {
+                throw new types_1.EmbeddingUnavailableError(`Vertex predict failed after ${attempt} attempts (HTTP ${response.status})`);
+            }
+            const waitMs = this.retryWaitMs(attempt);
+            if (Date.now() + waitMs > call.deadline) {
+                throw new types_1.EmbeddingUnavailableError(`Vertex predict refused (HTTP ${response.status}); the retry is later than this call may wait`);
+            }
+            process.stderr.write(`[vertex] HTTP ${response.status} on ${contents.length} inputs; retry ${attempt}/${maxAttempts - 1} in ${Math.round(waitMs / 1000)}s\n`);
+            await sleepUnlessAborted(waitMs, call.signal);
+        }
+        if (!response.ok) {
+            const bodyText = await response.text();
+            // The status travels with the error (the message is unchanged): a 400 names an input the service refuses.
+            throw Object.assign(new Error(`Vertex predict failed (${response.status}): ${bodyText.slice(0, 400)}`), { status: response.status });
+        }
+        const data = await response.json();
+        const predictions = data.predictions ?? [];
+        if (predictions.length !== contents.length
+            || predictions.some(p => !Array.isArray(p.embeddings?.values) || p.embeddings.values.length !== this.dimensions)) {
+            throw new Error('Vertex predict returned malformed or incomplete embeddings');
+        }
+        this.usage.requests += 1;
+        for (const p of predictions)
+            this.usage.inputTokens += p.embeddings?.statistics?.token_count ?? 0;
+        if (this.usage.inputTokens > this.maxInputTokens) {
+            throw new Error(`Embedding cost guard tripped: ${this.usage.inputTokens} input tokens exceeds the ${this.maxInputTokens} cap (set RENOR_EVAL_MAX_EMBED_TOKENS to raise it)`);
+        }
+        return predictions.map(p => p.embeddings.values);
+    }
+    async embedAll(texts, taskType, options) {
+        if (texts.length === 0)
+            return [];
+        const call = {
+            deadline: options?.budgetMs === undefined ? Infinity : Date.now() + options.budgetMs,
+            signal: options?.signal,
+        };
+        const out = [];
+        for (let i = 0; i < texts.length; i += this.maxInstancesPerRequest) {
+            const batch = texts.slice(i, i + this.maxInstancesPerRequest).map(text => ({ text, taskType }));
+            out.push(...await this.predict(batch, call));
+        }
+        return out;
+    }
+    async embed(texts, options) {
+        return this.embedAll(texts, 'RETRIEVAL_DOCUMENT', options);
+    }
+    async embedQuery(text, options) {
+        const [vector] = await this.embedAll([text], 'RETRIEVAL_QUERY', options);
+        return vector;
+    }
+    describe() {
+        // Credentials and project IDs never leave the process through logs.
+        return {
+            kind: 'vertex',
+            model: this.model,
+            dimensions: this.dimensions,
+            location: this.location,
+            taskTypes: { document: 'RETRIEVAL_DOCUMENT', query: 'RETRIEVAL_QUERY' },
+            usage: { ...this.usage },
+        };
+    }
+}
+exports.VertexEmbeddingProvider = VertexEmbeddingProvider;

package/dist/eval/artifacts.js

@@ -0,0 +1,42 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.sha256 = sha256;
+exports.claimDirectory = claimDirectory;
+exports.assertRegularTree = assertRegularTree;
+exports.safeEndpoint = safeEndpoint;
+const node_fs_1 = __importDefault(require("node:fs"));
+const node_path_1 = __importDefault(require("node:path"));
+const node_crypto_1 = __importDefault(require("node:crypto"));
+/**
+ * Measurement-side helpers shared by the eval runners. Evaluator provenance
+ * (source hash, git state) lives in src/eval/runManifest.ts — the single
+ * authority — so only artifact-safety utilities live here.
+ */
+function sha256(data) {
+    return node_crypto_1.default.createHash('sha256').update(data).digest('hex');
+}
+/** Exclusive claim: never overwrite another worker's run, even after failure. */
+function claimDirectory(directory) {
+    node_fs_1.default.mkdirSync(node_path_1.default.dirname(directory), { recursive: true });
+    node_fs_1.default.mkdirSync(directory, { mode: 0o700 });
+}
+/** Fixtures must consist only of regular files/directories. No external links. */
+function assertRegularTree(root) {
+    const stat = node_fs_1.default.lstatSync(root);
+    if (stat.isSymbolicLink() || (!stat.isDirectory() && !stat.isFile())) {
+        throw new Error(`Non-regular fixture entry: ${root}`);
+    }
+    if (stat.isDirectory())
+        for (const child of node_fs_1.default.readdirSync(root))
+            assertRegularTree(node_path_1.default.join(root, child));
+}
+function safeEndpoint(value) {
+    const url = new URL(value);
+    if (!['https:', 'http:'].includes(url.protocol) || url.username || url.password || url.search || url.hash) {
+        throw new Error('Embedding base URL must be HTTP(S) without credentials, query, or fragment');
+    }
+    return value;
+}

package/dist/eval/cacheReport.js

@@ -0,0 +1,139 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PER_RUN_EVIDENCE_BLOCKER = exports.UNCACHED_MODEL_BLOCKER = exports.UNCACHED_EMBEDDING_BLOCKER = exports.UNPINNED_SCOPE_BLOCKER = void 0;
+exports.cacheHitRate = cacheHitRate;
+exports.buildCacheReport = buildCacheReport;
+exports.formatCacheReport = formatCacheReport;
+exports.emitCacheReport = emitCacheReport;
+exports.readProxyCounters = readProxyCounters;
+exports.proxyWindow = proxyWindow;
+const node_fs_1 = __importDefault(require("node:fs"));
+const node_path_1 = __importDefault(require("node:path"));
+function cacheHitRate(counts) {
+    const lookups = counts.hits + counts.misses;
+    return lookups === 0 ? null : counts.hits / lookups;
+}
+const rate = (counts) => ({ hits: counts.hits, misses: counts.misses, hitRate: cacheHitRate(counts) });
+exports.UNPINNED_SCOPE_BLOCKER = "answer/judge keys are scoped to this process's routing session; pin RENOR_QA_CACHE_SCOPE (and share --cache-dir) to replay across runs";
+exports.UNCACHED_EMBEDDING_BLOCKER = 'embedding calls ran with the cache off (RENOR_EVAL_EMBED_CACHE=off): turn it on to replay them';
+exports.UNCACHED_MODEL_BLOCKER = 'answer/judge calls ran with the eval cache off (--no-cache or RENOR_EVAL_CACHE=off): turn it on to replay them';
+exports.PER_RUN_EVIDENCE_BLOCKER = "the evidence cache lives in this run's output directory; set RENOR_EVIDENCE_CACHE to a shared path to replay across runs";
+function buildCacheReport(input) {
+    const report = { liveProviderCalls: 0, healthProbeCalls: input.healthProbes ?? 0, repeatRunBlockers: [] };
+    const uncached = {};
+    if (input.embedding) {
+        const { sent } = input.embedding;
+        const calls = sent ? sent.calls : input.embedding.misses;
+        report.liveProviderCalls += calls;
+        if (sent)
+            report.embeddingSent = { texts: sent.texts, calls: sent.calls };
+        if (input.embedding.cached === false) {
+            if (calls > 0) {
+                uncached.embedding = calls;
+                report.repeatRunBlockers.push(exports.UNCACHED_EMBEDDING_BLOCKER);
+            }
+        }
+        else {
+            report.embedding = rate(input.embedding);
+            if (!sent)
+                report.embeddingCallsUpperBound = true;
+        }
+    }
+    if (input.evidence) {
+        report.evidence = rate(input.evidence);
+        report.liveProviderCalls += input.evidence.misses;
+        if (!input.evidenceCacheShared && input.evidence.hits + input.evidence.misses > 0)
+            report.repeatRunBlockers.push(exports.PER_RUN_EVIDENCE_BLOCKER);
+    }
+    if (input.evalCache) {
+        const { answerHits, answerMisses, judgeHits, judgeMisses } = input.evalCache;
+        report.answer = rate({ hits: answerHits, misses: answerMisses });
+        report.judge = rate({ hits: judgeHits, misses: judgeMisses });
+        if (!input.pinnedScope && answerHits + answerMisses + judgeHits + judgeMisses > 0)
+            report.repeatRunBlockers.push(exports.UNPINNED_SCOPE_BLOCKER);
+    }
+    const modelCalls = input.liveModelCalls ?? (input.evalCache ? input.evalCache.answerMisses + input.evalCache.judgeMisses : 0);
+    report.liveProviderCalls += modelCalls;
+    if (!input.evalCache && modelCalls > 0) {
+        uncached.model = modelCalls;
+        report.repeatRunBlockers.push(exports.UNCACHED_MODEL_BLOCKER);
+    }
+    if (Object.keys(uncached).length > 0)
+        report.uncached = uncached;
+    return report;
+}
+const part = (label, r) => r ? `${label} ${r.hits}/${r.hits + r.misses} (${r.hitRate === null ? 'n/a' : `${(r.hitRate * 100).toFixed(1)}%`})` : null;
+function formatCacheReport(report) {
+    const count = (n, noun) => `${n} ${noun}${n === 1 ? '' : 's'}`;
+    const sent = report.embeddingSent
+        ? `, ${count(report.embeddingSent.texts, 'text')} in ${count(report.embeddingSent.calls, 'call')}`
+        : report.embeddingCallsUpperBound ? ', calls counted as lookup misses (upper bound)' : '';
+    const layers = [
+        report.embedding ? `${part('embedding', report.embedding)}${sent}` : null,
+        report.uncached?.embedding ? `embedding uncached${sent}` : null,
+        part('evidence', report.evidence),
+        part('answer', report.answer),
+        part('judge', report.judge),
+        report.uncached?.model ? `answer/judge uncached ${report.uncached.model}` : null,
+    ].filter(Boolean);
+    const probes = report.healthProbeCalls > 0 ? ` (+${report.healthProbeCalls} health probes, never cached)` : '';
+    const lines = [`[cache] ${layers.length > 0 ? layers.join(' | ') : 'no caches in use'} | live provider calls ${report.liveProviderCalls}${probes}`];
+    for (const blocker of report.repeatRunBlockers)
+        lines.push(`[cache] a repeat run will miss: ${blocker}`);
+    return lines.join('\n');
+}
+/** Print the report to stderr and, given an output dir, keep it beside the run as cache-report.json. */
+function emitCacheReport(report, outputDir) {
+    process.stderr.write(`${formatCacheReport(report)}\n`);
+    if (outputDir)
+        node_fs_1.default.writeFileSync(node_path_1.default.join(outputDir, 'cache-report.json'), JSON.stringify(report, null, 2) + '\n', 'utf8');
+}
+/** Counters from an embed-proxy stats file, or null when it is absent or unreadable. */
+function readProxyCounters(statsPath) {
+    if (!statsPath || !node_fs_1.default.existsSync(statsPath))
+        return null;
+    try {
+        const raw = JSON.parse(node_fs_1.default.readFileSync(statsPath, 'utf8'));
+        if (typeof raw.cacheHits !== 'number' || typeof raw.cacheMisses !== 'number')
+            return null;
+        const counters = { startedAt: raw.startedAt, cacheHits: raw.cacheHits, cacheMisses: raw.cacheMisses };
+        if (typeof raw.upstreamTexts === 'number' && typeof raw.upstreamCalls === 'number') {
+            counters.upstreamTexts = raw.upstreamTexts;
+            counters.upstreamCalls = raw.upstreamCalls;
+        }
+        return counters;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * The embed-proxy is a long-lived service shared by every run (and shard), and
+ * its stats file holds lifetime counters, so one run's embedding lookups (and what
+ * the proxy sent upstream for them) are the change in those counters between its
+ * start and its end. That window also counts any other client of the same proxy.
+ * Unknown (undefined) when the proxy restarted in between, because its counters
+ * were reset.
+ */
+function proxyWindow(before, after) {
+    if (!after)
+        return undefined;
+    if (before && before.startedAt !== after.startedAt)
+        return undefined;
+    const window = {
+        hits: after.cacheHits - (before?.cacheHits ?? 0),
+        misses: after.cacheMisses - (before?.cacheMisses ?? 0),
+    };
+    // The sent figures are known only when the proxy counted them on both ends of the window; otherwise unknown, never zero.
+    if (after.upstreamTexts !== undefined && after.upstreamCalls !== undefined) {
+        if (!before)
+            window.sent = { texts: after.upstreamTexts, calls: after.upstreamCalls };
+        else if (before.upstreamTexts !== undefined && before.upstreamCalls !== undefined) {
+            window.sent = { texts: after.upstreamTexts - before.upstreamTexts, calls: after.upstreamCalls - before.upstreamCalls };
+        }
+    }
+    return window;
+}

package/dist/eval/comparison.js

@@ -0,0 +1,182 @@
+"use strict";
+/**
+ * Pure comparison policy between two eval runs. Extracted from the CLI
+ * wrapper so the promotion logic itself lives in the frozen evaluator
+ * trust domain (its hash is recorded in run manifests) and is unit tested.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.QUALITY_METRICS = void 0;
+exports.compareRuns = compareRuns;
+exports.QUALITY_METRICS = [
+    { key: 'hitAt1', higherBetter: true },
+    { key: 'hitAt5', higherBetter: true },
+    { key: 'recallAt5', higherBetter: true },
+    { key: 'recallAllAt10', higherBetter: true },
+    { key: 'mrr', higherBetter: true },
+    { key: 'ndcgAt5', higherBetter: true },
+    { key: 'contradictionFirstRate', higherBetter: false },
+    { key: 'forbiddenRetrievalRate', higherBetter: false },
+    { key: 'abstentionEmptyRate', higherBetter: true },
+];
+const EFFICIENCY_METRICS = ['meanLatencyMs', 'p50LatencyMs', 'p95LatencyMs', 'meanContextTokens'];
+const MATERIAL_EFFICIENCY_METRICS = ['p95LatencyMs', 'meanContextTokens'];
+function same(a, b) {
+    return JSON.stringify(a) === JSON.stringify(b);
+}
+function num(map, key) {
+    const value = map[key];
+    return typeof value === 'number' && Number.isFinite(value) ? value : null;
+}
+function delta(candidate, baseline) {
+    return candidate === null || baseline === null ? null : Math.round((candidate - baseline) * 10000) / 10000;
+}
+/** Relative improvement of candidate over baseline; null when not computable. */
+function relativeGain(baseline, candidate) {
+    if (baseline === null || candidate === null || baseline <= 0)
+        return null;
+    return (baseline - candidate) / baseline;
+}
+function compareRuns(baseline, candidate, options = {}) {
+    const tolerance = options.maxQualityRegression ?? 0;
+    const metricList = options.metrics ?? exports.QUALITY_METRICS;
+    const minEfficiencyGain = options.minEfficiencyGain ?? 0.05;
+    if (!Number.isFinite(tolerance) || tolerance < 0 || tolerance > 1) {
+        throw new Error('maxQualityRegression must be between 0 and 1');
+    }
+    if (!Number.isFinite(minEfficiencyGain) || minEfficiencyGain < 0 || minEfficiencyGain > 1) {
+        throw new Error('minEfficiencyGain must be between 0 and 1');
+    }
+    const invariantChecks = {
+        benchmark: same(baseline.manifest.benchmark, candidate.manifest.benchmark),
+        cases: same(baseline.manifest.casesSha256, candidate.manifest.casesSha256),
+        fixture: same(baseline.manifest.fixtureSha256, candidate.manifest.fixtureSha256),
+        topK: same(baseline.manifest.topK, candidate.manifest.topK),
+        provider: same(baseline.manifest.provider, candidate.manifest.provider),
+        chunking: same(baseline.manifest.chunking, candidate.manifest.chunking),
+        evaluator: same(baseline.manifest.evaluatorSha256, candidate.manifest.evaluatorSha256),
+        cleanTree: baseline.manifest.gitDirty === false && candidate.manifest.gitDirty === false,
+        qualityEligible: baseline.manifest.qualityEligible === true && candidate.manifest.qualityEligible === true,
+    };
+    const reasons = [];
+    const datasetCompatible = invariantChecks.benchmark && invariantChecks.cases && invariantChecks.fixture;
+    if (!datasetCompatible) {
+        throw new Error(`Cannot compare runs from different benchmark data: ${JSON.stringify(invariantChecks)}`);
+    }
+    if (!invariantChecks.evaluator) {
+        if (!options.allowEvaluatorChange) {
+            throw new Error('Evaluator changed between baseline and candidate (evaluatorSha256 mismatch). ' +
+                'The candidate cannot grade its own exam: re-run the candidate against the frozen evaluator, ' +
+                'or pass --allow-evaluator-change to accept a DESCRIPTIVE_ONLY comparison during evaluator development.');
+        }
+        reasons.push('Evaluator code changed between runs; measurement meaning differs.');
+    }
+    if (!invariantChecks.cleanTree) {
+        if (!options.allowDirtyTree) {
+            reasons.push('Working tree was dirty in at least one run; executed code does not match the recorded git SHA.');
+        }
+        else {
+            reasons.push('Dirty working tree tolerated (--allow-dirty-tree).');
+        }
+    }
+    if (!invariantChecks.qualityEligible) {
+        reasons.push('At least one run used a non-quality provider (e.g. hash embeddings); infrastructure evidence only.');
+    }
+    const configCompatible = invariantChecks.topK && invariantChecks.provider && invariantChecks.chunking;
+    if (!configCompatible) {
+        reasons.push('Provider, topK, or chunking changed between runs.');
+    }
+    const regressionGate = options.regressionGate === true;
+    const experimentCompatible = Boolean(invariantChecks.evaluator
+        && (regressionGate || (invariantChecks.cleanTree && invariantChecks.qualityEligible))
+        && (configCompatible || (!regressionGate && options.allowConfigChange)));
+    const quality = Object.fromEntries(metricList.map(metric => {
+        const baselineValue = num(baseline.summary.overall, metric.key);
+        const candidateValue = num(candidate.summary.overall, metric.key);
+        return [metric.key, { baseline: baselineValue, candidate: candidateValue, delta: delta(candidateValue, baselineValue) }];
+    }));
+    const efficiency = Object.fromEntries(EFFICIENCY_METRICS.map(key => {
+        const baselineValue = num(baseline.summary.overall, key);
+        const candidateValue = num(candidate.summary.overall, key);
+        return [key, { baseline: baselineValue, candidate: candidateValue, delta: delta(candidateValue, baselineValue) }];
+    }));
+    let verdict;
+    if (!experimentCompatible) {
+        verdict = 'DESCRIPTIVE_ONLY';
+    }
+    else {
+        const hasRegression = metricList.some(metric => {
+            const { delta: d } = quality[metric.key];
+            if (typeof d !== 'number')
+                return false;
+            return metric.higherBetter ? d < -tolerance : d > tolerance;
+        });
+        const hasQualityGain = metricList.some(metric => {
+            const { delta: d } = quality[metric.key];
+            if (typeof d !== 'number')
+                return false;
+            return metric.higherBetter ? d > tolerance : d < -tolerance;
+        });
+        const hasMaterialEfficiencyGain = MATERIAL_EFFICIENCY_METRICS.some(key => {
+            const gain = relativeGain(efficiency[key].baseline, efficiency[key].candidate);
+            return gain !== null && gain >= minEfficiencyGain;
+        });
+        if (hasRegression)
+            verdict = 'REGRESSED';
+        else if (hasQualityGain || hasMaterialEfficiencyGain)
+            verdict = 'IMPROVED';
+        else
+            verdict = 'NEUTRAL';
+    }
+    // Category-collapse guard: an aggregate improvement must not hide one
+    // category regressing beyond tolerance (independent-review finding).
+    const categoryRegressions = [];
+    const baselineCategories = baseline.summary.byCategory ?? {};
+    const candidateCategories = candidate.summary.byCategory ?? {};
+    // Both sides' categories: a category the baseline has and the candidate lost is a regression too.
+    for (const name of new Set([...Object.keys(baselineCategories), ...Object.keys(candidateCategories)])) {
+        const baselineMetrics = baselineCategories[name];
+        const metrics = candidateCategories[name];
+        if (!baselineMetrics)
+            continue; // new category: nothing to regress against
+        if (!metrics) {
+            categoryRegressions.push(`${name}.missing`);
+            continue;
+        }
+        for (const metric of metricList) {
+            const b = num(baselineMetrics, metric.key);
+            const c = num(metrics, metric.key);
+            if (b === null || c === null)
+                continue;
+            const d = delta(c, b);
+            if (typeof d === 'number' && (metric.higherBetter ? d < -tolerance : d > tolerance)) {
+                categoryRegressions.push(`${name}.${metric.key}`);
+            }
+        }
+    }
+    for (const name of categoryRegressions) {
+        reasons.push(`Category regression beyond tolerance: ${name}`);
+    }
+    if (categoryRegressions.length > 0) {
+        // A category regression beyond tolerance is a rejection regardless of the
+        // aggregate verdict: an overall gain must not hide a collapsing category.
+        verdict = 'REGRESSED';
+    }
+    const note = regressionGate
+        ? 'Regression gate against a committed baseline: this is a no-regression check, never a clean ablation or a promotion.'
+        : experimentCompatible
+            ? 'IMPROVED requires no quality regression beyond tolerance and a quality gain or a material (>= threshold) efficiency gain.'
+            : 'Not a clean ablation: this comparison is descriptive only and cannot support promotion.';
+    return {
+        verdict,
+        cleanAblation: experimentCompatible && !regressionGate,
+        regressionGate,
+        invariantChecks,
+        tolerance,
+        minEfficiencyGain,
+        quality,
+        efficiency,
+        categoryRegressions,
+        reasons,
+        note,
+    };
+}