webrecipe 0.1.0

package/dist/src/benchmark/health.js

@@ -0,0 +1,96 @@
+import { stat } from 'node:fs/promises';
+import { join } from 'node:path';
+import { RecipeRegistry } from '../recipes/registry.js';
+import { HttpHtmlStrategy } from '../executor/strategies/http-html.js';
+import { HttpJsonStrategy } from '../executor/strategies/http-json.js';
+import { validate } from '../validator/index.js';
+/**
+ * Whether the recipes already on disk still work.
+ *
+ * `bench run` cannot answer this. It pairs every task with a fresh browser
+ * baseline, which doubles what the site is asked for, and it grades against
+ * goldens captured days earlier, which reads a site editing its content as a
+ * recipe that broke. Durability is a narrower question — does the stored
+ * recipe still fetch and validate — and it is asked by replaying the recipe
+ * and nothing else.
+ */
+/** The statuses `src/executor/index.ts` treats as a refusal, for the same reason. */
+const BLOCK_STATUSES = new Set([401, 403, 429, 503]);
+export async function checkRecipes(opts) {
+    const registry = new RecipeRegistry(opts.recipeDir);
+    const html = new HttpHtmlStrategy(opts.net, opts.sites);
+    const json = new HttpJsonStrategy(opts.net, opts.sites);
+    const wanted = new Map();
+    for (const task of opts.tasks) {
+        const key = `${task.site}\u0000${task.intent}`;
+        const run = wanted.get(key);
+        if (run)
+            run.push(task);
+        else
+            wanted.set(key, [task]);
+    }
+    const health = [];
+    for (const [key, all] of wanted) {
+        const [site, intent] = key.split('\u0000');
+        const recipe = await registry.load(site, intent);
+        // A browser recipe has no HTTP path to outlive, so there is nothing here
+        // to measure.
+        if (recipe === null || recipe.strategy.type === 'browser')
+            continue;
+        const strategy = recipe.strategy.type === 'http-json' ? json : html;
+        const statuses = [];
+        const reasons = [];
+        let valid = 0;
+        const samples = all.slice(0, opts.samples);
+        for (const task of samples) {
+            try {
+                const result = await strategy.execute(recipe, task);
+                if (result.status !== undefined)
+                    statuses.push(result.status);
+                const outcome = validate(recipe, {
+                    status: result.status ?? recipe.validation.status,
+                    payload: result.payload,
+                    items: result.items,
+                });
+                if (outcome.valid)
+                    valid += 1;
+                else
+                    reasons.push(`${task.id}: ${outcome.reasons.join('; ')}`);
+            }
+            catch (err) {
+                reasons.push(`${task.id}: ${err instanceof Error ? err.message : String(err)}`);
+            }
+        }
+        // A refusal outranks a mismatch: a site that answered 403 has stopped
+        // serving this client, and whatever the recipe would have parsed is moot.
+        const verdict = statuses.some((s) => BLOCK_STATUSES.has(s))
+            ? 'blocked'
+            : valid === samples.length ? 'alive' : 'broken';
+        health.push({
+            site,
+            intent,
+            compiledAt: (await stat(join(opts.recipeDir, site, `${intent}.yaml`))).mtime,
+            samples: samples.length,
+            valid,
+            statuses: [...new Set(statuses)],
+            verdict,
+            reasons,
+        });
+    }
+    return health;
+}
+const age = (from, now) => {
+    const hours = Math.round((now.getTime() - from.getTime()) / 3_600_000);
+    return hours < 48 ? `${hours}h` : `${Math.round(hours / 24)}d`;
+};
+export function formatHealth(health, now = new Date()) {
+    const lines = health.map((h) => `  ${h.verdict.padEnd(8)} ${`${h.site} ${h.intent}`.padEnd(34)} ` +
+        `${h.valid}/${h.samples} valid  age ${age(h.compiledAt, now).padStart(4)}` +
+        `${h.statuses.length > 0 ? `  status ${h.statuses.join(',')}` : ''}`);
+    const count = (v) => health.filter((h) => h.verdict === v).length;
+    return [
+        ...lines,
+        '',
+        `  alive ${count('alive')}   blocked ${count('blocked')}   broken ${count('broken')}   of ${health.length}`,
+    ].join('\n');
+}

package/dist/src/benchmark/labels.js

@@ -0,0 +1,49 @@
+import { z } from 'zod';
+import { elementsOf, isEl, loadPage, normalize } from '../authoring/candidates.js';
+/**
+ * What a human read off the rendered page, and nothing else.
+ *
+ * There is deliberately nowhere in this schema to put a selector. The whole
+ * harness rests on the label being content while the candidate is structure; a
+ * label derived from a selector would score that selector against itself, which
+ * is the circle the compile-time equivalence check is already stuck in.
+ */
+export const LabelSchema = z.object({
+    snapshot: z.string().min(1),
+    url: z.string().url(),
+    capturedAt: z.string().min(1),
+    note: z.string().default(''),
+    /** Which of the two a human used to write the items down. */
+    identifier: z.enum(['text', 'href']),
+    items: z.array(z.string().min(1)).min(1),
+});
+/**
+ * The page's text in document order, so a human can mark the item boundaries
+ * without a selector having proposed them first.
+ */
+export function visibleRuns(html) {
+    const $ = loadPage(html);
+    const runs = [];
+    const walk = (node) => {
+        for (const child of $(node).contents().toArray()) {
+            if (child.type === 'text') {
+                const text = normalize($(child).text());
+                if (text !== '')
+                    runs.push(text);
+            }
+            else if (isEl(child)) {
+                // An image speaks through its alt, and on a page of covers that is the
+                // only thing the item says.
+                const alt = normalize(child.tagName.toLowerCase() === 'img' ? $(child).attr('alt') ?? '' : '');
+                if (alt !== '')
+                    runs.push(alt);
+                walk(child);
+            }
+        }
+    };
+    const root = elementsOf($, 'body')[0] ?? elementsOf($, 'html')[0];
+    if (root === undefined)
+        return [];
+    walk(root);
+    return runs;
+}

package/dist/src/benchmark/oracle.js

@@ -0,0 +1,55 @@
+import { z } from 'zod';
+/**
+ * `property-based` is reserved in the spec but deliberately unimplemented, so
+ * it is absent here: a task naming it fails at load rather than being quietly
+ * graded as something else.
+ */
+export const OracleModeSchema = z.enum(['golden', 'paired-live']);
+export const VolatilitySchema = z.enum(['low', 'medium', 'high']);
+// .strict() throughout: a misspelled key (e.g. `field` for `fields`) must fail
+// at load, the same reasoning that kept `property-based` out of the mode enum
+// rather than letting it fall through to a default silently.
+export const OracleSpecSchema = z.object({
+    mode: OracleModeSchema.optional(),
+    volatility: VolatilitySchema.optional(),
+    compare: z.object({
+        entities: z.literal('set').optional(),
+        ordering: z.enum(['ignore', 'strict']).optional(),
+        /** Restricts comparison to the fields that carry meaning. */
+        fields: z.array(z.string()).optional(),
+    }).strict().optional(),
+    /**
+     * Opt-in. Declared only where a site is known to be able to ignore its own
+     * search input, as arbeitnow.com does. Absent, query semantics is reported
+     * as unjudged rather than as a pass.
+     */
+    semantics: z.object({
+        input: z.string(),
+        fields: z.array(z.string()),
+        match: z.literal('contains-token'),
+        minShare: z.number().min(0).max(1),
+    }).strict().optional(),
+}).strict();
+/** What every run before this contract effectively used. */
+export const DEFAULT_ORACLE = {
+    mode: 'golden',
+    volatility: 'low',
+    compare: { entities: 'set', ordering: 'strict' },
+};
+/**
+ * Merges field by field, so an override may name only `mode` and inherit the
+ * rest. Volatility is not uniformly a property of a site: the same host serves
+ * a search that rotates in minutes and a detail page that does not.
+ */
+export function resolveOracle(site, task) {
+    return {
+        mode: task?.mode ?? site?.mode ?? DEFAULT_ORACLE.mode,
+        volatility: task?.volatility ?? site?.volatility ?? DEFAULT_ORACLE.volatility,
+        compare: {
+            entities: 'set',
+            ordering: task?.compare?.ordering ?? site?.compare?.ordering ?? DEFAULT_ORACLE.compare.ordering,
+            fields: task?.compare?.fields ?? site?.compare?.fields,
+        },
+        semantics: task?.semantics ?? site?.semantics,
+    };
+}

package/dist/src/benchmark/report.js

@@ -0,0 +1,191 @@
+import { isEquivalent } from './grade.js';
+const LEVELS = {
+    'http-html': 'L0',
+    'http-json': 'L1',
+    'warm-browser': 'L2',
+    browser: 'L3',
+};
+export const levelOf = (strategy) => LEVELS[strategy];
+function median(values) {
+    if (values.length === 0)
+        return 0;
+    const sorted = [...values].sort((a, b) => a - b);
+    const mid = Math.floor(sorted.length / 2);
+    return sorted.length % 2 === 1 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2;
+}
+const share = (rows, predicate) => rows.length === 0 ? 0 : rows.filter(predicate).length / rows.length;
+export function summarize(runs, baseline) {
+    const sets = ['controlled', 'wild'];
+    return sets.map((set) => {
+        const mine = runs.filter((r) => r.set === set);
+        const theirs = baseline.filter((r) => r.set === set);
+        const medianLatencyMs = median(mine.map((r) => r.meta.latencyMs));
+        const baselineMedianLatencyMs = median(theirs.map((r) => r.meta.latencyMs));
+        const at = (level) => share(mine, (r) => levelOf(r.meta.strategy) === level);
+        const sites = [...new Set(mine.map((r) => r.site))];
+        const siteSpeedups = sites.map((site) => {
+            const ours = median(mine.filter((r) => r.site === site).map((r) => r.meta.latencyMs));
+            const theirsHere = median(theirs.filter((r) => r.site === site).map((r) => r.meta.latencyMs));
+            return ours === 0 ? 0 : theirsHere / ours;
+        });
+        return {
+            set,
+            tasks: mine.length,
+            browserAvoidance: share(mine, (r) => r.meta.browserLaunches === 0 && !r.threw),
+            medianSiteSpeedup: median(siteSpeedups),
+            levels: { L0: at('L0'), L1: at('L1'), L2: at('L2'), L3: at('L3') },
+            browserFree: share(mine, (r) => ['L0', 'L1'].includes(levelOf(r.meta.strategy))),
+            fullBrowserAvoidance: share(mine, (r) => r.meta.browserLaunches === 0 && !r.threw),
+            blockedTasks: mine.filter((r) => r.blocked).length,
+            medianElapsedMs: mine.length > 0 && mine.every((r) => r.meta.elapsedMs !== undefined)
+                ? median(mine.map((r) => r.meta.elapsedMs)) : null,
+            baselineMedianElapsedMs: theirs.length > 0 && theirs.every((r) => r.meta.elapsedMs !== undefined)
+                ? median(theirs.map((r) => r.meta.elapsedMs)) : null,
+            medianLatencyMs,
+            baselineMedianLatencyMs,
+            speedup: medianLatencyMs === 0 ? 0 : baselineMedianLatencyMs / medianLatencyMs,
+            successRate: share(mine, (r) => r.success),
+            schemaRate: share(mine, (r) => r.schema !== false),
+            // This path predates the oracle contract: it never marks a baseline
+            // unusable, and it never declares a query-semantics rule.
+            usableTasks: mine.length,
+            excludedTasks: 0,
+            excludedBaselineTasks: 0,
+            excludedNoGoldenTasks: 0,
+            entityRate: share(mine, (r) => r.schema !== false),
+            querySemanticsRate: null,
+            querySemanticsJudged: 0,
+            orderingRate: (() => {
+                const judged = mine.filter((r) => r.ordering !== null && r.ordering !== undefined);
+                return judged.length === 0 ? null : share(judged, (r) => r.ordering === true);
+            })(),
+            medianPolitenessWaitMs: median(mine.map((r) => r.meta.politenessWaitMs)),
+            medianBytes: median(mine.map((r) => r.meta.bytesDownloaded)),
+            baselineMedianBytes: median(theirs.map((r) => r.meta.bytesDownloaded)),
+            medianRequests: median(mine.map((r) => r.meta.networkRequests)),
+            baselineMedianRequests: median(theirs.map((r) => r.meta.networkRequests)),
+            medianAgentTokens: median(mine.map((r) => r.meta.llmTokens)),
+            baselineMedianAgentTokens: median(theirs.map((r) => r.meta.llmTokens)),
+        };
+    });
+}
+/**
+ * Summarises pairs rather than two independent lists.
+ *
+ * A task whose baseline was unusable leaves the equivalence denominator — the
+ * engine cannot be blamed for a browser failure, and counting it as a pass
+ * would be worse — but it never leaves the report.
+ */
+export function summarizePairs(pairs) {
+    const sets = ['controlled', 'wild'];
+    return sets.map((set) => {
+        const mine = pairs.filter((p) => p.task.set === set);
+        const usable = mine.filter((p) => p.grade.usable);
+        const engineRuns = mine.map((p) => p.engine);
+        const baselineRuns = mine.map((p) => p.baseline);
+        const base = summarize(engineRuns, baselineRuns).find((s) => s.set === set);
+        const judgedOrdering = usable.filter((p) => p.grade.ordering !== null);
+        const judgedQuery = usable.filter((p) => p.grade.query !== null);
+        return {
+            ...base,
+            tasks: mine.length,
+            usableTasks: usable.length,
+            excludedTasks: mine.length - usable.length,
+            // Two different reasons look identical as "not usable": a task with no
+            // stored golden had a perfectly healthy browser run, and blaming that on
+            // baseline validity is the exact confusion this split exists to end.
+            excludedBaselineTasks: mine.filter((p) => p.grade.excludeReason === 'baseline').length,
+            excludedNoGoldenTasks: mine.filter((p) => p.grade.excludeReason === 'no-golden').length,
+            entityRate: share(usable, (p) => p.grade.entities === true),
+            // Denominator is the tasks an oracle actually asked about, so an
+            // undeclared rule cannot inflate the rate with unexamined passes.
+            querySemanticsJudged: judgedQuery.length,
+            querySemanticsRate: judgedQuery.length === 0 ? null : share(judgedQuery, (p) => p.grade.query === true),
+            successRate: share(usable, (p) => isEquivalent(p.grade)),
+            orderingRate: judgedOrdering.length === 0 ? null : share(judgedOrdering, (p) => p.grade.ordering === true),
+        };
+    });
+}
+/**
+ * The tasks that were judged and did not pass, in the same terms the CLI
+ * prints — extracted so the CLI cannot hand-copy `isEquivalent` and drift, the
+ * way its old failure filter did.
+ */
+export function failedPairs(pairs) {
+    return pairs
+        .filter((p) => p.grade.usable && !isEquivalent(p.grade))
+        .map((p) => ({
+        taskId: p.task.id,
+        // `grade.reasons` holds only the oracle's verdict text; an engine that
+        // timed out or fell back leaves that reason on the run itself, and a
+        // printed failure line is useless without it.
+        reasons: [...new Set([...p.grade.reasons, ...p.engine.reasons])],
+    }));
+}
+/** Tasks left out of the denominator for one specific cause, so the two never print under one heading. */
+export function excludedBy(pairs, cause) {
+    return pairs
+        .filter((p) => p.grade.excludeReason === cause)
+        .map((p) => ({ taskId: p.task.id, reason: p.grade.validityReason }));
+}
+const pct = (n) => `${(n * 100).toFixed(0)}%`;
+/**
+ * One cost axis, engine against baseline. Tokens is the first axis where the
+ * engine can come out worse, so the direction is read off the numbers rather
+ * than assumed to be a saving.
+ */
+const ratio = (engine, baseline) => {
+    if (engine === 0 || baseline === 0)
+        return '—';
+    return baseline >= engine ? `${(baseline / engine).toFixed(1)}x less` : `${(engine / baseline).toFixed(1)}x more`;
+};
+export function formatReport(summaries, robots, opts = {}) {
+    const { correctness = true } = opts;
+    const lines = [];
+    for (const s of summaries) {
+        if (s.tasks === 0)
+            continue;
+        lines.push(s.set === 'controlled' ? 'Controlled' : 'Wild');
+        lines.push(`  Tasks:             ${s.tasks}`);
+        lines.push(`  Browser-free:      ${pct(s.browserFree)}   (L0 + L1)`);
+        lines.push(`  Full-browser avoidance: ${pct(s.fullBrowserAvoidance)}   (observed zero launches)`);
+        if (s.medianElapsedMs !== null) {
+            lines.push(`  Actual elapsed:    ${s.medianElapsedMs}ms median (baseline ${s.baselineMedianElapsedMs ?? 'unavailable'}ms)`);
+            if (s.medianElapsedMs > 0 && s.baselineMedianElapsedMs !== null) {
+                lines.push(`  Actual speedup:    ${(s.baselineMedianElapsedMs / s.medianElapsedMs).toFixed(1)}x`);
+            }
+        }
+        lines.push(`  Levels:            L0 ${pct(s.levels.L0)}  L1 ${pct(s.levels.L1)}  L2 ${pct(s.levels.L2)}  L3 ${pct(s.levels.L3)}`);
+        lines.push(`  Blocked:           ${s.blockedTasks} (site refused the recipe; ran on the browser)`);
+        lines.push(`  Median latency:    ${s.medianLatencyMs}ms (baseline ${s.baselineMedianLatencyMs}ms; legacy attempt timing)`);
+        lines.push(`  Median speedup:    ${s.speedup.toFixed(1)}x (legacy timing)`);
+        lines.push(`  Median site speedup: ${s.medianSiteSpeedup.toFixed(1)}x`);
+        lines.push(`  Politeness wait:   ${s.medianPolitenessWaitMs}ms median (excluded from latency above)`);
+        // Latency is the noisiest of the three cost axes and was the only one
+        // reported. Bytes and requests were instrumented from the first commit and
+        // discarded on every run until held-out 3.
+        const kb = (b) => `${(b / 1024).toFixed(0)}KB`;
+        lines.push(`  Data fetched:      ${kb(s.medianBytes)} median (baseline ${kb(s.baselineMedianBytes)})  ${ratio(s.medianBytes, s.baselineMedianBytes)}`);
+        lines.push(`  Requests:          ${s.medianRequests} median (baseline ${s.baselineMedianRequests})`);
+        lines.push(`  Agent reads:       ${s.medianAgentTokens} tokens median (baseline ${s.baselineMedianAgentTokens})  ${ratio(s.medianAgentTokens, s.baselineMedianAgentTokens)}`);
+        // A raw browser-only run (`bench run --baseline`) never grades an engine
+        // against anything, so it has none of these figures to report — printing
+        // them anyway would be four false labels, not one.
+        if (correctness) {
+            lines.push('  Correctness');
+            lines.push(`    Baseline validity:  ${s.tasks - s.excludedBaselineTasks}/${s.tasks} usable          (${s.excludedBaselineTasks} excluded)`);
+            lines.push(`    No stored golden:   ${s.excludedNoGoldenTasks} excluded`);
+            lines.push(`    Entity equivalence: ${pct(s.entityRate)}`);
+            lines.push(`    Query semantics:    ${s.querySemanticsRate === null ? 'not declared' : `${pct(s.querySemanticsRate)} of ${s.querySemanticsJudged} judgeable`}`);
+            lines.push(`    Ordering:           ${s.orderingRate === null ? 'not judgeable' : `${pct(s.orderingRate)} where meaningful`}`);
+        }
+        lines.push('');
+    }
+    if (Object.keys(robots).length > 0) {
+        lines.push('robots.txt status of wild sites');
+        for (const [site, status] of Object.entries(robots))
+            lines.push(`  ${site}: ${status}`);
+        lines.push('');
+    }
+    return lines.join('\n');
+}

package/dist/src/benchmark/runner.js

@@ -0,0 +1,201 @@
+import { ExecutionFailure } from '../measurement.js';
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { z } from 'zod';
+import { gradeGolden } from './golden.js';
+import { gradePair, planAging, querySemantics } from './grade.js';
+import { OracleSpecSchema, resolveOracle } from './oracle.js';
+const BenchTaskSchema = z.object({
+    id: z.string(),
+    set: z.enum(['controlled', 'wild']),
+    site: z.string(),
+    intent: z.enum(['search', 'list', 'detail']),
+    input: z.record(z.union([z.string(), z.number()])),
+    volatile: z.array(z.string()).optional(),
+    /** A search that legitimately returns nothing; an empty golden is correct here. */
+    expectEmpty: z.boolean().optional(),
+    oracle: OracleSpecSchema.optional(),
+});
+export async function loadTasks(path) {
+    const raw = JSON.parse(await readFile(path, 'utf8'));
+    return z.array(BenchTaskSchema).parse(raw);
+}
+function goldenPath(dir, taskId) {
+    return join(dir, `${taskId}.json`);
+}
+function failedRun(task, error) {
+    const reason = message(error);
+    return {
+        taskId: task.id,
+        site: task.site,
+        set: task.set,
+        meta: error instanceof ExecutionFailure ? error.meta : {
+            strategy: 'browser', latencyMs: 0, browserLaunches: 0,
+            pageNavigations: 0, networkRequests: 0, bytesDownloaded: 0, llmTokens: 0, politenessWaitMs: 0,
+        },
+        success: false,
+        reasons: [reason],
+        items: [],
+        threw: true,
+        blocked: false,
+    };
+}
+const message = (err) => (err instanceof Error ? err.message : String(err));
+/** Explicit, never implicit: a golden is only written when this is called. */
+export async function captureGoldens(tasks, deps) {
+    await mkdir(deps.goldenDir, { recursive: true });
+    const goldens = [];
+    for (const task of tasks) {
+        const result = await deps.browser.execute({}, task);
+        const golden = { taskId: task.id, capturedAt: new Date().toISOString(), items: result.items };
+        await writeFile(goldenPath(deps.goldenDir, task.id), JSON.stringify(golden, null, 2), 'utf8');
+        goldens.push(golden);
+    }
+    return goldens;
+}
+export async function loadGoldens(dir, tasks) {
+    const goldens = [];
+    for (const task of tasks) {
+        try {
+            goldens.push(JSON.parse(await readFile(goldenPath(dir, task.id), 'utf8')));
+        }
+        catch {
+            // A missing golden means this task has never been captured; skip it.
+        }
+    }
+    return goldens;
+}
+export async function runBaseline(tasks, goldens, deps) {
+    const byId = new Map(goldens.map((g) => [g.taskId, g]));
+    const runs = [];
+    for (const task of tasks) {
+        try {
+            const result = await deps.browser.execute({}, task);
+            const golden = byId.get(task.id);
+            const grade = golden
+                ? gradeGolden(golden, result.items, task.volatile)
+                : { schema: true, semantic: true, ordering: null, reasons: [] };
+            runs.push({
+                taskId: task.id, site: task.site, set: task.set, meta: result.meta,
+                success: grade.schema && grade.semantic, schema: grade.schema, ordering: grade.ordering,
+                reasons: grade.reasons,
+                items: result.items, threw: false, blocked: false,
+            });
+        }
+        catch (err) {
+            runs.push(failedRun(task, err));
+        }
+    }
+    return runs;
+}
+async function runBenchmark(tasks, goldens, deps) {
+    const byId = new Map(goldens.map((g) => [g.taskId, g]));
+    const runs = [];
+    for (const task of tasks) {
+        try {
+            const outcome = await deps.executor.run(task);
+            const golden = byId.get(task.id);
+            const grade = golden
+                ? gradeGolden(golden, outcome.items, task.volatile)
+                : { schema: true, semantic: true, ordering: null, reasons: [] };
+            runs.push({
+                taskId: task.id, site: task.site, set: task.set, meta: outcome.meta,
+                success: grade.schema && grade.semantic, schema: grade.schema, ordering: grade.ordering,
+                reasons: [...outcome.reasons, ...grade.reasons],
+                items: outcome.items, threw: false, blocked: outcome.blocked,
+            });
+        }
+        catch (err) {
+            runs.push(failedRun(task, err));
+        }
+    }
+    return runs;
+}
+/** An engine that threw produced no items to grade; scoring it against an
+ * empty reference (an `expectEmpty` baseline, an empty golden) would read as a
+ * pass, which is worse than the failure it stands in for. Decided before
+ * either mode's layer two runs, so the two never diverge on it. */
+function engineException(engine) {
+    return {
+        entities: false, query: null, ordering: null,
+        reasons: engine.reasons.length > 0 ? engine.reasons : ['engine threw'],
+    };
+}
+/**
+ * Grades by mode, because the two modes ask different questions.
+ *
+ * `golden` asks whether the engine matches a stored capture. A live browser
+ * that times out says nothing about that, so it must not gate it — routing
+ * every mode through `gradePair` would have excluded a task whose engine
+ * matched its golden exactly, on the strength of an unrelated timeout. The
+ * baseline is still run, because the speedup figure needs it.
+ *
+ * `paired-live` asks whether the engine matches the browser beside it, so there
+ * the baseline's validity is exactly the right gate.
+ */
+export function gradeByOracle(oracle, stored, baseline, engine, task) {
+    if (oracle.mode === 'paired-live') {
+        const grade = gradePair(baseline.items, engine.items, oracle, {
+            threw: baseline.threw,
+            expectEmpty: task.expectEmpty === true,
+            input: task.input,
+        });
+        return grade.usable && engine.threw ? { ...grade, ...engineException(engine) } : grade;
+    }
+    if (stored === undefined) {
+        return {
+            usable: false, validityReason: 'no stored golden for this task', excludeReason: 'no-golden',
+            entities: null, query: null, ordering: null,
+            reasons: ['no stored golden for this task'],
+        };
+    }
+    if (engine.threw) {
+        return { usable: true, validityReason: null, excludeReason: null, ...engineException(engine) };
+    }
+    const graded = gradeGolden(stored, engine.items, task.volatile, oracle.compare.fields);
+    const query = querySemantics(engine.items, task.input, oracle);
+    return {
+        usable: true,
+        validityReason: null,
+        excludeReason: null,
+        entities: graded.schema && graded.semantic,
+        query: query.match,
+        ordering: oracle.compare.ordering === 'ignore' ? null : graded.ordering,
+        reasons: [...graded.reasons, ...query.reasons],
+    };
+}
+/**
+ * Runs both sides of each task before moving to the next, alternating which
+ * goes first.
+ *
+ * Running the whole baseline and then the whole engine put every engine
+ * measurement later in time than its baseline, which on a job board cost 25
+ * points of apparent correctness. Pairing removes that; alternating removes the
+ * residual bias of one side always going first. Comparison happens once both
+ * halves are in hand, so order within a pair does not decide which is the
+ * reference.
+ */
+export async function runPairs(tasks, goldens, deps, oracles) {
+    const byId = new Map(goldens.map((g) => [g.taskId, g]));
+    const pairs = [];
+    for (const [index, task] of tasks.entries()) {
+        const oracle = resolveOracle(OracleSpecSchema.optional().parse(oracles[task.site]), task.oracle);
+        const engineFirst = index % 2 === 1;
+        let baseline;
+        let engine;
+        if (engineFirst) {
+            engine = (await runBenchmark([task], goldens, deps))[0];
+            baseline = (await runBaseline([task], goldens, deps))[0];
+        }
+        else {
+            baseline = (await runBaseline([task], goldens, deps))[0];
+            engine = (await runBenchmark([task], goldens, deps))[0];
+        }
+        const grade = gradeByOracle(oracle, byId.get(task.id), baseline, engine, task);
+        const aging = oracle.mode === 'paired-live'
+            ? planAging(byId.get(task.id)?.items, baseline.items, oracle)
+            : null;
+        pairs.push({ task, oracle, baseline, engine, grade, aging });
+    }
+    return pairs;
+}